Compare commits
69 Commits
5ccfa4a345
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| a4264bf4b4 | |||
| 81f41564a2 | |||
| 6b5a2497cc | |||
| 422cc6c978 | |||
| f48e042565 | |||
| d9442850ef | |||
| 2e828157b6 | |||
| 4189146536 | |||
| 59be507d2f | |||
| b45cea4084 | |||
| efe885c6b2 | |||
| 404a4c331d | |||
| 174516cb39 | |||
| b0999112cc | |||
| ed0f7527d5 | |||
| 5064cf833d | |||
| 8745d256bd | |||
| 2e76003fd5 | |||
| 36ce5a5099 | |||
| 465dc1389d | |||
| ae6f1a9865 | |||
| ee99dc9478 | |||
| 00ee5893e9 | |||
| 64d1cffd89 | |||
| f9e8afa2c3 | |||
| e827631d8c | |||
| 115fe8ba58 | |||
| c53250f023 | |||
| 3d99483219 | |||
| 67f788b1e2 | |||
| 764103a2e2 | |||
| a08dd83d1f | |||
| e8922d8ec5 | |||
| 2d44305a8a | |||
| a11c80291e | |||
| 3239567297 | |||
| c239304c2a | |||
| 159b02116f | |||
| e5b7adfb49 | |||
| fa2384e696 | |||
| af0bd3f31a | |||
| 5fff8cd623 | |||
| b2a6c47778 | |||
| a1805fe550 | |||
| 93af155254 | |||
| d783b687a5 | |||
| 4ca3be2c14 | |||
| 227fb35f99 | |||
| e291b8bfe9 | |||
| 2b6a7f83c4 | |||
| 3a43550f70 | |||
| c281f721bc | |||
| 350b0e76d9 | |||
| e43350fd0d | |||
| 20d3e3b5ee | |||
| a93b799236 | |||
| e83a3ce179 | |||
| a04a3bbc5f | |||
| 731b66cff5 | |||
| 70e0ea0cf0 | |||
| d45c474c1e | |||
| 25f1ba0b30 | |||
| a718762da1 | |||
| 58ac3ce298 | |||
| 57f2ce1ce4 | |||
| c8b6d5c490 | |||
| abeb50b525 | |||
| 1cb07c7d91 | |||
| 8cfc71c351 |
14
README.md
14
README.md
@@ -33,7 +33,19 @@ boundary and constraints that framework work must preserve.
|
|||||||
|
|
||||||
## Release Guidance
|
## Release Guidance
|
||||||
|
|
||||||
Consumers upgrading from `v0.4.0` to `v0.5.0` should read the
|
Consumers upgrading from `v0.8.0` to `v0.9.0` should read the
|
||||||
|
[v0.9.0 changelog and migration guide](docs/releases/v0.9.0.md).
|
||||||
|
|
||||||
|
Consumers upgrading from `v0.7.0` to `v0.8.0` can consult the
|
||||||
|
[v0.8.0 changelog and migration guide](docs/releases/v0.8.0.md).
|
||||||
|
|
||||||
|
Earlier adopters can consult the
|
||||||
|
[v0.7.0 changelog and migration guide](docs/releases/v0.7.0.md).
|
||||||
|
|
||||||
|
Consumers upgrading from `v0.5.0` to `v0.6.0` can consult the
|
||||||
|
[v0.6.0 changelog and migration guide](docs/releases/v0.6.0.md).
|
||||||
|
|
||||||
|
Consumers upgrading from `v0.4.0` to `v0.5.0` can consult the
|
||||||
[v0.5.0 changelog and migration guide](docs/releases/v0.5.0.md).
|
[v0.5.0 changelog and migration guide](docs/releases/v0.5.0.md).
|
||||||
|
|
||||||
Consumers upgrading from `v0.3.0` to `v0.4.0` should read the
|
Consumers upgrading from `v0.3.0` to `v0.4.0` should read the
|
||||||
|
|||||||
145
appended_messages_contract_test.go
Normal file
145
appended_messages_contract_test.go
Normal file
@@ -0,0 +1,145 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAppendedMessageRoleConstantsAreStrings(t *testing.T) {
|
||||||
|
var (
|
||||||
|
developer string = promptkit.RoleDeveloper
|
||||||
|
system string = promptkit.RoleSystem
|
||||||
|
user string = promptkit.RoleUser
|
||||||
|
assistant string = promptkit.RoleAssistant
|
||||||
|
)
|
||||||
|
if developer != "developer" || system != "system" || user != "user" || assistant != "assistant" {
|
||||||
|
t.Fatalf("unexpected role constants: %q %q %q %q", developer, system, user, assistant)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAppendedMessagesComposeFrozenEffectivePrompt(t *testing.T) {
|
||||||
|
cacheControl := &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}
|
||||||
|
appended := []promptkit.RenderedMessage{
|
||||||
|
{Role: " \tAsSiStAnT\n", Content: "previous response"},
|
||||||
|
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: cacheControl},
|
||||||
|
}
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "output"}}
|
||||||
|
engine, err := promptkit.NewEngine(
|
||||||
|
promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "configured prompt"), "."),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
request := promptkit.RunRequest{PromptID: "prompt", AppendedMessages: appended}
|
||||||
|
|
||||||
|
plain, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare plain request: %v", err)
|
||||||
|
}
|
||||||
|
prepared, err := engine.Prepare(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare appended request: %v", err)
|
||||||
|
}
|
||||||
|
expected := []promptkit.RenderedMessage{
|
||||||
|
{Role: promptkit.RoleUser, Content: "configured prompt"},
|
||||||
|
{Role: promptkit.RoleAssistant, Content: "previous response"},
|
||||||
|
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(prepared.Messages, expected) || prepared.PromptHash != plain.PromptHash || prepared.RenderedPromptHash == plain.RenderedPromptHash {
|
||||||
|
t.Fatalf("prepared effective prompt = %#v, plain = %#v", prepared, plain)
|
||||||
|
}
|
||||||
|
|
||||||
|
equivalent, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{
|
||||||
|
{Role: promptkit.RoleAssistant, Content: "previous response"},
|
||||||
|
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}},
|
||||||
|
}})
|
||||||
|
if err != nil || equivalent.RenderedPromptHash != prepared.RenderedPromptHash {
|
||||||
|
t.Fatalf("normalized-equivalent request = (%#v, %v), want matching rendered hash %q", equivalent, err, prepared.RenderedPromptHash)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, variant := range []promptkit.RunRequest{
|
||||||
|
{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleAssistant, Content: "changed"}, expected[2]}},
|
||||||
|
{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{expected[2], expected[1]}},
|
||||||
|
{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleAssistant, Content: "previous response"}, {Role: promptkit.RoleUser, Content: "corrective request"}}},
|
||||||
|
} {
|
||||||
|
variantPrepared, prepareErr := engine.Prepare(context.Background(), variant)
|
||||||
|
if prepareErr != nil || variantPrepared.RenderedPromptHash == prepared.RenderedPromptHash || variantPrepared.PromptHash != prepared.PromptHash {
|
||||||
|
t.Fatalf("variant preparation = (%#v, %v)", variantPrepared, prepareErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := engine.Run(context.Background(), request)
|
||||||
|
if err != nil || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(client.requests[0].Prompt.Messages, expected) {
|
||||||
|
t.Fatalf("run result = (%#v, %v), request = %#v", result, err, client.requests)
|
||||||
|
}
|
||||||
|
|
||||||
|
execution, err := engine.PrepareExecution(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
appended[0].Content = "changed caller content"
|
||||||
|
cacheControl.TTL = ""
|
||||||
|
firstDetails := execution.Details()
|
||||||
|
firstDetails.Messages[1].Content = "changed details content"
|
||||||
|
firstDetails.Messages[2].CacheControl.Type = "changed details cache"
|
||||||
|
secondDetails := execution.Details()
|
||||||
|
if !reflect.DeepEqual(secondDetails.Messages, expected) || secondDetails.RenderedPromptHash != prepared.RenderedPromptHash {
|
||||||
|
t.Fatalf("prepared execution details = %#v, want frozen %#v", secondDetails, expected)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err = engine.RunPrepared(context.Background(), execution)
|
||||||
|
if err != nil || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(client.requests[1].Prompt.Messages, expected) {
|
||||||
|
t.Fatalf("prepared result = (%#v, %v), requests = %#v", result, err, client.requests)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInvalidAppendedMessagesFailBeforeSourceOrModelWork(t *testing.T) {
|
||||||
|
promptSource := &inspectionCountingFS{}
|
||||||
|
var modelCalls atomic.Int64
|
||||||
|
engine, err := promptkit.NewEngine(
|
||||||
|
promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(promptSource, "."),
|
||||||
|
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
request := promptkit.RunRequest{
|
||||||
|
PromptID: "unreached",
|
||||||
|
AppendedMessages: []promptkit.RenderedMessage{{
|
||||||
|
Role: "unsupported-role",
|
||||||
|
Content: "sensitive appended content",
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
operations := []struct {
|
||||||
|
name string
|
||||||
|
run func() error
|
||||||
|
}{
|
||||||
|
{name: "Prepare", run: func() error { _, err := engine.Prepare(context.Background(), request); return err }},
|
||||||
|
{name: "PrepareExecution", run: func() error { _, err := engine.PrepareExecution(context.Background(), request); return err }},
|
||||||
|
{name: "Run", run: func() error { _, err := engine.Run(context.Background(), request); return err }},
|
||||||
|
}
|
||||||
|
for _, operation := range operations {
|
||||||
|
t.Run(operation.name, func(t *testing.T) {
|
||||||
|
err := operation.run()
|
||||||
|
if !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("error = %v, want ErrInvalidRequest", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if promptSource.opens.Load() != 0 {
|
||||||
|
t.Fatalf("invalid appended message opened prompt sources %d times", promptSource.opens.Load())
|
||||||
|
}
|
||||||
|
if modelCalls.Load() != 0 {
|
||||||
|
t.Fatalf("invalid appended message invoked the model %d times", modelCalls.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
28
backends.go
28
backends.go
@@ -9,6 +9,10 @@ import (
|
|||||||
// backend.
|
// backend.
|
||||||
const BackendOpenRouter = backend.OpenRouterID
|
const BackendOpenRouter = backend.OpenRouterID
|
||||||
|
|
||||||
|
// BackendRakestrawHome is the reserved ID of Promptkit's built-in
|
||||||
|
// Rakestrawhome backend.
|
||||||
|
const BackendRakestrawHome = backend.RakestrawHomeID
|
||||||
|
|
||||||
// BackendLocal is the case-sensitive conventional ID used by [LocalBackend].
|
// BackendLocal is the case-sensitive conventional ID used by [LocalBackend].
|
||||||
// It is not a built-in or reserved backend and must be registered with
|
// It is not a built-in or reserved backend and must be registered with
|
||||||
// [WithBackend].
|
// [WithBackend].
|
||||||
@@ -20,21 +24,25 @@ const BackendLocal = "local"
|
|||||||
// to this configuration value do not break source compatibility.
|
// to this configuration value do not break source compatibility.
|
||||||
type Backend struct {
|
type Backend struct {
|
||||||
// ID is the stable, case-sensitive registry key. NewEngine trims it and
|
// ID is the stable, case-sensitive registry key. NewEngine trims it and
|
||||||
// requires a non-blank value. BackendOpenRouter is reserved.
|
// requires a non-blank value. Built-in backend IDs are reserved.
|
||||||
ID string
|
ID string
|
||||||
// Endpoint is the OpenAI-compatible base endpoint. NewEngine trims it and
|
// Endpoint is the OpenAI-compatible base endpoint. NewEngine trims it and
|
||||||
// requires an absolute HTTP or HTTPS URL with a host and without user
|
// requires an absolute HTTP or HTTPS URL with a host and without user
|
||||||
// information, a query string, or a fragment. Paths are allowed.
|
// information, a query string, or a fragment. Paths are allowed.
|
||||||
Endpoint string
|
Endpoint string
|
||||||
// APIKeyEnv optionally names the environment variable containing the API
|
// APIKeyEnv optionally names an environment lookup source for an API key.
|
||||||
// key. NewEngine trims it and requires the portable form
|
// NewEngine trims it and requires the portable form [A-Za-z_][A-Za-z0-9_]*.
|
||||||
// [A-Za-z_][A-Za-z0-9_]*. Store only the name, never a credential value.
|
// A direct RunRequest.APIKey takes precedence. When no usable credential is
|
||||||
|
// available, the built-in client omits Authorization; injected clients own
|
||||||
|
// their own credential-resolution behavior. Store only the name, never a
|
||||||
|
// credential value.
|
||||||
APIKeyEnv string
|
APIKeyEnv string
|
||||||
// ExtraParams contains backend-wide request defaults. Values must be
|
// ExtraParams contains backend-wide request defaults. Values must be
|
||||||
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
|
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
|
||||||
// must not be model, session_id, messages, temperature, max_tokens, top_p,
|
// must not be model, session_id, messages, temperature, max_tokens, top_p,
|
||||||
// service_tier, reasoning_effort, or response_format. An empty map supplies
|
// service_tier, reasoning_effort, or response_format. An empty map supplies
|
||||||
// no defaults. NewEngine deeply copies the map.
|
// no defaults. NewEngine deeply copies the map and rejects excessively deep
|
||||||
|
// or large values for safety.
|
||||||
ExtraParams map[string]any
|
ExtraParams map[string]any
|
||||||
// ConcurrencyLimit is the maximum number of simultaneous model-generation
|
// ConcurrencyLimit is the maximum number of simultaneous model-generation
|
||||||
// calls allowed for this backend within one Engine. Zero leaves the backend
|
// calls allowed for this backend within one Engine. Zero leaves the backend
|
||||||
@@ -73,11 +81,11 @@ func LocalBackend(endpoint string, concurrencyLimit int) Backend {
|
|||||||
//
|
//
|
||||||
// Registrations accumulate in option order. Every normalized ID must be unique
|
// Registrations accumulate in option order. Every normalized ID must be unique
|
||||||
// across consumer registrations and built-ins; a duplicate or invalid
|
// across consumer registrations and built-ins; a duplicate or invalid
|
||||||
// definition makes NewEngine fail with ErrInvalidConfig. In particular,
|
// definition makes NewEngine fail with ErrInvalidConfig. Built-in IDs,
|
||||||
// BackendOpenRouter cannot be replaced. The immutable registration is scoped
|
// including [BackendOpenRouter] and [BackendRakestrawHome], cannot be
|
||||||
// to the resulting Engine and cannot be enumerated, replaced, removed, or
|
// replaced. The immutable registration is scoped to the resulting Engine and
|
||||||
// mutated after construction. WithBackend does not install package-global
|
// cannot be enumerated, replaced, removed, or mutated after construction.
|
||||||
// state.
|
// WithBackend does not install package-global state.
|
||||||
func WithBackend(backend Backend) Option {
|
func WithBackend(backend Backend) Option {
|
||||||
queueCapacity := 0
|
queueCapacity := 0
|
||||||
queueCapacitySet := backend.QueueCapacity != nil
|
queueCapacitySet := backend.QueueCapacity != nil
|
||||||
|
|||||||
63
convert.go
63
convert.go
@@ -1,7 +1,9 @@
|
|||||||
package promptkit
|
package promptkit
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||||
@@ -12,19 +14,52 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return domain.RunRequest{}, err
|
return domain.RunRequest{}, err
|
||||||
}
|
}
|
||||||
|
appendedMessages, err := toDomainAppendedMessages(req.AppendedMessages)
|
||||||
|
if err != nil {
|
||||||
|
return domain.RunRequest{}, err
|
||||||
|
}
|
||||||
return domain.RunRequest{
|
return domain.RunRequest{
|
||||||
PromptID: req.PromptID,
|
PromptID: req.PromptID,
|
||||||
PromptVersion: req.PromptVersion,
|
PromptVersion: req.PromptVersion,
|
||||||
ProfileID: req.ProfileID,
|
ProfileID: req.ProfileID,
|
||||||
SessionID: req.SessionID,
|
SessionID: req.SessionID,
|
||||||
APIKey: req.APIKey,
|
APIKey: req.APIKey,
|
||||||
Inputs: toDomainArtifactRefMap(req.Inputs),
|
Inputs: toDomainArtifactRefMap(req.Inputs),
|
||||||
Vars: copyStringMap(req.Vars),
|
Vars: copyStringMap(req.Vars),
|
||||||
Execution: execution,
|
Execution: execution,
|
||||||
Validation: toDomainOutputContractPtr(req.Validation),
|
Validation: toDomainOutputContractPtr(req.Validation),
|
||||||
|
AppendedMessages: appendedMessages,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func toDomainAppendedMessages(messages []RenderedMessage) ([]domain.RenderedMessage, error) {
|
||||||
|
if messages == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
converted := make([]domain.RenderedMessage, len(messages))
|
||||||
|
for index, message := range messages {
|
||||||
|
if !utf8.ValidString(message.Content) {
|
||||||
|
return nil, fmt.Errorf("appended message %d content must be valid UTF-8", index)
|
||||||
|
}
|
||||||
|
role, err := domain.NormalizeMessageRole(message.Role)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("appended message %d role: %w", index, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cacheControl, err := domain.NormalizeCacheControl(toDomainCacheControl(message.CacheControl))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("appended message %d cache_control: %w", index, err)
|
||||||
|
}
|
||||||
|
converted[index] = domain.RenderedMessage{
|
||||||
|
Role: role,
|
||||||
|
Content: message.Content,
|
||||||
|
CacheControl: cacheControl,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return converted, nil
|
||||||
|
}
|
||||||
|
|
||||||
func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
||||||
if prepared == nil {
|
if prepared == nil {
|
||||||
return nil
|
return nil
|
||||||
@@ -285,6 +320,16 @@ func fromDomainRenderedMessages(messages []domain.RenderedMessage) []RenderedMes
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func toDomainCacheControl(cacheControl *CacheControl) *domain.CacheControl {
|
||||||
|
if cacheControl == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlType(cacheControl.Type),
|
||||||
|
TTL: cacheControl.TTL,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl {
|
func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl {
|
||||||
if cacheControl == nil {
|
if cacheControl == nil {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
108
convert_appended_messages_test.go
Normal file
108
convert_appended_messages_test.go
Normal file
@@ -0,0 +1,108 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestToDomainAppendedMessages(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
messages []RenderedMessage
|
||||||
|
want []domain.RenderedMessage
|
||||||
|
wantErr string
|
||||||
|
privateRole string
|
||||||
|
privateText string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "normalizes supported roles",
|
||||||
|
messages: []RenderedMessage{
|
||||||
|
{Role: " DeVeLoPeR ", Content: "developer"},
|
||||||
|
{Role: "SyStEm", Content: "system"},
|
||||||
|
{Role: "\tUsEr\n", Content: "user"},
|
||||||
|
{Role: "assistant", Content: "assistant"},
|
||||||
|
},
|
||||||
|
want: []domain.RenderedMessage{
|
||||||
|
{Role: domain.RoleDeveloper, Content: "developer"},
|
||||||
|
{Role: domain.RoleSystem, Content: "system"},
|
||||||
|
{Role: domain.RoleUser, Content: "user"},
|
||||||
|
{Role: domain.RoleAssistant, Content: "assistant"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid role UTF-8",
|
||||||
|
messages: []RenderedMessage{{Role: string([]byte{0xff}), Content: "private-content"}},
|
||||||
|
wantErr: "appended message 0 role",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid content UTF-8",
|
||||||
|
messages: []RenderedMessage{{Role: "private-role", Content: string([]byte{0xff})}},
|
||||||
|
wantErr: "appended message 0 content",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unsupported role",
|
||||||
|
messages: []RenderedMessage{{Role: "private-role", Content: "private-content"}},
|
||||||
|
wantErr: "appended message 0 role",
|
||||||
|
privateRole: "private-role",
|
||||||
|
privateText: "private-content",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid cache control",
|
||||||
|
messages: []RenderedMessage{{Role: RoleUser, Content: "private-content", CacheControl: &CacheControl{Type: "private-cache"}}},
|
||||||
|
wantErr: "appended message 0 cache_control",
|
||||||
|
privateText: "private-content",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty and whitespace content",
|
||||||
|
messages: []RenderedMessage{
|
||||||
|
{Role: RoleUser, Content: ""},
|
||||||
|
{Role: RoleAssistant, Content: " \t\n "},
|
||||||
|
},
|
||||||
|
want: []domain.RenderedMessage{
|
||||||
|
{Role: domain.RoleUser, Content: ""},
|
||||||
|
{Role: domain.RoleAssistant, Content: " \t\n "},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
got, err := toDomainAppendedMessages(test.messages)
|
||||||
|
if test.wantErr != "" {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), test.wantErr) {
|
||||||
|
t.Fatalf("error = %v, want %q", err, test.wantErr)
|
||||||
|
}
|
||||||
|
for _, privateValue := range []string{test.privateRole, test.privateText} {
|
||||||
|
if privateValue != "" && strings.Contains(err.Error(), privateValue) {
|
||||||
|
t.Fatalf("error exposed appended message data %q: %v", privateValue, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("toDomainAppendedMessages() error = %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, test.want) {
|
||||||
|
t.Fatalf("toDomainAppendedMessages() = %#v, want %#v", got, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestToDomainAppendedMessagesCopiesCacheControl(t *testing.T) {
|
||||||
|
cacheControl := &CacheControl{Type: CacheControlEphemeral, TTL: "1h"}
|
||||||
|
messages := []RenderedMessage{{Role: RoleUser, Content: "content", CacheControl: cacheControl}}
|
||||||
|
request, err := toDomainRunRequest(RunRequest{AppendedMessages: messages})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("toDomainRunRequest() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
messages[0].Content = "changed"
|
||||||
|
cacheControl.TTL = ""
|
||||||
|
if request.AppendedMessages[0].Content != "content" || request.AppendedMessages[0].CacheControl.TTL != "1h" {
|
||||||
|
t.Fatalf("domain request did not retain an independent appended-message copy: %#v", request.AppendedMessages)
|
||||||
|
}
|
||||||
|
}
|
||||||
16
doc.go
16
doc.go
@@ -23,8 +23,9 @@
|
|||||||
// InspectProfile return copied inspection values. Returned values and values
|
// InspectProfile return copied inspection values. Returned values and values
|
||||||
// passed to extension interfaces are likewise isolated from engine state.
|
// passed to extension interfaces are likewise isolated from engine state.
|
||||||
// Callers own those copies and may mutate them after the call that supplied or
|
// Callers own those copies and may mutate them after the call that supplied or
|
||||||
// returned them. Returned structured errors are likewise caller-owned and may
|
// returned them. [CapacityError] values are caller-owned and may be mutated
|
||||||
// be mutated without affecting engine state or another error.
|
// without affecting engine state or another error. Immutable [GenerationError]
|
||||||
|
// values are also caller-owned and do not retain shared engine state.
|
||||||
//
|
//
|
||||||
// # Security and sensitive data
|
// # Security and sensitive data
|
||||||
//
|
//
|
||||||
@@ -53,9 +54,14 @@
|
|||||||
// Construction, inspection, handle, and error values, including [Config],
|
// Construction, inspection, handle, and error values, including [Config],
|
||||||
// [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile],
|
// [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile],
|
||||||
// [OpenAICompatibleProfileConfig], [ProfileInspection],
|
// [OpenAICompatibleProfileConfig], [ProfileInspection],
|
||||||
// [PromptInputDefinition], [PromptInspection], [PreparedExecution], and
|
// [PromptInputDefinition], [PromptInspection], [PreparedExecution],
|
||||||
// [CapacityError], do not have stable JSON representations. Direct API keys
|
// [CapacityError], and [GenerationError], do not have stable JSON
|
||||||
// are nevertheless excluded from JSON for every public value.
|
// representations. Direct API keys are nevertheless excluded from JSON for
|
||||||
|
// every public value.
|
||||||
|
|
||||||
|
// Provider-derived [GenerationError] accessor values are untrusted and can
|
||||||
|
// contain sensitive request or schema fragments. Applications must apply their
|
||||||
|
// own disclosure policy before logging, displaying, or returning them.
|
||||||
//
|
//
|
||||||
// JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
|
// JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
|
||||||
// PreparedRun and RunResult durations are encoded as integer milliseconds in
|
// PreparedRun and RunResult durations are encoded as integer milliseconds in
|
||||||
|
|||||||
@@ -173,6 +173,49 @@ semantics. The
|
|||||||
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
|
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
|
||||||
owns the built-in client's outbound HTTP behavior.
|
owns the built-in client's outbound HTTP behavior.
|
||||||
|
|
||||||
|
### Repair A Structured Result
|
||||||
|
|
||||||
|
Set a small additional-call budget when a structurally invalid result can be
|
||||||
|
corrected automatically:
|
||||||
|
|
||||||
|
```go
|
||||||
|
request.Validation = &promptkit.OutputContract{
|
||||||
|
Format: promptkit.FormatJSON,
|
||||||
|
ValidationMode: promptkit.ValidationJSONSchema,
|
||||||
|
SchemaPath: "events.schema.json",
|
||||||
|
RepairAttempts: 1,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Each repair attempt is another model call, so it can increase latency and
|
||||||
|
usage; `RunResult.Usage` is cumulative and `Validation.RepairAttempts` reports
|
||||||
|
calls actually started. Exhaustion still returns the final failed validation
|
||||||
|
result. `basic` validation can also repair an empty candidate, but structural
|
||||||
|
validity is not evidence of factual or domain correctness. See the
|
||||||
|
[output-contract format reference](../formats.md#output-contract) and
|
||||||
|
[`OutputContract` GoDoc](../../types.go) for the exact budget and eligibility
|
||||||
|
rules.
|
||||||
|
|
||||||
|
### Append Already-Rendered Messages
|
||||||
|
|
||||||
|
An application can include an earlier assistant response and its own corrective
|
||||||
|
instruction in a fresh request without changing the configured prompt:
|
||||||
|
|
||||||
|
```go
|
||||||
|
request.AppendedMessages = []promptkit.RenderedMessage{
|
||||||
|
{Role: promptkit.RoleAssistant, Content: previousResponse},
|
||||||
|
{Role: promptkit.RoleUser, Content: correction},
|
||||||
|
}
|
||||||
|
result, err := engine.Run(ctx, request)
|
||||||
|
```
|
||||||
|
|
||||||
|
These messages are already rendered: Promptkit does not template or resolve
|
||||||
|
files in them, and they can contain sensitive model output or application
|
||||||
|
feedback. Promptkit remains stateless; every `Run` call re-resolves its current
|
||||||
|
sources and the application owns any semantic retry budget. When a
|
||||||
|
pre-execution equality check is required, use `PrepareExecution` and compare
|
||||||
|
its opaque rendered-prompt hash before invoking `RunPrepared`.
|
||||||
|
|
||||||
## Inputs, Profiles, And Overrides
|
## Inputs, Profiles, And Overrides
|
||||||
|
|
||||||
Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request
|
Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request
|
||||||
@@ -189,6 +232,53 @@ For programmatic profiles,
|
|||||||
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
|
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
|
||||||
OpenAI-compatible settings into a value accepted by `WithProfiles`.
|
OpenAI-compatible settings into a value accepted by `WithProfiles`.
|
||||||
|
|
||||||
|
### Alias A Built-In Profile
|
||||||
|
|
||||||
|
Give an application-owned profile ID a built-in base when prompts should select
|
||||||
|
the application ID while inheriting the built-in target. The child can override
|
||||||
|
only the setting it owns:
|
||||||
|
|
||||||
|
```go
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "weather-light",
|
||||||
|
BaseProfileID: "deepseek-4-flash",
|
||||||
|
ReasoningEffort: "high",
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
Select `weather-light` in a prompt or `RunRequest.ProfileID`; it remains the
|
||||||
|
reported selected profile. See the [profile inheritance format
|
||||||
|
reference](../formats.md#profile-inheritance) and the
|
||||||
|
[`Profile` GoDoc](../../types.go) for exact lookup, merging, and validation
|
||||||
|
behavior.
|
||||||
|
|
||||||
|
### Use The Rakestrawhome Built-In Profile
|
||||||
|
|
||||||
|
Set `RAKESTRAWHOME_INFERENCE_API_KEY` in the application environment, then
|
||||||
|
select `rakestrawhome-gemma-4-31b` as an ordinary profile ID. For example, a
|
||||||
|
prepared result identifies the selected built-in through
|
||||||
|
`BackendRakestrawHome`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
|
||||||
|
PromptID: "meeting.summary",
|
||||||
|
ProfileID: "rakestrawhome-gemma-4-31b",
|
||||||
|
Inputs: inputs,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if prepared.SelectedBackendID != promptkit.BackendRakestrawHome {
|
||||||
|
return fmt.Errorf("unexpected backend %q", prepared.SelectedBackendID)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not register `rakestrawhome` manually. When adopting this built-in, remove
|
||||||
|
an existing `WithBackend` registration with that exact ID; retaining it causes
|
||||||
|
the intentional duplicate-ID configuration error. Direct request credentials
|
||||||
|
and runtime endpoint overrides remain supported under their ordinary GoDoc and
|
||||||
|
format contracts.
|
||||||
|
|
||||||
### Inspect A Profile Before Prompt Work
|
### Inspect A Profile Before Prompt Work
|
||||||
|
|
||||||
Use [`Engine.InspectProfile`](../../engine.go) to validate one configured
|
Use [`Engine.InspectProfile`](../../engine.go) to validate one configured
|
||||||
@@ -204,7 +294,7 @@ if err != nil {
|
|||||||
|
|
||||||
target := inspection.EffectiveModelParams
|
target := inspection.EffectiveModelParams
|
||||||
if target.APIKeyEnv != "" {
|
if target.APIKeyEnv != "" {
|
||||||
// Apply application policy for the named environment variable.
|
// This is a configured optional environment lookup source.
|
||||||
} else if inspection.APIKeyRequired {
|
} else if inspection.APIKeyRequired {
|
||||||
// Arrange a direct credential before later execution.
|
// Arrange a direct credential before later execution.
|
||||||
}
|
}
|
||||||
@@ -213,9 +303,11 @@ if target.APIKeyEnv != "" {
|
|||||||
Use this configuration-time boundary when only the profile and its target need
|
Use this configuration-time boundary when only the profile and its target need
|
||||||
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
||||||
or rendering work; use prepared execution when that work must remain tied to a
|
or rendering work; use prepared execution when that work must remain tied to a
|
||||||
later execution. Inspection reports credential requirements but leaves the
|
later execution. A reported `APIKeyEnv` is a configured optional source, while
|
||||||
timing of credential enforcement to the application. The method's
|
`APIKeyRequired` is the explicit local requirement. The
|
||||||
[GoDoc](../../engine.go) owns its exact result and error contract.
|
[credential format reference](../formats.md#credentials) and the method's
|
||||||
|
[GoDoc](../../engine.go) own the exact precedence, timing, result, and error
|
||||||
|
contracts.
|
||||||
|
|
||||||
### Set A Per-Run Session And Reasoning
|
### Set A Per-Run Session And Reasoning
|
||||||
|
|
||||||
@@ -423,6 +515,24 @@ status; those choices remain with the consuming application. The
|
|||||||
contract, while the [`Engine.Run` and error GoDoc](../../engine.go) owns broad
|
contract, while the [`Engine.Run` and error GoDoc](../../engine.go) owns broad
|
||||||
error and cancellation identities.
|
error and cancellation identities.
|
||||||
|
|
||||||
|
For a non-2xx response from the built-in OpenAI-compatible client, inspect the
|
||||||
|
status and deliberately selected provider diagnostic when useful:
|
||||||
|
|
||||||
|
```go
|
||||||
|
var generationErr *promptkit.GenerationError
|
||||||
|
if errors.As(err, &generationErr) {
|
||||||
|
status := generationErr.StatusCode()
|
||||||
|
message := generationErr.ProviderMessage()
|
||||||
|
_, _ = status, message // Apply application retry and presentation policy.
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
All provider fields are untrusted and can contain sensitive request or schema
|
||||||
|
fragments. Do not log, display, or return them without an application-specific
|
||||||
|
disclosure policy. Promptkit does not assign retry or presentation behavior.
|
||||||
|
The [`GenerationError` GoDoc](../../generation_error.go) owns the exact typed
|
||||||
|
error contract.
|
||||||
|
|
||||||
## Application Boundary
|
## Application Boundary
|
||||||
|
|
||||||
Promptkit is an importable library. It does not own a command, inbound HTTP
|
Promptkit is an importable library. It does not own a command, inbound HTTP
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ Start with:
|
|||||||
| Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. |
|
| Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. |
|
||||||
| Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). |
|
| Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). |
|
||||||
| Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. |
|
| Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. |
|
||||||
|
| Maintained external catalogs or built-ins | The [internal source document](internal/sources.md), [framework format reference](formats.md), [testing policy](policy/testing.md), both catalog module repositories, and each catalog's release procedure. |
|
||||||
| Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. |
|
| Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. |
|
||||||
| Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. |
|
| Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. |
|
||||||
| Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. |
|
| Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. |
|
||||||
@@ -44,3 +45,190 @@ Start with:
|
|||||||
For cross-cutting changes, follow every applicable row. Do not create
|
For cross-cutting changes, follow every applicable row. Do not create
|
||||||
placeholder documents for packages, APIs, or integrations that do not yet
|
placeholder documents for packages, APIs, or integrations that do not yet
|
||||||
exist.
|
exist.
|
||||||
|
|
||||||
|
## Maintainer Validation
|
||||||
|
|
||||||
|
This section is the canonical local validation workflow for Promptkit. Run
|
||||||
|
every command from the repository root before accepting a change. The test
|
||||||
|
suite and maintained examples are deterministic, offline, and require no real
|
||||||
|
provider credentials.
|
||||||
|
|
||||||
|
### Tests, Analysis, Build, And Examples
|
||||||
|
|
||||||
|
Run the ordinary and race-enabled suites, static analysis, the build, and both
|
||||||
|
maintained consumer examples:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go test ./...
|
||||||
|
go test -race ./...
|
||||||
|
go vet ./...
|
||||||
|
go build ./...
|
||||||
|
go run ./examples/go-library/prepare
|
||||||
|
go run ./examples/go-library/run
|
||||||
|
```
|
||||||
|
|
||||||
|
Both examples must exit successfully. Review their JSON output: preparation
|
||||||
|
must report the selected offline prompt, profile, model, and message count;
|
||||||
|
execution must report the deterministic generated output, successful
|
||||||
|
validation, selected offline model, and usage. Neither command may contact a
|
||||||
|
provider or require credentials.
|
||||||
|
|
||||||
|
### Go Formatting
|
||||||
|
|
||||||
|
Check every tracked Go file. The final command must succeed and the captured
|
||||||
|
list must be empty:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
unformatted=$(
|
||||||
|
git ls-files '*.go' |
|
||||||
|
while IFS= read -r go_file
|
||||||
|
do
|
||||||
|
gofmt -l "$go_file"
|
||||||
|
done
|
||||||
|
)
|
||||||
|
test -z "$unformatted"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Local Markdown Links
|
||||||
|
|
||||||
|
Use the Python standard library to verify every repository-relative Markdown
|
||||||
|
target and local heading fragment. The check is offline and prints nothing on
|
||||||
|
success:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
python3 - <<'PY'
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from urllib.parse import unquote
|
||||||
|
|
||||||
|
root = Path.cwd().resolve()
|
||||||
|
markdown_files = [
|
||||||
|
root / name
|
||||||
|
for name in subprocess.check_output(
|
||||||
|
["git", "ls-files", "*.md"], text=True
|
||||||
|
).splitlines()
|
||||||
|
]
|
||||||
|
link_pattern = re.compile(r"!?\[[^]]*\]\(([^)]+)\)")
|
||||||
|
heading_pattern = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$")
|
||||||
|
scheme_pattern = re.compile(r"^[a-z][a-z0-9+.-]*:", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def markdown_lines(path):
|
||||||
|
in_fence = False
|
||||||
|
fence = ""
|
||||||
|
for line in path.read_text(encoding="utf-8").splitlines():
|
||||||
|
stripped = line.lstrip()
|
||||||
|
marker = stripped[:3]
|
||||||
|
if marker in {"```", "~~~"}:
|
||||||
|
if not in_fence:
|
||||||
|
in_fence = True
|
||||||
|
fence = marker
|
||||||
|
elif marker == fence:
|
||||||
|
in_fence = False
|
||||||
|
fence = ""
|
||||||
|
continue
|
||||||
|
if not in_fence:
|
||||||
|
yield line
|
||||||
|
|
||||||
|
|
||||||
|
anchor_cache = {}
|
||||||
|
|
||||||
|
|
||||||
|
def anchors(path):
|
||||||
|
if path in anchor_cache:
|
||||||
|
return anchor_cache[path]
|
||||||
|
found = set()
|
||||||
|
counts = {}
|
||||||
|
for line in markdown_lines(path):
|
||||||
|
match = heading_pattern.match(line)
|
||||||
|
if not match:
|
||||||
|
continue
|
||||||
|
heading = re.sub(r"<[^>]+>", "", match.group(1)).replace("`", "")
|
||||||
|
base = re.sub(r"[^\w\- ]", "", heading.lower()).replace(" ", "-")
|
||||||
|
count = counts.get(base, 0)
|
||||||
|
counts[base] = count + 1
|
||||||
|
found.add(base if count == 0 else f"{base}-{count}")
|
||||||
|
anchor_cache[path] = found
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
failures = []
|
||||||
|
for source in markdown_files:
|
||||||
|
text = "\n".join(markdown_lines(source))
|
||||||
|
for match in link_pattern.finditer(text):
|
||||||
|
target = match.group(1).strip()
|
||||||
|
if target.startswith("<") and target.endswith(">"):
|
||||||
|
target = target[1:-1]
|
||||||
|
if scheme_pattern.match(target) or target.startswith("//"):
|
||||||
|
continue
|
||||||
|
path_text, separator, fragment = target.partition("#")
|
||||||
|
destination = source if not path_text else source.parent / unquote(path_text)
|
||||||
|
try:
|
||||||
|
destination = destination.resolve()
|
||||||
|
destination.relative_to(root)
|
||||||
|
except ValueError:
|
||||||
|
failures.append(f"{source.relative_to(root)}: escapes repository: {target}")
|
||||||
|
continue
|
||||||
|
if not destination.exists():
|
||||||
|
failures.append(f"{source.relative_to(root)}: missing target: {target}")
|
||||||
|
continue
|
||||||
|
if separator and destination.suffix.lower() == ".md":
|
||||||
|
fragment = unquote(fragment).lower()
|
||||||
|
if fragment not in anchors(destination):
|
||||||
|
failures.append(f"{source.relative_to(root)}: missing anchor: {target}")
|
||||||
|
|
||||||
|
if failures:
|
||||||
|
print("\n".join(failures), file=sys.stderr)
|
||||||
|
raise SystemExit(1)
|
||||||
|
PY
|
||||||
|
```
|
||||||
|
|
||||||
|
### Repository Hygiene And Review
|
||||||
|
|
||||||
|
Reject an active Go workspace, tracked workspace files, a vendor tree, or a
|
||||||
|
module replacement:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
case "$(go env GOWORK)" in
|
||||||
|
''|off) ;;
|
||||||
|
*) printf '%s\n' 'an active Go workspace is not allowed' >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
test -z "$(git ls-files go.work go.work.sum)"
|
||||||
|
test ! -e vendor
|
||||||
|
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||||
|
then
|
||||||
|
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
```
|
||||||
|
|
||||||
|
Check whitespace in both unstaged and staged changes. List ignored files and
|
||||||
|
scan tracked content for common credential forms:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git diff --check
|
||||||
|
git diff --cached --check
|
||||||
|
test -z "$(git ls-files --others --ignored --exclude-standard)"
|
||||||
|
credential_pattern='-----BEGIN ([A-Z0-9]+ )?PRIV''ATE KEY-----|AKI''A[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{36,}|sk-[A-Za-z0-9]{32,}'
|
||||||
|
if git grep -nEI -e "$credential_pattern" -- .
|
||||||
|
then
|
||||||
|
printf '%s\n' 'possible credential found' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
```
|
||||||
|
|
||||||
|
Inspect `git status --short --untracked-files=all` and the complete diff before
|
||||||
|
accepting a change. The status may contain only the intended source changes
|
||||||
|
during development. Reject credentials, private keys, environment files,
|
||||||
|
generated binaries, test or coverage output, downloaded assets, template
|
||||||
|
residue, and any other artifact that does not belong in source control. The
|
||||||
|
credential scan catches common forms but does not replace inspection of the
|
||||||
|
actual change.
|
||||||
|
|
||||||
|
After committing the accepted change, require a clean candidate:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
test -z "$(git status --porcelain)"
|
||||||
|
```
|
||||||
|
|||||||
187
docs/formats.md
187
docs/formats.md
@@ -9,9 +9,10 @@ explains how to select these sources and invoke the engine. The
|
|||||||
owns the resulting outbound wire behavior.
|
owns the resulting outbound wire behavior.
|
||||||
|
|
||||||
Prompt and profile sources recursively discover files ending in `.yaml` or
|
Prompt and profile sources recursively discover files ending in `.yaml` or
|
||||||
`.yml`. YAML decoding is strict: unknown fields are errors for the selected
|
`.yml`. Each prompt-definition and profile file contains exactly one YAML
|
||||||
definition. Definitions are selected by their YAML `id`, not their file name
|
document; comments and trailing whitespace are allowed. YAML decoding is
|
||||||
or directory.
|
strict: unknown fields are errors for the selected definition. Definitions are
|
||||||
|
selected by their YAML `id`, not their file name or directory.
|
||||||
|
|
||||||
## Prompt Definitions
|
## Prompt Definitions
|
||||||
|
|
||||||
@@ -81,14 +82,27 @@ allowed.
|
|||||||
|
|
||||||
### Messages And Templates
|
### Messages And Templates
|
||||||
|
|
||||||
Each message has a non-empty `role` and exactly one of:
|
Each message has a `role` that Promptkit trims and lowercases. It must then be
|
||||||
|
exactly one of `developer`, `system`, `user`, or `assistant`; blank, custom,
|
||||||
|
`tool`, and `function` roles are invalid. This intentionally tightens the
|
||||||
|
previous nonblank-string rule. Consumers migrating to the next minor release
|
||||||
|
must update any nonstandard prompt-definition roles before upgrading.
|
||||||
|
|
||||||
|
Each message also has exactly one of:
|
||||||
|
|
||||||
- `content`, containing an inline Go template; or
|
- `content`, containing an inline Go template; or
|
||||||
- `content_file`, naming a file whose contents are the Go template.
|
- `content_file`, naming a file whose contents are the Go template.
|
||||||
|
|
||||||
For directory and `fs.FS` prompt sources, `content_file` resolves relative to
|
`content_file` must be a relative path. It resolves from the directory that
|
||||||
the prompt file and remains within the source root. `WithPromptFile` also
|
contains the prompt file and must remain within the configured prompt source
|
||||||
resolves it relative to that file.
|
root; parent components are allowed only when the resolved target remains
|
||||||
|
inside that root. Absolute paths and paths that escape the root are rejected.
|
||||||
|
Operating-system directory and single-file sources also reject symlink targets
|
||||||
|
outside the root, while injected `fs.FS` sources apply containment in that
|
||||||
|
filesystem's relative path namespace. For `WithPromptFile`, the source root is
|
||||||
|
the directory containing the selected prompt file. Promptkit uses the parsed
|
||||||
|
path text exactly after checking separately that it is not blank, so leading
|
||||||
|
and trailing whitespace can name real filesystem entries.
|
||||||
|
|
||||||
Request variables are the template data, so a variable named `audience` is
|
Request variables are the template data, so a variable named `audience` is
|
||||||
referenced as `{{.audience}}`. The `{{input "note"}}` helper renders the body
|
referenced as `{{.audience}}`. The `{{input "note"}}` helper renders the body
|
||||||
@@ -118,7 +132,7 @@ outbound integration determines its wire representation.
|
|||||||
| `format` | yes | `text`, `markdown`, or `json`. |
|
| `format` | yes | `text`, `markdown`, or `json`. |
|
||||||
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema`. |
|
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema`. |
|
||||||
| `schema_path` | for `json_schema` | Path to a schema in the configured schema source. |
|
| `schema_path` | for `json_schema` | Path to a schema in the configured schema source. |
|
||||||
| `repair_attempts` | no | Integer zero or greater; omitted means zero. |
|
| `repair_attempts` | no | Integer from zero through three; omitted means zero. A positive value requires `basic`, `json`, or `json_schema` validation. |
|
||||||
|
|
||||||
The validation modes behave as follows:
|
The validation modes behave as follows:
|
||||||
|
|
||||||
@@ -128,14 +142,31 @@ The validation modes behave as follows:
|
|||||||
- `json_schema` requires valid JSON that satisfies the selected schema.
|
- `json_schema` requires valid JSON that satisfies the selected schema.
|
||||||
|
|
||||||
`format` controls output artifact metadata. JSON Schema mode also supplies the
|
`format` controls output artifact metadata. JSON Schema mode also supplies the
|
||||||
schema to compatible model clients as structured-output metadata. The public
|
schema to compatible model clients as structured-output metadata. Plain `json`
|
||||||
engine does not install an output repairer, so its validation is single-pass
|
validation accepts every valid JSON value and does not request a provider-native
|
||||||
even when a positive `repair_attempts` value is present.
|
JSON-object constraint.
|
||||||
|
|
||||||
|
`repair_attempts` counts additional generation calls after a failed validation.
|
||||||
|
Zero is single-pass. With a positive eligible budget, Promptkit stops at the
|
||||||
|
first valid candidate. If the budget is exhausted, it returns the final
|
||||||
|
candidate and its complete failed validation result; generation and operational
|
||||||
|
validation failures remain errors. `none` never permits repair.
|
||||||
|
|
||||||
A request-level `OutputContract` replaces the complete prompt output contract.
|
A request-level `OutputContract` replaces the complete prompt output contract.
|
||||||
It does not merge individual fields. If its format is empty, Promptkit uses
|
It does not merge individual fields. If its format is empty, Promptkit uses
|
||||||
`text`.
|
`text`.
|
||||||
|
|
||||||
|
## Built-In Backends
|
||||||
|
|
||||||
|
Every engine provides these reserved OpenAI-compatible backend IDs. Consumers
|
||||||
|
must not register either ID with `WithBackend`; exact registration and
|
||||||
|
reservation behavior belongs to the [`Backend` GoDoc](../backends.go).
|
||||||
|
|
||||||
|
| ID | Base endpoint | API-key environment variable | Active generation limit | Default queue capacity |
|
||||||
|
| --- | --- | --- | ---: | ---: |
|
||||||
|
| `openrouter` | `https://openrouter.ai/api/v1` | `OPENROUTER_API_KEY` | 16 | 1024 |
|
||||||
|
| `rakestrawhome` | `https://inference.ai.rakestrawhome.com/v1` | `RAKESTRAWHOME_INFERENCE_API_KEY` | 4 | 1024 |
|
||||||
|
|
||||||
## Profile Definitions
|
## Profile Definitions
|
||||||
|
|
||||||
A profile supplies model execution settings:
|
A profile supplies model execution settings:
|
||||||
@@ -154,11 +185,21 @@ extra_params:
|
|||||||
provider_option: enabled
|
provider_option: enabled
|
||||||
```
|
```
|
||||||
|
|
||||||
|
A derived profile can use a named base and override only the settings it owns:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
id: local-summary-fast
|
||||||
|
base_profile: local-summary
|
||||||
|
timeout_seconds: 30
|
||||||
|
reasoning_effort: low
|
||||||
|
```
|
||||||
|
|
||||||
| Field | Required | Meaning |
|
| Field | Required | Meaning |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `id` | yes | Non-empty profile identifier. IDs must be unique within one source. |
|
| `id` | yes | Profile identifier, trimmed before selection and publication. It must be non-empty after trimming and unique within one source after normalization. |
|
||||||
|
| `base_profile` | no | One optional parent profile ID. A derived profile may inherit target fields from it. |
|
||||||
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared or inspected. |
|
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared or inspected. |
|
||||||
| `endpoint` | unless `backend` is present | Non-empty OpenAI-compatible base URL, including an API version path when required. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
|
| `endpoint` | unless `backend` is present | OpenAI-compatible base URL, including an API version path when required. A nonempty value is trimmed and must be absolute HTTP or HTTPS with a host and without user information, a query, or a fragment. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
|
||||||
| `model` | yes | Non-empty provider model name. |
|
| `model` | yes | Non-empty provider model name. |
|
||||||
| `temperature` | no | Number from 0 through 2. |
|
| `temperature` | no | Number from 0 through 2. |
|
||||||
| `max_tokens` | no | Integer zero or greater. |
|
| `max_tokens` | no | Integer zero or greater. |
|
||||||
@@ -166,16 +207,21 @@ extra_params:
|
|||||||
| `timeout_seconds` | no | Per-generation deadline in whole seconds; integer zero or greater. |
|
| `timeout_seconds` | no | Per-generation deadline in whole seconds; integer zero or greater. |
|
||||||
| `service_tier` | no | Provider-specific request tier. |
|
| `service_tier` | no | Provider-specific request tier. |
|
||||||
| `reasoning_effort` | no | Provider-specific reasoning setting. |
|
| `reasoning_effort` | no | Provider-specific reasoning setting. |
|
||||||
| `api_key_env` | no | Name of an environment variable containing the API key. |
|
| `api_key_env` | no | Optional environment-variable lookup source for an API key. |
|
||||||
| `extra_params` | no | JSON-compatible provider-specific outbound fields. |
|
| `extra_params` | no | JSON-compatible provider-specific outbound fields. |
|
||||||
|
|
||||||
Raw `api_key` is prohibited in profile YAML. Store only an environment
|
Raw `api_key` is prohibited in profile YAML. Store only an environment
|
||||||
variable name in `api_key_env`.
|
variable name in `api_key_env`.
|
||||||
|
|
||||||
|
A standalone profile must provide a model and at least one of `backend` or
|
||||||
|
`endpoint`. A derived profile may omit those target fields because its selected
|
||||||
|
base chain can provide them. Local parsing still validates a derived profile's
|
||||||
|
own ID, supplied endpoint, execution-setting bounds, and `extra_params`.
|
||||||
|
|
||||||
Promptkit does not infer a backend from a model or endpoint. Endpoint-only
|
Promptkit does not infer a backend from a model or endpoint. Endpoint-only
|
||||||
profiles remain supported and have no effective backend ID.
|
profiles remain supported and have no effective backend ID.
|
||||||
The engine always provides the built-in `openrouter` ID. Consumers can add
|
The engine always provides the built-in `openrouter` and `rakestrawhome` IDs.
|
||||||
engine-scoped IDs with
|
Consumers can add engine-scoped IDs with
|
||||||
[`WithBackend`](../backends.go); exact registration validation belongs to its
|
[`WithBackend`](../backends.go); exact registration validation belongs to its
|
||||||
GoDoc.
|
GoDoc.
|
||||||
|
|
||||||
@@ -183,6 +229,7 @@ GoDoc.
|
|||||||
objects with string keys. Keys must be non-empty. With the built-in client,
|
objects with string keys. Keys must be non-empty. With the built-in client,
|
||||||
they also cannot collide with the standard fields listed in the
|
they also cannot collide with the standard fields listed in the
|
||||||
[outbound request contract](integrations/openai-compatible-chat.md#request-body).
|
[outbound request contract](integrations/openai-compatible-chat.md#request-body).
|
||||||
|
Excessively deep or large JSON-shaped values are rejected for safety.
|
||||||
|
|
||||||
### Defaults And Overrides
|
### Defaults And Overrides
|
||||||
|
|
||||||
@@ -236,7 +283,7 @@ Profile sources resolve matching IDs in this order:
|
|||||||
2. the ordinary configured source selected by a profile file, `fs.FS`, or
|
2. the ordinary configured source selected by a profile file, `fs.FS`, or
|
||||||
configured profile directory;
|
configured profile directory;
|
||||||
3. application fallback profiles supplied with `WithFallbackProfileFS`; and
|
3. application fallback profiles supplied with `WithFallbackProfileFS`; and
|
||||||
4. embedded built-in profiles.
|
4. maintained external catalog profiles.
|
||||||
|
|
||||||
A profile source supplies a complete definition; definitions and their fields
|
A profile source supplies a complete definition; definitions and their fields
|
||||||
are not merged across sources. A higher-precedence source falls back only when
|
are not merged across sources. A higher-precedence source falls back only when
|
||||||
@@ -246,41 +293,66 @@ profiles. They use `APIKeyRequired` for request-scoped credentials instead of
|
|||||||
`api_key_env`. Preparation and exact profile inspection use this same source
|
`api_key_env`. Preparation and exact profile inspection use this same source
|
||||||
precedence.
|
precedence.
|
||||||
|
|
||||||
|
When a selected definition names `base_profile`, every profile ID in that
|
||||||
|
chain is looked up through this same precedence order. A higher-precedence
|
||||||
|
definition therefore shadows a lower-precedence definition of the same base
|
||||||
|
ID, including a built-in. References are not source-qualified.
|
||||||
|
|
||||||
|
### Profile Inheritance
|
||||||
|
|
||||||
|
Promptkit resolves one linear base chain of at most 32 profiles, including the
|
||||||
|
selected profile. It merges settings from the root base to the selected leaf.
|
||||||
|
The leaf's `id` remains the selected profile identity. Nonblank string fields
|
||||||
|
(`backend`, `endpoint`, `model`, `service_tier`, `reasoning_effort`, and
|
||||||
|
`api_key_env`) and nonzero numeric fields replace inherited values. A nonempty
|
||||||
|
`extra_params` map replaces the complete inherited map rather than merging
|
||||||
|
keys, and `APIKeyRequired: true` remains true through the chain. Backend and
|
||||||
|
endpoint are independent: replacing one does not clear the other.
|
||||||
|
|
||||||
|
There is no profile-level clearing syntax. Blank strings, zero numbers, false,
|
||||||
|
and empty maps remain unspecified and inherit from a base. Use existing
|
||||||
|
presence-aware request overrides where an execution needs an explicit zero or
|
||||||
|
empty reasoning setting.
|
||||||
|
|
||||||
|
An absent directly selected profile reports the ordinary not-found error. Once
|
||||||
|
the selected profile exists, a missing base, cycle, overlong chain, or
|
||||||
|
incomplete resolved target is a profile-load failure. Ordinary operations
|
||||||
|
resolve chains afresh; prepared execution retains the fully resolved target.
|
||||||
|
|
||||||
## Built-In Profile Catalog
|
## Built-In Profile Catalog
|
||||||
|
|
||||||
Every built-in selects the `openrouter` backend. The engine's built-in backend
|
Every built-in profile selects one maintained built-in backend and inherits
|
||||||
registry supplies `https://openrouter.ai/api/v1` and the environment-variable
|
that backend's connection and credential metadata. Profile files do not repeat
|
||||||
name `OPENROUTER_API_KEY`, so individual profiles contain only model and
|
those values. A configured, application fallback, or in-memory profile with
|
||||||
generation settings. Built-in profile files do not repeat those connection
|
the same profile ID takes precedence.
|
||||||
values. A configured, application fallback, or in-memory profile with the same
|
|
||||||
profile ID takes precedence.
|
|
||||||
|
|
||||||
| Provider | ID | Model |
|
| Provider | ID | Backend | Model |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| aion-labs | `aion-2` | `aion-labs/aion-2.0` |
|
| aion-labs | `aion-2` | `openrouter` | `aion-labs/aion-2.0` |
|
||||||
| anthropic | `claude-fable-latest` | `~anthropic/claude-fable-latest` |
|
| anthropic | `claude-fable-latest` | `openrouter` | `~anthropic/claude-fable-latest` |
|
||||||
| anthropic | `claude-haiku-latest` | `~anthropic/claude-haiku-latest` |
|
| anthropic | `claude-haiku-latest` | `openrouter` | `~anthropic/claude-haiku-latest` |
|
||||||
| anthropic | `claude-opus-latest` | `~anthropic/claude-opus-latest` |
|
| anthropic | `claude-opus-latest` | `openrouter` | `~anthropic/claude-opus-latest` |
|
||||||
| anthropic | `claude-sonnet-latest` | `~anthropic/claude-sonnet-latest` |
|
| anthropic | `claude-sonnet-latest` | `openrouter` | `~anthropic/claude-sonnet-latest` |
|
||||||
| deepseek | `deepseek-3-2` | `deepseek/deepseek-v3.2` |
|
| deepseek | `deepseek-3-2` | `openrouter` | `deepseek/deepseek-v3.2` |
|
||||||
| deepseek | `deepseek-4-flash` | `deepseek/deepseek-v4-flash` |
|
| deepseek | `deepseek-4-flash` | `openrouter` | `deepseek/deepseek-v4-flash` |
|
||||||
| deepseek | `deepseek-4-pro` | `deepseek/deepseek-v4-pro` |
|
| deepseek | `deepseek-4-pro` | `openrouter` | `deepseek/deepseek-v4-pro` |
|
||||||
| google | `gemini-2-flash` | `google/gemini-2.5-flash` |
|
| google | `gemini-2-flash` | `openrouter` | `google/gemini-2.5-flash` |
|
||||||
| google | `gemini-2-flash-lite` | `google/gemini-2.5-flash-lite` |
|
| google | `gemini-2-flash-lite` | `openrouter` | `google/gemini-2.5-flash-lite` |
|
||||||
| google | `gemini-2-pro` | `google/gemini-2.5-pro` |
|
| google | `gemini-2-pro` | `openrouter` | `google/gemini-2.5-pro` |
|
||||||
| google | `gemini-3-flash-lite` | `google/gemini-3.1-flash-lite` |
|
| google | `gemini-3-flash-lite` | `openrouter` | `google/gemini-3.1-flash-lite` |
|
||||||
| google | `gemini-flash-latest` | `~google/gemini-flash-latest` |
|
| google | `gemini-flash-latest` | `openrouter` | `~google/gemini-flash-latest` |
|
||||||
| google | `gemini-pro-latest` | `~google/gemini-pro-latest` |
|
| google | `gemini-pro-latest` | `openrouter` | `~google/gemini-pro-latest` |
|
||||||
| google | `gemma-4-31b` | `google/gemma-4-31b-it:exacto` |
|
| google | `gemma-4-31b` | `openrouter` | `google/gemma-4-31b-it:exacto` |
|
||||||
| minimax | `minimax-m2` | `minimax/minimax-m2.5` |
|
| google | `rakestrawhome-gemma-4-31b` | `rakestrawhome` | `google/gemma-4-31b-it` |
|
||||||
| minimax | `minimax-m3` | `minimax/minimax-m3` |
|
| minimax | `minimax-m2` | `openrouter` | `minimax/minimax-m2.5` |
|
||||||
| mistral | `mistral-large-2512` | `mistralai/mistral-large-2512` |
|
| minimax | `minimax-m3` | `openrouter` | `minimax/minimax-m3` |
|
||||||
| mistral | `mistral-medium-3-5` | `mistralai/mistral-medium-3-5` |
|
| mistral | `mistral-large-2512` | `openrouter` | `mistralai/mistral-large-2512` |
|
||||||
| mistral | `mistral-small-3` | `mistralai/mistral-small-3.2-24b-instruct` |
|
| mistral | `mistral-medium-3-5` | `openrouter` | `mistralai/mistral-medium-3-5` |
|
||||||
| mistral | `mistral-small-4` | `mistralai/mistral-small-2603` |
|
| mistral | `mistral-small-3` | `openrouter` | `mistralai/mistral-small-3.2-24b-instruct` |
|
||||||
| nvidia | `nemotron-3-ultra` | `nvidia/nemotron-3-ultra-550b-a55b` |
|
| mistral | `mistral-small-4` | `openrouter` | `mistralai/mistral-small-2603` |
|
||||||
| openai | `gpt-5-mini` | `openai/gpt-5.4-mini` |
|
| nvidia | `nemotron-3-ultra` | `openrouter` | `nvidia/nemotron-3-ultra-550b-a55b` |
|
||||||
| openai | `gpt-5-nano` | `openai/gpt-5.4-nano` |
|
| openai | `gpt-5-mini` | `openrouter` | `openai/gpt-5.4-mini` |
|
||||||
|
| openai | `gpt-5-nano` | `openrouter` | `openai/gpt-5.4-nano` |
|
||||||
|
|
||||||
## Schemas
|
## Schemas
|
||||||
|
|
||||||
@@ -299,16 +371,25 @@ schema produces a failed validation result.
|
|||||||
Credential values belong at the request or environment boundary, never in
|
Credential values belong at the request or environment boundary, never in
|
||||||
prompt, profile, schema, or example files:
|
prompt, profile, schema, or example files:
|
||||||
|
|
||||||
- a file profile names an environment variable with `api_key_env`;
|
- a backend or file profile can name an optional environment lookup source
|
||||||
- an in-memory profile may set `APIKeyRequired`;
|
with `APIKeyEnv` or `api_key_env`;
|
||||||
- a request can provide a direct `APIKey` or override `APIKeyEnv`; and
|
- an in-memory profile may set `APIKeyRequired` as an explicit local
|
||||||
|
requirement;
|
||||||
|
- a request can provide a direct `APIKey` or override the optional `APIKeyEnv`
|
||||||
|
source; and
|
||||||
- a direct request key takes precedence over environment lookup.
|
- a direct request key takes precedence over environment lookup.
|
||||||
|
|
||||||
After a direct request key, the credential-source precedence is request
|
After a direct request key, the credential-source precedence is request
|
||||||
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
|
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
|
||||||
profile with `APIKeyRequired` clears an inherited backend environment name and
|
profile with `APIKeyRequired` clears an inherited backend environment name and
|
||||||
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
|
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
|
||||||
Promptkit validates required credential availability during preparation.
|
Named environment sources are optional: when the selected source is absent,
|
||||||
|
empty, or whitespace-only, the built-in client omits the `Authorization`
|
||||||
|
header and handles the provider response normally. `APIKeyRequired` is the
|
||||||
|
only explicit local availability requirement. Promptkit validates required
|
||||||
|
credential availability during preparation and rechecks it when a prepared
|
||||||
|
execution runs. Injected clients receive resolved source metadata but define
|
||||||
|
their own credential-resolution behavior.
|
||||||
Direct keys are excluded from JSON results and redacted by public string
|
Direct keys are excluded from JSON results and redacted by public string
|
||||||
formatters. Environment-variable names may appear in prepared metadata, but
|
formatters. Environment-variable names may appear in prepared metadata, but
|
||||||
their values do not.
|
their values do not.
|
||||||
|
|||||||
@@ -14,10 +14,16 @@ that produce these outbound settings.
|
|||||||
|
|
||||||
Generation sends an HTTP `POST` with `Content-Type: application/json`.
|
Generation sends an HTTP `POST` with `Content-Type: application/json`.
|
||||||
Before the client is called, the engine resolves framework, backend, profile,
|
Before the client is called, the engine resolves framework, backend, profile,
|
||||||
and request values into one execution target. A non-empty endpoint from that
|
and request values into one execution target. Endpoint configuration is trimmed
|
||||||
target overrides the client's configured base URL. After trailing slashes are
|
and must be an absolute HTTP or HTTPS URL with a host and without user
|
||||||
removed, `/chat/completions` is appended. Generation fails before sending when
|
information, a query, or a fragment. A non-empty endpoint from the target
|
||||||
neither source supplies an endpoint.
|
overrides the client's configured base URL. The final selected endpoint is
|
||||||
|
validated again before transport.
|
||||||
|
|
||||||
|
The completion URL is composed through parsed URL path operations. Nested base
|
||||||
|
paths are retained, repeated trailing slashes are normalized, and the result
|
||||||
|
has exactly one appended `/chat/completions` suffix. Generation fails before
|
||||||
|
sending when neither source supplies a valid endpoint.
|
||||||
|
|
||||||
The target's backend ID is routing metadata for prepared values, results, and
|
The target's backend ID is routing metadata for prepared values, results, and
|
||||||
injected clients. The built-in client does not derive the URL from that ID and
|
injected clients. The built-in client does not derive the URL from that ID and
|
||||||
@@ -25,11 +31,13 @@ does not serialize it in the provider request.
|
|||||||
|
|
||||||
## Authentication
|
## Authentication
|
||||||
|
|
||||||
A non-empty API key supplied directly on the execution target takes
|
A usable API key supplied directly on the execution target takes precedence.
|
||||||
precedence. Otherwise, when an API-key environment-variable name is supplied,
|
Otherwise, when an API-key environment-variable name is supplied, the client
|
||||||
the client reads that variable and requires a non-empty value. The selected
|
reads and trims that variable. A bearer header is sent only when the resolved
|
||||||
key is sent as `Authorization: Bearer <key>`. No authorization header is sent
|
direct or environment credential is non-empty. When neither source is usable,
|
||||||
when neither mechanism is configured.
|
the client omits `Authorization` and handles the provider response normally.
|
||||||
|
An explicitly required target with no usable source is rejected before
|
||||||
|
transport.
|
||||||
|
|
||||||
The target contains the already resolved environment-variable name: an
|
The target contains the already resolved environment-variable name: an
|
||||||
explicit request override takes precedence over profile metadata, which takes
|
explicit request override takes precedence over profile metadata, which takes
|
||||||
@@ -47,6 +55,12 @@ Each ordinary message contains its `role` and string `content`. A
|
|||||||
cache-controlled message instead uses a text content block containing `type`,
|
cache-controlled message instead uses a text content block containing `type`,
|
||||||
`text`, and `cache_control`; an empty cache-control TTL is omitted.
|
`text`, and `cache_control`; an empty cache-control TTL is omitted.
|
||||||
|
|
||||||
|
Promptkit sends only `developer`, `system`, `user`, and `assistant` roles and
|
||||||
|
does so without provider-specific translation. Tool and deprecated function
|
||||||
|
payloads are outside this text-message contract. A backend or model that
|
||||||
|
rejects an otherwise supported role or context returns its ordinary provider
|
||||||
|
error, which follows the normal generation-error path.
|
||||||
|
|
||||||
The effective direct or prompt-rendered session ID is trimmed, limited to 256
|
The effective direct or prompt-rendered session ID is trimmed, limited to 256
|
||||||
Unicode code points, and sent when nonempty as top-level `session_id`. It is
|
Unicode code points, and sent when nonempty as top-level `session_id`. It is
|
||||||
never also sent as a session header.
|
never also sent as a session header.
|
||||||
@@ -59,7 +73,8 @@ The client conditionally includes:
|
|||||||
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly
|
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly
|
||||||
disabled reasoning setting is empty and therefore omitted; and
|
disabled reasoning setting is empty and therefore omitted; and
|
||||||
- `response_format` for JSON Schema structured output, including its name,
|
- `response_format` for JSON Schema structured output, including its name,
|
||||||
strict flag, and schema document.
|
strict flag, and schema document. Plain JSON validation does not add an
|
||||||
|
object-only response constraint.
|
||||||
|
|
||||||
The engine resolves backend, profile, and request extra-parameter maps by
|
The engine resolves backend, profile, and request extra-parameter maps by
|
||||||
whole-map replacement rather than key merging. The resulting effective map is
|
whole-map replacement rather than key merging. The resulting effective map is
|
||||||
@@ -82,13 +97,47 @@ request fields.
|
|||||||
|
|
||||||
## Response Handling
|
## Response Handling
|
||||||
|
|
||||||
Any 2xx response is decoded as an OpenAI-compatible chat response. The client
|
Any 2xx response body is limited to 16 MiB (16,777,216 bytes). A larger
|
||||||
returns the first choice's non-empty message content and maps prompt,
|
declared `Content-Length` is rejected before the body is read, and streamed,
|
||||||
completion, total, cached, and cache-write token counts.
|
chunked, or underreported bodies are read through the same bound with at most
|
||||||
|
one additional byte used to detect overflow. A body exactly at the limit is
|
||||||
|
allowed. The body is closed on every outcome and an oversized stream is not
|
||||||
|
drained.
|
||||||
|
|
||||||
Invalid JSON, absent choices, and empty first-choice content are malformed
|
The bounded body must contain exactly one OpenAI-compatible JSON response
|
||||||
responses. For a non-2xx status, the error includes the status code but never
|
object followed only by JSON whitespace and EOF. The client returns the first
|
||||||
the provider response body.
|
choice's explicitly present string message content, including an empty or
|
||||||
|
whitespace-only string, and maps prompt, completion, total, cached, and
|
||||||
|
cache-write token counts. Invalid or truncated JSON, trailing non-whitespace
|
||||||
|
data, a second JSON value, absent choices, missing content, `null` content,
|
||||||
|
non-string content, and size overflow are malformed responses and return no
|
||||||
|
partial result.
|
||||||
|
|
||||||
|
For a non-2xx status, Promptkit recognizes one JSON document with a top-level
|
||||||
|
object-valued `error` member. Its optional `message` and `type` fields must be
|
||||||
|
strings, and `code` may be a string or JSON number. Valid supported fields are
|
||||||
|
handled independently, numeric codes retain their JSON number text, and
|
||||||
|
unknown fields are ignored. Missing, invalid, malformed, or multiply framed
|
||||||
|
envelopes contribute no provider detail.
|
||||||
|
|
||||||
|
Non-success bodies have a 65,536-byte limit. A larger declared
|
||||||
|
`Content-Length` is not read; otherwise the client reads at most one additional
|
||||||
|
byte to detect streamed or underreported overflow. Empty, unreadable,
|
||||||
|
oversized, malformed, and unrecognized bodies retain only the received status.
|
||||||
|
The body is always closed and no oversized stream is drained beyond that probe.
|
||||||
|
|
||||||
|
Extracted strings are made valid UTF-8, trimmed, and converted to one line by
|
||||||
|
collapsing Unicode whitespace, control, and format-character runs. Blank
|
||||||
|
values are omitted. Codes and types longer than 256 Unicode code points are
|
||||||
|
omitted; messages longer than 4,096 code points are truncated at a code-point
|
||||||
|
boundary with an ellipsis inside the limit. Promptkit never exposes raw bodies,
|
||||||
|
headers, endpoints, credentials, request data, schemas, generated content, or
|
||||||
|
unsupported provider metadata through this handling.
|
||||||
|
|
||||||
|
An outbound `http.Client.Do` failure retains both Promptkit's request-failure
|
||||||
|
identity and the exact transport error for `errors.Is` and `errors.As` checks.
|
||||||
|
The rendered error does not include the selected endpoint, request headers,
|
||||||
|
request content, credentials, or provider body.
|
||||||
|
|
||||||
## Timeout And Cancellation
|
## Timeout And Cancellation
|
||||||
|
|
||||||
@@ -103,5 +152,7 @@ Timeouts are layered:
|
|||||||
timeout when the supplied value is not positive.
|
timeout when the supplied value is not positive.
|
||||||
|
|
||||||
The earliest applicable caller, generation, or transport deadline controls the
|
The earliest applicable caller, generation, or transport deadline controls the
|
||||||
request. Constructing the internal client does not mutate a supplied
|
request. Caller cancellation retains `context.Canceled`; caller, generation,
|
||||||
`http.Client`.
|
and whole-request timeout failures retain `context.DeadlineExceeded`, together
|
||||||
|
with the request-failure identity. Constructing the internal client does not
|
||||||
|
mutate a supplied `http.Client`.
|
||||||
|
|||||||
@@ -56,7 +56,10 @@ reserve another bounded slot.
|
|||||||
|
|
||||||
`NewClient` wraps the engine's selected internal model client after public
|
`NewClient` wraps the engine's selected internal model client after public
|
||||||
client adaptation or built-in client construction. Initial generation and the
|
client adaptation or built-in client construction. Initial generation and the
|
||||||
default repairer receive the same wrapper.
|
default repairer receive the same wrapper. Their requests retain the same
|
||||||
|
effective backend, credential, numeric-presence metadata, and structured-output
|
||||||
|
settings, so scheduling does not change provider omission semantics between
|
||||||
|
calls.
|
||||||
|
|
||||||
For each `Generate` call, the wrapper selects a pool from the request's
|
For each `Generate` call, the wrapper selects a pool from the request's
|
||||||
effective backend ID. An unlimited call passes directly to the next client. A
|
effective backend ID. An unlimited call passes directly to the next client. A
|
||||||
@@ -71,7 +74,9 @@ other backend IDs.
|
|||||||
|
|
||||||
The wrapper passes generation requests, responses, and collaborator errors
|
The wrapper passes generation requests, responses, and collaborator errors
|
||||||
through unchanged. It owns scheduling only; the concrete model client remains
|
through unchanged. It owns scheduling only; the concrete model client remains
|
||||||
responsible for provider transport behavior.
|
responsible for provider transport behavior. The runner, rather than the
|
||||||
|
capacity layer, sums all five usage fields from the initial response and every
|
||||||
|
completed repair response into the successful run result.
|
||||||
|
|
||||||
## Cancellation And Release
|
## Cancellation And Release
|
||||||
|
|
||||||
|
|||||||
@@ -21,20 +21,26 @@ uses internal domain values for rendered prompts, execution targets,
|
|||||||
structured output, responses, and token usage.
|
structured output, responses, and token usage.
|
||||||
|
|
||||||
The runner supplies a fully resolved target after applying backend, profile,
|
The runner supplies a fully resolved target after applying backend, profile,
|
||||||
and request precedence. The client uses its endpoint, credential metadata,
|
and request precedence, plus canonical provider-bound text messages. The
|
||||||
generation fields, and extra parameters. `BackendID` remains routing metadata
|
client uses its endpoint, credential metadata, generation fields, and extra
|
||||||
|
parameters. `BackendID` remains routing metadata
|
||||||
for the generation boundary and is not mapped into the provider payload.
|
for the generation boundary and is not mapped into the provider payload.
|
||||||
|
|
||||||
Construction validates the configured base URL and clones any supplied
|
Construction trims and validates a nonempty configured base URL and clones any
|
||||||
`http.Client` so Promptkit can apply its timeout default without mutating the
|
supplied `http.Client` so Promptkit can apply its timeout default without
|
||||||
caller's client. Generation then:
|
mutating the caller's client. An empty configured base remains valid because a
|
||||||
|
resolved request target may supply the endpoint. Generation then:
|
||||||
|
|
||||||
1. validates request-level timeout and endpoint requirements;
|
1. validates shared execution-setting invariants and the final selected base
|
||||||
|
endpoint;
|
||||||
2. maps the internal request into the OpenAI-compatible chat payload;
|
2. maps the internal request into the OpenAI-compatible chat payload;
|
||||||
3. validates and merges extra parameters;
|
3. validates and merges extra parameters;
|
||||||
4. resolves authentication;
|
4. composes `/chat/completions` through parsed URL path operations;
|
||||||
5. performs the outbound request under the applicable deadlines; and
|
5. resolves authentication;
|
||||||
6. decodes the first response choice and token usage.
|
6. performs the outbound request under the applicable deadlines; and
|
||||||
|
7. decodes one strictly framed, size-bounded successful response object and
|
||||||
|
maps its first choice and token usage, or decodes bounded structured
|
||||||
|
non-success detail.
|
||||||
|
|
||||||
`internal/llm` owns the set of reserved OpenAI-compatible request fields used
|
`internal/llm` owns the set of reserved OpenAI-compatible request fields used
|
||||||
when validating extra parameters. Backend registration consumes the same rule
|
when validating extra parameters. Backend registration consumes the same rule
|
||||||
@@ -50,12 +56,14 @@ the target, rendered messages, and structured-output constraint retained by
|
|||||||
executable preparation. Execution does not reopen or rerender consumer
|
executable preparation. Execution does not reopen or rerender consumer
|
||||||
sources.
|
sources.
|
||||||
|
|
||||||
Before backend admission, the runner rechecks that the frozen credential
|
Before backend admission, the runner rechecks a frozen credential
|
||||||
environment-variable name is available. The handle does not retain the
|
environment-variable name only when the target explicitly requires a
|
||||||
environment value; the model client resolves the value visible when generation
|
credential. The handle does not retain the environment value; the model client
|
||||||
begins. A direct request key remains in private execution state only until the
|
resolves the value visible when generation begins. For optional sources with no
|
||||||
claimed execution finishes or an unclaimed handle is discarded. Exact public
|
usable value, the built-in client omits `Authorization` and continues to the
|
||||||
ownership and redaction semantics belong to the
|
provider. A direct request key remains in private execution state only until
|
||||||
|
the claimed execution finishes or an unclaimed handle is discarded. Exact
|
||||||
|
public ownership and redaction semantics belong to the
|
||||||
[`PreparedExecution` GoDoc](../../prepared_execution.go).
|
[`PreparedExecution` GoDoc](../../prepared_execution.go).
|
||||||
|
|
||||||
## Failure Categories
|
## Failure Categories
|
||||||
@@ -63,11 +71,47 @@ ownership and redaction semantics belong to the
|
|||||||
The package preserves distinct error identities for invalid client
|
The package preserves distinct error identities for invalid client
|
||||||
configuration, invalid generation requests, request execution failures,
|
configuration, invalid generation requests, request execution failures,
|
||||||
non-success provider statuses, and malformed successful responses. Provider
|
non-success provider statuses, and malformed successful responses. Provider
|
||||||
response bodies are not included in non-success errors.
|
response bodies are never exposed in raw form through non-success errors.
|
||||||
|
|
||||||
Caller cancellation and deadline failures during the outbound request are
|
Invalid nonempty configured endpoints are configuration failures. A missing or
|
||||||
reported as request execution failures. The runner classifies these identities
|
invalid final selected endpoint is an invalid generation request and is
|
||||||
without depending on HTTP status mapping.
|
rejected before transport.
|
||||||
|
|
||||||
|
Authentication resolves a trimmed direct key before a trimmed configured
|
||||||
|
environment value. Optional missing, empty, or whitespace-only sources do not
|
||||||
|
block transport and produce no `Authorization` header. An explicitly required
|
||||||
|
target with no usable source is rejected before transport with the existing
|
||||||
|
invalid-request diagnostics.
|
||||||
|
|
||||||
|
Successful response bodies have a fixed 16 MiB limit enforced by declared
|
||||||
|
length and by reading at most one byte beyond the boundary. The decoder accepts
|
||||||
|
exactly one JSON object plus trailing whitespace and EOF. Size overflow,
|
||||||
|
truncation, malformed JSON, trailing data, and a second value are malformed
|
||||||
|
responses with no partial result or provider content in the error. Every body
|
||||||
|
is closed, and an unbounded oversized stream is not drained.
|
||||||
|
|
||||||
|
After framing succeeds, the first choice must contain an explicitly present
|
||||||
|
string `message.content`. The string is returned exactly, including empty or
|
||||||
|
whitespace-only content. Missing choices, missing or `null` content, and
|
||||||
|
non-string content are malformed responses. Output validation and correction
|
||||||
|
eligibility remain outside this package.
|
||||||
|
|
||||||
|
For a non-success response, `ProviderHTTPError` retains the HTTP status and
|
||||||
|
only normalized detail from the bounded recognized envelope. It retains
|
||||||
|
`ErrUnexpectedStatus` through unwrapping. The client owns response closure;
|
||||||
|
its bounded reader and parser never close or drain a body themselves. The root
|
||||||
|
facade converts this concrete internal error into the public
|
||||||
|
[`GenerationError`](../../generation_error.go), while arbitrary injected-client
|
||||||
|
errors continue through the ordinary generation-error mapping unchanged.
|
||||||
|
|
||||||
|
An `http.Client.Do` failure is represented by a redacting multi-cause error:
|
||||||
|
the package request-failure sentinel and the exact returned transport error are
|
||||||
|
both available through `errors.Is` and `errors.As`, while the rendered text
|
||||||
|
does not expose the endpoint, headers, request content, credential, transport
|
||||||
|
detail, or provider body. Caller cancellation retains `context.Canceled`;
|
||||||
|
caller deadlines, generation deadlines, and whole-request client timeouts
|
||||||
|
retain `context.DeadlineExceeded`. The runner adds its generation category
|
||||||
|
without discarding those identities or depending on HTTP status mapping.
|
||||||
|
|
||||||
## Test Ownership
|
## Test Ownership
|
||||||
|
|
||||||
@@ -75,7 +119,15 @@ The
|
|||||||
[OpenAI-compatible client tests](../../internal/llm/openai_compatible_client_test.go)
|
[OpenAI-compatible client tests](../../internal/llm/openai_compatible_client_test.go)
|
||||||
own configuration, client cloning, deterministic deadline precedence,
|
own configuration, client cloning, deterministic deadline precedence,
|
||||||
authentication, request and response mapping, malformed data, error identity,
|
authentication, request and response mapping, malformed data, error identity,
|
||||||
cancellation, and response-body suppression. The root transport contract test
|
cancellation, endpoint selection and composition, pre-transport rejection, and
|
||||||
also verifies that resolved backend settings reach this client without
|
bounded single-document successful-response framing, closure, and
|
||||||
serializing backend identity. All use local test servers or test transports;
|
response-body suppression. The focused
|
||||||
the default suite makes no live or paid provider requests.
|
[provider HTTP error tests](../../internal/llm/provider_http_error_test.go)
|
||||||
|
own envelope parsing, normalization, and bounded-reader cases; their
|
||||||
|
[transport tests](../../internal/llm/provider_http_error_transport_test.go)
|
||||||
|
own non-success response closure and integration. Root transport contract tests
|
||||||
|
own public `GenerationError` conversion, while also verifying that resolved
|
||||||
|
backend settings reach this client without serializing backend identity and
|
||||||
|
that ordinary-run cancellation retains its public generation and context
|
||||||
|
identities. All use local test servers or controlled test transports; the
|
||||||
|
default suite makes no live or paid provider requests.
|
||||||
|
|||||||
@@ -11,23 +11,23 @@ contributor workflow and validation.
|
|||||||
|
|
||||||
| Component | Implemented responsibility | References |
|
| Component | Implemented responsibility | References |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, and profile-inspection values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, typed capacity errors, public error mapping, and engine-local profile-source assembly including application fallbacks. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
|
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, and profile-inspection values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, typed capacity and generation error mapping, engine-local profile-source assembly including application fallbacks, and bounded output-repair assembly. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
|
||||||
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
||||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||||
| `internal/backend` | Constructs each engine's immutable registry from the built-in OpenRouter definition and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
|
| `internal/backend` | Constructs each engine's immutable registry from maintained definitions and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
|
||||||
|
| `internal/catalog` | Strictly validates imported immutable maintained backend and profile catalog assets before engine assembly uses them. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) |
|
||||||
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation. | [Domain declarations](../../internal/domain/domain.go) |
|
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||||
| `internal/jsonvalue` | Validates and deeply copies JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||||
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
|
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
|
||||||
| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
|
| `internal/profile` | Loads strictly decoded, locally validated execution profiles from filesystem and `fs.FS` sources, overlays raw sources with error-preserving fallback, and resolves inherited profiles. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go), [internal sources](sources.md#profiles-and-built-ins) |
|
||||||
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
|
||||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
||||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates frozen validation plans for prepared execution. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||||
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, generation, validation, capacity, and optional repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||||
|
|
||||||
The root package assembles these internal components without exposing their
|
The root package assembles these internal components without exposing their
|
||||||
representations. Consumers depend only on the root facade.
|
representations. Consumers depend only on the root facade.
|
||||||
|
|||||||
@@ -20,10 +20,13 @@ and override semantics consumed by the runner.
|
|||||||
profiles, backend resolution, artifacts, rendering, model generation, and
|
profiles, backend resolution, artifacts, rendering, model generation, and
|
||||||
validation. The root engine supplies one immutable registry containing the
|
validation. The root engine supplies one immutable registry containing the
|
||||||
built-in backend and validated consumer additions, one engine-local run
|
built-in backend and validated consumer additions, one engine-local run
|
||||||
admitter, and a model client wrapped by the same capacity manager. Schema
|
admitter, and a model client wrapped by the same capacity manager. Validation
|
||||||
documents are loaded through the validator's optional schema-loader interface.
|
plans and provider-facing schema metadata come from the validator's preparation
|
||||||
An output repairer can be injected internally, but the ordinary runner
|
interface.
|
||||||
constructor does not enable one.
|
The root engine supplies one default output repairer through the explicit
|
||||||
|
runner constructor, using the same capacity-wrapped client as initial
|
||||||
|
generation. The no-repair runner constructor remains available for focused
|
||||||
|
internal callers and tests.
|
||||||
|
|
||||||
Each invocation carries its state in request, prepared-run, and result values.
|
Each invocation carries its state in request, prepared-run, and result values.
|
||||||
The runner has no durable run or session store.
|
The runner has no durable run or session store.
|
||||||
@@ -69,17 +72,20 @@ performs only the work needed to validate routing and admission:
|
|||||||
5. resolve application-neutral defaults, backend defaults, profile values,
|
5. resolve application-neutral defaults, backend defaults, profile values,
|
||||||
and explicit request overrides in that order;
|
and explicit request overrides in that order;
|
||||||
6. validate endpoint, model, numeric overrides, and credential requirements;
|
6. validate endpoint, model, numeric overrides, and credential requirements;
|
||||||
7. resolve the effective output contract without loading its schema; and
|
7. resolve and validate the effective output contract without loading its
|
||||||
|
schema; and
|
||||||
8. retain the definition, source identities, effective settings, output
|
8. retain the definition, source identities, effective settings, output
|
||||||
contract, and preparation start time in invocation-local state.
|
contract, and preparation start time in invocation-local state.
|
||||||
|
|
||||||
The completion phase consumes that state without reloading the prompt,
|
The completion phase consumes that state without reloading the prompt,
|
||||||
profile, or backend:
|
profile, or backend:
|
||||||
|
|
||||||
1. load structured-output schema metadata when required;
|
1. create one operation-local validation plan and derive structured-output
|
||||||
|
schema metadata from it when required;
|
||||||
2. load and hash input artifacts;
|
2. load and hash input artifacts;
|
||||||
3. render messages and the prompt-defined session;
|
3. render messages and the prompt-defined session;
|
||||||
4. apply any direct session ID;
|
4. apply any direct session ID, then append already-normalized request messages
|
||||||
|
after the rendered definition messages;
|
||||||
5. hash the effective rendered prompt; and
|
5. hash the effective rendered prompt; and
|
||||||
6. construct the prepared value and preparation timing.
|
6. construct the prepared value and preparation timing.
|
||||||
|
|
||||||
@@ -87,7 +93,10 @@ profile, or backend:
|
|||||||
`Run` performs backend admission between the phases. This structure preserves
|
`Run` performs backend admission between the phases. This structure preserves
|
||||||
one execution-precedence and error-ordering implementation while allowing a
|
one execution-precedence and error-ordering implementation while allowing a
|
||||||
full backend pool to reject work before expensive schema, artifact, and
|
full backend pool to reject work before expensive schema, artifact, and
|
||||||
rendering operations.
|
rendering operations. `Prepare` discards the plan after returning its public
|
||||||
|
metadata. `Run` retains the plan through initial and repaired-output validation
|
||||||
|
and discards it when the operation ends. Prepared execution stores the same
|
||||||
|
kind of plan only in its private payload.
|
||||||
|
|
||||||
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
||||||
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
||||||
@@ -105,7 +114,10 @@ the prompt session template, and is applied after ordinary message rendering.
|
|||||||
A blank direct value retains prompt-template behavior. The runner clears the
|
A blank direct value retains prompt-template behavior. The runner clears the
|
||||||
template only on a value copy of the definition, so the definition hash always
|
template only on a value copy of the definition, so the definition hash always
|
||||||
describes the original source while the rendered-prompt hash includes the
|
describes the original source while the rendered-prompt hash includes the
|
||||||
effective direct or rendered session.
|
effective direct or rendered session and the complete effective message
|
||||||
|
sequence. The rendered-prompt hash uses a versioned, length-framed SHA-256
|
||||||
|
encoding of that session, every role and content value, and cache-control
|
||||||
|
presence and values; its hexadecimal value is opaque.
|
||||||
|
|
||||||
The registry is read-only after engine construction. Concurrent `Prepare` and
|
The registry is read-only after engine construction. Concurrent `Prepare` and
|
||||||
`Run` calls resolve independent defensive backend values and keep all
|
`Run` calls resolve independent defensive backend values and keep all
|
||||||
@@ -118,8 +130,17 @@ its `RunAdmitter` to reserve capacity for the effective backend ID. A nil
|
|||||||
admitter is an internal unlimited fallback. After successful admission, `Run`
|
admitter is an internal unlimited fallback. After successful admission, `Run`
|
||||||
immediately defers the returned release function, performs the completion
|
immediately defers the returned release function, performs the completion
|
||||||
phase, makes one initial generation call, builds the named output artifact,
|
phase, makes one initial generation call, builds the named output artifact,
|
||||||
and validates that artifact. Invalid generated content remains a validation
|
and validates that artifact with the plan compiled during completion. Invalid
|
||||||
result; an inability to perform validation is an operational error.
|
generated content remains a validation result; an inability to generate or
|
||||||
|
perform validation is an operational error.
|
||||||
|
|
||||||
|
Validation preparation and execution honor cancellation at every
|
||||||
|
Promptkit-controlled boundary and do not publish a partial plan or result.
|
||||||
|
Schema reads are bounded and context-checked between chunks; JSON decoding,
|
||||||
|
schema compilation, and schema execution are checked immediately before and
|
||||||
|
after their synchronous calls. Promptkit does not move arbitrary filesystem or
|
||||||
|
JSON Schema work to background goroutines, so an already-blocked dependency
|
||||||
|
method must return before cancellation can take precedence over its outcome.
|
||||||
|
|
||||||
The admission lease covers completion-phase preparation, initial generation,
|
The admission lease covers completion-phase preparation, initial generation,
|
||||||
validation, every repair, and every exit. It bounds accepted work without
|
validation, every repair, and every exit. It bounds accepted work without
|
||||||
@@ -127,22 +148,38 @@ serializing preparation or validation behind the active-generation limit.
|
|||||||
The wrapped model client separately acquires a FIFO active permit only around
|
The wrapped model client separately acquires a FIFO active permit only around
|
||||||
each actual generation call.
|
each actual generation call.
|
||||||
|
|
||||||
When an internal repairer is present, a JSON or JSON Schema content failure can
|
After a failed `basic`, JSON, or JSON Schema validation with a positive frozen
|
||||||
trigger bounded repair attempts. Repair receives the effective execution
|
budget, the installed repairer can make a bounded corrective call. Each request
|
||||||
target and session ID, validation errors, prior output, and structured-output
|
starts with a fresh copy of the complete effective message sequence (the
|
||||||
specification. The default repairer uses the same wrapped client as initial
|
configured prefix followed by the request suffix), includes only the latest
|
||||||
|
nonempty candidate as an assistant message, and appends one corrective user
|
||||||
|
message. Empty candidates omit that assistant message. The
|
||||||
|
correction carries validation diagnostics as JSON data bounded to 64 KiB; the
|
||||||
|
full diagnostics remain in the validation result.
|
||||||
|
|
||||||
|
Repair receives the effective execution target, explicit numeric-presence bits,
|
||||||
|
credential, backend identity, session ID, and structured-output specification.
|
||||||
|
The same request constructor supplies those common fields to initial and repair
|
||||||
|
generation. The default repairer uses the same wrapped client as initial
|
||||||
generation, so each repair reacquires the selected backend's active permit
|
generation, so each repair reacquires the selected backend's active permit
|
||||||
while remaining inside its original admission lease. Repair never performs a
|
while remaining inside its original admission lease. Repair never performs a
|
||||||
second bounded admission. This capability remains internal and is not a public
|
second bounded admission, and repaired outputs use the operation's existing
|
||||||
option.
|
validation plan. The runner stops at the first valid candidate, sums completed
|
||||||
|
generation usage, reports calls actually started, and returns the final failed
|
||||||
|
validation result on exhaustion. A repair generation failure follows the
|
||||||
|
ordinary generation-error category rather than becoming a validation error.
|
||||||
|
|
||||||
A successful result includes the output artifact and raw output, validation
|
A successful result includes the output artifact and raw output, validation
|
||||||
state, effective session ID, prompt and rendered-prompt hashes, selected
|
state, effective session ID, prompt and rendered-prompt hashes, selected
|
||||||
profile and backend, effective settings, input hashes, token usage, a generated
|
profile and backend, effective settings, input hashes, token usage, a generated
|
||||||
run identifier, and UTC timing. The same effective session reaches initial
|
run identifier, and UTC timing. The same effective session reaches initial
|
||||||
generation and any repair attempt through the rendered prompt. The same
|
generation and any repair attempt through the rendered prompt. The same
|
||||||
effective target, including backend identity, reaches generation and any
|
effective target and presence metadata, including backend identity and direct
|
||||||
repair attempt.
|
credential during execution, reaches generation and every repair attempt.
|
||||||
|
Result usage is the field-wise sum of all five usage values from the initial
|
||||||
|
response and every completed repair response. Final raw output, artifact, and
|
||||||
|
validation state still come from the last candidate. A repair error returns no
|
||||||
|
partial run result or partial usage.
|
||||||
|
|
||||||
## Failure Categories
|
## Failure Categories
|
||||||
|
|
||||||
@@ -165,7 +202,8 @@ category. Deferred release restores the admission lease on preparation,
|
|||||||
generation, validation, repair, and cancellation failures.
|
generation, validation, repair, and cancellation failures.
|
||||||
|
|
||||||
Other context cancellation propagates through the invoked collaborator and is
|
Other context cancellation propagates through the invoked collaborator and is
|
||||||
classified by the owning operation.
|
classified by the owning operation. In particular, cancellation observed by
|
||||||
|
validation retains the context identity through the validation error category.
|
||||||
An overlong direct session is an invalid request before source loading, while
|
An overlong direct session is an invalid request before source loading, while
|
||||||
an invalid or overlong prompt session template remains a prompt-render failure.
|
an invalid or overlong prompt session template remains a prompt-render failure.
|
||||||
An unknown selected backend, or a selected backend with no configured resolver,
|
An unknown selected backend, or a selected backend with no configured resolver,
|
||||||
@@ -177,8 +215,9 @@ The [runner tests](../../internal/usecase/runner_test.go) own preparation order,
|
|||||||
selection and override precedence, the two-phase boundary, early admission,
|
selection and override precedence, the two-phase boundary, early admission,
|
||||||
lease lifetime and release, direct-session resolution, schema-before-generation
|
lease lifetime and release, direct-session resolution, schema-before-generation
|
||||||
behavior, hashing, generation and validation outcomes, backend propagation,
|
behavior, hashing, generation and validation outcomes, backend propagation,
|
||||||
bounded repair, shared initial/repair capacity, credentials and redaction,
|
bounded repair progression, initial/repair request parity, cumulative usage,
|
||||||
error categories, artifact metadata, usage, and timing. The
|
shared initial/repair capacity, credentials and redaction, error categories,
|
||||||
|
artifact metadata, and timing. The
|
||||||
[capacity subsystem document](capacity.md) identifies the focused pool,
|
[capacity subsystem document](capacity.md) identifies the focused pool,
|
||||||
waiter, and wrapped-client tests.
|
waiter, and wrapped-client tests.
|
||||||
|
|
||||||
|
|||||||
@@ -12,9 +12,27 @@ validation modes, built-in catalog, and source precedence.
|
|||||||
|
|
||||||
## Prompt Definitions
|
## Prompt Definitions
|
||||||
|
|
||||||
`internal/promptdef` discovers YAML deterministically, decodes and validates
|
`internal/promptdef` uses one source-neutral flow for prompt selection and
|
||||||
definitions, selects an ID and optional version, and resolves file-backed
|
normalization. That flow scans normalized YAML ID and version metadata,
|
||||||
message content within the selected operating-system or `fs.FS` source.
|
requires one strictly decoded document per file, classifies errors for the
|
||||||
|
selected definition, detects duplicates, and normalizes the exact match.
|
||||||
|
Small operating-system and `fs.FS` adapters own discovery, byte reads, display
|
||||||
|
paths, content opening, and root containment. Each lookup remains a
|
||||||
|
point-in-time scan: definitions and catalogs are not cached, and file-backed
|
||||||
|
message content is opened only for the exact selected candidate.
|
||||||
|
|
||||||
|
Message roles are normalized through the shared domain owner by trimming
|
||||||
|
Unicode whitespace and lowercasing. Only `developer`, `system`, `user`, and
|
||||||
|
`assistant` are published; invalid roles remain selected prompt-definition
|
||||||
|
failures rather than becoming request errors. Cache-control metadata uses the
|
||||||
|
same shared domain normalization and defensive-copy rule.
|
||||||
|
|
||||||
|
Operating-system sources enforce containment against canonical roots and
|
||||||
|
targets so symlinks cannot escape. Injected `fs.FS` sources enforce containment
|
||||||
|
in their clean relative path namespace. A single-file source uses the selected
|
||||||
|
prompt file's containing directory as its root. Every content path must be
|
||||||
|
relative and is opened from its exact parsed text after a separate blank check;
|
||||||
|
contained parent components and whitespace-bearing names remain valid.
|
||||||
|
|
||||||
Exact prompt inspection performs one point-in-time lookup through that same
|
Exact prompt inspection performs one point-in-time lookup through that same
|
||||||
repository and validates referenced message content before returning declared
|
repository and validates referenced message content before returning declared
|
||||||
@@ -27,39 +45,78 @@ duplicate detection, and source containment:
|
|||||||
|
|
||||||
## Profiles And Built-Ins
|
## Profiles And Built-Ins
|
||||||
|
|
||||||
`internal/profile` loads and validates execution profiles from an
|
`internal/profile` loads, locally validates, overlays, and resolves execution
|
||||||
operating-system filesystem or an `fs.FS`. Its overlay repository consults the
|
profiles from an operating-system filesystem or an `fs.FS`. A file contains
|
||||||
next repository only when the higher-precedence repository reports that a
|
exactly one YAML document and its trimmed YAML `id` is its only selection
|
||||||
profile is absent. Strict YAML decoding recognizes the optional `backend`
|
identity; filenames do not confer authority. Each point lookup reads discovered
|
||||||
field, trims its value, and requires a model plus at least one non-blank
|
files once for their metadata and reuses the selected file's bytes for strict
|
||||||
backend or endpoint. Loading does not check registry membership because the
|
decoding; unrelated profiles are not fully decoded. Strict selected decoding
|
||||||
available registry belongs to the assembled engine; the runner checks
|
recognizes `base_profile` and the optional `backend` field, trims their values,
|
||||||
membership during preparation and exact profile inspection.
|
and permits inherited target fields only when a base is named. File-backed
|
||||||
|
`extra_params` values are validated and defensively copied through the shared
|
||||||
|
bounded JSON-value owner before a profile is published. OpenAI-compatible
|
||||||
|
reserved-field policy remains with the model-client and backend-registry owners.
|
||||||
|
|
||||||
The root engine assembles profile repositories in precedence order: in-memory
|
`LoadFSRepository` is the eager immutable loading boundary for internal
|
||||||
profiles, one ordinary configured source, an application fallback source, then
|
catalog consumers. It discovers and strictly validates every raw profile once,
|
||||||
the embedded built-in catalog. An explicit file or `fs.FS` profile source
|
preserves safe source metadata including explicitly present YAML fields, and
|
||||||
replaces `Config.ProfileDir` within the ordinary configured-source category.
|
publishes independently copied values from memory. It does not resolve profile
|
||||||
|
inheritance. Configured consumer sources continue to use the lazy point lookup
|
||||||
|
repositories described above.
|
||||||
|
|
||||||
Exact profile inspection performs one point-in-time lookup through those
|
`internal/catalog` validates the imported immutable OpenRouter and
|
||||||
profile sources and checks the resolved target without reading prompt, input,
|
Rakestrawhome asset modules as one private adapter boundary. It enforces their
|
||||||
|
manifest, layout, profile ownership, inheritance, and secret-safety rules
|
||||||
|
before returning raw catalog sources. Root assembly uses those validated
|
||||||
|
catalogs as the maintained lowest-precedence profile source.
|
||||||
|
|
||||||
|
The overlay repository consults the next repository only when the
|
||||||
|
higher-precedence repository reports that a profile is absent. A reliably
|
||||||
|
selected malformed profile stops fallback, while an unrelated malformed file
|
||||||
|
does not become authoritative through its filename. Loading does not check
|
||||||
|
backend registry membership because the available registry belongs to the
|
||||||
|
assembled engine; the runner checks membership during preparation and exact
|
||||||
|
profile inspection.
|
||||||
|
|
||||||
|
The root engine assembles one raw composite catalog in precedence order:
|
||||||
|
in-memory profiles, one ordinary configured source, an application fallback
|
||||||
|
source, then the maintained external catalog. An explicit file or `fs.FS` profile
|
||||||
|
source replaces `Config.ProfileDir` within the ordinary configured-source
|
||||||
|
category. One outer resolving repository wraps that complete raw catalog, so
|
||||||
|
each base lookup observes the same precedence and shadowing rules.
|
||||||
|
|
||||||
|
The resolving repository traverses every selected chain afresh, retains no
|
||||||
|
cache, detects cycles, limits a chain to 32 profiles, merges root-to-leaf into a
|
||||||
|
new caller-owned value, and validates the final target before publishing it. It
|
||||||
|
does not check backend registry membership. Exact `base_profile` syntax, merge
|
||||||
|
rules, and consumer-visible failure behavior belong to the [framework format
|
||||||
|
reference](../formats.md#profile-inheritance).
|
||||||
|
|
||||||
|
Exact profile inspection performs one point-in-time resolved lookup through
|
||||||
|
those profile sources and checks the final target without reading prompt, input,
|
||||||
or schema sources. It does not retain that lookup for a later execution.
|
or schema sources. It does not retain that lookup for a later execution.
|
||||||
|
Prepared execution instead freezes the fully resolved target; a later ordinary
|
||||||
|
operation performs a fresh traversal.
|
||||||
|
|
||||||
`internal/profile/builtin` embeds the maintained built-in profile catalog.
|
The maintained external catalog provides every built-in profile and its
|
||||||
Every embedded profile selects `openrouter` and inherits its endpoint and
|
matching backend definition. Profile loading and overlay behavior are owned by the
|
||||||
credential environment-variable name from the built-in backend registry rather
|
[profile repository tests](../../internal/profile/repository_test.go), while
|
||||||
than repeating those values. Profile loading and overlay behavior are owned by
|
catalog completeness, backend selection, and duplicate IDs are owned by the
|
||||||
the [profile repository tests](../../internal/profile/repository_test.go),
|
[catalog adapter tests](../../internal/catalog/catalog_test.go).
|
||||||
while catalog completeness, the backend-selection invariant, and duplicate IDs
|
|
||||||
are owned by the
|
|
||||||
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
|
|
||||||
|
|
||||||
## Ordinary Artifacts
|
## Ordinary Artifacts
|
||||||
|
|
||||||
`internal/artifact` resolves inline references and unrestricted,
|
`internal/artifact` accepts explicitly typed inline references even when their
|
||||||
caller-selected file paths. It copies content into an artifact, records
|
body is empty. It also resolves unrestricted, caller-selected paths only when
|
||||||
metadata and a content hash, applies a content-type fallback, and honors
|
they identify regular operating-system files, checking that condition before
|
||||||
context cancellation.
|
and after opening the file. It copies content into an artifact, records
|
||||||
|
metadata and an opaque content-equality value, and applies a content-type
|
||||||
|
fallback.
|
||||||
|
|
||||||
|
Regular files are read synchronously in bounded chunks. Cancellation is
|
||||||
|
checked before opening, before and after every read, and before publishing the
|
||||||
|
artifact, so a canceled read never publishes partial content. The ordinary
|
||||||
|
reader does not detach file reads into background goroutines.
|
||||||
|
|
||||||
This ordinary reader does not implement an inbound HTTP security boundary. In
|
This ordinary reader does not implement an inbound HTTP security boundary. In
|
||||||
particular, it does not constrain files to an application root or impose an
|
particular, it does not constrain files to an application root or impose an
|
||||||
@@ -71,7 +128,17 @@ implemented reader behavior and failures.
|
|||||||
## Rendering
|
## Rendering
|
||||||
|
|
||||||
`internal/prompt` renders definition messages as Go templates using named
|
`internal/prompt` renders definition messages as Go templates using named
|
||||||
artifacts and variables. It carries message roles, session IDs, and cache
|
artifacts and variables. Within one render, each referenced artifact body is
|
||||||
|
converted to text lazily and cached by input name for reuse across the session
|
||||||
|
and every message; the cache is not shared across renders. Conversion uses
|
||||||
|
bounded chunks and preserves the artifact bytes exactly.
|
||||||
|
|
||||||
|
Session and message parsing and execution remain synchronous. The renderer
|
||||||
|
checks cancellation before and after each parse and execution boundary,
|
||||||
|
between artifact conversion chunks, around each message, and before publishing
|
||||||
|
the complete prompt. It cannot interrupt template work already in progress and
|
||||||
|
never publishes a partial prompt after observing cancellation. It validates and
|
||||||
|
canonicalizes message roles before carrying roles, session IDs, and cache
|
||||||
control into the rendered prompt. The
|
control into the rendered prompt. The
|
||||||
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
|
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
|
||||||
|
|
||||||
@@ -82,22 +149,35 @@ filesystem or an `fs.FS`. Invalid generated content is returned as a validation
|
|||||||
result; inability to load, register, or compile a schema is an operational
|
result; inability to load, register, or compile a schema is an operational
|
||||||
error.
|
error.
|
||||||
|
|
||||||
For executable preparation, the built-in validators create a frozen validation
|
Every preparation operation creates one operation-local validation plan. None,
|
||||||
plan. None, basic, and JSON modes retain the effective output contract without
|
basic, and JSON modes retain the effective output contract without source
|
||||||
source access. JSON Schema mode loads the root document, resolves and compiles
|
access. JSON Schema mode loads the root document once, resolves and compiles
|
||||||
every transitive reference during preparation, and retains the compiled
|
each transitive reference, and retains the compiled validator. Schema compiler
|
||||||
validator. The provider-facing structured-output metadata uses that same
|
resources use canonical escaped file or private-scheme URLs; loaders decode
|
||||||
captured root document.
|
their paths once and enforce the configured source boundary. The
|
||||||
|
provider-facing structured-output metadata uses the root document captured by
|
||||||
|
the same plan.
|
||||||
|
|
||||||
`PrepareExecution` also completes prompt and profile selection, artifact
|
Schema preparation and execution remain synchronous. Promptkit checks
|
||||||
loading and hashing, session and message rendering, and target resolution.
|
cancellation before and after source resolution, JSON decoding, compilation,
|
||||||
`RunPrepared` uses the retained source-derived state and validation plan; it
|
and validation, and between bounded schema-read chunks. Once cancellation is
|
||||||
does not reopen prompt, profile, input, or schema sources and does not rerender
|
observed it returns the context error without publishing a partial plan or
|
||||||
the request. By contrast, ordinary `Prepare` produces a preparation value only:
|
validation result, even when a compiler or validator has just returned a
|
||||||
a later `Run` performs its own source resolution and preparation.
|
different error or a successful result. An `fs.FS` method or JSON Schema
|
||||||
|
dependency call already in progress cannot be preempted; Promptkit waits for
|
||||||
|
that call to return and then gives cancellation precedence. Validation does
|
||||||
|
not detach dependency work into background goroutines.
|
||||||
|
|
||||||
|
`Prepare` discards its validation plan after returning metadata. `Run` retains
|
||||||
|
its plan for initial and repaired-output validation, then discards it with the
|
||||||
|
operation. `PrepareExecution` retains the plan in its private frozen payload;
|
||||||
|
`RunPrepared` uses that plan without reopening prompt, profile, input, or
|
||||||
|
schema sources or rerendering the request. A later ordinary `Run` always
|
||||||
|
performs fresh source resolution and preparation.
|
||||||
|
|
||||||
The [validator tests](../../internal/validate/standard_validator_test.go) own
|
The [validator tests](../../internal/validate/standard_validator_test.go) own
|
||||||
basic, JSON, JSON Schema, source resolution, schema loading, compilation,
|
basic, JSON, JSON Schema, source resolution, schema loading, compilation,
|
||||||
frozen-reference behavior, and content-failure behavior. Prepared execution
|
frozen-reference behavior, content-failure behavior, and the synchronous
|
||||||
|
cancellation boundary. Prepared execution
|
||||||
orchestration is owned by the
|
orchestration is owned by the
|
||||||
[use-case tests](../../internal/usecase/prepared_execution_test.go).
|
[use-case tests](../../internal/usecase/prepared_execution_test.go).
|
||||||
|
|||||||
@@ -19,24 +19,24 @@ results, public values, extension interfaces, profiles, and error sentinels.
|
|||||||
|
|
||||||
The implemented internal components consist of:
|
The implemented internal components consist of:
|
||||||
|
|
||||||
- `internal/domain`, which owns framework data values shared by later internal
|
- `internal/domain`, which owns framework data values and source-neutral
|
||||||
components;
|
invariants shared by later internal components;
|
||||||
- `internal/backend`, which owns validated immutable OpenAI-compatible backend
|
- `internal/backend`, which owns validated immutable OpenAI-compatible backend
|
||||||
definitions and the built-in OpenRouter definition;
|
definitions;
|
||||||
- `internal/capacity`, which owns engine-local bounded run admission and
|
- `internal/capacity`, which owns engine-local bounded run admission and
|
||||||
model-generation scheduling for limited backends;
|
model-generation scheduling for limited backends;
|
||||||
- `internal/defaults`, which owns application-neutral framework defaults and
|
- `internal/defaults`, which owns application-neutral framework defaults and
|
||||||
constructs the default execution target;
|
constructs the default execution target;
|
||||||
- `internal/filecatalog`, which discovers YAML files and provides source-path
|
- `internal/filecatalog`, which discovers YAML files and provides source-path
|
||||||
helpers for filesystem and `fs.FS` consumers;
|
helpers for filesystem and `fs.FS` consumers;
|
||||||
|
- `internal/catalog`, which validates immutable external maintained-catalog
|
||||||
|
assets before they are eligible for engine assembly;
|
||||||
- `internal/jsonvalue`, which validates and defensively copies JSON-compatible
|
- `internal/jsonvalue`, which validates and defensively copies JSON-compatible
|
||||||
extra-parameter trees;
|
extra-parameter trees;
|
||||||
- `internal/promptdef`, which loads and validates prompt definitions from
|
- `internal/promptdef`, which loads and validates prompt definitions from
|
||||||
filesystem and `fs.FS` sources;
|
filesystem and `fs.FS` sources;
|
||||||
- `internal/profile`, which loads, validates, and overlays execution profiles
|
- `internal/profile`, which loads, validates, and overlays execution profiles
|
||||||
from filesystem and `fs.FS` sources;
|
from filesystem and `fs.FS` sources;
|
||||||
- `internal/profile/builtin`, which embeds the built-in execution profile
|
|
||||||
catalog;
|
|
||||||
- `internal/prompt`, which renders prompt messages from Go templates;
|
- `internal/prompt`, which renders prompt messages from Go templates;
|
||||||
- `internal/artifact`, which resolves ordinary inline and unrestricted
|
- `internal/artifact`, which resolves ordinary inline and unrestricted
|
||||||
caller-selected file references;
|
caller-selected file references;
|
||||||
@@ -54,13 +54,15 @@ packages or participate in internal assembly.
|
|||||||
The root facade assembles one immutable backend registry, one capacity manager,
|
The root facade assembles one immutable backend registry, one capacity manager,
|
||||||
the internal repositories, renderer, validator, outbound client, and use-case
|
the internal repositories, renderer, validator, outbound client, and use-case
|
||||||
runner while translating public values and errors at the library boundary. The
|
runner while translating public values and errors at the library boundary. The
|
||||||
registry contains built-ins plus validated engine-scoped consumer additions.
|
registry contains validated maintained definitions plus engine-scoped consumer
|
||||||
|
additions.
|
||||||
The facade constructs the capacity manager from the registry's immutable
|
The facade constructs the capacity manager from the registry's immutable
|
||||||
policy snapshot, wraps the selected built-in or injected model client, and
|
policy snapshot, wraps the selected built-in or injected model client, and
|
||||||
supplies bounded admission to the runner. The defaults and renderer depend on
|
supplies bounded admission to the runner. The defaults and renderer depend on
|
||||||
the domain model. Prompt-definition and profile repositories use the domain
|
the domain model. Prompt-definition and profile repositories use the domain
|
||||||
model, file catalog, and YAML decoder. The built-in profile repository supplies
|
model, file catalog, and YAML decoder. The catalog adapter validates imported
|
||||||
an embedded `fs.FS` to the profile package. Artifact reading uses the domain
|
immutable maintained data before root assembly supplies it to the profile
|
||||||
|
package. Artifact reading uses the domain
|
||||||
model and application-neutral defaults. Validation uses the domain model, file
|
model and application-neutral defaults. Validation uses the domain model, file
|
||||||
catalog, and JSON Schema implementation. The model client uses the domain
|
catalog, and JSON Schema implementation. The model client uses the domain
|
||||||
model, application-neutral defaults, and an injected or standard-library HTTP
|
model, application-neutral defaults, and an injected or standard-library HTTP
|
||||||
@@ -92,6 +94,13 @@ coordinates internal components and adapts the supported public extension
|
|||||||
interfaces to narrow internal abstractions. Internal components must not depend
|
interfaces to narrow internal abstractions. Internal components must not depend
|
||||||
on consumers or on Scriptorium.
|
on consumers or on Scriptorium.
|
||||||
|
|
||||||
|
`internal/domain` owns source-neutral invariants for values shared across
|
||||||
|
multiple input and execution boundaries, including execution-setting bounds,
|
||||||
|
OpenAI-compatible base endpoints, session identifiers, and output-contract
|
||||||
|
legality. Callers retain source parsing, required-field rules, other
|
||||||
|
source-specific normalization, defaulting, error classification, and policy
|
||||||
|
specific to their own boundary.
|
||||||
|
|
||||||
## Repository And Consumer Boundary
|
## Repository And Consumer Boundary
|
||||||
|
|
||||||
Scriptorium is a downstream application that consumes Promptkit through
|
Scriptorium is a downstream application that consumes Promptkit through
|
||||||
|
|||||||
@@ -50,19 +50,18 @@ Examples of appropriate seams include clocks, randomness, subprocesses, remote A
|
|||||||
## Test execution requirements
|
## Test execution requirements
|
||||||
|
|
||||||
Promptkit currently uses maintainer-run validation rather than hosted CI.
|
Promptkit currently uses maintainer-run validation rather than hosted CI.
|
||||||
Maintainers run the repository-documented test, vet, build, formatting,
|
Maintainers run the complete local workflow in the
|
||||||
documentation-link, and repository-hygiene checks before accepting changes.
|
[development guide](../development.md#maintainer-validation) before accepting
|
||||||
|
changes. That guide is the canonical owner of exact commands, formatting,
|
||||||
|
documentation-link validation, and repository-hygiene checks.
|
||||||
Introducing hosted CI later would supplement, not silently redefine, this
|
Introducing hosted CI later would supplement, not silently redefine, this
|
||||||
documented validation model.
|
documented validation model.
|
||||||
|
|
||||||
The complete test sequence includes ordinary and race-enabled package tests.
|
Maintainer validation must include ordinary and race-enabled package tests,
|
||||||
The maintained offline consumer workflow is also run from the repository root:
|
static analysis, a complete build, and execution of both maintained offline
|
||||||
|
consumer examples. The preparation example protects assembled preparation and
|
||||||
```sh
|
inspection behavior. The execution example separately protects assembled
|
||||||
go test ./...
|
`Run`, injected-client, validation, usage, and result behavior.
|
||||||
go test -race ./...
|
|
||||||
go run ./examples/go-library/prepare
|
|
||||||
```
|
|
||||||
|
|
||||||
Tests in the default suite must be deterministic, offline, and independent of
|
Tests in the default suite must be deterministic, offline, and independent of
|
||||||
real credentials. They must not invoke paid APIs, use live network
|
real credentials. They must not invoke paid APIs, use live network
|
||||||
@@ -88,8 +87,9 @@ Use each test type where it protects a distinct risk:
|
|||||||
interaction, while replacing live or nondeterministic external boundaries.
|
interaction, while replacing live or nondeterministic external boundaries.
|
||||||
- External-package root tests exercise the public facade as a Go consumer,
|
- External-package root tests exercise the public facade as a Go consumer,
|
||||||
while internal package tests own focused implementation behavior.
|
while internal package tests own focused implementation behavior.
|
||||||
- The maintained offline preparation example protects one representative
|
- The maintained offline preparation and execution examples protect distinct
|
||||||
assembled consumer workflow without contacting a model provider.
|
representative assembled consumer workflows without contacting a model
|
||||||
|
provider.
|
||||||
- Fixtures should be minimal, synthetic, versioned with the behavior they
|
- Fixtures should be minimal, synthetic, versioned with the behavior they
|
||||||
exercise, and free of credentials or private data.
|
exercise, and free of credentials or private data.
|
||||||
- Golden files are appropriate only when the complete output is intentionally
|
- Golden files are appropriate only when the complete output is intentionally
|
||||||
|
|||||||
@@ -106,48 +106,12 @@ gitea.maximumdirect.net/eric/promptkit 1.25.5
|
|||||||
promptkit gitea.maximumdirect.net/eric/promptkit
|
promptkit gitea.maximumdirect.net/eric/promptkit
|
||||||
```
|
```
|
||||||
|
|
||||||
Run the complete maintainer validation required by the
|
As a release prerequisite, run the complete
|
||||||
[development guide](development.md):
|
[maintainer validation workflow](development.md#maintainer-validation) against
|
||||||
|
the clean candidate. Do not substitute a partial command list: the development
|
||||||
```sh
|
guide owns the tests, race checks, analysis, build, both offline examples,
|
||||||
go test ./...
|
formatting, Markdown links, generated-output and credential review, and
|
||||||
go test -race ./...
|
repository hygiene. Record the successful workflow result with the candidate.
|
||||||
go vet ./...
|
|
||||||
go build ./...
|
|
||||||
go run ./examples/go-library/prepare
|
|
||||||
```
|
|
||||||
|
|
||||||
Check every tracked Go file. This command must produce no output:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
unformatted=$(
|
|
||||||
git ls-files '*.go' |
|
|
||||||
while IFS= read -r go_file
|
|
||||||
do
|
|
||||||
gofmt -l "$go_file"
|
|
||||||
done
|
|
||||||
)
|
|
||||||
test -z "$unformatted"
|
|
||||||
```
|
|
||||||
|
|
||||||
Follow every maintained Markdown link and confirm that its local or published
|
|
||||||
target exists. Review the repository for generated binaries, test or coverage
|
|
||||||
output, credentials, template residue, downloaded assets, and other files that
|
|
||||||
do not belong in source control.
|
|
||||||
|
|
||||||
Recheck module and repository hygiene, whitespace, and the clean checkout:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
test -z "$(git ls-files go.work go.work.sum)"
|
|
||||||
test ! -e vendor
|
|
||||||
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
|
||||||
then
|
|
||||||
printf '%s\n' 'go.mod contains a replacement' >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
git diff --check
|
|
||||||
test -z "$(git status --porcelain)"
|
|
||||||
```
|
|
||||||
|
|
||||||
## Write The Release Note
|
## Write The Release Note
|
||||||
|
|
||||||
@@ -182,6 +146,13 @@ grep -F 'Consumer action:' "$RELEASE_NOTES_FILE"
|
|||||||
Inspect the complete message and confirm that it accurately records the
|
Inspect the complete message and confirm that it accurately records the
|
||||||
compatibility impact, public API changes, and required consumer action.
|
compatibility impact, public API changes, and required consumer action.
|
||||||
|
|
||||||
|
When a release introduces or updates the maintained external catalogs, state
|
||||||
|
that it selects two independently versioned data dependencies, preserves the
|
||||||
|
public API and configuration, requires no consumer migration, and guarantees
|
||||||
|
only the catalog versions selected and tested by that Promptkit release. Link
|
||||||
|
to the current architecture and format documentation rather than duplicating
|
||||||
|
catalog contracts in the release note.
|
||||||
|
|
||||||
## Create And Inspect The Tag
|
## Create And Inspect The Tag
|
||||||
|
|
||||||
Run the candidate guard again immediately before tag creation. This ensures
|
Run the candidate guard again immediately before tag creation. This ensures
|
||||||
|
|||||||
131
docs/releases/v0.6.0.md
Normal file
131
docs/releases/v0.6.0.md
Normal file
@@ -0,0 +1,131 @@
|
|||||||
|
# Promptkit v0.6.0
|
||||||
|
|
||||||
|
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||||
|
changes from `v0.5.0` to `v0.6.0`. The annotated `v0.6.0` tag is the
|
||||||
|
authoritative release record. Exact current contracts belong to the linked
|
||||||
|
GoDoc and durable documentation.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`v0.6.0` is a broad correctness, safety, efficiency, and maintainability
|
||||||
|
release. It does not add or remove public declarations. The release:
|
||||||
|
|
||||||
|
- centralizes shared execution-setting, output-contract, endpoint, and
|
||||||
|
JSON-compatible-value rules;
|
||||||
|
- unifies prompt repository behavior and avoids unnecessary prompt and profile
|
||||||
|
decoding;
|
||||||
|
- bounds consumer-controlled JSON trees and successful provider responses;
|
||||||
|
- hardens prompt content paths, artifact files, provider URLs, JSON framing,
|
||||||
|
and error propagation;
|
||||||
|
- reuses compiled schema plans and rendered artifact text within an operation;
|
||||||
|
and
|
||||||
|
- improves cancellation behavior, prepared-value ownership, deterministic
|
||||||
|
transport testing, and maintainer validation.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
No public declaration was added, removed, or changed. Ordinary valid `v0.5.0`
|
||||||
|
configurations and requests should continue to compile and behave as before.
|
||||||
|
|
||||||
|
The release intentionally rejects or reports several inputs that were
|
||||||
|
previously accepted, altered, or misclassified:
|
||||||
|
|
||||||
|
- execution settings must be finite, within their documented ranges, and safe
|
||||||
|
to convert to Go durations;
|
||||||
|
- output formats, validation modes, repair counts, and JSON Schema dependencies
|
||||||
|
are validated consistently;
|
||||||
|
- file-backed prompt and profile identity comes from normalized YAML metadata,
|
||||||
|
not filenames;
|
||||||
|
- prompt `content_file` values must be exact relative paths contained by their
|
||||||
|
configured source root;
|
||||||
|
- built-in file artifacts must resolve to regular files;
|
||||||
|
- selected provider endpoints must be absolute HTTP or HTTPS URLs without user
|
||||||
|
information, query strings, or fragments;
|
||||||
|
- JSON documents and successful provider responses must contain exactly one
|
||||||
|
value, and successful provider bodies are limited to 16 MiB; and
|
||||||
|
- excessively deep or expansive JSON-compatible values fail with ordinary
|
||||||
|
validation errors.
|
||||||
|
|
||||||
|
These are compatibility corrections and safety boundaries rather than new
|
||||||
|
consumer configuration requirements. Consumers relying on an invalid or
|
||||||
|
ambiguous input should correct that input before upgrading.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
Update the module dependency with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.6.0
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the consuming project's ordinary and race-enabled tests after upgrading.
|
||||||
|
Applications with custom prompt/profile sources, local provider endpoints,
|
||||||
|
unusual artifact paths, or assertions over provider error identities should
|
||||||
|
pay particular attention to the compatibility notes below.
|
||||||
|
|
||||||
|
## Source Loading And Identity
|
||||||
|
|
||||||
|
Prompt definitions now share one source-neutral selection and normalization
|
||||||
|
flow across operating-system and `fs.FS` sources. YAML `id` and `version`
|
||||||
|
metadata are authoritative; filenames do not create a second identity system.
|
||||||
|
Only selected content bodies are loaded, malformed unrelated definitions do
|
||||||
|
not shadow valid exact matches, and per-file read failures are reported as
|
||||||
|
prompt-load failures rather than false absence.
|
||||||
|
|
||||||
|
File-backed profiles likewise use normalized YAML IDs, reuse their metadata
|
||||||
|
read for selected strict decoding, and avoid fully decoding unrelated files.
|
||||||
|
Selected malformed definitions remain authoritative and do not silently fall
|
||||||
|
through to a lower-precedence source.
|
||||||
|
|
||||||
|
Prompt `content_file` paths are opened exactly as declared after a separate
|
||||||
|
blank check. They must remain relative to and contained by the configured
|
||||||
|
prompt source root, including across operating-system symlinks.
|
||||||
|
|
||||||
|
See the [framework source and identity reference](../formats.md) and
|
||||||
|
[internal source overview](../internal/sources.md) for the current contracts.
|
||||||
|
|
||||||
|
## Validation, Cancellation, And Efficiency
|
||||||
|
|
||||||
|
JSON Schema documents preserve exact JSON-number representations. Schema
|
||||||
|
resource URLs safely escape legal filesystem names, and each operation loads
|
||||||
|
and compiles its schema graph once. `Run` and prepared execution reuse that
|
||||||
|
operation-local plan; Promptkit does not introduce a cross-operation cache.
|
||||||
|
|
||||||
|
Artifact reading, rendering, schema loading, compilation, and validation now
|
||||||
|
check cancellation at the synchronous boundaries Promptkit controls. Rendering
|
||||||
|
memoizes each artifact's text within one render operation, while plain JSON
|
||||||
|
validation avoids materializing an unnecessary generic tree.
|
||||||
|
|
||||||
|
The shared JSON-compatible-value owner now limits nesting and produced work so
|
||||||
|
unsafe consumer-controlled structures return errors instead of risking
|
||||||
|
unbounded recursion or allocation. See the
|
||||||
|
[architecture policy](../policy/architecture.md) for invariant ownership and
|
||||||
|
the [format reference](../formats.md) for validation behavior.
|
||||||
|
|
||||||
|
## Provider Transport Hardening
|
||||||
|
|
||||||
|
OpenAI-compatible endpoints are parsed and composed structurally, including
|
||||||
|
nested base paths. Underlying transport cancellation and deadline errors remain
|
||||||
|
discoverable with `errors.Is` through Promptkit's generation error category.
|
||||||
|
|
||||||
|
Successful provider bodies are read with a fixed 16 MiB bound and must contain
|
||||||
|
exactly one JSON response object followed only by whitespace. Oversized,
|
||||||
|
truncated, malformed, or multiply framed responses fail without returning a
|
||||||
|
partial result. See the
|
||||||
|
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
|
||||||
|
for the canonical request, endpoint, error, and response behavior.
|
||||||
|
|
||||||
|
## Public API Changes
|
||||||
|
|
||||||
|
None.
|
||||||
|
|
||||||
|
## Consumer Action
|
||||||
|
|
||||||
|
- Correct any configuration or request that depends on the formerly permissive
|
||||||
|
cases described under Compatibility.
|
||||||
|
- Confirm custom local provider endpoints are absolute HTTP or HTTPS base URLs
|
||||||
|
without credentials, queries, or fragments.
|
||||||
|
- Confirm prompt content paths remain within their configured source root and
|
||||||
|
file artifacts resolve to regular files.
|
||||||
|
- Run ordinary and race-enabled consumer tests after updating the dependency.
|
||||||
155
docs/releases/v0.7.0.md
Normal file
155
docs/releases/v0.7.0.md
Normal file
@@ -0,0 +1,155 @@
|
|||||||
|
# Promptkit v0.7.0
|
||||||
|
|
||||||
|
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||||
|
changes from `v0.6.0` to `v0.7.0`. The annotated `v0.7.0` tag is the
|
||||||
|
authoritative release record. Exact current contracts belong to the linked
|
||||||
|
GoDoc and durable documentation.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`v0.7.0` expands provider integration and profile composition while making
|
||||||
|
credential and generation-failure handling more flexible:
|
||||||
|
|
||||||
|
- Promptkit now includes the `rakestrawhome` backend and its Gemma profile;
|
||||||
|
- built-in generation failures expose bounded structured provider details;
|
||||||
|
- an unavailable optional API-key environment source no longer prevents a
|
||||||
|
request from reaching an upstream that permits unauthenticated access; and
|
||||||
|
- profiles can inherit from and selectively refine another profile.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
This release adds public declarations and fields but removes none. Existing
|
||||||
|
keyed configuration literals and ordinary `errors.Is` handling continue to
|
||||||
|
work.
|
||||||
|
|
||||||
|
Adding `BaseProfileID` to `Profile` and `OpenAICompatibleProfileConfig` changes
|
||||||
|
their struct shape. Consumers using positional composite literals for either
|
||||||
|
type must convert them to keyed literals. Existing keyed literals require no
|
||||||
|
change.
|
||||||
|
|
||||||
|
The `rakestrawhome` backend ID is now built in and reserved. A consumer that
|
||||||
|
previously registered that exact ID with `WithBackend` must remove its manual
|
||||||
|
registration before upgrading. Other custom backend registrations are
|
||||||
|
unchanged.
|
||||||
|
|
||||||
|
When an optional backend, profile, or request `APIKeyEnv` is unset, empty, or
|
||||||
|
whitespace-only, the built-in client now omits `Authorization` and sends the
|
||||||
|
request. Previously this condition could fail before transport. Set
|
||||||
|
`Profile.APIKeyRequired` when missing credentials must remain a local
|
||||||
|
preflight error.
|
||||||
|
|
||||||
|
Provider non-success responses continue to match `ErrLLMGenerate`. Their
|
||||||
|
rendered wording is not a compatibility contract; consumers can now use
|
||||||
|
`errors.As` with `*GenerationError` when structured status information is
|
||||||
|
needed.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
Update the module dependency with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.7.0
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Remove any manual `rakestrawhome` backend registration, convert positional
|
||||||
|
profile literals to keyed literals, and run the consuming project's ordinary
|
||||||
|
and race-enabled tests.
|
||||||
|
|
||||||
|
## Rakestrawhome Built-In Backend And Profile
|
||||||
|
|
||||||
|
Every engine now includes the reserved `rakestrawhome` backend, identified by
|
||||||
|
`BackendRakestrawHome`. The built-in `rakestrawhome-gemma-4-31b` profile
|
||||||
|
selects that backend. Consumers can use the maintained endpoint, credential,
|
||||||
|
capacity, and model defaults without registering either definition themselves.
|
||||||
|
|
||||||
|
See the [built-in backend and profile catalogs](../formats.md#built-in-backends)
|
||||||
|
and the [consumer adoption example](../consumers/pkg-promptkit.md#use-the-rakestrawhome-built-in-profile)
|
||||||
|
for the current contracts.
|
||||||
|
|
||||||
|
## Structured Generation Errors
|
||||||
|
|
||||||
|
Non-2xx responses from the built-in OpenAI-compatible client now return an
|
||||||
|
immutable `*GenerationError`. Consumers can inspect the HTTP status and any
|
||||||
|
safely extracted provider code, type, or message while retaining the ordinary
|
||||||
|
generation-error category:
|
||||||
|
|
||||||
|
```go
|
||||||
|
var generationErr *promptkit.GenerationError
|
||||||
|
if errors.As(err, &generationErr) {
|
||||||
|
status := generationErr.StatusCode()
|
||||||
|
_ = status
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Provider fields are bounded and normalized but remain untrusted and may
|
||||||
|
contain sensitive request or schema details. Default and Go-syntax formatting
|
||||||
|
omit those fields. Applications must apply their own disclosure policy before
|
||||||
|
logging or presenting accessor values.
|
||||||
|
|
||||||
|
See the [`GenerationError` GoDoc](../../generation_error.go), the
|
||||||
|
[consumer error-handling guide](../consumers/pkg-promptkit.md#handle-errors),
|
||||||
|
and the [OpenAI-compatible response contract](../integrations/openai-compatible-chat.md#response-handling).
|
||||||
|
|
||||||
|
## Optional Credential Sources
|
||||||
|
|
||||||
|
`APIKeyEnv` names an optional environment lookup source unless the selected
|
||||||
|
profile explicitly sets `APIKeyRequired`. When neither a direct request key nor
|
||||||
|
a usable environment value exists, the built-in client omits the bearer header
|
||||||
|
and handles the upstream response normally. This supports local and other
|
||||||
|
OpenAI-compatible providers that permit unauthenticated requests without
|
||||||
|
hiding an authentication error returned by a provider that requires one.
|
||||||
|
|
||||||
|
The [credential format reference](../formats.md#credentials), the
|
||||||
|
[`Backend` GoDoc](../../backends.go), the
|
||||||
|
[`ExecutionTargetOverride` GoDoc](../../types.go), and the
|
||||||
|
[authentication integration contract](../integrations/openai-compatible-chat.md#authentication)
|
||||||
|
define the current precedence and availability rules.
|
||||||
|
|
||||||
|
## Profile Inheritance
|
||||||
|
|
||||||
|
YAML profiles can name one parent with `base_profile`; in-memory profiles use
|
||||||
|
`Profile.BaseProfileID`, and `OpenAICompatibleProfileConfig` forwards the same
|
||||||
|
field. A profile can act as an application-owned alias of a built-in or refine
|
||||||
|
selected inherited settings:
|
||||||
|
|
||||||
|
```go
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "weather-light",
|
||||||
|
BaseProfileID: "deepseek-4-flash",
|
||||||
|
ReasoningEffort: "high",
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
Base lookup observes the existing source precedence. Chains are linear,
|
||||||
|
cycle-safe, and resolved afresh for ordinary operations. Prepared execution
|
||||||
|
freezes the fully resolved target. The selected leaf ID remains public while
|
||||||
|
effective execution settings reflect the resolved chain.
|
||||||
|
|
||||||
|
See the [profile inheritance format reference](../formats.md#profile-inheritance),
|
||||||
|
the [consumer alias example](../consumers/pkg-promptkit.md#alias-a-built-in-profile),
|
||||||
|
and the [`Profile` GoDoc](../../types.go) for exact merge and validation
|
||||||
|
behavior.
|
||||||
|
|
||||||
|
## Public API Changes
|
||||||
|
|
||||||
|
The release adds:
|
||||||
|
|
||||||
|
- `BackendRakestrawHome`;
|
||||||
|
- `GenerationError`, including `StatusCode`, `ProviderCode`, `ProviderType`,
|
||||||
|
`ProviderMessage`, `Error`, `GoString`, and `Unwrap`;
|
||||||
|
- `Profile.BaseProfileID`; and
|
||||||
|
- `OpenAICompatibleProfileConfig.BaseProfileID`.
|
||||||
|
|
||||||
|
No public declaration was removed.
|
||||||
|
|
||||||
|
## Consumer Action
|
||||||
|
|
||||||
|
- Remove a manual backend registration whose ID is exactly `rakestrawhome`.
|
||||||
|
- Convert positional `Profile` or `OpenAICompatibleProfileConfig` literals to
|
||||||
|
keyed literals.
|
||||||
|
- Set `Profile.APIKeyRequired` where a missing credential must fail locally
|
||||||
|
instead of reaching the provider unauthenticated.
|
||||||
|
- Treat `GenerationError` provider fields as untrusted and potentially
|
||||||
|
sensitive when adopting the new accessors.
|
||||||
|
- Run consumer ordinary and race-enabled tests after updating the module.
|
||||||
109
docs/releases/v0.8.0.md
Normal file
109
docs/releases/v0.8.0.md
Normal file
@@ -0,0 +1,109 @@
|
|||||||
|
# Promptkit v0.8.0
|
||||||
|
|
||||||
|
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||||
|
changes from `v0.7.0` to `v0.8.0`. The annotated `v0.8.0` tag is the
|
||||||
|
authoritative release record. Exact current contracts belong to the linked
|
||||||
|
GoDoc and durable documentation.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`v0.8.0` activates Promptkit's bounded output-repair workflow:
|
||||||
|
|
||||||
|
- failed nonempty-text, JSON, and JSON Schema validation can make a limited
|
||||||
|
number of corrective model calls;
|
||||||
|
- corrective calls preserve the original rendered conversation, effective
|
||||||
|
target, session, structured-output contract, and backend capacity policy;
|
||||||
|
- results report cumulative usage and the number of corrective calls actually
|
||||||
|
made; and
|
||||||
|
- explicitly empty OpenAI-compatible response content now reaches output
|
||||||
|
validation instead of being classified as a malformed provider envelope.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
This release adds no public declarations or fields and removes none. Existing
|
||||||
|
source code remains source-compatible.
|
||||||
|
|
||||||
|
The behavior of the existing `OutputContract.RepairAttempts` field and prompt
|
||||||
|
YAML `repair_attempts` field has changed. A positive value now authorizes real
|
||||||
|
additional model calls after eligible validation failures; earlier releases
|
||||||
|
accepted the field but the public engine remained single-pass. Consumers that
|
||||||
|
set a positive value should expect additional latency, token usage, and
|
||||||
|
provider cost when repair is needed.
|
||||||
|
|
||||||
|
Repair budgets must now be between zero and three. A positive budget requires
|
||||||
|
`basic`, `json`, or `json_schema` validation. Values above three and a positive
|
||||||
|
budget paired with `none` are invalid contracts rather than ignored settings.
|
||||||
|
|
||||||
|
An explicitly present empty or whitespace-only string returned by the built-in
|
||||||
|
OpenAI-compatible client is now a completed generation candidate. `none`
|
||||||
|
validation permits it, while `basic`, `json`, and `json_schema` classify it
|
||||||
|
under their ordinary validation rules and may repair it when configured.
|
||||||
|
Missing, `null`, or non-string content remains a malformed provider response.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
Update the module dependency with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.8.0
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Review every prompt definition and request override that sets a positive repair
|
||||||
|
budget. Use zero or omit the field to retain single-pass execution. Ensure each
|
||||||
|
positive budget is no greater than three and uses an eligible validation mode,
|
||||||
|
then run the consuming project's ordinary and race-enabled tests.
|
||||||
|
|
||||||
|
## Bounded Output Repair
|
||||||
|
|
||||||
|
`repair_attempts` counts corrective calls in addition to the initial model
|
||||||
|
call. Promptkit validates each completed candidate, stops at the first valid
|
||||||
|
one, and never exceeds the configured bound. If every candidate remains
|
||||||
|
invalid, the run completes successfully with the final candidate and its
|
||||||
|
failed validation result rather than returning an operational error.
|
||||||
|
|
||||||
|
Each correction starts from the original rendered messages and includes only
|
||||||
|
the latest invalid candidate and latest validation diagnostics. JSON Schema
|
||||||
|
mode retains the provider-native structured-output request as its first line of
|
||||||
|
defense. Promptkit performs only deterministic structural validation; a valid
|
||||||
|
response is not necessarily factual or correct for an application's domain.
|
||||||
|
|
||||||
|
Usage in the final result is cumulative across the initial response and every
|
||||||
|
completed corrective response. `ValidationResult.RepairAttempts` reports the
|
||||||
|
number of corrective calls actually made. Corrective generation failures use
|
||||||
|
the same public generation-error categories and structured provider details as
|
||||||
|
an initial generation failure.
|
||||||
|
|
||||||
|
See the [output-contract format reference](../formats.md#output-contract), the
|
||||||
|
[consumer repair example](../consumers/pkg-promptkit.md#repair-a-structured-result),
|
||||||
|
and the [`OutputContract` and `ValidationResult` GoDoc](../../types.go) for the
|
||||||
|
current contracts.
|
||||||
|
|
||||||
|
## Explicit Empty Content
|
||||||
|
|
||||||
|
The built-in OpenAI-compatible client now distinguishes an explicitly present
|
||||||
|
empty string from a missing or malformed `content` field. This aligns built-in
|
||||||
|
and injected clients by letting the selected output contract decide whether an
|
||||||
|
empty candidate is acceptable, invalid, or eligible for repair.
|
||||||
|
|
||||||
|
See the
|
||||||
|
[OpenAI-compatible response contract](../integrations/openai-compatible-chat.md#response-handling)
|
||||||
|
for the exact envelope behavior.
|
||||||
|
|
||||||
|
## Public API Changes
|
||||||
|
|
||||||
|
None. This release activates and tightens the documented behavior of existing
|
||||||
|
fields.
|
||||||
|
|
||||||
|
## Consumer Action
|
||||||
|
|
||||||
|
- Remove or set `repair_attempts` to zero where execution must remain
|
||||||
|
single-pass.
|
||||||
|
- Keep every positive repair budget at three or fewer and pair it with
|
||||||
|
`basic`, `json`, or `json_schema` validation.
|
||||||
|
- Account for additional latency, usage, and provider cost when enabling
|
||||||
|
repair.
|
||||||
|
- Continue checking the returned validation status because bounded repair can
|
||||||
|
exhaust without producing a valid candidate.
|
||||||
|
- Review workflows that previously treated explicit empty provider content as
|
||||||
|
a generation error.
|
||||||
125
docs/releases/v0.9.0.md
Normal file
125
docs/releases/v0.9.0.md
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
# Promptkit v0.9.0
|
||||||
|
|
||||||
|
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||||
|
changes from `v0.8.0` to `v0.9.0`. The annotated `v0.9.0` tag is the
|
||||||
|
authoritative release record. Exact current contracts belong to the linked
|
||||||
|
GoDoc and durable documentation.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`v0.9.0` adds stateless request-message composition and separates maintained
|
||||||
|
provider data from Promptkit's core implementation:
|
||||||
|
|
||||||
|
- callers can append already-rendered messages to a configured prompt for
|
||||||
|
application-owned conversations and semantic correction workflows;
|
||||||
|
- the public package now publishes constants for the four supported text-chat
|
||||||
|
roles, and prompt definitions use the same normalized role vocabulary; and
|
||||||
|
- the OpenRouter and Rakestrawhome backend/profile catalogs now come from two
|
||||||
|
independently versioned Go module dependencies.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
This release adds one field and four constants to the public API and removes no
|
||||||
|
public declarations. Existing keyed `RunRequest` literals that omit
|
||||||
|
`AppendedMessages` retain their behavior. Adding the field changes the struct
|
||||||
|
shape, so consumers using positional `RunRequest` literals must convert them
|
||||||
|
to keyed literals.
|
||||||
|
|
||||||
|
Message roles in prompt definitions are now trimmed, lowercased, and required
|
||||||
|
to be `developer`, `system`, `user`, or `assistant`. Definitions using another
|
||||||
|
role that earlier releases accepted as an arbitrary nonblank string now fail
|
||||||
|
prompt loading. In particular, the text-only message contract does not support
|
||||||
|
`tool` or the deprecated `function` role. Otherwise valid roles with different
|
||||||
|
case or surrounding whitespace are normalized rather than rejected.
|
||||||
|
|
||||||
|
The catalog extraction preserves Promptkit's public API, built-in backend and
|
||||||
|
profile IDs, configuration, precedence, credential handling, and capacity
|
||||||
|
behavior. Consumers do not import or register either catalog themselves, and
|
||||||
|
no configuration migration is required. Promptkit now selects two
|
||||||
|
independently versioned data dependencies and guarantees only the catalog
|
||||||
|
versions selected and tested by this Promptkit release.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
Update the module dependency with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.9.0
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Convert any positional `RunRequest` literals to keyed literals. Review prompt
|
||||||
|
definitions for unsupported roles, then run the consuming project's ordinary
|
||||||
|
and race-enabled tests.
|
||||||
|
|
||||||
|
## Appended Request Messages
|
||||||
|
|
||||||
|
`RunRequest.AppendedMessages` accepts caller-owned `RenderedMessage` values
|
||||||
|
that Promptkit validates, copies, and appends after every rendered
|
||||||
|
prompt-definition message in caller order. Promptkit does not template this
|
||||||
|
content, retain conversation state between calls, impose a retry policy, or
|
||||||
|
apply a message-count, byte-size, token, or context-window limit. Upstream
|
||||||
|
rejections continue through the ordinary generation-error boundary.
|
||||||
|
|
||||||
|
This primitive supports application-owned conversation continuations and
|
||||||
|
domain-aware correction loops while preserving Promptkit's existing
|
||||||
|
preparation, hashing, prepared-execution, structural repair, backend-capacity,
|
||||||
|
credential, and cancellation behavior. Appended content can include sensitive
|
||||||
|
model output or application feedback; prepared values expose the complete
|
||||||
|
effective messages by design, while default request formatting reports only
|
||||||
|
the appended-message count.
|
||||||
|
|
||||||
|
See the
|
||||||
|
[consumer appended-message example](../consumers/pkg-promptkit.md#append-already-rendered-messages),
|
||||||
|
the [`RunRequest`, `RenderedMessage`, and `CacheControl` GoDoc](../../types.go),
|
||||||
|
and the [OpenAI-compatible request contract](../integrations/openai-compatible-chat.md#request-body)
|
||||||
|
for current behavior.
|
||||||
|
|
||||||
|
## Supported Message Roles
|
||||||
|
|
||||||
|
The new `RoleDeveloper`, `RoleSystem`, `RoleUser`, and `RoleAssistant`
|
||||||
|
constants identify the complete role vocabulary accepted by Promptkit's
|
||||||
|
text-chat message model. The same validation and normalization now apply to
|
||||||
|
prompt-definition messages and request-supplied appended messages. Promptkit
|
||||||
|
does not translate between roles; provider- or model-specific rejection of an
|
||||||
|
otherwise supported role remains an upstream generation error.
|
||||||
|
|
||||||
|
See the [message format reference](../formats.md#messages-and-templates) for
|
||||||
|
the canonical prompt-definition contract.
|
||||||
|
|
||||||
|
## Independently Versioned Backend Catalogs
|
||||||
|
|
||||||
|
Promptkit imports immutable catalog data for its maintained OpenRouter and
|
||||||
|
Rakestrawhome backends and profiles. Engine construction validates and
|
||||||
|
assembles both catalogs behind the existing built-in registry and profile
|
||||||
|
precedence rules. Promptkit no longer keeps duplicate embedded profile assets
|
||||||
|
or hard-coded definitions for those maintained backends.
|
||||||
|
|
||||||
|
The module versions in Promptkit's `go.mod` identify the catalog releases
|
||||||
|
tested with this release. The [built-in backend and profile format
|
||||||
|
reference](../formats.md#built-in-backends) remains the canonical consumer
|
||||||
|
contract, while the [internal source documentation](../internal/sources.md#profiles-and-built-ins)
|
||||||
|
describes the dependency boundary.
|
||||||
|
|
||||||
|
## Public API Changes
|
||||||
|
|
||||||
|
The release adds:
|
||||||
|
|
||||||
|
- `RunRequest.AppendedMessages`;
|
||||||
|
- `RoleDeveloper`;
|
||||||
|
- `RoleSystem`;
|
||||||
|
- `RoleUser`; and
|
||||||
|
- `RoleAssistant`.
|
||||||
|
|
||||||
|
No public declaration was removed.
|
||||||
|
|
||||||
|
## Consumer Action
|
||||||
|
|
||||||
|
- Convert positional `RunRequest` literals to keyed literals.
|
||||||
|
- Replace unsupported prompt-definition roles with an appropriate supported
|
||||||
|
role, or keep richer tool-call protocols in an application-owned client.
|
||||||
|
- Treat appended messages and prepared effective messages according to the
|
||||||
|
application's sensitive-data policy.
|
||||||
|
- Do not add direct catalog imports or registration calls; existing Promptkit
|
||||||
|
construction and configuration remain correct.
|
||||||
|
- Run consumer ordinary and race-enabled tests after updating the module.
|
||||||
@@ -1,635 +0,0 @@
|
|||||||
# Codebase Audit Sequence
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
This document defines the staged sequence for auditing Promptkit before further
|
|
||||||
feature development. The audit is intended to identify high-confidence
|
|
||||||
opportunities to improve correctness, efficiency, duplication, implementation
|
|
||||||
clarity, and test-suite quality without changing production behavior during the
|
|
||||||
review itself.
|
|
||||||
|
|
||||||
The audit findings belong in `audit.md`. A later, separate planning pass will
|
|
||||||
translate accepted findings into a staged remediation plan in
|
|
||||||
`implementation.md`. Neither this sequence nor the findings log owns current
|
|
||||||
behavior; the canonical sources identified by the
|
|
||||||
[documentation policy](../policy/documentation.md) remain authoritative.
|
|
||||||
|
|
||||||
Each stage below is deliberately scoped for one LLM coding-agent prompt. Run
|
|
||||||
the stages in order and do not combine them. A stage may discover a concern
|
|
||||||
outside its scope, but it should record that concern for the owning later stage
|
|
||||||
rather than expanding its own review.
|
|
||||||
|
|
||||||
## Governing Policies And Boundaries
|
|
||||||
|
|
||||||
Every stage must follow:
|
|
||||||
|
|
||||||
- the [development guide](../development.md), including its task-specific
|
|
||||||
reading guide;
|
|
||||||
- the [architecture policy](../policy/architecture.md), especially the public
|
|
||||||
facade, internal-package, dependency-direction, and consumer boundaries;
|
|
||||||
- the [testing policy](../policy/testing.md), including its risk-based,
|
|
||||||
behavior-oriented standard; and
|
|
||||||
- the [documentation policy](../policy/documentation.md), including canonical
|
|
||||||
ownership and the temporary nature of roadmap documents.
|
|
||||||
|
|
||||||
This is an audit, not an implementation pass:
|
|
||||||
|
|
||||||
- Do not change production code, tests, examples, fixtures, public contracts,
|
|
||||||
or current-state documentation.
|
|
||||||
- Limit repository edits to the audit artifacts explicitly authorized for the
|
|
||||||
stage.
|
|
||||||
- Do not silently repair an issue while investigating it.
|
|
||||||
- Do not treat coverage, complexity, similarity, lint, or graph output as a
|
|
||||||
finding without confirming the underlying behavior in source and tests.
|
|
||||||
- Do not recommend centralization merely because code looks similar. The code
|
|
||||||
must implement the same semantic rule, and consolidation must improve
|
|
||||||
ownership or reduce a credible drift risk.
|
|
||||||
- Do not recommend performance work without identifying a relevant execution
|
|
||||||
path and establishing a defensible cost model, measurement, or complexity
|
|
||||||
problem.
|
|
||||||
- Preserve unrelated working-tree changes. Record the audit baseline rather
|
|
||||||
than requiring an otherwise unrelated dirty tree to be cleaned.
|
|
||||||
|
|
||||||
## Finding Standard
|
|
||||||
|
|
||||||
Record each actionable finding in `audit.md` with:
|
|
||||||
|
|
||||||
- a stable ID in the form `SNN-FNN`, where the first number is the stage;
|
|
||||||
- category: correctness, efficiency, duplication, clarity, testing, or
|
|
||||||
contract-documentation consistency;
|
|
||||||
- severity: critical, high, medium, or low;
|
|
||||||
- confidence: confirmed, high, medium, or low;
|
|
||||||
- affected packages, files, symbols, and tests;
|
|
||||||
- the contract, invariant, policy, or maintenance concern at issue;
|
|
||||||
- concrete evidence and a concise explanation of the failure mode or cost;
|
|
||||||
- the recommended direction, without implementation-level sequencing;
|
|
||||||
- the verification or regression protection that remediation would require;
|
|
||||||
and
|
|
||||||
- status: accepted, deferred, rejected, superseded, or resolved.
|
|
||||||
|
|
||||||
Use **confirmed** confidence when the problem is reproduced or follows
|
|
||||||
unavoidably from a complete trace. Use **high** confidence when direct source
|
|
||||||
and test evidence establishes the problem but a safe reproduction is not
|
|
||||||
practical. Medium- and low-confidence concerns belong in a separate
|
|
||||||
observations section until a later stage confirms or rejects them; they must
|
|
||||||
not enter the remediation plan as if they were findings.
|
|
||||||
|
|
||||||
Severity describes impact, not implementation effort:
|
|
||||||
|
|
||||||
- **Critical:** credible data disclosure, data corruption, deadlock, unbounded
|
|
||||||
resource consumption, or a broadly unusable public contract.
|
|
||||||
- **High:** violation of an important public contract or invariant, a likely
|
|
||||||
concurrency or resource-lifecycle defect, or a failure with substantial
|
|
||||||
downstream impact.
|
|
||||||
- **Medium:** a real but narrower behavioral defect, meaningful avoidable cost,
|
|
||||||
duplicated policy with credible drift risk, or a material testing gap.
|
|
||||||
- **Low:** a bounded clarity, maintainability, or testing-friction problem with
|
|
||||||
a concrete improvement and little behavioral risk.
|
|
||||||
|
|
||||||
When a reviewed area yields no finding, record the important behavior or risk
|
|
||||||
that was inspected and found adequately implemented or tested. This coverage
|
|
||||||
ledger prevents later reviewers from mistaking silence for omission.
|
|
||||||
|
|
||||||
## Per-Stage Procedure
|
|
||||||
|
|
||||||
Unless a stage says otherwise, its single agent prompt should:
|
|
||||||
|
|
||||||
1. Read the required policies, focused internal documentation, production
|
|
||||||
files, and tests for that stage.
|
|
||||||
2. Use the code knowledge graph for symbol discovery, callers, callees, and
|
|
||||||
cross-package traces; confirm important conclusions against source.
|
|
||||||
3. Trace normal, boundary, and failure paths through the narrowest relevant
|
|
||||||
public or package contract.
|
|
||||||
4. Review correctness, meaningful runtime cost, semantic duplication,
|
|
||||||
responsibility clarity, and the value and ownership of tests in scope.
|
|
||||||
5. Run the narrowest existing tests needed to validate conclusions. Use
|
|
||||||
race-enabled or repeated focused tests when concurrency or nondeterminism is
|
|
||||||
in scope. Do not add permanent tests during the audit.
|
|
||||||
6. Add the stage result to `audit.md`: accepted findings, unresolved
|
|
||||||
observations, areas verified, commands run, and any handoff to a later
|
|
||||||
stage.
|
|
||||||
7. Recheck the working tree and confirm that only the authorized audit artifact
|
|
||||||
changed.
|
|
||||||
|
|
||||||
## Stage 0: Initialize The Audit And Establish The Baseline
|
|
||||||
|
|
||||||
Create `audit.md` and establish a reproducible starting point before reviewing
|
|
||||||
individual components.
|
|
||||||
|
|
||||||
Record:
|
|
||||||
|
|
||||||
- the audited commit, branch, Go version, module identity, and working-tree
|
|
||||||
state;
|
|
||||||
- unrelated pre-existing changes that all later stages must preserve;
|
|
||||||
- the implemented package and public-facade inventory;
|
|
||||||
- the baseline validation results; and
|
|
||||||
- the finding template, status vocabulary, and coverage ledger used by later
|
|
||||||
stages.
|
|
||||||
|
|
||||||
Refresh the code knowledge graph for the recorded commit. Run the repository's
|
|
||||||
ordinary tests, race tests, vet, build, maintained offline preparation example,
|
|
||||||
Go formatting check, Markdown link check, and repository-hygiene checks. Run
|
|
||||||
package coverage once as a diagnostic and record the result without defining a
|
|
||||||
coverage target or committing generated output. Measure coarse package test
|
|
||||||
duration only if it can be done without adding tooling or changing tests.
|
|
||||||
|
|
||||||
Compare the validation requirements stated by the testing policy, development
|
|
||||||
guide, and release procedure. Record a finding if their ownership or command
|
|
||||||
sets are materially inconsistent; do not edit those documents in this stage.
|
|
||||||
|
|
||||||
**Exit condition:** `audit.md` contains the baseline, ledger structure, and
|
|
||||||
validation result, and no component-level audit has begun.
|
|
||||||
|
|
||||||
## Stage 1: Public Values, Conversion, Errors, And Formatting
|
|
||||||
|
|
||||||
Review the root facade's public request, result, inspection, prepared-run, and
|
|
||||||
error values together with public-to-internal and internal-to-public
|
|
||||||
conversion. Scope the review to `doc.go`, `types.go`, `convert.go`, `errors.go`,
|
|
||||||
`capacity_error.go`, `formatting.go`, and `prepared_execution.go`, plus the
|
|
||||||
directly relevant portions of root tests.
|
|
||||||
|
|
||||||
Focus on:
|
|
||||||
|
|
||||||
- zero-value and nil behavior;
|
|
||||||
- defensive copying, aliasing, and immutable snapshots;
|
|
||||||
- lossless conversion and field precedence;
|
|
||||||
- error identity through `errors.Is` and `errors.As`;
|
|
||||||
- containment of internal representations;
|
|
||||||
- safe `String`, `GoString`, and diagnostic formatting;
|
|
||||||
- accidental disclosure of credentials, prompt content, generated content, or
|
|
||||||
other private state; and
|
|
||||||
- conversion or copying logic that represents the same rule in multiple
|
|
||||||
places.
|
|
||||||
|
|
||||||
Review only tests that own these value and boundary contracts. Defer engine
|
|
||||||
assembly, execution coordination, and transport behavior to their later
|
|
||||||
stages.
|
|
||||||
|
|
||||||
**Exit condition:** all root value-conversion and error-formatting paths have a
|
|
||||||
recorded audit result without evaluating engine orchestration.
|
|
||||||
|
|
||||||
## Stage 2: Public Configuration And Extension Adapters
|
|
||||||
|
|
||||||
Review the smaller public construction and extension surfaces in
|
|
||||||
`backends.go`, `profiles.go`, `artifact_reader.go`, `json.go`, and
|
|
||||||
`llm_adapter.go`, together with their directly relevant root and internal
|
|
||||||
adapter tests.
|
|
||||||
|
|
||||||
Focus on:
|
|
||||||
|
|
||||||
- validation performed at the public boundary;
|
|
||||||
- ownership and copying of caller-supplied maps, slices, filesystems, readers,
|
|
||||||
and clients;
|
|
||||||
- adapter error propagation and cancellation;
|
|
||||||
- consistency between convenience constructors and general configuration;
|
|
||||||
- whether extension interfaces are as narrow as their consumers require;
|
|
||||||
- whether public helpers duplicate internal policy or merely translate it;
|
|
||||||
and
|
|
||||||
- whether tests protect consumer-visible behavior rather than private adapter
|
|
||||||
choreography.
|
|
||||||
|
|
||||||
Do not review how `NewEngine` combines these values; that belongs to Stage 3.
|
|
||||||
|
|
||||||
**Exit condition:** every non-engine public configuration helper and adapter
|
|
||||||
has a recorded result and any assembly questions are handed to Stage 3.
|
|
||||||
|
|
||||||
## Stage 3: Engine Construction, Options, And Source Assembly
|
|
||||||
|
|
||||||
Review the construction and configuration portions of `engine.go` and the
|
|
||||||
corresponding tests in `engine_test.go`. Limit the scope to `NewEngine`, option
|
|
||||||
application, dependency defaults, backend registration, profile and prompt
|
|
||||||
source composition, fallback-profile placement, validator and client
|
|
||||||
selection, capacity-manager construction, and construction-time validation.
|
|
||||||
|
|
||||||
Focus on:
|
|
||||||
|
|
||||||
- deterministic option precedence;
|
|
||||||
- required versus optional dependencies;
|
|
||||||
- isolation between engine instances;
|
|
||||||
- freezing or copying consumer configuration at the correct boundary;
|
|
||||||
- correct dependency direction and absence of process-global mutable state;
|
|
||||||
- failure atomicity and useful public errors;
|
|
||||||
- consistency between configured backends and capacity policies; and
|
|
||||||
- assembly logic that is repeated or split across unclear owners.
|
|
||||||
|
|
||||||
Do not audit the runtime behavior of `Run`, `Prepare`, or inspection methods;
|
|
||||||
that belongs to Stage 4 and the internal use-case stages.
|
|
||||||
|
|
||||||
**Exit condition:** engine construction and source assembly are fully accounted
|
|
||||||
for, including tests, without expanding into runtime orchestration.
|
|
||||||
|
|
||||||
## Stage 4: Engine Operations And Root Contract Coverage
|
|
||||||
|
|
||||||
Review the remaining public methods in `engine.go` and their directly relevant
|
|
||||||
root tests, including the external-package contracts in
|
|
||||||
`public_contract_test.go` and `prepared_execution_contract_test.go` only where
|
|
||||||
they exercise the engine boundary under review.
|
|
||||||
|
|
||||||
Focus on:
|
|
||||||
|
|
||||||
- request translation and context propagation;
|
|
||||||
- ordinary run, preparation, inspection, and prepared-execution entry points;
|
|
||||||
- public error mapping and preservation of injected dependency errors;
|
|
||||||
- result and prepared-state ownership;
|
|
||||||
- consistency between method and package-level convenience functions;
|
|
||||||
- public behavior that is asserted redundantly in root internal tests and
|
|
||||||
external-package contract tests; and
|
|
||||||
- important public behavior that is tested only through internal packages.
|
|
||||||
|
|
||||||
Treat internal runner, transport, validation, and capacity mechanics as black
|
|
||||||
boxes in this stage. Hand questions about their implementation to their owning
|
|
||||||
later stages.
|
|
||||||
|
|
||||||
**Exit condition:** the public execution boundary and its contract-test
|
|
||||||
ownership are recorded without duplicating internal component audits.
|
|
||||||
|
|
||||||
## Stage 5: Internal Domain And JSON-Compatible Values
|
|
||||||
|
|
||||||
Review `internal/domain` and `internal/jsonvalue`, including all of their tests.
|
|
||||||
|
|
||||||
Focus on:
|
|
||||||
|
|
||||||
- domain invariants and invalid states;
|
|
||||||
- session normalization;
|
|
||||||
- prepared-run and schema immutability;
|
|
||||||
- deep-copy correctness for every supported JSON-compatible shape;
|
|
||||||
- numeric-type preservation and rejection policy;
|
|
||||||
- cycles, excessive nesting, unsupported values, and nil distinctions;
|
|
||||||
- avoidable repeated copying on execution paths; and
|
|
||||||
- whether generic value machinery has a single clear owner.
|
|
||||||
|
|
||||||
Trace important callers to confirm that these packages enforce the invariants
|
|
||||||
their consumers assume, but do not audit the callers' broader behavior.
|
|
||||||
|
|
||||||
**Exit condition:** shared value semantics and their test ownership are fully
|
|
||||||
recorded.
|
|
||||||
|
|
||||||
## Stage 6: Backend Registry, Defaults, And Built-In Profiles
|
|
||||||
|
|
||||||
Review `internal/backend`, `internal/defaults`, and
|
|
||||||
`internal/profile/builtin`, including their focused tests and the relevant
|
|
||||||
backend-policy traces into engine assembly and the LLM reserved-field rule.
|
|
||||||
|
|
||||||
Focus on:
|
|
||||||
|
|
||||||
- immutable registry construction and lookup;
|
|
||||||
- built-in versus consumer ID collision rules;
|
|
||||||
- endpoint, credential-environment, header, parameter, and concurrency
|
|
||||||
validation;
|
|
||||||
- defensive copies at registry boundaries;
|
|
||||||
- application-neutral default ownership;
|
|
||||||
- built-in profile/backend consistency;
|
|
||||||
- reserved request-field ownership without dependency inversion; and
|
|
||||||
- duplicated validation or default policy across public and internal layers.
|
|
||||||
|
|
||||||
Defer scheduling mechanics to Stage 15 and actual HTTP request construction to
|
|
||||||
Stage 14.
|
|
||||||
|
|
||||||
**Exit condition:** registry and default-policy correctness are recorded, with
|
|
||||||
transport and scheduling questions handed to their owning stages.
|
|
||||||
|
|
||||||
## Stage 7: File Discovery And Prompt Definitions
|
|
||||||
|
|
||||||
Review `internal/filecatalog` and `internal/promptdef`, including their tests
|
|
||||||
and fixtures. Read the framework format reference and internal source document
|
|
||||||
before evaluating behavior.
|
|
||||||
|
|
||||||
Focus on:
|
|
||||||
|
|
||||||
- deterministic discovery and duplicate handling;
|
|
||||||
- filesystem and `fs.FS` parity;
|
|
||||||
- root and relative-path normalization;
|
|
||||||
- strict YAML decoding and version selection;
|
|
||||||
- prompt ID, message, input, cache-control, and validation declarations;
|
|
||||||
- inline versus file-backed content rules;
|
|
||||||
- containment of referenced files where promised;
|
|
||||||
- malformed input and contextual error behavior;
|
|
||||||
- unnecessary repeated directory scans or file reads; and
|
|
||||||
- fixture and case duplication that does not protect distinct parser risks.
|
|
||||||
|
|
||||||
Do not audit rendering, artifact loading, profile loading, or schema validation
|
|
||||||
in this stage.
|
|
||||||
|
|
||||||
**Exit condition:** discovery and prompt-definition parsing have complete
|
|
||||||
findings and coverage-ledger entries.
|
|
||||||
|
|
||||||
## Stage 8: Profile Sources And Repository Composition
|
|
||||||
|
|
||||||
Review `internal/profile` excluding its built-in subpackage, including all
|
|
||||||
repository tests and profile fixtures. Read the profile format contract first.
|
|
||||||
|
|
||||||
Focus on:
|
|
||||||
|
|
||||||
- strict decoding and profile validation;
|
|
||||||
- filesystem and `fs.FS` parity;
|
|
||||||
- repository overlay and fallback precedence;
|
|
||||||
- distinction between absence and a malformed authoritative source;
|
|
||||||
- preservation of useful error identity and context;
|
|
||||||
- conversion to immutable execution profiles;
|
|
||||||
- duplicate IDs and deterministic selection;
|
|
||||||
- repeated parsing, validation, or copying; and
|
|
||||||
- whether tests at repository, engine, and public-contract layers have clear,
|
|
||||||
nonduplicative ownership.
|
|
||||||
|
|
||||||
Defer resolution of a profile with runtime overrides and backend definitions to
|
|
||||||
Stage 11.
|
|
||||||
|
|
||||||
**Exit condition:** profile-source and repository-composition behavior are
|
|
||||||
fully recorded.
|
|
||||||
|
|
||||||
## Stage 9: Artifact Loading And Prompt Rendering
|
|
||||||
|
|
||||||
Review `internal/artifact` and `internal/prompt`, including all focused tests.
|
|
||||||
Read the internal source document and format reference first.
|
|
||||||
|
|
||||||
Focus on:
|
|
||||||
|
|
||||||
- inline and file artifact ownership, metadata, hashing, and error behavior;
|
|
||||||
- copied versus shared byte storage;
|
|
||||||
- caller-selected path semantics and architecture-policy boundaries;
|
|
||||||
- template parsing and execution;
|
|
||||||
- artifact, variable, session, and cache-control rendering;
|
|
||||||
- missing, extra, nil, and malformed input behavior;
|
|
||||||
- deterministic output and safe diagnostics;
|
|
||||||
- unnecessary repeated reads, hashes, parses, or allocations on common paths;
|
|
||||||
and
|
|
||||||
- tests coupled to incidental template or struct implementation.
|
|
||||||
|
|
||||||
Do not audit the runner's decision about when rendering occurs.
|
|
||||||
|
|
||||||
**Exit condition:** input materialization and rendering are accounted for
|
|
||||||
through their package boundaries.
|
|
||||||
|
|
||||||
## Stage 10: Output Validation And Frozen Validation Plans
|
|
||||||
|
|
||||||
Review `internal/validate`, including all tests, schema fixtures used by the
|
|
||||||
root contract suite, and traces from preparation into frozen validation plans.
|
|
||||||
Read the format and internal source documents first.
|
|
||||||
|
|
||||||
Focus on:
|
|
||||||
|
|
||||||
- basic, JSON, and JSON Schema mode semantics;
|
|
||||||
- schema-path resolution and filesystem/`fs.FS` parity;
|
|
||||||
- schema compilation, transitive references, and source-lifetime independence;
|
|
||||||
- output normalization and preservation;
|
|
||||||
- malformed schema and malformed model-output errors;
|
|
||||||
- thread safety of reusable validators and prepared plans;
|
|
||||||
- expensive recompilation or copying on repeated execution; and
|
|
||||||
- whether parser, validator, runner, and public tests each own distinct risks.
|
|
||||||
|
|
||||||
Do not audit repair decisions or provider request construction.
|
|
||||||
|
|
||||||
**Exit condition:** validation behavior, plan lifetime, and focused test value
|
|
||||||
are fully recorded.
|
|
||||||
|
|
||||||
## Stage 11: Inspection And Execution-Target Resolution
|
|
||||||
|
|
||||||
Review `internal/usecase/profile_inspection.go`,
|
|
||||||
`internal/usecase/prompt_inspection.go`, and the preparation and target-
|
|
||||||
resolution portions of `internal/usecase/runner.go`, together with their
|
|
||||||
focused tests. Use graph traces to define the exact helper and call-path scope
|
|
||||||
before reviewing.
|
|
||||||
|
|
||||||
Focus on:
|
|
||||||
|
|
||||||
- prompt and profile selection;
|
|
||||||
- backend lookup and endpoint overrides;
|
|
||||||
- reasoning, session, and other runtime precedence;
|
|
||||||
- merge semantics for default, profile, backend, and per-run values;
|
|
||||||
- inspection fidelity versus actual execution;
|
|
||||||
- credential-name versus credential-value handling;
|
|
||||||
- prompt-definition and schema freezing during preparation;
|
|
||||||
- stable error identity and context; and
|
|
||||||
- duplicated resolution rules across inspection, preparation, and execution.
|
|
||||||
|
|
||||||
Do not review model invocation, repair execution, or prepared-handle lifecycle;
|
|
||||||
those belong to Stages 12 and 13.
|
|
||||||
|
|
||||||
**Exit condition:** all selection, merge, inspection, and preparation rules are
|
|
||||||
traced and recorded once.
|
|
||||||
|
|
||||||
## Stage 12: Ordinary Execution, Validation, And Repair Coordination
|
|
||||||
|
|
||||||
Review `internal/usecase/runner.go`, `internal/usecase/repairer.go`, and
|
|
||||||
`internal/usecase/capacity_error.go` only for the ordinary execution path after
|
|
||||||
preparation, together with the corresponding sections of `runner_test.go`.
|
|
||||||
Use the Stage 11 resolution result as an established input rather than
|
|
||||||
reauditing it.
|
|
||||||
|
|
||||||
Focus on:
|
|
||||||
|
|
||||||
- rendering, generation, validation, and optional repair transitions;
|
|
||||||
- context cancellation and dependency-error propagation;
|
|
||||||
- partial result and usage accounting;
|
|
||||||
- exact attempt count and repair eligibility;
|
|
||||||
- avoidance of unintended retries;
|
|
||||||
- capacity-error translation;
|
|
||||||
- cleanup and failure behavior on every exit path;
|
|
||||||
- repeated orchestration or request construction; and
|
|
||||||
- oversized tests, helpers, or case matrices that obscure distinct behavior.
|
|
||||||
|
|
||||||
Treat LLM transport and capacity scheduling as injected package contracts;
|
|
||||||
their mechanics belong to Stages 14 and 15.
|
|
||||||
|
|
||||||
**Exit condition:** the ordinary execution state machine and its test ownership
|
|
||||||
are fully recorded.
|
|
||||||
|
|
||||||
## Stage 13: Prepared Execution Lifecycle
|
|
||||||
|
|
||||||
Review `internal/usecase/prepared_execution.go`, its focused tests, and the
|
|
||||||
prepared-execution portions of the root facade and external contract tests.
|
|
||||||
Do not repeat the public value review from Stages 1 and 4 or the resolution
|
|
||||||
review from Stage 11.
|
|
||||||
|
|
||||||
Focus on:
|
|
||||||
|
|
||||||
- single-attempt or other lifecycle guarantees;
|
|
||||||
- concurrent use and synchronization;
|
|
||||||
- discard behavior and resource release;
|
|
||||||
- frozen source, target, credential, capacity, timing, and schema semantics;
|
|
||||||
- independence of returned details and results;
|
|
||||||
- context and error behavior;
|
|
||||||
- consistency between ordinary and prepared execution where promised;
|
|
||||||
- private-state containment in formatting; and
|
|
||||||
- redundant assertions across internal, root, and external-package tests.
|
|
||||||
|
|
||||||
Run focused race tests and repeated tests for lifecycle behavior where useful.
|
|
||||||
|
|
||||||
**Exit condition:** prepared execution has one complete lifecycle analysis and
|
|
||||||
a clear map of which test layer owns each guarantee.
|
|
||||||
|
|
||||||
## Stage 14: OpenAI-Compatible Transport
|
|
||||||
|
|
||||||
Review `internal/llm`, including all transport tests. Read the
|
|
||||||
OpenAI-compatible integration contract and internal LLM document first.
|
|
||||||
|
|
||||||
Focus on:
|
|
||||||
|
|
||||||
- request endpoint, headers, authentication, and JSON body construction;
|
|
||||||
- omission versus explicit zero-value behavior;
|
|
||||||
- reserved-field enforcement and extra-parameter collision handling;
|
|
||||||
- session ID and reasoning encoding;
|
|
||||||
- structured-output and cache-control translation;
|
|
||||||
- client and per-generation deadlines;
|
|
||||||
- cancellation, body closure, bounded response reads, and decode failures;
|
|
||||||
- non-success HTTP response behavior;
|
|
||||||
- response choices, usage, and malformed-success handling;
|
|
||||||
- wire-visible compatibility and safe error disclosure;
|
|
||||||
- unnecessary marshaling, copying, or buffering; and
|
|
||||||
- whether the large transport test file can be simplified without losing
|
|
||||||
protocol-risk coverage.
|
|
||||||
|
|
||||||
Use `httptest`-based existing tests; do not contact a live provider.
|
|
||||||
|
|
||||||
**Exit condition:** every outbound and inbound wire path has a recorded result,
|
|
||||||
including focused test ownership.
|
|
||||||
|
|
||||||
## Stage 15: Capacity, Admission, And Concurrency
|
|
||||||
|
|
||||||
Review `internal/capacity`, its tests, `capacity_contract_test.go`, and the
|
|
||||||
integration points already identified in engine and use-case stages. Read the
|
|
||||||
internal capacity document first.
|
|
||||||
|
|
||||||
Focus on:
|
|
||||||
|
|
||||||
- bounded run admission and queue-capacity enforcement;
|
|
||||||
- per-backend limited and unlimited scheduling;
|
|
||||||
- FIFO behavior and cancellation-safe waiter removal;
|
|
||||||
- permit release on success, error, panic-relevant boundaries, and
|
|
||||||
cancellation;
|
|
||||||
- goroutine, timer, and waiter lifecycle;
|
|
||||||
- starvation, deadlock, race, and engine-isolation risks;
|
|
||||||
- lock scope and meaningful contention or allocation costs;
|
|
||||||
- preservation of injected-client concurrency where promised;
|
|
||||||
- relational testing of configured limits rather than duplicated defaults;
|
|
||||||
and
|
|
||||||
- duplication between internal concurrency tests and public contract tests.
|
|
||||||
|
|
||||||
Run focused ordinary, race-enabled, and repeated tests. Repetition must remain
|
|
||||||
bounded and diagnostic; a test that passes many times is not proof of
|
|
||||||
correctness without a source-level synchronization analysis.
|
|
||||||
|
|
||||||
**Exit condition:** concurrency invariants have both a source trace and a
|
|
||||||
test-ownership assessment.
|
|
||||||
|
|
||||||
## Stage 16: Repository-Wide Test Strategy And Maintained Examples
|
|
||||||
|
|
||||||
Perform a suite-level review after every component has been audited. Review
|
|
||||||
the testing policy, test inventory, fixtures, external-package root tests,
|
|
||||||
`architecture_test.go`, and both maintained examples. Use the component-stage
|
|
||||||
coverage ledger instead of repeating every individual test assertion.
|
|
||||||
|
|
||||||
Construct a risk-to-owner matrix for:
|
|
||||||
|
|
||||||
- public compatibility and error identity;
|
|
||||||
- parsing, validation, and serialization;
|
|
||||||
- immutability and data integrity;
|
|
||||||
- external wire behavior;
|
|
||||||
- cancellation, failure propagation, and recovery;
|
|
||||||
- concurrency and resource lifecycle; and
|
|
||||||
- representative assembled consumer workflows.
|
|
||||||
|
|
||||||
Identify only evidence-backed cases of:
|
|
||||||
|
|
||||||
- consequential behavior with no credible test owner;
|
|
||||||
- the same semantic rule asserted redundantly at several layers;
|
|
||||||
- tests coupled to private helpers, internal constants, exact noncontractual
|
|
||||||
wording, or collaborator choreography;
|
|
||||||
- low-value or obsolete cases whose lifetime cost exceeds their protection;
|
|
||||||
- missing failure, cancellation, race, or boundary coverage;
|
|
||||||
- nondeterminism, shared state, environment dependence, fixed ports, or test
|
|
||||||
ordering assumptions;
|
|
||||||
- helpers and fixtures whose complexity is not justified; and
|
|
||||||
- maintained examples that duplicate one another without protecting distinct
|
|
||||||
workflows.
|
|
||||||
|
|
||||||
Use coverage and timing only to direct attention. Do not propose tests solely
|
|
||||||
to raise percentages or remove tests solely to shorten the suite.
|
|
||||||
|
|
||||||
**Exit condition:** every important risk has a named test owner or an accepted
|
|
||||||
finding, and every proposed test deletion or consolidation states what
|
|
||||||
protection remains.
|
|
||||||
|
|
||||||
## Stage 17: Cross-Cutting Duplication, Efficiency, And Architecture Review
|
|
||||||
|
|
||||||
Review the codebase as a whole using the completed component findings, graph
|
|
||||||
traces, complexity signals, similarity signals, and package dependency map.
|
|
||||||
Do not reopen settled package behavior without new cross-cutting evidence.
|
|
||||||
|
|
||||||
Focus on:
|
|
||||||
|
|
||||||
- one semantic policy implemented by multiple packages;
|
|
||||||
- repeated public/internal transformations with credible drift risk;
|
|
||||||
- interfaces broader than their actual consumers;
|
|
||||||
- responsibilities split across packages or concentrated in the facade
|
|
||||||
contrary to the architecture policy;
|
|
||||||
- repeated parsing, copying, schema compilation, request construction, or
|
|
||||||
source traversal on important paths;
|
|
||||||
- avoidable lock contention or serial work supported by the concurrency audit;
|
|
||||||
- abstractions that add indirection without enforcing a boundary; and
|
|
||||||
- discrepancies between implemented package responsibilities and their
|
|
||||||
canonical architecture or internal documentation.
|
|
||||||
|
|
||||||
For each possible consolidation, state why the code represents one rule, which
|
|
||||||
package should own it, and why the resulting dependency direction remains
|
|
||||||
valid. For each efficiency finding, state the path frequency, input scale,
|
|
||||||
complexity or measurement evidence, and the benchmark or invariant needed to
|
|
||||||
verify a remediation.
|
|
||||||
|
|
||||||
**Exit condition:** all cross-cutting opportunities are either accepted with
|
|
||||||
high confidence, retained as explicitly lower-confidence observations, or
|
|
||||||
rejected with a short rationale.
|
|
||||||
|
|
||||||
## Stage 18: Consolidate And Close The Audit
|
|
||||||
|
|
||||||
Perform a findings-only synthesis. Do not change code and do not write the
|
|
||||||
remediation plan yet.
|
|
||||||
|
|
||||||
- Recheck every accepted finding against the final audited tree.
|
|
||||||
- Merge duplicates and mark superseded IDs without erasing their history.
|
|
||||||
- Separate shared root causes from downstream symptoms.
|
|
||||||
- Confirm that every accepted item is confirmed or high confidence.
|
|
||||||
- Confirm that severity describes impact rather than effort.
|
|
||||||
- Reject speculative cleanup, coverage-driven test work, and centralization
|
|
||||||
without a clear owner or drift risk.
|
|
||||||
- Record dependencies and a recommended remediation order.
|
|
||||||
- Distinguish behavioral fixes, safe refactors, performance work, test gaps,
|
|
||||||
test consolidation, and documentation synchronization.
|
|
||||||
- Add an audit summary stating what was reviewed, what validation ran, the
|
|
||||||
accepted finding counts by category and severity, and any residual
|
|
||||||
uncertainty.
|
|
||||||
- Re-run baseline validation if audit-only investigation could have affected
|
|
||||||
repository state, and confirm that only authorized roadmap files differ from
|
|
||||||
the recorded baseline.
|
|
||||||
|
|
||||||
The recommended ordering should place correctness, data-integrity,
|
|
||||||
resource-lifecycle, and concurrency defects first; policy duplication and
|
|
||||||
missing protection for consequential behavior next; then clarity, test
|
|
||||||
consolidation, and demonstrated efficiency improvements. Actual implementation
|
|
||||||
stages must be decided in the later `implementation.md` planning pass, where
|
|
||||||
files, dependencies, acceptance criteria, and validation can be made
|
|
||||||
decision-complete.
|
|
||||||
|
|
||||||
**Exit condition:** `audit.md` is a complete, internally consistent input to a
|
|
||||||
separate remediation-planning prompt, with no code or test changes mixed into
|
|
||||||
the audit.
|
|
||||||
|
|
||||||
## Completion Criteria
|
|
||||||
|
|
||||||
The audit is complete only when:
|
|
||||||
|
|
||||||
- every production component and public boundary appears in the coverage
|
|
||||||
ledger;
|
|
||||||
- every test file and maintained example has been reviewed at its owning stage
|
|
||||||
or in the suite-wide stage;
|
|
||||||
- important cross-package paths have been traced end to end;
|
|
||||||
- concurrency-sensitive behavior has received source and race-test review;
|
|
||||||
- every accepted finding meets the evidence and confidence standard;
|
|
||||||
- lower-confidence observations are visibly separated from remediation
|
|
||||||
candidates;
|
|
||||||
- proposed test additions, deletions, and consolidations are justified against
|
|
||||||
the testing policy;
|
|
||||||
- proposed simplifications identify a durable responsibility owner;
|
|
||||||
- proposed efficiency work has a relevant cost model or measurement plan; and
|
|
||||||
- the repository remains unchanged except for the authorized audit roadmap
|
|
||||||
artifacts.
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -38,33 +38,8 @@ consumers.
|
|||||||
|
|
||||||
## Ideas
|
## Ideas
|
||||||
|
|
||||||
### Public bounded output repair
|
No ideas are currently awaiting selection. Active feature work belongs in its
|
||||||
|
focused roadmap rather than this catalog.
|
||||||
After the codebase-audit remediations are complete, Promptkit should make its
|
|
||||||
bounded output-repair capability available through the public engine. A
|
|
||||||
consumer should be able to request a limited number of corrective generation
|
|
||||||
attempts when JSON or JSON Schema output fails content validation, without
|
|
||||||
having to reproduce Promptkit's generation, validation, capacity, and result-
|
|
||||||
accounting orchestration.
|
|
||||||
|
|
||||||
- Repair is validation recovery, not a general provider retry, failover, or
|
|
||||||
backoff policy. Transport failures, cancellation, and operational schema or
|
|
||||||
validation errors must retain their ordinary error behavior.
|
|
||||||
- Repair must stop after the first valid result or the configured attempt
|
|
||||||
bound. Exhausting the bound should preserve the final invalid result and its
|
|
||||||
validation diagnostics rather than inventing success.
|
|
||||||
- Initial generation and every repair attempt must use the same resolved
|
|
||||||
backend, effective execution settings and presence semantics, session,
|
|
||||||
credential boundary, structured-output contract, and backend-capacity
|
|
||||||
policy.
|
|
||||||
- Results should report the number of repair attempts and cumulative usage for
|
|
||||||
every model call made by the run.
|
|
||||||
- Ordinary and prepared execution should expose coherent behavior, including
|
|
||||||
cancellation, frozen prepared state, error identity, and capacity lifetime.
|
|
||||||
|
|
||||||
Select this work only after the accepted audit findings affecting shared
|
|
||||||
execution invariants, validation, orchestration, transport, and repair
|
|
||||||
internals have been remediated.
|
|
||||||
|
|
||||||
## Entry Format
|
## Entry Format
|
||||||
|
|
||||||
|
|||||||
@@ -1,840 +0,0 @@
|
|||||||
# Audit Remediation Implementation Plan
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
This document is the decision-complete implementation plan for the accepted
|
|
||||||
findings in the [codebase audit](audit.md). It is written for a
|
|
||||||
`gpt-5.6-terra` coding agent that will implement one numbered stage per prompt,
|
|
||||||
in order.
|
|
||||||
|
|
||||||
The audit remains the evidence and rationale for each finding. This plan owns
|
|
||||||
implementation order, selected policy decisions, required code and test work,
|
|
||||||
and stage gates. It does not activate the future public output-repair feature
|
|
||||||
described in the [future feature catalog](future.md); it only corrects and
|
|
||||||
protects the retained internal repair machinery on which that later feature
|
|
||||||
may build.
|
|
||||||
|
|
||||||
## Implementation Policies
|
|
||||||
|
|
||||||
Every stage must follow the [development guide](../development.md),
|
|
||||||
[architecture policy](../policy/architecture.md),
|
|
||||||
[testing policy](../policy/testing.md), and
|
|
||||||
[documentation policy](../policy/documentation.md). Before changing a
|
|
||||||
subsystem, read the focused current-state documents identified by the
|
|
||||||
development guide and inspect the exact implementation and tests named by the
|
|
||||||
stage.
|
|
||||||
|
|
||||||
Apply these rules throughout:
|
|
||||||
|
|
||||||
- Implement exactly one stage per agent prompt. Do not combine stages or begin
|
|
||||||
a later stage early.
|
|
||||||
- Inspect the working tree before editing and preserve unrelated changes.
|
|
||||||
- Use the code knowledge graph to locate symbols, callers, and dependency
|
|
||||||
paths; confirm important conclusions against source.
|
|
||||||
- Keep the root package as the public facade and implementation under
|
|
||||||
`internal/`. Do not expose internal representations or add a public package.
|
|
||||||
- Put source-neutral invariants in their assigned internal owner while
|
|
||||||
preserving source-specific normalization, error classification, and public
|
|
||||||
translation at existing boundaries.
|
|
||||||
- Add regression protection at the narrowest stable owner in the same stage
|
|
||||||
as a behavioral fix. Retain only representative integration coverage at
|
|
||||||
higher layers.
|
|
||||||
- Do not add tests to raise coverage percentages. Do not preserve tests that
|
|
||||||
assert an incidental algorithm, private constant, dormant serialization
|
|
||||||
shape, or duplicated lower-layer truth table.
|
|
||||||
- Keep all tests deterministic, offline, race-safe, credential-free, and free
|
|
||||||
of fixed-port or mutable-service assumptions.
|
|
||||||
- Do not add engine-wide caches, generic facade abstractions, scheduler
|
|
||||||
changes, provider retry policy, or new consumer configuration unless a stage
|
|
||||||
explicitly requires it.
|
|
||||||
- Update canonical GoDoc and current-state documents in the same stage as the
|
|
||||||
behavior they describe. Do not describe a later stage as already
|
|
||||||
implemented.
|
|
||||||
- Format changed Go files. Run the stage's focused commands, then at least
|
|
||||||
`go test ./...` and `go vet ./...`. Run focused race tests wherever the
|
|
||||||
stage changes ownership, cancellation, shared state, or lifecycle behavior.
|
|
||||||
- Do not commit, push, tag, or publish unless separately instructed.
|
|
||||||
|
|
||||||
## Decisions Fixed By This Plan
|
|
||||||
|
|
||||||
The implementing agent must not reopen these choices:
|
|
||||||
|
|
||||||
1. **JSON-compatible numbers:** accept every value Go can faithfully encode as
|
|
||||||
a JSON number: every signed and unsigned integer width, finite `float32` and
|
|
||||||
`float64` values, and a `json.Number` whose text is valid JSON-number syntax.
|
|
||||||
Do not impose the current IEEE-754 safe-integer restriction. Reject NaN,
|
|
||||||
infinities, and malformed `json.Number` text. Preserve supported concrete
|
|
||||||
numeric types when copying.
|
|
||||||
2. **JSON-shaped traversal bounds:** allow at most 100 JSON container levels
|
|
||||||
and 100,000 produced JSON value nodes per `Copy` or `CopyMap` operation.
|
|
||||||
Count the root, each map/slice/array container, and every produced child
|
|
||||||
value; map keys are not separate nodes. Pointer and interface indirection
|
|
||||||
do not add JSON depth or an extra node. Repeated appearances of an acyclic
|
|
||||||
shared value count each produced occurrence. Continue rejecting active-path
|
|
||||||
cycles and return deterministic, path-aware validation errors on either
|
|
||||||
bound.
|
|
||||||
3. **Execution timeout bound:** a positive `TimeoutSeconds` must fit in
|
|
||||||
`time.Duration` after multiplication by `time.Second`. Derive the maximum
|
|
||||||
from `math.MaxInt64` and `time.Second`; do not duplicate its numeric literal
|
|
||||||
in tests or documentation.
|
|
||||||
4. **Output contracts:** the only valid formats are `text`, `markdown`, and
|
|
||||||
`json`; the only valid validation modes are `none`, `basic`, `json`, and
|
|
||||||
`json_schema`; repair attempts are non-negative; and `json_schema` requires
|
|
||||||
a nonblank schema path. A non-nil request replacement defaults an empty
|
|
||||||
format to `text` before shared validation. It does not default an empty
|
|
||||||
validation mode.
|
|
||||||
5. **Prompt content paths:** every `content_file` is an exact, relative path
|
|
||||||
resolved from its prompt file and contained by the configured prompt source
|
|
||||||
root. Directory, `fs.FS`, and single-file sources all reject absolute and
|
|
||||||
escaping paths. A single-file source's root is the containing directory of
|
|
||||||
that selected prompt file. Trimming determines only whether a value is
|
|
||||||
blank; it must not change the path opened. OS containment must account for
|
|
||||||
symlinks; containment inside an injected `fs.FS` remains expressed in that
|
|
||||||
filesystem's namespace.
|
|
||||||
6. **Profile IDs:** normalize file-backed IDs with `strings.TrimSpace` once,
|
|
||||||
just as in-memory IDs are normalized. Use the normalized value for
|
|
||||||
selection, duplicate detection, results, and diagnostics. A whitespace-only
|
|
||||||
ID is invalid, and IDs that become equal after normalization are
|
|
||||||
duplicates.
|
|
||||||
7. **Ordinary artifact files:** the built-in `File` reader supports regular
|
|
||||||
files, including symlinks whose targets are regular files. It rejects
|
|
||||||
directories, FIFOs, devices, sockets, and other non-regular targets before
|
|
||||||
consuming them. It remains unrestricted by an application root and does
|
|
||||||
not introduce an application-specific byte limit.
|
|
||||||
8. **Validation cancellation:** do not return early by abandoning goroutines
|
|
||||||
around `fs.FS` or the JSON Schema dependency. Promptkit must check
|
|
||||||
cancellation before, between, and after work it controls; read opened files
|
|
||||||
in context-checked chunks; and let a canceled context win before publishing
|
|
||||||
a result after synchronous decode, compile, or validation calls. Go's
|
|
||||||
`fs.FS` and the current JSON Schema library expose no general mechanism to
|
|
||||||
preempt a blocked `Open`, `Read`, compile, or validation method, so canonical
|
|
||||||
documentation must describe this synchronous limitation rather than claim
|
|
||||||
impossible asynchronous interruption.
|
|
||||||
9. **Successful provider-response limit:** the built-in OpenAI-compatible
|
|
||||||
client accepts at most 16 MiB (`16 << 20` bytes) for the complete successful
|
|
||||||
HTTP response body, including surrounding whitespace. The limit is fixed,
|
|
||||||
internal, and application-neutral. Exactly the limit is allowed; the first
|
|
||||||
byte beyond it fails as `internal/llm.ErrMalformedResponse`. Do not add a
|
|
||||||
public setting. Non-success response parsing remains outside this audit
|
|
||||||
remediation and belongs to the separate structured-generation-error
|
|
||||||
roadmap.
|
|
||||||
10. **Repair machinery:** retain and fix the internal repairer, cumulative
|
|
||||||
usage, and bounded repair state machine. The public engine must continue to
|
|
||||||
install no repairer and remain single-pass. Do not activate public repair
|
|
||||||
in this plan.
|
|
||||||
|
|
||||||
## Stage 1: Centralize Execution-Setting And Session Invariants
|
|
||||||
|
|
||||||
**Findings:** S05-F01, S17-F01, S14-F02. This stage also resolves the
|
|
||||||
source-specific evidence in S02-F02, S08-F01, and S11-F01.
|
|
||||||
|
|
||||||
Add a source-neutral execution-setting validator to `internal/domain`. It must
|
|
||||||
validate temperature, maximum tokens, top-p, and timeout on a domain execution
|
|
||||||
target: temperature and top-p must be finite and within their closed ranges,
|
|
||||||
maximum tokens must be non-negative, and timeout must be non-negative and no
|
|
||||||
greater than the derived duration-safe maximum. Keep optional-pointer presence,
|
|
||||||
profile required fields, normalization, and error wrapping outside this
|
|
||||||
validator.
|
|
||||||
|
|
||||||
Use that owner from:
|
|
||||||
|
|
||||||
- in-memory profile validation in the root package;
|
|
||||||
- OS and `fs.FS` profile validation;
|
|
||||||
- resolved request/target validation in `internal/usecase`; and
|
|
||||||
- the built-in model client as a defensive final boundary.
|
|
||||||
|
|
||||||
Remove the duplicated scalar comparisons from those callers. Preserve
|
|
||||||
`ErrInvalidConfig` for in-memory construction, profile-load identities for file
|
|
||||||
profiles, `ErrInvalidRequest` for runtime overrides, and the LLM package's
|
|
||||||
defensive invalid-request identity. Explicit numeric zero must retain its
|
|
||||||
presence semantics.
|
|
||||||
|
|
||||||
Update `internal/domain.NormalizeSessionID` to reject invalid UTF-8 before
|
|
||||||
trimming or rune counting. Preserve the existing blank and 256-code-point
|
|
||||||
rules. Direct requests must still map failures to `ErrInvalidRequest`, while
|
|
||||||
session-template failures remain renderer failures.
|
|
||||||
|
|
||||||
Add one domain-owned table for every exact setting boundary, finite neighbors,
|
|
||||||
NaN, both infinities, negative values, and the timeout representability edge.
|
|
||||||
Retain small boundary-integration cases for in-memory profiles, both file
|
|
||||||
source forms, request overrides through `Prepare` and `PrepareExecution`, and
|
|
||||||
the model-client defense. Add malformed UTF-8 session cases before, within,
|
|
||||||
and after otherwise valid content.
|
|
||||||
|
|
||||||
Update the architecture policy and internal component overview so
|
|
||||||
`internal/domain` explicitly owns source-neutral invariants for its shared
|
|
||||||
execution values, without claiming ownership of source-specific policy.
|
|
||||||
|
|
||||||
Run focused domain, profile, use-case, root, and LLM tests, including the
|
|
||||||
affected race-enabled request and profile cases, followed by the repository
|
|
||||||
test and vet gates.
|
|
||||||
|
|
||||||
## Stage 2: Centralize Output-Contract Legality
|
|
||||||
|
|
||||||
**Finding:** S17-F02, including the request-boundary symptom S11-F02.
|
|
||||||
|
|
||||||
Add one pure `internal/domain` validator for `OutputContract`. It must enforce
|
|
||||||
the format, validation-mode, non-negative repair-attempt, and JSON-Schema path
|
|
||||||
rules fixed above. It must not load schemas or apply source/request defaults.
|
|
||||||
|
|
||||||
Make prompt-definition normalization call the shared validator after its file-
|
|
||||||
specific normalization. Keep prompt-required fields and contextual
|
|
||||||
`ErrInvalidPromptDefinition` ownership in `internal/promptdef`. Make request
|
|
||||||
resolution default an empty replacement format to `text`, then call the same
|
|
||||||
validator and translate failure to `ErrInvalidRequest` before artifact,
|
|
||||||
rendering, validation, admission, or generation work. Keep schema loading and
|
|
||||||
compilation in `internal/validate`.
|
|
||||||
|
|
||||||
Add a domain table covering every supported and unsupported enum, empty values,
|
|
||||||
negative and non-negative repair counts, and schema-path relationships. Retain
|
|
||||||
small prompt-source and use-case integration tables that prove correct error
|
|
||||||
categories and parity between `Prepare` and `PrepareExecution`; do not repeat
|
|
||||||
the entire domain table at those layers.
|
|
||||||
|
|
||||||
Update the architecture and internal overview language added in Stage 1 to
|
|
||||||
include source-neutral output-contract invariants. Run focused domain,
|
|
||||||
prompt-definition, use-case, and root tests, then repository test and vet
|
|
||||||
gates.
|
|
||||||
|
|
||||||
## Stage 3: Make JSON-Compatible Value Handling Coherent And Bounded
|
|
||||||
|
|
||||||
**Findings:** S05-F02, S05-F03, S05-F04.
|
|
||||||
|
|
||||||
Refactor `internal/jsonvalue` around the numeric and traversal decisions fixed
|
|
||||||
by this plan. Remove the safe-integer restriction and apply one numeric rule to
|
|
||||||
all supported representations. Preserve concrete named and unnamed scalar,
|
|
||||||
map, slice, and array types where the existing contract promises preservation;
|
|
||||||
keep nil versus empty container distinctions and `Copy` versus `CopyMap` empty-
|
|
||||||
key behavior.
|
|
||||||
|
|
||||||
Extend the traversal state to track JSON container depth and produced-node
|
|
||||||
work. Enforce the 100-level and 100,000-node limits before allocation or
|
|
||||||
descent would cross them. Continue using active-path identity for cycle
|
|
||||||
detection; do not use alias memoization that would make distinct JSON paths
|
|
||||||
share mutable output. Errors must identify the structural path and whether the
|
|
||||||
depth or work budget was exceeded.
|
|
||||||
|
|
||||||
Expand the focused package tables by behavior branch: signed and unsigned
|
|
||||||
integer widths, ordinary and named finite floats, `json.Number`, pointers and
|
|
||||||
interfaces, named maps/slices/arrays, nil and empty values, mixed nested trees,
|
|
||||||
arrays, mutation isolation, active cycles, alternating just-below/at/over
|
|
||||||
depth, and shared acyclic subgraphs just below and over the work budget. Tests
|
|
||||||
must derive their edges from package constants or relationships instead of
|
|
||||||
copying unexplained literals.
|
|
||||||
|
|
||||||
Retain only representative public/backend/profile/prepared integration cases
|
|
||||||
that prove error translation and ownership. Update public GoDoc only if it
|
|
||||||
currently states the narrower safe-integer behavior; otherwise the existing
|
|
||||||
finite JSON-compatible-number contract remains canonical. Update the relevant
|
|
||||||
public value GoDoc and format/internal documentation to state that excessively
|
|
||||||
deep or large JSON-shaped values are rejected for safety; keep the exact
|
|
||||||
numeric limits owned by the internal constants rather than duplicating them
|
|
||||||
throughout consumer documentation. Run focused package and caller tests,
|
|
||||||
focused race tests, repository tests, and vet.
|
|
||||||
|
|
||||||
## Stage 4: Consolidate Stable Public JSON And Remove Dormant Internal JSON
|
|
||||||
|
|
||||||
**Findings:** S02-F01, S02-F05, S05-F05.
|
|
||||||
|
|
||||||
Refactor `json.go` so each public value has one ordinary field mapping. Use
|
|
||||||
private aliases or embedded wire representations for ordinary fields and keep
|
|
||||||
only timestamp, millisecond-duration, and intentional omission exceptions
|
|
||||||
explicit. Preserve every existing JSON name and omission rule.
|
|
||||||
|
|
||||||
Before converting `duration_ms`, reject values outside the millisecond range
|
|
||||||
that can be multiplied by `time.Millisecond` without overflow. Derive both
|
|
||||||
edges from `time.Duration` bounds. Return a contextual decode error and do not
|
|
||||||
partially update the receiver on failure.
|
|
||||||
|
|
||||||
Add fully populated `PreparedRun` and `RunResult` contract cases. Verify all
|
|
||||||
ordinary fields, intentional omissions, zero and nonzero timing, complete
|
|
||||||
round trips, the largest safe positive and negative millisecond values, and
|
|
||||||
their first unsafe neighbors.
|
|
||||||
|
|
||||||
Remove unused JSON tags and serialization tests from
|
|
||||||
`internal/domain.PreparedRun` after confirming production never marshals that
|
|
||||||
type. Keep credential absence protected at preparation/clone producers and
|
|
||||||
move any useful cache-control JSON assertion to the public `PreparedRun`
|
|
||||||
contract. Do not retain a parallel internal wire format.
|
|
||||||
|
|
||||||
Run focused domain and root JSON tests, repository tests, and vet.
|
|
||||||
|
|
||||||
## Stage 5: Harden Public Ownership And Diagnostic Contracts
|
|
||||||
|
|
||||||
**Findings:** S01-F01, S02-F03, S02-F04, S13-F01.
|
|
||||||
|
|
||||||
Extend the existing run-request formatting test with distinct input URI,
|
|
||||||
input-body, variable, and API-key sentinels. Require their absence from
|
|
||||||
`String`, `GoString`, `%v`, `%+v`, and `%#v` while retaining positive structural
|
|
||||||
summary assertions.
|
|
||||||
|
|
||||||
Add one focused public-LLM-adapter ownership test. Have the injected client
|
|
||||||
mutate and retain prompt messages, cache-control pointers, nested target extra
|
|
||||||
parameters, and structured-output schema values; prove the domain/prepared
|
|
||||||
source remains unchanged and later details or execution cannot race with those
|
|
||||||
mutations.
|
|
||||||
|
|
||||||
Add one direct all-field mapping test for `OpenAICompatibleProfile`. Populate
|
|
||||||
every field distinctly and compare the complete returned `Profile`. Keep only
|
|
||||||
the existing higher-level cases that prove normal validation and nested-value
|
|
||||||
ownership.
|
|
||||||
|
|
||||||
Make copied `PreparedExecution` values format opaquely by using value-receiver
|
|
||||||
formatting behavior shared by non-nil pointers and values. A nil pointer may
|
|
||||||
use Go's normal `<nil>` formatting, but formatting must never panic or expose
|
|
||||||
internal types, field names, addresses, credentials, or content. Cover original
|
|
||||||
pointers, copied values, zero values, and nil pointers under string, Go-string,
|
|
||||||
and ordinary fmt verbs, and prove formatting does not claim or discard a
|
|
||||||
handle.
|
|
||||||
|
|
||||||
Run focused root tests and the affected prepared/adapter race tests, followed
|
|
||||||
by repository tests and vet.
|
|
||||||
|
|
||||||
## Stage 6: Correct Engine Construction Edges And Immutable Defaults
|
|
||||||
|
|
||||||
**Findings:** S03-F01, S03-F02, S06-F01.
|
|
||||||
|
|
||||||
Change the shared single-file option helper so trimming is used only for the
|
|
||||||
blank-input check. Perform `Stat`, path decomposition, storage, diagnostics,
|
|
||||||
and later access with the exact caller path for prompt, profile, and schema
|
|
||||||
files. Add one compact table covering existing leading- and trailing-whitespace
|
|
||||||
names through all three options.
|
|
||||||
|
|
||||||
Strengthen engine construction tests with three discriminating cases:
|
|
||||||
|
|
||||||
- reverse the argument order of in-memory, ordinary, fallback, and built-in
|
|
||||||
profile categories while retaining fixed category precedence;
|
|
||||||
- collide `Config.ProfileDir` with an ordinary profile option and prove the
|
|
||||||
option replaces the configuration source; and
|
|
||||||
- place a valid same-category replacement after an invalid option and prove
|
|
||||||
construction still fails at the earlier invalid option.
|
|
||||||
|
|
||||||
Convert `internal/defaults.LLMRequestTimeoutDefault` from a variable to a
|
|
||||||
constant without changing its value or adding a setter. Do not add a test that
|
|
||||||
mutates or pins a noncontractual default; existing client deadline behavior is
|
|
||||||
the verification owner.
|
|
||||||
|
|
||||||
Run focused engine construction, default-client construction, and race tests,
|
|
||||||
then repository tests and vet.
|
|
||||||
|
|
||||||
## Stage 7: Contain And Preserve Prompt Content Paths
|
|
||||||
|
|
||||||
**Finding:** S07-F01.
|
|
||||||
|
|
||||||
Refactor prompt content resolution so both repository forms receive an
|
|
||||||
explicit source-root abstraction. Enforce the path decision fixed by this plan
|
|
||||||
before any content read. Use exact parsed path text after a separate blank
|
|
||||||
check. For OS sources, canonicalize the root and resolved target sufficiently
|
|
||||||
to reject symlink escape; for injected `fs.FS`, use its clean relative path
|
|
||||||
namespace. A parent component that remains inside the root is valid. Absolute,
|
|
||||||
escaping, and symlink-escaping targets are invalid.
|
|
||||||
|
|
||||||
Apply the same behavioral table to an OS directory, `WithPromptFS`, and a
|
|
||||||
single-file source: ordinary sibling, nested parent still within root, parent
|
|
||||||
escape, absolute path, symlink escape where supported, and existing names with
|
|
||||||
leading or trailing whitespace. Prove rejected targets cause no outside read
|
|
||||||
and public operations preserve `ErrPromptLoad`.
|
|
||||||
|
|
||||||
Update the framework format reference and internal source document to make the
|
|
||||||
single-file root and absolute-path rule explicit. Run focused prompt-definition
|
|
||||||
and public source tests, including race tests, then repository tests and vet.
|
|
||||||
|
|
||||||
## Stage 8: Correct Prompt Selection, Strictness, Coverage, And Lookup Cost
|
|
||||||
|
|
||||||
**Findings:** S07-F02, S07-F03, S07-F04, S07-F06.
|
|
||||||
|
|
||||||
Correct both existing prompt repository paths before consolidating them in the
|
|
||||||
next stage:
|
|
||||||
|
|
||||||
- Recover selector metadata from YAML `id` and `version`; never use a filename
|
|
||||||
stem as an identity.
|
|
||||||
- Apply normalized ID and requested-version selection before semantic
|
|
||||||
normalization or `content_file` reads.
|
|
||||||
- Associate strict YAML, semantic, and content errors only with a reliably
|
|
||||||
matching selected definition. An unidentifiable malformed file is unrelated
|
|
||||||
to point lookup; a reliably selected malformed file remains authoritative.
|
|
||||||
- Require exactly one YAML document. Comments and trailing whitespace are
|
|
||||||
allowed; a second empty or populated document and malformed trailing YAML
|
|
||||||
are `ErrInvalidYAML`.
|
|
||||||
- Continue scanning the YAML metadata required for duplicate detection, but
|
|
||||||
open content only for selected candidates. A selected content file is opened
|
|
||||||
once; unrelated and different-version bodies are never opened.
|
|
||||||
|
|
||||||
Add paired OS and `fs.FS` regressions for same-stem/different-ID malformed
|
|
||||||
files, same-ID/different-version invalid files, selected malformed definitions,
|
|
||||||
additional YAML documents, duplicates, and counting filesystem behavior.
|
|
||||||
Add a compact normalization table for the previously uncovered missing
|
|
||||||
version, blank input name, blank message role, invalid output format, negative
|
|
||||||
repair attempts, and explicit blank default profile. Output-contract rows
|
|
||||||
should exercise the shared Stage 2 owner rather than recreate its full table.
|
|
||||||
|
|
||||||
Run focused prompt-definition, use-case inspection, and root source tests,
|
|
||||||
focused race tests, repository tests, and vet.
|
|
||||||
|
|
||||||
## Stage 9: Unify Prompt Repository Semantics
|
|
||||||
|
|
||||||
**Finding:** S07-F05.
|
|
||||||
|
|
||||||
After Stage 8 establishes correct behavior in both paths, replace their
|
|
||||||
duplicated discovery-to-selection algorithms with one source-neutral prompt
|
|
||||||
selection and normalization flow. Introduce only the small internal source
|
|
||||||
adapter needed for YAML discovery, bytes, exact content opening, display paths,
|
|
||||||
and root containment. Keep genuine OS and `fs.FS` mechanics at the adapter
|
|
||||||
edge.
|
|
||||||
|
|
||||||
Move exact selection, version filtering, strict one-document decoding,
|
|
||||||
selected-error classification, normalization, duplicate handling, and
|
|
||||||
not-found behavior into the shared flow. Preserve point-in-time source access;
|
|
||||||
do not cache catalogs or definitions across operations.
|
|
||||||
|
|
||||||
Turn the Stage 8 behavior matrix into a shared suite over both adapters and
|
|
||||||
retain source-specific tests only for distinct path and I/O failures. Delete
|
|
||||||
superseded duplicate helpers and tests only after the shared suite protects
|
|
||||||
their meaningful behavior. Use a counting filesystem and a before/after
|
|
||||||
benchmark over small and large prompt catalogs to confirm unrelated content is
|
|
||||||
not read and the refactor adds no second scan; do not enforce wall-clock
|
|
||||||
thresholds.
|
|
||||||
|
|
||||||
Update the internal source document to describe the unified semantic owner.
|
|
||||||
Run focused package, integration, race, repository test, and vet gates.
|
|
||||||
|
|
||||||
## Stage 10: Correct Profile Source Validation And Identity
|
|
||||||
|
|
||||||
**Findings:** S08-F02, S08-F03, S08-F04, S08-F05. The non-finite scalar
|
|
||||||
symptom S08-F01 is already resolved by Stage 1.
|
|
||||||
|
|
||||||
Make file profile normalization pass `extra_params` through
|
|
||||||
`internal/jsonvalue.CopyMap` before publishing a domain profile. Preserve
|
|
||||||
`ErrInvalidProfile` and source path context for empty keys, non-finite values,
|
|
||||||
nested invalid data, or traversal-budget failures. Do not move reserved
|
|
||||||
OpenAI-compatible field policy into the profile package.
|
|
||||||
|
|
||||||
Use YAML metadata ID as the only selector; never infer authority from a
|
|
||||||
filename. Normalize the decoded ID once according to this plan. Require exactly
|
|
||||||
one YAML document in both metadata and strict selected decoding, so trailing
|
|
||||||
raw credentials, unknown fields, empty documents, and malformed YAML cannot be
|
|
||||||
ignored. Reliably selected malformed definitions must stop overlay fallback;
|
|
||||||
unrelated malformed files must not.
|
|
||||||
|
|
||||||
Add shared OS and `fs.FS` tables for invalid/valid extra parameters,
|
|
||||||
same-stem/different-ID malformed files beside a valid profile, fallback
|
|
||||||
behavior, additional documents, leading/trailing/blank IDs, normalized
|
|
||||||
duplicates, and exact inspection/preparation of the normalized ID. Retain only
|
|
||||||
representative public error-translation cases.
|
|
||||||
|
|
||||||
Update the framework format and internal source documents if needed to state
|
|
||||||
ID normalization and one-document behavior. Run focused profile, use-case,
|
|
||||||
root, and race tests, followed by repository tests and vet.
|
|
||||||
|
|
||||||
## Stage 11: Eliminate Duplicate Profile Decoding
|
|
||||||
|
|
||||||
**Finding:** S08-F06.
|
|
||||||
|
|
||||||
Refactor point lookup so each file receives one metadata pass and only
|
|
||||||
canonical ID matches receive strict full decoding and normalization. Reuse
|
|
||||||
bytes already read for metadata; do not decode every unrelated full profile or
|
|
||||||
turn the repository into a cache. Preserve deterministic duplicate detection,
|
|
||||||
strict selected errors, overlay fallthrough only on not-found, and fresh
|
|
||||||
point-in-time reads on every operation.
|
|
||||||
|
|
||||||
Add counting/parser-observation tests where stable behavior can be observed,
|
|
||||||
plus benchmarks for small and large catalogs reporting time and allocations.
|
|
||||||
Exercise valid selection, unrelated malformed files, selected malformed files,
|
|
||||||
duplicates, overlay fallthrough, and repeated lookup. Do not add brittle exact
|
|
||||||
allocation thresholds to ordinary tests.
|
|
||||||
|
|
||||||
Run focused profile and root integration tests, benchmarks for diagnostic
|
|
||||||
comparison, race tests, repository tests, and vet.
|
|
||||||
|
|
||||||
## Stage 12: Correct Artifact Semantics, Cancellation, And Hash Tests
|
|
||||||
|
|
||||||
**Findings:** S09-F01, S09-F02, S09-F05.
|
|
||||||
|
|
||||||
Treat an explicitly typed empty inline reference as a valid zero-byte artifact,
|
|
||||||
including `InlineWithURI`. Keep absence at the input map/reference boundary and
|
|
||||||
compute the same metadata and opaque equality value used for other bodies.
|
|
||||||
|
|
||||||
For ordinary file references, inspect the target before opening and again
|
|
||||||
after opening; reject anything that is not a regular file under the decision
|
|
||||||
above. Replace unbounded `io.ReadAll` with a normal synchronous chunked read
|
|
||||||
that checks `ctx.Err()` before open, before and after each read, and before
|
|
||||||
publishing the artifact. Do not return partial artifacts, add a hidden size
|
|
||||||
limit, or launch an abandoned reader goroutine.
|
|
||||||
|
|
||||||
Add source-parity cases for empty and nonempty inline, inline-with-URI, and file
|
|
||||||
content. Add a platform-appropriate FIFO regression proving the known FIFO is
|
|
||||||
rejected without requiring an external writer, and cancellation cases for a
|
|
||||||
pre-canceled file and a progressing regular-file read. Run them repeatedly and
|
|
||||||
under the race detector.
|
|
||||||
|
|
||||||
Replace exact SHA-256 literals with relational assertions: nonempty and stable
|
|
||||||
for repeat reads, equal for equal inline/file bodies, unequal for changed
|
|
||||||
bodies, and propagated opaquely through preparation. Do not document or test a
|
|
||||||
specific algorithm.
|
|
||||||
|
|
||||||
Update public GoDoc and the internal source document to describe regular-file
|
|
||||||
support and cancellation checkpoints. Run focused package, use-case, root,
|
|
||||||
race, repository test, and vet gates.
|
|
||||||
|
|
||||||
## Stage 13: Make Rendering Cancellation-Aware And Reuse Artifact Text
|
|
||||||
|
|
||||||
**Findings:** S09-F03, S09-F04.
|
|
||||||
|
|
||||||
Check context before session work, before and after each template parse and
|
|
||||||
execution, before and after every message, and before returning the completed
|
|
||||||
prompt. Make the `input` helper return an error when cancellation is observed.
|
|
||||||
Do not run template execution in a detached goroutine.
|
|
||||||
|
|
||||||
Within one `Render` call, lazily convert each named artifact body to text once
|
|
||||||
and memoize that string for the session and all messages. Build the cached
|
|
||||||
string in 64 KiB chunks with one pre-grown `strings.Builder`, checking the
|
|
||||||
context between chunks. Preserve bytes exactly, including invalid UTF-8; do not
|
|
||||||
cache across render calls or mutate artifacts. Unknown and nil inputs retain
|
|
||||||
their current errors, and a canceled conversion must not publish or cache a
|
|
||||||
partial string.
|
|
||||||
|
|
||||||
Add deterministic tests for pre-cancellation, cancellation during the chunked
|
|
||||||
input conversion, and cancellation observed after final-message execution.
|
|
||||||
Require the context identity and no partial prompt while preserving active-
|
|
||||||
context template errors. Add benchmarks for one and repeated references across
|
|
||||||
session and messages; report allocations without hard-coded timing limits.
|
|
||||||
|
|
||||||
Run focused renderer/use-case/root tests, benchmarks, repeated race tests,
|
|
||||||
repository tests, and vet.
|
|
||||||
|
|
||||||
## Stage 14: Preserve Exact JSON Validation Semantics
|
|
||||||
|
|
||||||
**Findings:** S10-F01, S10-F05.
|
|
||||||
|
|
||||||
Create one helper for decoding exactly one JSON value with
|
|
||||||
`json.Decoder.UseNumber` and required EOF after trailing whitespace. Use it for
|
|
||||||
schema documents and JSON Schema instance values so large integers, precise
|
|
||||||
decimals, and exponents retain exact `json.Number` semantics through
|
|
||||||
compilation, prepared metadata, copying, and validation.
|
|
||||||
|
|
||||||
For plain `ValidationJSON`, use a non-materializing complete-document syntax
|
|
||||||
check such as `json.Valid`; do not build a generic tree. Preserve the current
|
|
||||||
result distinction: malformed generated JSON is a completed failed validation,
|
|
||||||
not an operational error, and original output bytes remain unchanged.
|
|
||||||
|
|
||||||
Add focused OS and `fs.FS` cases around `2^53`, `1e400`, precise decimals,
|
|
||||||
ordinary numbers, malformed syntax, and trailing values. Exercise schema
|
|
||||||
`const`, minimum/maximum, and `multipleOf`, and verify the public structured
|
|
||||||
schema retains exact numeric values. Add benchmarks for scalar, object, and
|
|
||||||
large-array JSON validation with allocation reporting but no wall-clock
|
|
||||||
contract.
|
|
||||||
|
|
||||||
Update format/internal validation documentation only where it currently
|
|
||||||
implies float64-limited semantics. Run focused validator/use-case/root tests,
|
|
||||||
benchmarks, race tests, repository tests, and vet.
|
|
||||||
|
|
||||||
## Stage 15: Escape Schema Resources And Compile Once Per Operation
|
|
||||||
|
|
||||||
**Findings:** S10-F02, S10-F03.
|
|
||||||
|
|
||||||
Represent schema compiler resources with `url.URL` rather than string
|
|
||||||
concatenation. Use canonical escaped file URLs for OS paths and a private
|
|
||||||
scheme URL whose path segments are escaped for `fs.FS`. Preserve separators,
|
|
||||||
decode resource paths exactly once at the loader boundary, and continue
|
|
||||||
rejecting remote and escaping references. Legal filenames containing percent,
|
|
||||||
space, `#`, `?`, or Unicode must compile, including contained relative
|
|
||||||
references.
|
|
||||||
|
|
||||||
Unify JSON Schema preparation around `validate.PreparedValidation`:
|
|
||||||
|
|
||||||
- the shared preparation pipeline must create one operation-local compiled
|
|
||||||
plan and derive provider-facing root schema metadata from that plan;
|
|
||||||
- `Prepare` may discard the plan after returning metadata;
|
|
||||||
- `Run` must retain and use the plan for its one operation so the schema graph
|
|
||||||
is not loaded or compiled again during validation; and
|
|
||||||
- `PrepareExecution` must retain the same plan in its frozen payload.
|
|
||||||
|
|
||||||
Remove the document-only `SchemaDocumentLoader` capability if it has no
|
|
||||||
remaining production caller. Do not add an engine-wide or cross-operation
|
|
||||||
schema cache. Keep a clear private preparation carrier in `internal/usecase`
|
|
||||||
if needed so public `domain.PreparedRun` remains free of validator interfaces.
|
|
||||||
|
|
||||||
Replace the existing legal-filename expected failure with valid behavior and
|
|
||||||
retain a genuine compiler-registration failure only if reachable through a
|
|
||||||
valid source. Add public parity tests for invalid keywords, malformed and
|
|
||||||
missing direct/second-level references, unsupported dialects, escapes, remote
|
|
||||||
references, and valid multi-document graphs. A counting source must show each
|
|
||||||
document read once per operation and fresh reads across separate operations.
|
|
||||||
|
|
||||||
Update internal source, validator, and runner documentation for the unified
|
|
||||||
plan lifetime. Run focused validator/use-case/root tests, race tests,
|
|
||||||
repository tests, and vet.
|
|
||||||
|
|
||||||
## Stage 16: Make Validation Cancellation Authoritative
|
|
||||||
|
|
||||||
**Finding:** S10-F04.
|
|
||||||
|
|
||||||
Apply the cancellation decision fixed above. Thread context through schema
|
|
||||||
resource loaders and all Promptkit-controlled read/decode helpers. Read opened
|
|
||||||
schema files in context-checked chunks. Check the context immediately before
|
|
||||||
and after JSON decoding, schema compilation, and schema execution; if
|
|
||||||
cancellation occurred during a synchronous dependency call, return the context
|
|
||||||
error instead of a schema or successful validation result. Do not publish a
|
|
||||||
partial plan or validation result.
|
|
||||||
|
|
||||||
Do not place arbitrary `fs.FS` calls or JSON Schema work in goroutines merely
|
|
||||||
to race them against `ctx.Done()`. Tests must therefore distinguish:
|
|
||||||
|
|
||||||
- prompt cancellation before work;
|
|
||||||
- cancellation between controlled read chunks;
|
|
||||||
- cancellation that becomes authoritative immediately after a synchronous
|
|
||||||
compile or validation call returns; and
|
|
||||||
- the documented limitation that Promptkit cannot preempt a dependency method
|
|
||||||
that never returns.
|
|
||||||
|
|
||||||
Use deterministic controlled readers/contexts rather than sleeps. Assert no
|
|
||||||
goroutine growth or leaked work and preserve operational validation and public
|
|
||||||
context identities. Update validator GoDoc and internal source/runner documents
|
|
||||||
to state the synchronous cancellation boundary accurately.
|
|
||||||
|
|
||||||
Run focused cancellation tests normally, repeatedly, and under the race
|
|
||||||
detector, followed by repository tests and vet.
|
|
||||||
|
|
||||||
## Stage 17: Repair And Protect The Retained Internal Repair Path
|
|
||||||
|
|
||||||
**Findings:** S12-F01, S12-F02, S12-F03.
|
|
||||||
|
|
||||||
Retain the internal repair architecture. Extend `RepairRequest` with
|
|
||||||
`ExecutionTargetPresence` and carry the resolved presence bits unchanged into
|
|
||||||
the default repairer's `GenerateRequest`. Factor one use-case-local constructor
|
|
||||||
for common initial/repair generation fields—effective target, presence,
|
|
||||||
credential, backend identity, session, and structured output—while keeping the
|
|
||||||
initial and repair prompts intentionally separate.
|
|
||||||
|
|
||||||
Accumulate every completed generation response's five token-usage fields into
|
|
||||||
run-level usage. The final content/raw output/artifact continues to come from
|
|
||||||
the last candidate, while usage includes initial generation and every completed
|
|
||||||
repair exactly once. A repair call that returns an error still returns no
|
|
||||||
partial public result under current error semantics.
|
|
||||||
|
|
||||||
Replace the one-attempt-only repair coverage with a compact state-machine
|
|
||||||
table for:
|
|
||||||
|
|
||||||
- initial success with no repair;
|
|
||||||
- ineligible basic validation despite a positive budget;
|
|
||||||
- explicit zero and inherited-zero presence across initial and repair calls;
|
|
||||||
- success before a larger budget is exhausted;
|
|
||||||
- exact exhaustion of a larger budget; and
|
|
||||||
- advancement of attempt number, maximum, prior output, diagnostics, final
|
|
||||||
status, cumulative usage, and collaborator call count.
|
|
||||||
|
|
||||||
Retain the distinct capacity integration proving initial and repair generation
|
|
||||||
use the same backend pool and one whole-run admission lease, but simplify it if
|
|
||||||
the new focused table makes repair-state assertions redundant.
|
|
||||||
|
|
||||||
Update the internal runner and capacity documents for presence fidelity and
|
|
||||||
cumulative usage. Do not alter `NewRunner` to install a repairer, public GoDoc
|
|
||||||
that says the engine is single-pass, or the future public repair roadmap.
|
|
||||||
|
|
||||||
Run focused use-case, prepared, capacity, and race tests, followed by repository
|
|
||||||
tests and vet.
|
|
||||||
|
|
||||||
## Stage 18: Preserve Transport Error Identities And Test Deterministically
|
|
||||||
|
|
||||||
**Findings:** S14-F01, S04-F01, S16-F02. Timeout overflow S14-F02 is already
|
|
||||||
resolved through Stage 1's shared bound.
|
|
||||||
|
|
||||||
Preserve the underlying `http.Client.Do` error in the chain while retaining
|
|
||||||
`internal/llm.ErrRequestFailed` and the public generation category. Do not add
|
|
||||||
headers, request content, endpoints, or provider bodies to error text.
|
|
||||||
Cancellation and deadline identities must survive caller cancellation, caller
|
|
||||||
deadline, generation deadline, and whole-request client timeout.
|
|
||||||
|
|
||||||
Replace the port-9999 test with a controlled round tripper or `httptest`
|
|
||||||
endpoint that records the selected URL and returns a deliberate result. It must
|
|
||||||
make no host-dependent connection and must separately prove empty configured
|
|
||||||
base acceptance and request-endpoint precedence.
|
|
||||||
|
|
||||||
Extend the existing external ordinary-run cancellation test to require both
|
|
||||||
`ErrLLMGenerate` and `context.Canceled`; retain lower-layer tests only for their
|
|
||||||
distinct error owners.
|
|
||||||
|
|
||||||
Update the OpenAI-compatible integration and internal LLM documents for error
|
|
||||||
identity behavior. Run focused LLM and root tests with repetition and the race
|
|
||||||
detector, followed by repository tests and vet.
|
|
||||||
|
|
||||||
## Stage 19: Validate And Compose Effective Provider Endpoints
|
|
||||||
|
|
||||||
**Finding:** S14-F03.
|
|
||||||
|
|
||||||
Add one source-neutral OpenAI-compatible base-endpoint validator in
|
|
||||||
`internal/domain`, alongside the effective execution target invariant. It must
|
|
||||||
trim surrounding configuration whitespace, require absolute HTTP or HTTPS with
|
|
||||||
a host, and reject user information, query, and fragment. Use it from backend
|
|
||||||
registration, in-memory and file profiles, resolved request overrides, and the
|
|
||||||
built-in client defense while preserving each boundary's existing config,
|
|
||||||
profile-load, invalid-request, or LLM error category.
|
|
||||||
|
|
||||||
An empty configured base URL remains valid for a built-in client because a
|
|
||||||
resolved request endpoint may supply it later. Validate only a nonempty
|
|
||||||
configured base at construction, and always validate the final selected
|
|
||||||
endpoint before transport. Backends and endpoint-only profiles retain their
|
|
||||||
existing nonempty endpoint requirements.
|
|
||||||
|
|
||||||
Compose the completion URL through parsed URL operations (prefer
|
|
||||||
`url.JoinPath`) so nested paths and trailing slashes reach exactly one
|
|
||||||
`/chat/completions` suffix. Never append to a raw string.
|
|
||||||
|
|
||||||
Add endpoint tables for HTTP and HTTPS, hosts, nested paths, repeated trailing
|
|
||||||
slashes, queries, fragments, user information, relative paths, missing hosts,
|
|
||||||
unsupported schemes, and request/profile/config error mapping. Require every
|
|
||||||
invalid selected endpoint to fail before transport.
|
|
||||||
|
|
||||||
Update the architecture/internal overview for domain endpoint invariants and
|
|
||||||
the OpenAI-compatible integration and internal LLM documents for URL behavior.
|
|
||||||
Run focused domain/backend/profile/use-case/LLM/root tests, repetition and race
|
|
||||||
tests where ownership crosses packages, followed by repository tests and vet.
|
|
||||||
|
|
||||||
## Stage 20: Bound And Strictly Frame Successful Provider Responses
|
|
||||||
|
|
||||||
**Findings:** S14-F04, S14-F05.
|
|
||||||
|
|
||||||
Enforce the 16 MiB successful-response decision without first copying the
|
|
||||||
entire body. Reject an over-limit `Content-Length` immediately, but also wrap
|
|
||||||
the body in a counting/limited reader that reads at most one byte beyond the
|
|
||||||
limit so chunked or dishonest responses cannot bypass it. Exactly-limit bodies
|
|
||||||
remain valid. Always close the body; do not drain an unbounded oversized
|
|
||||||
stream.
|
|
||||||
|
|
||||||
Decode exactly one response object. After the first decode, require only
|
|
||||||
trailing JSON whitespace and EOF. A second value, non-whitespace suffix,
|
|
||||||
truncated body, malformed JSON, or size overflow returns
|
|
||||||
`ErrMalformedResponse` with no partial response and no provider content in the
|
|
||||||
error.
|
|
||||||
|
|
||||||
Add streaming tests just below, at, and one byte over the limit with and
|
|
||||||
without `Content-Length`, plus a continuing oversized stream. Assert bounded
|
|
||||||
bytes read, timely return, no partial result, and closure. Add trailing
|
|
||||||
whitespace success and trailing garbage/second-value failures. Retain ordinary
|
|
||||||
response mapping and redaction cases.
|
|
||||||
|
|
||||||
Document the fixed successful-response boundary and strict one-document rule
|
|
||||||
in the integration and internal LLM documents. Explicitly leave bounded
|
|
||||||
non-success error-envelope parsing to the structured-generation-error roadmap.
|
|
||||||
Run focused transport tests normally, repeatedly, and under race, followed by
|
|
||||||
repository tests and vet.
|
|
||||||
|
|
||||||
## Stage 21: Consolidate Transport Test Scaffolding
|
|
||||||
|
|
||||||
**Finding:** S14-F06.
|
|
||||||
|
|
||||||
After transport behavior is stable, introduce one small recording-provider
|
|
||||||
fixture for common request capture and successful/error response setup.
|
|
||||||
Organize focused tables around request mapping, authentication, endpoint
|
|
||||||
composition, timeout/error identity, and response framing. Keep specialized
|
|
||||||
round trippers/readers for cancellation, deadlines, byte counts, continuing
|
|
||||||
streams, and body closure.
|
|
||||||
|
|
||||||
Retain every existing durable assertion for method, path, headers,
|
|
||||||
authentication, omission and explicit presence, reserved fields, cache
|
|
||||||
control, structured output, response mapping, usage, error redaction, and
|
|
||||||
timeout precedence. Retain all Stage 18 through 20 regressions. Delete repeated
|
|
||||||
servers, generic-map decoding, and response literals only where the fixture
|
|
||||||
makes the owning behavior clearer; do not replace wire assertions with a broad
|
|
||||||
snapshot.
|
|
||||||
|
|
||||||
Run the LLM suite normally, with shuffle/repetition, and under the race
|
|
||||||
detector. Deliberately inspect the resulting test inventory against the audit's
|
|
||||||
transport matrix before running repository tests and vet.
|
|
||||||
|
|
||||||
## Stage 22: Restore One Canonical Maintainer Validation Workflow
|
|
||||||
|
|
||||||
**Finding:** S16-F01.
|
|
||||||
|
|
||||||
Make `docs/development.md` the canonical owner of the complete local maintainer
|
|
||||||
workflow, as assigned by the documentation policy. Its validation section must
|
|
||||||
include, from the repository root:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./...
|
|
||||||
go test -race ./...
|
|
||||||
go vet ./...
|
|
||||||
go build ./...
|
|
||||||
go run ./examples/go-library/prepare
|
|
||||||
go run ./examples/go-library/run
|
|
||||||
```
|
|
||||||
|
|
||||||
It must also own the Go formatting, local Markdown link, `git diff --check`,
|
|
||||||
workspace/vendor/replacement, generated-output, credential, and working-tree
|
|
||||||
hygiene checks used before accepting changes.
|
|
||||||
|
|
||||||
Change the testing policy to state the semantic requirements and link to that
|
|
||||||
canonical workflow instead of maintaining a partial competing command list.
|
|
||||||
Change the release procedure to invoke the development-guide validation as a
|
|
||||||
release prerequisite rather than presenting a separately maintained copy;
|
|
||||||
retain release-specific metadata, candidate, tag, and publication commands in
|
|
||||||
the release document.
|
|
||||||
|
|
||||||
Run both examples offline and confirm that a missing or invalid Run example
|
|
||||||
fixture makes its command fail. Validate all changed Markdown links and ensure
|
|
||||||
current-state documentation describes only the implemented workflow.
|
|
||||||
|
|
||||||
## Stage 23: Complete Traceability And Final Validation
|
|
||||||
|
|
||||||
This final stage introduces no new behavior. Review the final tree against the
|
|
||||||
finding-to-stage table below and the evidence in `audit.md`. Confirm every
|
|
||||||
canonical group is implemented and every source-specific symptom retains its
|
|
||||||
required regression and error boundary. Do not mark a finding resolved merely
|
|
||||||
because a nearby refactor landed.
|
|
||||||
|
|
||||||
Run the complete development-guide workflow, including both examples, all
|
|
||||||
formatting and link checks, and repository hygiene. Also run shuffled ordinary
|
|
||||||
tests and repeated race-enabled tests for the changed concurrency,
|
|
||||||
cancellation, prepared, validation, repair, and transport packages. Run the
|
|
||||||
accepted performance benchmarks for prompt lookup, profile lookup, rendering,
|
|
||||||
and JSON validation and record only qualitative before/after conclusions; do
|
|
||||||
not establish release timing promises.
|
|
||||||
|
|
||||||
Inspect canonical GoDoc, formats, integration, architecture, and internal
|
|
||||||
documents against the final implementation. Confirm the public engine still
|
|
||||||
performs no output repair and the future repair entry remains future work.
|
|
||||||
Confirm the structured-generation-error feature was not implemented as part of
|
|
||||||
transport remediation.
|
|
||||||
|
|
||||||
Leave `audit-sequence.md`, `audit.md`, and this plan in place for maintainer
|
|
||||||
review. Retire them only in a separately authorized roadmap-cleanup pass after
|
|
||||||
the remediation has been reviewed and accepted.
|
|
||||||
|
|
||||||
## Finding-To-Stage Traceability
|
|
||||||
|
|
||||||
| Stage | Canonical findings | Historical or source-specific records handled with the canonical owner |
|
|
||||||
| ---: | --- | --- |
|
|
||||||
| 1 | S05-F01, S17-F01, S14-F02 | S02-F02, S08-F01, S11-F01 |
|
|
||||||
| 2 | S17-F02 | S11-F02 |
|
|
||||||
| 3 | S05-F02, S05-F03, S05-F04 | None |
|
|
||||||
| 4 | S02-F01, S02-F05, S05-F05 | None |
|
|
||||||
| 5 | S01-F01, S02-F03, S02-F04, S13-F01 | None |
|
|
||||||
| 6 | S03-F01, S03-F02, S06-F01 | None |
|
|
||||||
| 7 | S07-F01 | None |
|
|
||||||
| 8 | S07-F02, S07-F03, S07-F04, S07-F06 | None |
|
|
||||||
| 9 | S07-F05 | None |
|
|
||||||
| 10 | S08-F02, S08-F03, S08-F04, S08-F05 | S08-F01 was handled in Stage 1 |
|
|
||||||
| 11 | S08-F06 | None |
|
|
||||||
| 12 | S09-F01, S09-F02, S09-F05 | None |
|
|
||||||
| 13 | S09-F03, S09-F04 | None |
|
|
||||||
| 14 | S10-F01, S10-F05 | None |
|
|
||||||
| 15 | S10-F02, S10-F03 | None |
|
|
||||||
| 16 | S10-F04 | None |
|
|
||||||
| 17 | S12-F01, S12-F02, S12-F03 | None |
|
|
||||||
| 18 | S14-F01, S04-F01, S16-F02 | S14-F02 was handled in Stage 1 |
|
|
||||||
| 19 | S14-F03 | None |
|
|
||||||
| 20 | S14-F04, S14-F05 | None |
|
|
||||||
| 21 | S14-F06 | None |
|
|
||||||
| 22 | S16-F01 | None |
|
|
||||||
|
|
||||||
The table maps all 49 canonical remediation groups exactly once. S02-F02 is
|
|
||||||
the one superseded historical finding retained as evidence under S17-F01;
|
|
||||||
S08-F01, S11-F01, and S11-F02 retain their source-specific regression
|
|
||||||
responsibilities without being double-counted as canonical groups.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
None. The numeric contract, resource bounds, path and identity rules,
|
|
||||||
validation-cancellation limitation, repair retention, transport response
|
|
||||||
limit, and documentation ownership required to implement these stages are
|
|
||||||
fixed above.
|
|
||||||
@@ -1,71 +0,0 @@
|
|||||||
# Structured Generation Errors
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Promptkit should give downstream applications actionable, machine-readable
|
|
||||||
details when the built-in OpenAI-compatible client receives a non-success HTTP
|
|
||||||
response. Today the client reports only the status code and discards the
|
|
||||||
provider response body. This makes ordinary configuration failures—such as an
|
|
||||||
unsupported strict JSON Schema keyword—unnecessarily difficult to diagnose.
|
|
||||||
|
|
||||||
## Target End State
|
|
||||||
|
|
||||||
Failures from the built-in transport are available through a public typed error
|
|
||||||
that works with `errors.As` while continuing to match `ErrLLMGenerate` through
|
|
||||||
`errors.Is`. The error should expose:
|
|
||||||
|
|
||||||
- the HTTP status code;
|
|
||||||
- a normalized provider error code or type when supplied; and
|
|
||||||
- a bounded provider message extracted from a recognized OpenAI-compatible
|
|
||||||
JSON error envelope.
|
|
||||||
|
|
||||||
The ordinary `Error()` string should remain safe and concise: it should include
|
|
||||||
the status and provider code or type, but not automatically include the
|
|
||||||
provider message. Consumers that deliberately want the provider's diagnostic
|
|
||||||
text can retrieve it from the typed error and apply their own disclosure and
|
|
||||||
logging policy.
|
|
||||||
|
|
||||||
This contract should be available for both ordinary and prepared execution.
|
|
||||||
Errors returned by injected model clients must continue to preserve their own
|
|
||||||
identity and should not be converted into fabricated HTTP details.
|
|
||||||
|
|
||||||
## Safety And Compatibility Boundaries
|
|
||||||
|
|
||||||
- Never expose the raw response body, response headers, endpoint, credentials,
|
|
||||||
request messages, schema document, or generated content through this API.
|
|
||||||
- Read only a small fixed maximum response body, reject malformed or
|
|
||||||
unrecognized envelopes, normalize invalid UTF-8 and control characters, and
|
|
||||||
cap every retained diagnostic field independently.
|
|
||||||
- Treat the extracted provider message as untrusted and potentially sensitive:
|
|
||||||
its GoDoc must tell consumers not to log or display it without applying their
|
|
||||||
own policy.
|
|
||||||
- Preserve the existing generic behavior when a response is empty, non-JSON,
|
|
||||||
oversized, or does not match a recognized error envelope.
|
|
||||||
- Do not assign retryability from an HTTP status. Promptkit supplies facts;
|
|
||||||
downstream applications retain retry and presentation policy.
|
|
||||||
|
|
||||||
## Recommended API Direction
|
|
||||||
|
|
||||||
Prefer one immutable public `GenerationError` value, constructed internally and
|
|
||||||
carrying accessors for HTTP status, provider code or type, and provider message.
|
|
||||||
This keeps the exact representation evolvable while giving consumers an
|
|
||||||
idiomatic `errors.As` contract. Public Go declarations and GoDoc should own the
|
|
||||||
final exact names and semantics.
|
|
||||||
|
|
||||||
The internal OpenAI-compatible client should parse only the conventional
|
|
||||||
top-level `error` envelope and pass normalized details through the use-case and
|
|
||||||
public error-mapping layers. The integration documentation should continue to
|
|
||||||
own wire behavior; the public declarations should own the consumer contract.
|
|
||||||
|
|
||||||
## Acceptance Criteria
|
|
||||||
|
|
||||||
- A downstream consumer can distinguish a provider HTTP 400 from other
|
|
||||||
generation failures and obtain a bounded provider explanation when present.
|
|
||||||
- The typed error still satisfies `errors.Is(err, ErrLLMGenerate)`.
|
|
||||||
- Existing cancellation, capacity, validation, and injected-client error
|
|
||||||
identities remain unchanged.
|
|
||||||
- Tests cover recognized string and numeric provider codes, absent and malformed
|
|
||||||
envelopes, oversized bodies and fields, control characters, and error-chain
|
|
||||||
behavior without making live provider requests.
|
|
||||||
- Current-state GoDoc and the OpenAI-compatible integration and internal-client
|
|
||||||
documents are updated only when the implementation lands.
|
|
||||||
108
engine.go
108
engine.go
@@ -11,14 +11,16 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter"
|
||||||
|
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome"
|
||||||
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/catalog"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile/builtin"
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
|
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||||
@@ -53,9 +55,9 @@ var (
|
|||||||
// an execution profile or resolve its backend, except for the profile
|
// an execution profile or resolve its backend, except for the profile
|
||||||
// not-found case represented by ErrProfileNotFound.
|
// not-found case represented by ErrProfileNotFound.
|
||||||
ErrProfileLoad = errors.New("failed to load execution profile")
|
ErrProfileLoad = errors.New("failed to load execution profile")
|
||||||
// ErrAPIKeyEnvMissing identifies an APIKeyEnv whose environment variable is
|
// ErrAPIKeyEnvMissing identifies an explicitly required APIKeyEnv whose
|
||||||
// unset or empty when no direct RunRequest.APIKey takes precedence. Such an
|
// environment variable is unset or empty after direct RunRequest.APIKey
|
||||||
// error also matches ErrInvalidRequest.
|
// precedence is applied. Such an error also matches ErrInvalidRequest.
|
||||||
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
||||||
// ErrArtifactLoad identifies a failure to resolve an input artifact. Errors
|
// ErrArtifactLoad identifies a failure to resolve an input artifact. Errors
|
||||||
// returned by an injected ArtifactReader remain available through errors.Is.
|
// returned by an injected ArtifactReader remain available through errors.Is.
|
||||||
@@ -69,8 +71,9 @@ var (
|
|||||||
// request, an LLM or provider rate-limit response, or ErrLLMGenerate.
|
// request, an LLM or provider rate-limit response, or ErrLLMGenerate.
|
||||||
ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
||||||
// ErrLLMGenerate identifies a model-client failure or a nil successful
|
// ErrLLMGenerate identifies a model-client failure or a nil successful
|
||||||
// response. Errors returned by an injected LLMClient remain available
|
// response. A built-in OpenAI-compatible non-2xx response is available as a
|
||||||
// through errors.Is.
|
// [GenerationError]. Errors returned by an injected LLMClient remain
|
||||||
|
// available through errors.Is.
|
||||||
ErrLLMGenerate = errors.New("failed to generate output")
|
ErrLLMGenerate = errors.New("failed to generate output")
|
||||||
// ErrValidation identifies an operational failure to load or compile a
|
// ErrValidation identifies an operational failure to load or compile a
|
||||||
// schema or validate output. A completed validation whose Status is
|
// schema or validate output. A completed validation whose Status is
|
||||||
@@ -99,7 +102,7 @@ type Config struct {
|
|||||||
// prompt source.
|
// prompt source.
|
||||||
PromptDir string
|
PromptDir string
|
||||||
// ProfileDir is an optional ordinary configured source whose profiles take
|
// ProfileDir is an optional ordinary configured source whose profiles take
|
||||||
// precedence over application fallback and embedded built-in profiles. An
|
// precedence over application fallback and maintained catalog profiles. An
|
||||||
// empty value selects the lower-precedence sources unless a profile-source
|
// empty value selects the lower-precedence sources unless a profile-source
|
||||||
// option supplies the ordinary source.
|
// option supplies the ordinary source.
|
||||||
ProfileDir string
|
ProfileDir string
|
||||||
@@ -219,7 +222,7 @@ func WithPromptFile(path string) Option {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
options.promptDefs = promptdef.NewFSRepository(fsys, root)
|
options.promptDefs = promptdef.NewFileRepository(fsys, root, filepath.Dir(path))
|
||||||
options.promptSource = true
|
options.promptSource = true
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
@@ -272,7 +275,7 @@ func WithProfileFile(path string) Option {
|
|||||||
//
|
//
|
||||||
// Profile lookup checks, in order, profiles supplied by WithProfiles; the
|
// Profile lookup checks, in order, profiles supplied by WithProfiles; the
|
||||||
// ordinary configured source selected by WithProfileFile, WithProfileFS, or
|
// ordinary configured source selected by WithProfileFile, WithProfileFS, or
|
||||||
// Config.ProfileDir; this fallback source; and Promptkit's embedded built-in
|
// Config.ProfileDir; this fallback source; and Promptkit's maintained catalog
|
||||||
// profiles. Each source supplies a complete profile definition; profile fields
|
// profiles. Each source supplies a complete profile definition; profile fields
|
||||||
// are not merged between sources. Only an absent profile ID proceeds to the
|
// are not merged between sources. Only an absent profile ID proceeds to the
|
||||||
// next source. A matching read, parse, duplicate, validation, or credential
|
// next source. A matching read, parse, duplicate, validation, or credential
|
||||||
@@ -303,10 +306,12 @@ func WithFallbackProfileFS(fsys fs.FS, root string) Option {
|
|||||||
// WithProfiles configures in-memory profiles that take precedence over
|
// WithProfiles configures in-memory profiles that take precedence over
|
||||||
// ordinary configured, application fallback, and built-in profiles.
|
// ordinary configured, application fallback, and built-in profiles.
|
||||||
//
|
//
|
||||||
// NewEngine validates and copies every profile. IDs must be unique within one
|
// NewEngine locally validates and copies every profile. IDs must be unique
|
||||||
// call. An invalid profile, duplicate ID, or unsupported ExtraParams value
|
// within one call. An invalid local definition, duplicate ID, or unsupported
|
||||||
// makes construction fail with ErrInvalidConfig. Repeating WithProfiles
|
// ExtraParams value makes construction fail with ErrInvalidConfig. A derived
|
||||||
// replaces the complete earlier in-memory set rather than merging it.
|
// profile's base reference and resolved target completeness are checked when it
|
||||||
|
// is selected or inspected. Repeating WithProfiles replaces the complete
|
||||||
|
// earlier in-memory set rather than merging it.
|
||||||
func WithProfiles(profiles ...Profile) Option {
|
func WithProfiles(profiles ...Profile) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
repo, err := newMemoryProfileRepository(profiles)
|
repo, err := newMemoryProfileRepository(profiles)
|
||||||
@@ -387,9 +392,17 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
|
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
|
||||||
}
|
}
|
||||||
|
|
||||||
profiles := newProfileRepository(cfg.ProfileDir, options)
|
maintainedCatalogs, err := catalog.Load(
|
||||||
|
catalog.Source{Name: "OpenRouter", ExpectedBackendID: backend.OpenRouterID, FS: openrouter.FS(), Root: openrouter.Root},
|
||||||
|
catalog.Source{Name: "Rakestrawhome", ExpectedBackendID: backend.RakestrawHomeID, FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: failed to load maintained catalogs: %v", ErrInvalidConfig, err)
|
||||||
|
}
|
||||||
|
|
||||||
backendRegistry, err := backend.NewRegistry(options.backends)
|
profiles := newProfileRepository(cfg.ProfileDir, options, maintainedCatalogs.Profiles)
|
||||||
|
|
||||||
|
backendRegistry, err := backend.NewRegistry(maintainedCatalogs.Backends, options.backends)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err)
|
return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err)
|
||||||
}
|
}
|
||||||
@@ -427,7 +440,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &Engine{
|
return &Engine{
|
||||||
runner: usecase.NewRunner(
|
runner: usecase.NewRunnerWithRepairer(
|
||||||
promptDefs,
|
promptDefs,
|
||||||
profiles,
|
profiles,
|
||||||
backendRegistry,
|
backendRegistry,
|
||||||
@@ -435,13 +448,14 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
prompt.NewGoRenderer(),
|
prompt.NewGoRenderer(),
|
||||||
llmClient,
|
llmClient,
|
||||||
validator,
|
validator,
|
||||||
|
usecase.NewDefaultOutputRepairer(llmClient),
|
||||||
capacityManager,
|
capacityManager,
|
||||||
),
|
),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func newProfileRepository(profileDir string, options engineOptions) profile.Repository {
|
func newProfileRepository(profileDir string, options engineOptions, maintained profile.Repository) profile.Repository {
|
||||||
repository := builtin.NewRepository()
|
repository := maintained
|
||||||
|
|
||||||
if options.fallbackProfileSource {
|
if options.fallbackProfileSource {
|
||||||
repository = profile.NewOverlayRepository(options.fallbackProfiles, repository)
|
repository = profile.NewOverlayRepository(options.fallbackProfiles, repository)
|
||||||
@@ -457,25 +471,24 @@ func newProfileRepository(profileDir string, options engineOptions) profile.Repo
|
|||||||
repository = profile.NewOverlayRepository(options.memoryProfiles, repository)
|
repository = profile.NewOverlayRepository(options.memoryProfiles, repository)
|
||||||
}
|
}
|
||||||
|
|
||||||
return repository
|
return profile.NewResolvingRepository(repository)
|
||||||
}
|
}
|
||||||
|
|
||||||
func fileSource(name string) (fs.FS, string, error) {
|
func fileSource(name string) (fs.FS, string, error) {
|
||||||
cleanName := strings.TrimSpace(name)
|
if strings.TrimSpace(name) == "" {
|
||||||
if cleanName == "" {
|
|
||||||
return nil, "", ErrInvalidConfig
|
return nil, "", ErrInvalidConfig
|
||||||
}
|
}
|
||||||
dir := filepath.Dir(cleanName)
|
dir := filepath.Dir(name)
|
||||||
base := filepath.Base(cleanName)
|
base := filepath.Base(name)
|
||||||
if base == "." || base == string(filepath.Separator) || strings.TrimSpace(base) == "" {
|
if base == "." || base == string(filepath.Separator) {
|
||||||
return nil, "", ErrInvalidConfig
|
return nil, "", ErrInvalidConfig
|
||||||
}
|
}
|
||||||
info, err := os.Stat(cleanName)
|
info, err := os.Stat(name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, cleanName, err)
|
return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, name, err)
|
||||||
}
|
}
|
||||||
if info.IsDir() {
|
if info.IsDir() {
|
||||||
return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, cleanName)
|
return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, name)
|
||||||
}
|
}
|
||||||
return os.DirFS(dir), filepath.ToSlash(base), nil
|
return os.DirFS(dir), filepath.ToSlash(base), nil
|
||||||
}
|
}
|
||||||
@@ -568,6 +581,8 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Prepare resolves and renders a prompt request without calling an LLM.
|
// Prepare resolves and renders a prompt request without calling an LLM.
|
||||||
|
// It appends any validated RunRequest.AppendedMessages after rendered
|
||||||
|
// definition messages in the returned caller-owned snapshot.
|
||||||
//
|
//
|
||||||
// Prepare selects the prompt and profile, resolves any selected backend and
|
// Prepare selects the prompt and profile, resolves any selected backend and
|
||||||
// effective execution settings, resolves the output contract, loads and hashes
|
// effective execution settings, resolves the output contract, loads and hashes
|
||||||
@@ -602,7 +617,8 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
|||||||
}
|
}
|
||||||
|
|
||||||
// PrepareExecution completely prepares a prompt request without calling the
|
// PrepareExecution completely prepares a prompt request without calling the
|
||||||
// configured LLMClient or reserving backend admission capacity.
|
// configured LLMClient or reserving backend admission capacity. Validated
|
||||||
|
// RunRequest.AppendedMessages are included in the frozen effective messages.
|
||||||
//
|
//
|
||||||
// The returned opaque handle is bound to this Engine and permits one
|
// The returned opaque handle is bound to this Engine and permits one
|
||||||
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
|
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
|
||||||
@@ -634,24 +650,29 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Run prepares a request, invokes the configured LLMClient, and validates the
|
// Run prepares a request, invokes the configured LLMClient, and validates the
|
||||||
// generated output.
|
// generated output. Each call resolves current sources and composes a fresh,
|
||||||
|
// stateless effective prompt with any validated RunRequest.AppendedMessages.
|
||||||
//
|
//
|
||||||
// A content-validation failure is a successful run whose
|
// A content-validation failure is a successful run whose
|
||||||
// RunResult.Validation has Status ValidationFailed. An inability to perform
|
// RunResult.Validation has Status ValidationFailed. When its output contract
|
||||||
// validation returns an error matching ErrValidation and no partial result.
|
// has a positive repair budget, a failed eligible validation can make bounded
|
||||||
// The public Engine does not perform output repair, so validation is
|
// additional model calls and stops at the first valid candidate. Exhaustion
|
||||||
// single-pass even when OutputContract.RepairAttempts is positive.
|
// returns the final failed validation result with cumulative usage and actual
|
||||||
|
// repair attempts. An inability to generate or validate returns an error and
|
||||||
|
// no partial result.
|
||||||
//
|
//
|
||||||
// Run can return every error category documented by [Engine.Prepare], plus
|
// Run can return every error category documented by [Engine.Prepare], plus
|
||||||
// ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is
|
// ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is
|
||||||
// discoverable as [CapacityError] and still matches ErrCapacityExceeded. It
|
// discoverable as [CapacityError] and still matches ErrCapacityExceeded. It
|
||||||
// occurs before artifacts, schemas, rendering, or model generation because the
|
// occurs before artifacts, schemas, rendering, or model generation because the
|
||||||
// selected backend's admission capacity is full; it does not match
|
// selected backend's admission capacity is full; it does not match
|
||||||
// ErrInvalidRequest or ErrLLMGenerate. Errors from injected clients remain
|
// ErrInvalidRequest or ErrLLMGenerate. A built-in OpenAI-compatible non-2xx
|
||||||
// available through errors.Is. Cancellation while waiting for model-generation
|
// response is discoverable as [GenerationError]. Errors from injected clients
|
||||||
// capacity matches both ErrLLMGenerate and the context error. Cancellation
|
// remain available through errors.Is. Cancellation while waiting for
|
||||||
// otherwise follows the active collaborator's documented behavior. A nil
|
// model-generation capacity matches both ErrLLMGenerate and the context error.
|
||||||
// Engine returns ErrInvalidConfig. Run returns no partial result on error.
|
// Cancellation otherwise follows the active collaborator's documented
|
||||||
|
// behavior. A nil Engine returns ErrInvalidConfig. Run returns no partial
|
||||||
|
// result on error.
|
||||||
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||||
if e == nil || e.runner == nil {
|
if e == nil || e.runner == nil {
|
||||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
@@ -682,16 +703,17 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
|||||||
//
|
//
|
||||||
// The supplied context governs this execution attempt independently of the
|
// The supplied context governs this execution attempt independently of the
|
||||||
// preparation context. It covers credential revalidation, admission,
|
// preparation context. It covers credential revalidation, admission,
|
||||||
// generation, validation, and any internal repair. Result timing begins after
|
// generation, validation, and any bounded output repair. Result timing begins
|
||||||
// the claim and excludes preparation and consumer-held delay.
|
// after the claim and excludes preparation and consumer-held delay.
|
||||||
//
|
//
|
||||||
// RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing,
|
// RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing,
|
||||||
// ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while
|
// ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while
|
||||||
// preserving documented collaborator and context identities. An engine
|
// preserving documented collaborator and context identities. An engine
|
||||||
// admission rejection is discoverable as [CapacityError] and still matches
|
// admission rejection is discoverable as [CapacityError] and still matches
|
||||||
// ErrCapacityExceeded. A completed content-validation rejection is returned
|
// ErrCapacityExceeded. A built-in OpenAI-compatible non-2xx response is
|
||||||
// in RunResult, not as an operational error. An operational error returns no
|
// discoverable as [GenerationError]. A completed content-validation rejection,
|
||||||
// partial RunResult.
|
// including repair exhaustion, is returned in RunResult, not as an operational
|
||||||
|
// error. An operational error returns no partial RunResult.
|
||||||
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
|
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
|
||||||
if e == nil || e.runner == nil {
|
if e == nil || e.runner == nil {
|
||||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
|||||||
1289
engine_test.go
1289
engine_test.go
File diff suppressed because it is too large
Load Diff
14
errors.go
14
errors.go
@@ -6,6 +6,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||||
@@ -21,6 +22,19 @@ func mapPublicError(err error) error {
|
|||||||
return &CapacityError{BackendID: internalCapacityError.BackendID}
|
return &CapacityError{BackendID: internalCapacityError.BackendID}
|
||||||
}
|
}
|
||||||
publicErr := publicErrorFor(err)
|
publicErr := publicErrorFor(err)
|
||||||
|
var providerHTTPError *llm.ProviderHTTPError
|
||||||
|
if errors.As(err, &providerHTTPError) && providerHTTPError != nil {
|
||||||
|
generationErr := newGenerationError(
|
||||||
|
providerHTTPError.StatusCode(),
|
||||||
|
providerHTTPError.ProviderCode(),
|
||||||
|
providerHTTPError.ProviderType(),
|
||||||
|
providerHTTPError.ProviderMessage(),
|
||||||
|
)
|
||||||
|
if publicErr != nil && !errors.Is(publicErr, ErrLLMGenerate) {
|
||||||
|
return fmt.Errorf("%w: %w", publicErr, generationErr)
|
||||||
|
}
|
||||||
|
return generationErr
|
||||||
|
}
|
||||||
if publicErr == nil {
|
if publicErr == nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -48,3 +49,27 @@ func TestMapPublicErrorTranslatesCapacityError(t *testing.T) {
|
|||||||
t.Fatalf("mapped backend ID changed with source error: %q", publicErr.BackendID)
|
t.Fatalf("mapped backend ID changed with source error: %q", publicErr.BackendID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMapPublicErrorPreservesValidationAroundGenerationError(t *testing.T) {
|
||||||
|
internalErr := fmt.Errorf(
|
||||||
|
"%w: %w",
|
||||||
|
usecase.ErrValidation,
|
||||||
|
&llm.ProviderHTTPError{},
|
||||||
|
)
|
||||||
|
|
||||||
|
err := mapPublicError(internalErr)
|
||||||
|
if !errors.Is(err, ErrValidation) {
|
||||||
|
t.Fatalf("mapped error=%v, want ErrValidation", err)
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrLLMGenerate) {
|
||||||
|
t.Fatalf("mapped error=%v, want ErrLLMGenerate", err)
|
||||||
|
}
|
||||||
|
var generationErr *GenerationError
|
||||||
|
if !errors.As(err, &generationErr) || generationErr == nil {
|
||||||
|
t.Fatalf("mapped error=%v, want GenerationError", err)
|
||||||
|
}
|
||||||
|
var leakedInternalErr *llm.ProviderHTTPError
|
||||||
|
if errors.As(err, &leakedInternalErr) {
|
||||||
|
t.Fatalf("mapped error exposes internal ProviderHTTPError: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ func (r RunRequest) GoString() string {
|
|||||||
|
|
||||||
func (r RunRequest) redactedString() string {
|
func (r RunRequest) redactedString() string {
|
||||||
return fmt.Sprintf(
|
return fmt.Sprintf(
|
||||||
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t}",
|
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t AppendedMessages:%d}",
|
||||||
r.PromptID,
|
r.PromptID,
|
||||||
r.PromptVersion,
|
r.PromptVersion,
|
||||||
r.ProfileID,
|
r.ProfileID,
|
||||||
@@ -27,6 +27,7 @@ func (r RunRequest) redactedString() string {
|
|||||||
len(r.Vars),
|
len(r.Vars),
|
||||||
r.Execution != nil,
|
r.Execution != nil,
|
||||||
r.Validation != nil,
|
r.Validation != nil,
|
||||||
|
len(r.AppendedMessages),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
87
generation_error.go
Normal file
87
generation_error.go
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import "fmt"
|
||||||
|
|
||||||
|
// GenerationError reports a non-2xx response from Promptkit's built-in
|
||||||
|
// OpenAI-compatible client during [Engine.Run] or [Engine.RunPrepared].
|
||||||
|
//
|
||||||
|
// Engine-produced values are immutable, caller-owned values. Use errors.Is to
|
||||||
|
// match [ErrLLMGenerate] and errors.As with a *GenerationError target to obtain
|
||||||
|
// this type. The four provider accessors expose untrusted provider-controlled
|
||||||
|
// values that can contain sensitive request or schema fragments. Applications
|
||||||
|
// must apply their own disclosure policy before logging, displaying, or
|
||||||
|
// returning them to another caller.
|
||||||
|
//
|
||||||
|
// Accessors, Error, GoString, and Unwrap are safe on a nil receiver and a zero
|
||||||
|
// value. Default and Go-syntax formatting deliberately redact provider details.
|
||||||
|
// GenerationError has no stable JSON representation.
|
||||||
|
type GenerationError struct {
|
||||||
|
statusCode int
|
||||||
|
providerCode string
|
||||||
|
providerType string
|
||||||
|
providerMessage string
|
||||||
|
}
|
||||||
|
|
||||||
|
func newGenerationError(statusCode int, providerCode, providerType, providerMessage string) *GenerationError {
|
||||||
|
return &GenerationError{
|
||||||
|
statusCode: statusCode,
|
||||||
|
providerCode: providerCode,
|
||||||
|
providerType: providerType,
|
||||||
|
providerMessage: providerMessage,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// StatusCode returns the received provider HTTP status code, or zero for a nil
|
||||||
|
// receiver or zero value.
|
||||||
|
func (e *GenerationError) StatusCode() int {
|
||||||
|
if e == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return e.statusCode
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProviderCode returns the normalized provider error code, if present. Its
|
||||||
|
// value is untrusted and may contain sensitive data.
|
||||||
|
func (e *GenerationError) ProviderCode() string {
|
||||||
|
if e == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return e.providerCode
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProviderType returns the normalized provider error type, if present. Its
|
||||||
|
// value is untrusted and may contain sensitive data.
|
||||||
|
func (e *GenerationError) ProviderType() string {
|
||||||
|
if e == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return e.providerType
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProviderMessage returns the bounded normalized provider diagnostic, if
|
||||||
|
// present. Its value is untrusted and may contain sensitive data.
|
||||||
|
func (e *GenerationError) ProviderMessage() string {
|
||||||
|
if e == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return e.providerMessage
|
||||||
|
}
|
||||||
|
|
||||||
|
// Error returns a redacted diagnostic that is not a parsing contract.
|
||||||
|
func (e *GenerationError) Error() string {
|
||||||
|
if e == nil || e.statusCode == 0 {
|
||||||
|
return ErrLLMGenerate.Error()
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s: provider returned HTTP status %d", ErrLLMGenerate, e.statusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GoString returns the same redacted diagnostic as Error.
|
||||||
|
func (e *GenerationError) GoString() string {
|
||||||
|
return e.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unwrap returns ErrLLMGenerate. It is safe to call on a nil receiver or zero
|
||||||
|
// value.
|
||||||
|
func (e *GenerationError) Unwrap() error {
|
||||||
|
return ErrLLMGenerate
|
||||||
|
}
|
||||||
150
generation_error_contract_test.go
Normal file
150
generation_error_contract_test.go
Normal file
@@ -0,0 +1,150 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBuiltInGenerationError(t *testing.T) {
|
||||||
|
const (
|
||||||
|
codeMarker = "provider-code-marker"
|
||||||
|
typeMarker = "provider-type-marker"
|
||||||
|
messageMarker = "provider-message-marker"
|
||||||
|
)
|
||||||
|
engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity,
|
||||||
|
`{"error":{"code":"`+codeMarker+`","type":"`+typeMarker+`","message":"`+messageMarker+`"}}`)
|
||||||
|
|
||||||
|
result, err := engine.Run(context.Background(), generationErrorRunRequest())
|
||||||
|
if result != nil {
|
||||||
|
t.Fatalf("Run result = %#v, want nil", result)
|
||||||
|
}
|
||||||
|
assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker)
|
||||||
|
|
||||||
|
preparedEngine := newBuiltInGenerationErrorEngine(t, http.StatusServiceUnavailable, `{"error":{}}`)
|
||||||
|
prepared, err := preparedEngine.PrepareExecution(context.Background(), generationErrorRunRequest())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PrepareExecution: %v", err)
|
||||||
|
}
|
||||||
|
result, err = preparedEngine.RunPrepared(context.Background(), prepared)
|
||||||
|
if result != nil {
|
||||||
|
t.Fatalf("RunPrepared result = %#v, want nil", result)
|
||||||
|
}
|
||||||
|
assertGenerationError(t, err, http.StatusServiceUnavailable, "", "", "")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuiltInGenerationErrorWithAppendedMessages(t *testing.T) {
|
||||||
|
const messageMarker = "combined-request-provider-marker"
|
||||||
|
engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity,
|
||||||
|
`{"error":{"message":"`+messageMarker+`"}}`)
|
||||||
|
request := generationErrorRunRequest()
|
||||||
|
request.AppendedMessages = []promptkit.RenderedMessage{
|
||||||
|
{Role: promptkit.RoleAssistant, Content: "previous response"},
|
||||||
|
{Role: promptkit.RoleUser, Content: "consumer correction"},
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := engine.Run(context.Background(), request)
|
||||||
|
if result != nil {
|
||||||
|
t.Fatalf("Run result = %#v, want nil", result)
|
||||||
|
}
|
||||||
|
assertGenerationError(t, err, http.StatusUnprocessableEntity, "", "", messageMarker)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuiltInRepairGenerationError(t *testing.T) {
|
||||||
|
const (
|
||||||
|
codeMarker = "repair-code-marker"
|
||||||
|
typeMarker = "repair-type-marker"
|
||||||
|
messageMarker = "repair-message-marker"
|
||||||
|
)
|
||||||
|
calls := 0
|
||||||
|
config := contractConfig(frameworkSchemaDir)
|
||||||
|
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
calls++
|
||||||
|
if calls == 1 {
|
||||||
|
body := `{"choices":[{"message":{"content":"not-json"}}]}`
|
||||||
|
return &http.Response{StatusCode: http.StatusOK, ContentLength: int64(len(body)), Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||||
|
}
|
||||||
|
body := `{"error":{"code":"` + codeMarker + `","type":"` + typeMarker + `","message":"` + messageMarker + `"}}`
|
||||||
|
return &http.Response{StatusCode: http.StatusUnprocessableEntity, ContentLength: int64(len(body)), Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(config)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewEngine: %v", err)
|
||||||
|
}
|
||||||
|
req := generationErrorRunRequest()
|
||||||
|
req.Validation = &promptkit.OutputContract{
|
||||||
|
Format: promptkit.FormatJSON,
|
||||||
|
ValidationMode: promptkit.ValidationJSON,
|
||||||
|
RepairAttempts: 1,
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := engine.Run(context.Background(), req)
|
||||||
|
if result != nil {
|
||||||
|
t.Fatalf("Run result = %#v, want nil", result)
|
||||||
|
}
|
||||||
|
if calls != 2 {
|
||||||
|
t.Fatalf("provider calls = %d, want 2", calls)
|
||||||
|
}
|
||||||
|
assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertGenerationError(t *testing.T, err error, statusCode int, code, providerType, message string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
if !errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||||
|
t.Fatalf("errors.Is(%v, ErrLLMGenerate) = false", err)
|
||||||
|
}
|
||||||
|
var generationErr *promptkit.GenerationError
|
||||||
|
if !errors.As(err, &generationErr) || generationErr == nil {
|
||||||
|
t.Fatalf("error = %T, want *GenerationError", err)
|
||||||
|
}
|
||||||
|
if generationErr.StatusCode() != statusCode || generationErr.ProviderCode() != code || generationErr.ProviderType() != providerType || generationErr.ProviderMessage() != message {
|
||||||
|
t.Fatalf("GenerationError = %#v", generationErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantFormatted := fmt.Sprintf("failed to generate output: provider returned HTTP status %d", statusCode)
|
||||||
|
for _, rendered := range []string{fmt.Sprintf("%v", generationErr), fmt.Sprintf("%+v", generationErr), fmt.Sprintf("%#v", generationErr)} {
|
||||||
|
if rendered != wantFormatted {
|
||||||
|
t.Fatalf("formatted error = %q, want %q", rendered, wantFormatted)
|
||||||
|
}
|
||||||
|
for _, marker := range []string{code, providerType, message} {
|
||||||
|
if marker != "" && strings.Contains(rendered, marker) {
|
||||||
|
t.Fatalf("formatted error exposed provider marker %q: %q", marker, rendered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newBuiltInGenerationErrorEngine(t *testing.T, statusCode int, body string) *promptkit.Engine {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
config := contractConfig(frameworkSchemaDir)
|
||||||
|
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{
|
||||||
|
StatusCode: statusCode,
|
||||||
|
ContentLength: int64(len(body)),
|
||||||
|
Body: io.NopCloser(strings.NewReader(body)),
|
||||||
|
}, nil
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(config)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewEngine: %v", err)
|
||||||
|
}
|
||||||
|
return engine
|
||||||
|
}
|
||||||
|
|
||||||
|
func generationErrorRunRequest() promptkit.RunRequest {
|
||||||
|
return promptkit.RunRequest{
|
||||||
|
PromptID: frameworkMarkdownSummaryPromptID,
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||||
|
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
32
generation_error_internal_test.go
Normal file
32
generation_error_internal_test.go
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGenerationErrorNilAndZeroValue(t *testing.T) {
|
||||||
|
var nilError *GenerationError
|
||||||
|
zeroError := &GenerationError{}
|
||||||
|
|
||||||
|
for name, err := range map[string]*GenerationError{
|
||||||
|
"nil": nilError,
|
||||||
|
"zero": zeroError,
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
if err.StatusCode() != 0 || err.ProviderCode() != "" || err.ProviderType() != "" || err.ProviderMessage() != "" {
|
||||||
|
t.Fatalf("accessors returned provider details: %#v", err)
|
||||||
|
}
|
||||||
|
if err.Error() != "failed to generate output" || err.GoString() != "failed to generate output" {
|
||||||
|
t.Fatalf("redacted formatting = (%q, %q)", err.Error(), err.GoString())
|
||||||
|
}
|
||||||
|
if fmt.Sprintf("%v", err) != "failed to generate output" || fmt.Sprintf("%#v", err) != "failed to generate output" {
|
||||||
|
t.Fatalf("formatted error = (%q, %q)", fmt.Sprintf("%v", err), fmt.Sprintf("%#v", err))
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrLLMGenerate) {
|
||||||
|
t.Fatalf("errors.Is(%v, ErrLLMGenerate) = false", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
2
go.mod
2
go.mod
@@ -3,6 +3,8 @@ module gitea.maximumdirect.net/eric/promptkit
|
|||||||
go 1.25.5
|
go 1.25.5
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
|
|||||||
4
go.sum
4
go.sum
@@ -1,3 +1,7 @@
|
|||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 h1:lc062euk2qseO//D762i3JaFyulDNML3eQQX7DkYTho=
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0/go.mod h1:AIa7kAu2mfrRQgcspe4L+DW51WqgnALQT60lqkEywJI=
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 h1:j9YY7wsTVjzke2kHH4YAzpU0oUpM+x+nXwl1IeS+2eg=
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0/go.mod h1:4RNS+LILDg4JbS4Ts9Lwy1C92wauXJIbeQaalps4Koo=
|
||||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||||
|
|||||||
@@ -16,10 +16,12 @@ import (
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
ErrUnsupportedRefType = errors.New("unsupported artifact reference type")
|
ErrUnsupportedRefType = errors.New("unsupported artifact reference type")
|
||||||
ErrMissingInlineBody = errors.New("missing body for inline artifact")
|
|
||||||
ErrMissingFilePath = errors.New("missing file path for file artifact")
|
ErrMissingFilePath = errors.New("missing file path for file artifact")
|
||||||
|
ErrUnsupportedFile = errors.New("file artifact path is not a regular file")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const fileReadChunkSize = 64 * 1024
|
||||||
|
|
||||||
// Reader resolves artifact references into actual artifacts.
|
// Reader resolves artifact references into actual artifacts.
|
||||||
type Reader interface {
|
type Reader interface {
|
||||||
Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error)
|
Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error)
|
||||||
@@ -34,7 +36,7 @@ type CompositeReader struct {
|
|||||||
func NewCompositeReader() Reader {
|
func NewCompositeReader() Reader {
|
||||||
return &CompositeReader{
|
return &CompositeReader{
|
||||||
inlineReader: &inlineReader{},
|
inlineReader: &inlineReader{},
|
||||||
fileReader: &fileReader{},
|
fileReader: &fileReader{open: openArtifactFile},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -64,10 +66,6 @@ func (r *inlineReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domai
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
if ref.Body == "" {
|
|
||||||
return nil, ErrMissingInlineBody
|
|
||||||
}
|
|
||||||
|
|
||||||
body := []byte(ref.Body)
|
body := []byte(ref.Body)
|
||||||
return &domain.Artifact{
|
return &domain.Artifact{
|
||||||
ContentType: defaults.ContentTypeTextPlain,
|
ContentType: defaults.ContentTypeTextPlain,
|
||||||
@@ -78,7 +76,15 @@ func (r *inlineReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domai
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type fileReader struct{}
|
type artifactFile interface {
|
||||||
|
Read([]byte) (int, error)
|
||||||
|
Stat() (os.FileInfo, error)
|
||||||
|
Close() error
|
||||||
|
}
|
||||||
|
|
||||||
|
type fileReader struct {
|
||||||
|
open func(string) (artifactFile, error)
|
||||||
|
}
|
||||||
|
|
||||||
func (r *fileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
func (r *fileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
||||||
select {
|
select {
|
||||||
@@ -91,25 +97,71 @@ func (r *fileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.
|
|||||||
return nil, ErrMissingFilePath
|
return nil, ErrMissingFilePath
|
||||||
}
|
}
|
||||||
|
|
||||||
return readFileArtifact(ref.URI)
|
return readFileArtifact(ctx, ref.URI, r.open)
|
||||||
}
|
}
|
||||||
|
|
||||||
func readFileArtifact(path string) (*domain.Artifact, error) {
|
func openArtifactFile(path string) (artifactFile, error) {
|
||||||
file, err := os.Open(path)
|
return os.Open(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
func readFileArtifact(ctx context.Context, path string, open func(string) (artifactFile, error)) (*domain.Artifact, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path)
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
|
||||||
data, err := io.ReadAll(file)
|
openedInfo, err := file.Stat()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
return nil, fmt.Errorf("failed to inspect opened file %s: %w", path, err)
|
||||||
|
}
|
||||||
|
if !openedInfo.Mode().IsRegular() {
|
||||||
|
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
data := make([]byte, 0)
|
||||||
|
chunk := make([]byte, fileReadChunkSize)
|
||||||
|
for {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
n, readErr := file.Read(chunk)
|
||||||
|
if n > 0 {
|
||||||
|
data = append(data, chunk[:n]...)
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if errors.Is(readErr, io.EOF) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if readErr != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read file %s: %w", path, readErr)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
contentType := mime.TypeByExtension(filepath.Ext(path))
|
contentType := mime.TypeByExtension(filepath.Ext(path))
|
||||||
if contentType == "" {
|
if contentType == "" {
|
||||||
contentType = defaults.ContentTypeTextPlain
|
contentType = defaults.ContentTypeTextPlain
|
||||||
}
|
}
|
||||||
|
hash := fmt.Sprintf("%x", sha256.Sum256(data))
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
return &domain.Artifact{
|
return &domain.Artifact{
|
||||||
Name: filepath.Base(path),
|
Name: filepath.Base(path),
|
||||||
@@ -117,6 +169,6 @@ func readFileArtifact(path string) (*domain.Artifact, error) {
|
|||||||
Body: data,
|
Body: data,
|
||||||
URI: path,
|
URI: path,
|
||||||
Size: int64(len(data)),
|
Size: int64(len(data)),
|
||||||
Hash: fmt.Sprintf("%x", sha256.Sum256(data)),
|
Hash: hash,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
43
internal/artifact/reader_fifo_linux_test.go
Normal file
43
internal/artifact/reader_fifo_linux_test.go
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
//go:build linux
|
||||||
|
|
||||||
|
package artifact
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"path/filepath"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFileReaderRejectsFIFOBeforeOpen(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "artifact.fifo")
|
||||||
|
if err := syscall.Mkfifo(path, 0o600); err != nil {
|
||||||
|
t.Fatalf("create fifo: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
type result struct {
|
||||||
|
artifact *domain.Artifact
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
done := make(chan result, 1)
|
||||||
|
go func() {
|
||||||
|
artifact, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{
|
||||||
|
Type: domain.ArtifactRefFile,
|
||||||
|
URI: path,
|
||||||
|
})
|
||||||
|
done <- result{artifact: artifact, err: err}
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case got := <-done:
|
||||||
|
if got.artifact != nil || !errors.Is(got.err, ErrUnsupportedFile) {
|
||||||
|
t.Fatalf("artifact=%#v err=%v, want nil/ErrUnsupportedFile", got.artifact, got.err)
|
||||||
|
}
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("FIFO read blocked instead of rejecting the non-regular file")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
package artifact
|
package artifact
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
@@ -11,54 +12,97 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCompositeReader_Read(t *testing.T) {
|
func TestCompositeReaderRejectsUnsupportedReferences(t *testing.T) {
|
||||||
|
_, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{
|
||||||
|
Type: domain.ArtifactRefType("unsupported"),
|
||||||
|
URI: "unsupported://bucket/key",
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrUnsupportedRefType) {
|
||||||
|
t.Fatalf("expected ErrUnsupportedRefType, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompositeReaderSourceParityAndOpaqueHashes(t *testing.T) {
|
||||||
reader := NewCompositeReader()
|
reader := NewCompositeReader()
|
||||||
ctx := context.Background()
|
hashes := make(map[string]string)
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
content string
|
||||||
|
}{
|
||||||
|
{name: "empty", content: ""},
|
||||||
|
{name: "ordinary", content: "same content"},
|
||||||
|
{name: "changed", content: "changed content"},
|
||||||
|
}
|
||||||
|
|
||||||
t.Run("inline artifact", func(t *testing.T) {
|
for _, tc := range tests {
|
||||||
ref := domain.ArtifactRef{
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
Type: domain.ArtifactRefInline,
|
filePath := filepath.Join(t.TempDir(), "artifact.txt")
|
||||||
Body: "hello world",
|
if err := os.WriteFile(filePath, []byte(tc.content), 0o600); err != nil {
|
||||||
}
|
t.Fatal(err)
|
||||||
art, err := reader.Read(ctx, ref)
|
}
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if string(art.Body) != "hello world" {
|
|
||||||
t.Errorf("expected 'hello world', got %s", string(art.Body))
|
|
||||||
}
|
|
||||||
if art.ContentType != "text/plain" {
|
|
||||||
t.Errorf("expected text/plain content type, got %q", art.ContentType)
|
|
||||||
}
|
|
||||||
if art.Hash != "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" {
|
|
||||||
t.Errorf("unexpected hash: %s", art.Hash)
|
|
||||||
}
|
|
||||||
if art.Size != int64(len(ref.Body)) {
|
|
||||||
t.Errorf("expected size %d, got %d", len(ref.Body), art.Size)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("inline artifact missing body", func(t *testing.T) {
|
sources := []struct {
|
||||||
ref := domain.ArtifactRef{
|
name string
|
||||||
Type: domain.ArtifactRefInline,
|
ref domain.ArtifactRef
|
||||||
Body: "",
|
wantURI string
|
||||||
}
|
}{
|
||||||
_, err := reader.Read(ctx, ref)
|
{
|
||||||
if !errors.Is(err, ErrMissingInlineBody) {
|
name: "inline",
|
||||||
t.Errorf("expected ErrMissingInlineBody, got %v", err)
|
ref: domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: tc.content},
|
||||||
}
|
},
|
||||||
})
|
{
|
||||||
|
name: "inline with uri",
|
||||||
|
ref: domain.ArtifactRef{Type: domain.ArtifactRefInline, URI: "memory://input", Body: tc.content},
|
||||||
|
wantURI: "memory://input",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "file",
|
||||||
|
ref: domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: filePath},
|
||||||
|
wantURI: filePath,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
t.Run("unsupported ref type", func(t *testing.T) {
|
var sourceHash string
|
||||||
ref := domain.ArtifactRef{
|
for _, source := range sources {
|
||||||
Type: domain.ArtifactRefType("unsupported"),
|
t.Run(source.name, func(t *testing.T) {
|
||||||
URI: "unsupported://bucket/key",
|
first, err := reader.Read(context.Background(), source.ref)
|
||||||
}
|
if err != nil {
|
||||||
_, err := reader.Read(ctx, ref)
|
t.Fatalf("first read: %v", err)
|
||||||
if !errors.Is(err, ErrUnsupportedRefType) {
|
}
|
||||||
t.Error("expected error for unsupported type")
|
second, err := reader.Read(context.Background(), source.ref)
|
||||||
}
|
if err != nil {
|
||||||
})
|
t.Fatalf("second read: %v", err)
|
||||||
|
}
|
||||||
|
if string(first.Body) != tc.content || first.Size != int64(len(tc.content)) {
|
||||||
|
t.Fatalf("body=%q size=%d, want %q/%d", first.Body, first.Size, tc.content, len(tc.content))
|
||||||
|
}
|
||||||
|
if first.URI != source.wantURI {
|
||||||
|
t.Fatalf("URI = %q, want %q", first.URI, source.wantURI)
|
||||||
|
}
|
||||||
|
if first.Hash == "" || first.Hash != second.Hash {
|
||||||
|
t.Fatalf("hashes are not non-empty and stable: %q/%q", first.Hash, second.Hash)
|
||||||
|
}
|
||||||
|
if sourceHash == "" {
|
||||||
|
sourceHash = first.Hash
|
||||||
|
} else if first.Hash != sourceHash {
|
||||||
|
t.Fatalf("equal content hashes differ: %q/%q", sourceHash, first.Hash)
|
||||||
|
}
|
||||||
|
if source.ref.Type == domain.ArtifactRefFile {
|
||||||
|
if first.Name != filepath.Base(filePath) || !strings.HasPrefix(first.ContentType, "text/plain") {
|
||||||
|
t.Fatalf("unexpected file metadata: %+v", first)
|
||||||
|
}
|
||||||
|
} else if first.ContentType != "text/plain" {
|
||||||
|
t.Fatalf("inline content type = %q", first.ContentType)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
hashes[tc.name] = sourceHash
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if hashes["empty"] == hashes["ordinary"] || hashes["ordinary"] == hashes["changed"] {
|
||||||
|
t.Fatalf("changed content did not change opaque hash: %#v", hashes)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCompositeReaderCopiesInlineData(t *testing.T) {
|
func TestCompositeReaderCopiesInlineData(t *testing.T) {
|
||||||
@@ -87,94 +131,154 @@ func TestCompositeReaderCopiesInlineData(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCompositeReaderHonorsCancellation(t *testing.T) {
|
func TestCompositeReaderHonorsPreCancellation(t *testing.T) {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
filePath := filepath.Join(t.TempDir(), "artifact.txt")
|
||||||
cancel()
|
if err := os.WriteFile(filePath, []byte("ignored"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
ref domain.ArtifactRef
|
||||||
|
}{
|
||||||
|
{name: "inline", ref: domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: "ignored"}},
|
||||||
|
{name: "file", ref: domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: filePath}},
|
||||||
|
}
|
||||||
|
|
||||||
_, err := NewCompositeReader().Read(ctx, domain.ArtifactRef{
|
for _, tc := range tests {
|
||||||
Type: domain.ArtifactRefInline,
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
Body: "ignored",
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
})
|
cancel()
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("expected context cancellation, got %v", err)
|
artifact, err := NewCompositeReader().Read(ctx, tc.ref)
|
||||||
|
if artifact != nil || !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("artifact=%#v err=%v, want nil/context.Canceled", artifact, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFileReader_Read(t *testing.T) {
|
func TestFileReaderFailuresAndMetadata(t *testing.T) {
|
||||||
content := []byte("test file content")
|
|
||||||
filePath := filepath.Join(t.TempDir(), "artifact.txt")
|
|
||||||
if err := os.WriteFile(filePath, content, 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
reader := NewCompositeReader()
|
reader := NewCompositeReader()
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
t.Run("file artifact loading", func(t *testing.T) {
|
|
||||||
ref := domain.ArtifactRef{
|
|
||||||
Type: domain.ArtifactRefFile,
|
|
||||||
URI: filePath,
|
|
||||||
}
|
|
||||||
art, err := reader.Read(ctx, ref)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if string(art.Body) != string(content) {
|
|
||||||
t.Errorf("expected %s, got %s", string(content), string(art.Body))
|
|
||||||
}
|
|
||||||
if art.Name != filepath.Base(filePath) {
|
|
||||||
t.Errorf("expected name %q, got %q", filepath.Base(filePath), art.Name)
|
|
||||||
}
|
|
||||||
if !strings.HasPrefix(art.ContentType, "text/plain") {
|
|
||||||
t.Errorf("expected text content type, got %q", art.ContentType)
|
|
||||||
}
|
|
||||||
if art.URI != filePath {
|
|
||||||
t.Errorf("expected URI %q, got %q", filePath, art.URI)
|
|
||||||
}
|
|
||||||
if art.Size != int64(len(content)) {
|
|
||||||
t.Errorf("expected size %d, got %d", len(content), art.Size)
|
|
||||||
}
|
|
||||||
if art.Hash != "60f5237ed4049f0382661ef009d2bc42e48c3ceb3edb6600f7024e7ab3b838f3" {
|
|
||||||
t.Errorf("unexpected hash: %s", art.Hash)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("missing file path", func(t *testing.T) {
|
t.Run("missing file path", func(t *testing.T) {
|
||||||
ref := domain.ArtifactRef{
|
_, err := reader.Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefFile})
|
||||||
Type: domain.ArtifactRefFile,
|
|
||||||
URI: "",
|
|
||||||
}
|
|
||||||
_, err := reader.Read(ctx, ref)
|
|
||||||
if !errors.Is(err, ErrMissingFilePath) {
|
if !errors.Is(err, ErrMissingFilePath) {
|
||||||
t.Errorf("expected ErrMissingFilePath, got %v", err)
|
t.Fatalf("expected ErrMissingFilePath, got %v", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("missing file", func(t *testing.T) {
|
t.Run("missing file", func(t *testing.T) {
|
||||||
ref := domain.ArtifactRef{
|
_, err := reader.Read(context.Background(), domain.ArtifactRef{
|
||||||
Type: domain.ArtifactRefFile,
|
Type: domain.ArtifactRefFile,
|
||||||
URI: filepath.Join(t.TempDir(), "missing.txt"),
|
URI: filepath.Join(t.TempDir(), "missing.txt"),
|
||||||
}
|
})
|
||||||
if _, err := reader.Read(ctx, ref); err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected missing file error")
|
t.Fatal("expected missing file error")
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("directory rejected before open", func(t *testing.T) {
|
||||||
|
artifact, err := reader.Read(context.Background(), domain.ArtifactRef{
|
||||||
|
Type: domain.ArtifactRefFile,
|
||||||
|
URI: t.TempDir(),
|
||||||
|
})
|
||||||
|
if artifact != nil || !errors.Is(err, ErrUnsupportedFile) {
|
||||||
|
t.Fatalf("artifact=%#v err=%v, want nil/ErrUnsupportedFile", artifact, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("non-regular opened target rejected", func(t *testing.T) {
|
||||||
|
filePath := filepath.Join(t.TempDir(), "artifact.txt")
|
||||||
|
if err := os.WriteFile(filePath, []byte("content"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
directoryInfo, err := os.Stat(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fileReader := &fileReader{open: func(path string) (artifactFile, error) {
|
||||||
|
file, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &reportedInfoFile{artifactFile: file, info: directoryInfo}, nil
|
||||||
|
}}
|
||||||
|
|
||||||
|
artifact, err := fileReader.Read(context.Background(), domain.ArtifactRef{
|
||||||
|
Type: domain.ArtifactRefFile,
|
||||||
|
URI: filePath,
|
||||||
|
})
|
||||||
|
if artifact != nil || !errors.Is(err, ErrUnsupportedFile) {
|
||||||
|
t.Fatalf("artifact=%#v err=%v, want nil/ErrUnsupportedFile", artifact, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("unknown extension uses text fallback", func(t *testing.T) {
|
t.Run("unknown extension uses text fallback", func(t *testing.T) {
|
||||||
path := filepath.Join(t.TempDir(), "artifact.unknownextension")
|
filePath := filepath.Join(t.TempDir(), "artifact.unknownextension")
|
||||||
if err := os.WriteFile(path, content, 0o600); err != nil {
|
if err := os.WriteFile(filePath, []byte("content"), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
art, err := reader.Read(ctx, domain.ArtifactRef{
|
artifact, err := reader.Read(context.Background(), domain.ArtifactRef{
|
||||||
Type: domain.ArtifactRefFile,
|
Type: domain.ArtifactRefFile,
|
||||||
URI: path,
|
URI: filePath,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("read artifact: %v", err)
|
||||||
}
|
}
|
||||||
if art.ContentType != "text/plain" {
|
if artifact.ContentType != "text/plain" {
|
||||||
t.Errorf("expected text/plain fallback, got %q", art.ContentType)
|
t.Fatalf("content type = %q", artifact.ContentType)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFileReaderCancelsAfterReadProgress(t *testing.T) {
|
||||||
|
filePath := filepath.Join(t.TempDir(), "artifact.bin")
|
||||||
|
content := bytes.Repeat([]byte("x"), fileReadChunkSize*2)
|
||||||
|
if err := os.WriteFile(filePath, content, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
var opened *cancelAfterProgressFile
|
||||||
|
reader := &fileReader{open: func(path string) (artifactFile, error) {
|
||||||
|
file, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
opened = &cancelAfterProgressFile{artifactFile: file, cancel: cancel}
|
||||||
|
return opened, nil
|
||||||
|
}}
|
||||||
|
|
||||||
|
artifact, err := reader.Read(ctx, domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: filePath})
|
||||||
|
if artifact != nil || !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("artifact=%#v err=%v, want nil/context.Canceled", artifact, err)
|
||||||
|
}
|
||||||
|
if opened == nil || opened.reads != 1 {
|
||||||
|
t.Fatalf("read count = %v, want one progressing read", opened)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type reportedInfoFile struct {
|
||||||
|
artifactFile
|
||||||
|
info os.FileInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *reportedInfoFile) Stat() (os.FileInfo, error) {
|
||||||
|
return f.info, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type cancelAfterProgressFile struct {
|
||||||
|
artifactFile
|
||||||
|
cancel context.CancelFunc
|
||||||
|
reads int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *cancelAfterProgressFile) Read(buffer []byte) (int, error) {
|
||||||
|
n, err := f.artifactFile.Read(buffer)
|
||||||
|
if n > 0 {
|
||||||
|
f.reads++
|
||||||
|
f.cancel()
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ package backend
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/url"
|
|
||||||
"regexp"
|
"regexp"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -19,12 +18,11 @@ const (
|
|||||||
// OpenRouterID is the reserved ID of Promptkit's built-in OpenRouter
|
// OpenRouterID is the reserved ID of Promptkit's built-in OpenRouter
|
||||||
// backend.
|
// backend.
|
||||||
OpenRouterID = "openrouter"
|
OpenRouterID = "openrouter"
|
||||||
|
// RakestrawHomeID is the reserved ID of Promptkit's built-in Rakestrawhome
|
||||||
|
// backend.
|
||||||
|
RakestrawHomeID = "rakestrawhome"
|
||||||
|
|
||||||
openRouterEndpoint = "https://openrouter.ai/api/v1"
|
defaultQueueCapacity = 1024
|
||||||
openRouterAPIKeyEnv = "OPENROUTER_API_KEY"
|
|
||||||
|
|
||||||
openRouterConcurrencyLimit = 16
|
|
||||||
defaultQueueCapacity = 1024
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrBackendNotFound identifies a registry lookup for an unknown backend ID.
|
// ErrBackendNotFound identifies a registry lookup for an unknown backend ID.
|
||||||
@@ -37,20 +35,15 @@ type Registry struct {
|
|||||||
backends map[string]domain.Backend
|
backends map[string]domain.Backend
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewRegistry constructs a registry containing the built-in OpenRouter
|
// NewRegistry constructs a registry containing maintained definitions followed
|
||||||
// definition followed by the supplied additions. Every ID must be unique.
|
// by consumer additions. Every ID must be unique across both groups.
|
||||||
func NewRegistry(additions []domain.Backend) (*Registry, error) {
|
func NewRegistry(maintained, additions []domain.Backend) (*Registry, error) {
|
||||||
registry := &Registry{
|
registry := &Registry{
|
||||||
backends: make(map[string]domain.Backend, len(additions)+1),
|
backends: make(map[string]domain.Backend, len(maintained)+len(additions)),
|
||||||
}
|
}
|
||||||
|
|
||||||
definitions := make([]domain.Backend, 0, len(additions)+1)
|
definitions := make([]domain.Backend, 0, len(maintained)+len(additions))
|
||||||
definitions = append(definitions, domain.Backend{
|
definitions = append(definitions, maintained...)
|
||||||
ID: OpenRouterID,
|
|
||||||
Endpoint: openRouterEndpoint,
|
|
||||||
APIKeyEnv: openRouterAPIKeyEnv,
|
|
||||||
ConcurrencyLimit: openRouterConcurrencyLimit,
|
|
||||||
})
|
|
||||||
definitions = append(definitions, additions...)
|
definitions = append(definitions, additions...)
|
||||||
|
|
||||||
for _, definition := range definitions {
|
for _, definition := range definitions {
|
||||||
@@ -62,7 +55,7 @@ func NewRegistry(additions []domain.Backend) (*Registry, error) {
|
|||||||
return nil, fmt.Errorf("backend ID %q is already registered", definition.ID)
|
return nil, fmt.Errorf("backend ID %q is already registered", definition.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
normalized, err := normalizeBackend(definition)
|
normalized, err := NormalizeDefinition(definition)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -108,11 +101,13 @@ func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy {
|
|||||||
return policies
|
return policies
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeBackend(definition domain.Backend) (domain.Backend, error) {
|
// NormalizeDefinition validates and defensively copies one backend definition.
|
||||||
definition.Endpoint = strings.TrimSpace(definition.Endpoint)
|
func NormalizeDefinition(definition domain.Backend) (domain.Backend, error) {
|
||||||
if err := validateEndpoint(definition.Endpoint); err != nil {
|
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(definition.Endpoint)
|
||||||
|
if err != nil {
|
||||||
return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err)
|
return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err)
|
||||||
}
|
}
|
||||||
|
definition.Endpoint = endpoint
|
||||||
|
|
||||||
definition.APIKeyEnv = strings.TrimSpace(definition.APIKeyEnv)
|
definition.APIKeyEnv = strings.TrimSpace(definition.APIKeyEnv)
|
||||||
if definition.APIKeyEnv != "" && !environmentVariableName.MatchString(definition.APIKeyEnv) {
|
if definition.APIKeyEnv != "" && !environmentVariableName.MatchString(definition.APIKeyEnv) {
|
||||||
@@ -182,31 +177,3 @@ func normalizeBackend(definition domain.Backend) (domain.Backend, error) {
|
|||||||
definition.ExtraParams = extraParams
|
definition.ExtraParams = extraParams
|
||||||
return definition, nil
|
return definition, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateEndpoint(endpoint string) error {
|
|
||||||
if endpoint == "" {
|
|
||||||
return errors.New("must not be blank")
|
|
||||||
}
|
|
||||||
if strings.Contains(endpoint, "#") {
|
|
||||||
return errors.New("must not contain a fragment")
|
|
||||||
}
|
|
||||||
|
|
||||||
parsed, err := url.Parse(endpoint)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("must be a valid URL: %w", err)
|
|
||||||
}
|
|
||||||
scheme := strings.ToLower(parsed.Scheme)
|
|
||||||
if scheme != "http" && scheme != "https" {
|
|
||||||
return errors.New("must use http or https")
|
|
||||||
}
|
|
||||||
if !parsed.IsAbs() || parsed.Hostname() == "" {
|
|
||||||
return errors.New("must be absolute and include a host")
|
|
||||||
}
|
|
||||||
if parsed.User != nil {
|
|
||||||
return errors.New("must not contain user information")
|
|
||||||
}
|
|
||||||
if parsed.RawQuery != "" || parsed.ForceQuery {
|
|
||||||
return errors.New("must not contain a query string")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package backend_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -11,32 +12,39 @@ import (
|
|||||||
|
|
||||||
const validEndpoint = "https://backend.example/v1"
|
const validEndpoint = "https://backend.example/v1"
|
||||||
|
|
||||||
func TestRegistryIncludesExactOpenRouterDefinition(t *testing.T) {
|
func TestRegistryIncludesMaintainedDefinitions(t *testing.T) {
|
||||||
registry, err := backend.NewRegistry(nil)
|
maintained := []domain.Backend{
|
||||||
|
{ID: backend.OpenRouterID, Endpoint: validEndpoint, ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true},
|
||||||
|
{ID: backend.RakestrawHomeID, Endpoint: "https://second.example/v1", ConcurrencyLimit: 4, QueueCapacity: 5, QueueCapacitySet: true},
|
||||||
|
}
|
||||||
|
registry, err := backend.NewRegistry(maintained, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct registry: %v", err)
|
t.Fatalf("construct registry: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
definition, err := registry.GetBackend(backend.OpenRouterID)
|
for _, expected := range maintained {
|
||||||
if err != nil {
|
t.Run(expected.ID, func(t *testing.T) {
|
||||||
t.Fatalf("look up OpenRouter: %v", err)
|
definition, err := registry.GetBackend(expected.ID)
|
||||||
}
|
if err != nil {
|
||||||
if definition.ID != "openrouter" ||
|
t.Fatalf("look up maintained definition: %v", err)
|
||||||
definition.Endpoint != "https://openrouter.ai/api/v1" ||
|
}
|
||||||
definition.APIKeyEnv != "OPENROUTER_API_KEY" ||
|
if !reflect.DeepEqual(definition, expected) {
|
||||||
definition.ConcurrencyLimit != 16 ||
|
t.Fatalf("unexpected maintained definition: %#v", definition)
|
||||||
definition.QueueCapacity != 1024 ||
|
}
|
||||||
!definition.QueueCapacitySet ||
|
})
|
||||||
definition.ExtraParams != nil {
|
|
||||||
t.Fatalf("unexpected OpenRouter definition: %#v", definition)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
policies := registry.CapacityPolicies()
|
policies := registry.CapacityPolicies()
|
||||||
if len(policies) != 1 ||
|
if len(policies) != 2 ||
|
||||||
policies["openrouter"] != (domain.BackendCapacityPolicy{
|
policies[backend.OpenRouterID] != (domain.BackendCapacityPolicy{
|
||||||
ConcurrencyLimit: 16,
|
ConcurrencyLimit: 2,
|
||||||
QueueCapacity: 1024,
|
QueueCapacity: 3,
|
||||||
|
}) ||
|
||||||
|
policies[backend.RakestrawHomeID] != (domain.BackendCapacityPolicy{
|
||||||
|
ConcurrencyLimit: 4,
|
||||||
|
QueueCapacity: 5,
|
||||||
}) {
|
}) {
|
||||||
t.Fatalf("unexpected OpenRouter capacity policies: %#v", policies)
|
t.Fatalf("unexpected built-in capacity policies: %#v", policies)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -46,7 +54,7 @@ func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
|
|||||||
"count": int64(7),
|
"count": int64(7),
|
||||||
"nested": nested,
|
"nested": nested,
|
||||||
}
|
}
|
||||||
registry, err := backend.NewRegistry([]domain.Backend{
|
registry, err := backend.NewRegistry(nil, []domain.Backend{
|
||||||
{
|
{
|
||||||
ID: " custom ",
|
ID: " custom ",
|
||||||
Endpoint: " https://custom.example/openai/v1 ",
|
Endpoint: " https://custom.example/openai/v1 ",
|
||||||
@@ -109,11 +117,11 @@ func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
policies := registry.CapacityPolicies()
|
policies := registry.CapacityPolicies()
|
||||||
if len(policies) != 2 {
|
if len(policies) != 1 {
|
||||||
t.Fatalf("unexpected capacity policy count: %#v", policies)
|
t.Fatalf("unexpected capacity policy count: %#v", policies)
|
||||||
}
|
}
|
||||||
policies["custom"] = domain.BackendCapacityPolicy{}
|
policies["custom"] = domain.BackendCapacityPolicy{}
|
||||||
delete(policies, backend.OpenRouterID)
|
delete(policies, "custom")
|
||||||
againPolicies := registry.CapacityPolicies()
|
againPolicies := registry.CapacityPolicies()
|
||||||
if againPolicies["custom"] != (domain.BackendCapacityPolicy{
|
if againPolicies["custom"] != (domain.BackendCapacityPolicy{
|
||||||
ConcurrencyLimit: 3,
|
ConcurrencyLimit: 3,
|
||||||
@@ -121,9 +129,6 @@ func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
|
|||||||
}) {
|
}) {
|
||||||
t.Fatalf("capacity policy map mutated registry state: %#v", againPolicies)
|
t.Fatalf("capacity policy map mutated registry state: %#v", againPolicies)
|
||||||
}
|
}
|
||||||
if _, ok := againPolicies[backend.OpenRouterID]; !ok {
|
|
||||||
t.Fatalf("capacity policy deletion mutated registry state: %#v", againPolicies)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) {
|
func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) {
|
||||||
@@ -199,7 +204,7 @@ func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
tc.definition.ID = "custom"
|
tc.definition.ID = "custom"
|
||||||
tc.definition.Endpoint = validEndpoint
|
tc.definition.Endpoint = validEndpoint
|
||||||
registry, err := backend.NewRegistry([]domain.Backend{tc.definition})
|
registry, err := backend.NewRegistry(nil, []domain.Backend{tc.definition})
|
||||||
if tc.wantError {
|
if tc.wantError {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected invalid capacity policy error")
|
t.Fatal("expected invalid capacity policy error")
|
||||||
@@ -242,11 +247,14 @@ func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
|
|||||||
wantID string
|
wantID string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "built-in collision after normalization",
|
name: "OpenRouter collision after normalization",
|
||||||
additions: []domain.Backend{{
|
additions: []domain.Backend{{ID: " openrouter "}},
|
||||||
ID: " openrouter ",
|
wantID: backend.OpenRouterID,
|
||||||
}},
|
},
|
||||||
wantID: "openrouter",
|
{
|
||||||
|
name: "Rakestrawhome collision after normalization",
|
||||||
|
additions: []domain.Backend{{ID: " rakestrawhome "}},
|
||||||
|
wantID: backend.RakestrawHomeID,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "consumer collision after normalization",
|
name: "consumer collision after normalization",
|
||||||
@@ -260,7 +268,7 @@ func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
|
|||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
_, err := backend.NewRegistry(tc.additions)
|
_, err := backend.NewRegistry(nil, tc.additions)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected duplicate ID error")
|
t.Fatal("expected duplicate ID error")
|
||||||
}
|
}
|
||||||
@@ -274,7 +282,7 @@ func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
|
|||||||
func TestNewRegistryValidatesIDs(t *testing.T) {
|
func TestNewRegistryValidatesIDs(t *testing.T) {
|
||||||
for _, id := range []string{"", " \t\n "} {
|
for _, id := range []string{"", " \t\n "} {
|
||||||
t.Run(id, func(t *testing.T) {
|
t.Run(id, func(t *testing.T) {
|
||||||
_, err := backend.NewRegistry([]domain.Backend{{
|
_, err := backend.NewRegistry(nil, []domain.Backend{{
|
||||||
ID: id,
|
ID: id,
|
||||||
Endpoint: validEndpoint,
|
Endpoint: validEndpoint,
|
||||||
}})
|
}})
|
||||||
@@ -303,7 +311,7 @@ func TestNewRegistryValidatesEndpoints(t *testing.T) {
|
|||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
_, err := backend.NewRegistry([]domain.Backend{{
|
_, err := backend.NewRegistry(nil, []domain.Backend{{
|
||||||
ID: "custom",
|
ID: "custom",
|
||||||
Endpoint: tc.endpoint,
|
Endpoint: tc.endpoint,
|
||||||
}})
|
}})
|
||||||
@@ -317,7 +325,7 @@ func TestNewRegistryValidatesEndpoints(t *testing.T) {
|
|||||||
func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) {
|
func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) {
|
||||||
for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} {
|
for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
_, err := backend.NewRegistry([]domain.Backend{{
|
_, err := backend.NewRegistry(nil, []domain.Backend{{
|
||||||
ID: "custom",
|
ID: "custom",
|
||||||
Endpoint: validEndpoint,
|
Endpoint: validEndpoint,
|
||||||
APIKeyEnv: name,
|
APIKeyEnv: name,
|
||||||
@@ -340,7 +348,7 @@ func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
|
|||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
_, err := backend.NewRegistry([]domain.Backend{{
|
_, err := backend.NewRegistry(nil, []domain.Backend{{
|
||||||
ID: "custom",
|
ID: "custom",
|
||||||
Endpoint: validEndpoint,
|
Endpoint: validEndpoint,
|
||||||
ExtraParams: tc.extraParams,
|
ExtraParams: tc.extraParams,
|
||||||
@@ -353,7 +361,7 @@ func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRegistryLookupReportsNotFound(t *testing.T) {
|
func TestRegistryLookupReportsNotFound(t *testing.T) {
|
||||||
registry, err := backend.NewRegistry(nil)
|
registry, err := backend.NewRegistry(nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct registry: %v", err)
|
t.Fatalf("construct registry: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
246
internal/catalog/catalog.go
Normal file
246
internal/catalog/catalog.go
Normal file
@@ -0,0 +1,246 @@
|
|||||||
|
// Package catalog validates immutable maintained backend catalog assets.
|
||||||
|
package catalog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/fs"
|
||||||
|
"path"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Source identifies one immutable backend catalog asset tree.
|
||||||
|
type Source struct {
|
||||||
|
Name string
|
||||||
|
ExpectedBackendID string
|
||||||
|
FS fs.FS
|
||||||
|
Root string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set is the validated maintained backend and raw profile catalog.
|
||||||
|
type Set struct {
|
||||||
|
Backends []domain.Backend
|
||||||
|
Profiles profile.Repository
|
||||||
|
profileIDs []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load validates and combines immutable catalog sources in source order.
|
||||||
|
func Load(sources ...Source) (Set, error) {
|
||||||
|
if len(sources) == 0 {
|
||||||
|
return Set{}, errors.New("at least one catalog source is required")
|
||||||
|
}
|
||||||
|
loaded := Set{Backends: make([]domain.Backend, 0, len(sources))}
|
||||||
|
names := map[string]bool{}
|
||||||
|
backendIDs := map[string]bool{}
|
||||||
|
profileIDs := map[string]bool{}
|
||||||
|
for _, source := range sources {
|
||||||
|
if err := validateSource(source, names); err != nil {
|
||||||
|
return Set{}, err
|
||||||
|
}
|
||||||
|
names[source.Name] = true
|
||||||
|
definition, err := loadBackend(source)
|
||||||
|
if err != nil {
|
||||||
|
return Set{}, err
|
||||||
|
}
|
||||||
|
if backendIDs[definition.ID] {
|
||||||
|
return Set{}, fmt.Errorf("catalog %s: backend ID duplicates an earlier catalog", source.Name)
|
||||||
|
}
|
||||||
|
backendIDs[definition.ID] = true
|
||||||
|
repository, metadata, err := profile.LoadFSRepository(context.Background(), source.FS, path.Join(source.Root, "profiles"))
|
||||||
|
if err != nil {
|
||||||
|
return Set{}, fmt.Errorf("catalog %s: %w", source.Name, err)
|
||||||
|
}
|
||||||
|
if len(metadata) == 0 {
|
||||||
|
return Set{}, fmt.Errorf("catalog %s: profiles must not be empty", source.Name)
|
||||||
|
}
|
||||||
|
resolvingRepository := profile.NewResolvingRepository(repository)
|
||||||
|
for _, entry := range metadata {
|
||||||
|
if profileIDs[entry.ID] {
|
||||||
|
return Set{}, fmt.Errorf("catalog %s: %s duplicates an earlier profile ID", source.Name, entry.Path)
|
||||||
|
}
|
||||||
|
if containsField(entry.ExplicitFields, "endpoint") || containsField(entry.ExplicitFields, "api_key_env") {
|
||||||
|
return Set{}, fmt.Errorf("catalog %s: %s contains connection metadata", source.Name, entry.Path)
|
||||||
|
}
|
||||||
|
value, err := repository.GetProfile(context.Background(), entry.ID)
|
||||||
|
if err != nil {
|
||||||
|
return Set{}, fmt.Errorf("catalog %s: %s: %w", source.Name, entry.Path, err)
|
||||||
|
}
|
||||||
|
if err := rejectSecretKeys(value.ExtraParams); err != nil {
|
||||||
|
return Set{}, fmt.Errorf("catalog %s: %s: prohibited extra parameter key", source.Name, entry.Path)
|
||||||
|
}
|
||||||
|
resolved, err := resolvingRepository.GetProfile(context.Background(), entry.ID)
|
||||||
|
if err != nil {
|
||||||
|
return Set{}, fmt.Errorf("catalog %s: %s has invalid profile inheritance", source.Name, entry.Path)
|
||||||
|
}
|
||||||
|
if resolved.BackendID != definition.ID {
|
||||||
|
return Set{}, fmt.Errorf("catalog %s: %s selects a different backend", source.Name, entry.Path)
|
||||||
|
}
|
||||||
|
profileIDs[entry.ID] = true
|
||||||
|
loaded.profileIDs = append(loaded.profileIDs, entry.ID)
|
||||||
|
}
|
||||||
|
if loaded.Profiles == nil {
|
||||||
|
loaded.Profiles = repository
|
||||||
|
} else {
|
||||||
|
loaded.Profiles = profile.NewOverlayRepository(loaded.Profiles, repository)
|
||||||
|
}
|
||||||
|
loaded.Backends = append(loaded.Backends, definition)
|
||||||
|
}
|
||||||
|
sort.Strings(loaded.profileIDs)
|
||||||
|
return loaded, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateSource(source Source, names map[string]bool) error {
|
||||||
|
if strings.TrimSpace(source.Name) == "" || source.Name != strings.TrimSpace(source.Name) {
|
||||||
|
return errors.New("catalog source name must not be blank")
|
||||||
|
}
|
||||||
|
if names[source.Name] {
|
||||||
|
return fmt.Errorf("duplicate catalog source name %q", source.Name)
|
||||||
|
}
|
||||||
|
if source.FS == nil {
|
||||||
|
return fmt.Errorf("catalog %s: filesystem is nil", source.Name)
|
||||||
|
}
|
||||||
|
if source.Root == "." || source.Root == "" || !fs.ValidPath(source.Root) {
|
||||||
|
return fmt.Errorf("catalog %s: asset root is invalid", source.Name)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(source.ExpectedBackendID) == "" {
|
||||||
|
return fmt.Errorf("catalog %s: expected backend ID is blank", source.Name)
|
||||||
|
}
|
||||||
|
return validateLayout(source)
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateLayout(source Source) error {
|
||||||
|
manifestPath := path.Join(source.Root, "backend.json")
|
||||||
|
profilesRoot := path.Join(source.Root, "profiles")
|
||||||
|
return fs.WalkDir(source.FS, source.Root, func(assetPath string, entry fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if assetPath == source.Root {
|
||||||
|
if !entry.IsDir() {
|
||||||
|
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if entry.IsDir() {
|
||||||
|
if assetPath == profilesRoot || strings.HasPrefix(assetPath, profilesRoot+"/") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
|
||||||
|
}
|
||||||
|
if assetPath == manifestPath && entry.Type().IsRegular() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(assetPath, profilesRoot+"/") && entry.Type().IsRegular() && strings.HasSuffix(assetPath, ".yml") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadBackend(source Source) (domain.Backend, error) {
|
||||||
|
data, err := fs.ReadFile(source.FS, path.Join(source.Root, "backend.json"))
|
||||||
|
if err != nil {
|
||||||
|
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: %w", source.Name, err)
|
||||||
|
}
|
||||||
|
type manifest struct {
|
||||||
|
SchemaVersion *int `json:"schema_version"`
|
||||||
|
ID *string `json:"id"`
|
||||||
|
Endpoint *string `json:"endpoint"`
|
||||||
|
APIKeyEnv *string `json:"api_key_env"`
|
||||||
|
ConcurrencyLimit *int `json:"concurrency_limit"`
|
||||||
|
QueueCapacity *int `json:"queue_capacity"`
|
||||||
|
ExtraParams json.RawMessage `json:"extra_params"`
|
||||||
|
}
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||||
|
decoder.DisallowUnknownFields()
|
||||||
|
var value manifest
|
||||||
|
if err := decoder.Decode(&value); err != nil {
|
||||||
|
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
|
||||||
|
}
|
||||||
|
var trailing any
|
||||||
|
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||||
|
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
|
||||||
|
}
|
||||||
|
if value.SchemaVersion == nil || *value.SchemaVersion != 1 || value.ID == nil || value.Endpoint == nil || value.APIKeyEnv == nil || value.ConcurrencyLimit == nil || value.QueueCapacity == nil || value.ExtraParams == nil {
|
||||||
|
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: required field is missing or unsupported", source.Name)
|
||||||
|
}
|
||||||
|
if *value.ID != source.ExpectedBackendID {
|
||||||
|
return domain.Backend{}, fmt.Errorf("catalog %s: backend ID does not match expected ID", source.Name)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(*value.APIKeyEnv) == "" {
|
||||||
|
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: api key environment variable must not be blank", source.Name)
|
||||||
|
}
|
||||||
|
extraParams, err := decodeExtraParams(value.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid extra parameters", source.Name)
|
||||||
|
}
|
||||||
|
if err := rejectSecretKeys(extraParams); err != nil {
|
||||||
|
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: prohibited extra parameter key", source.Name)
|
||||||
|
}
|
||||||
|
normalized, err := backend.NormalizeDefinition(domain.Backend{ID: *value.ID, Endpoint: *value.Endpoint, APIKeyEnv: *value.APIKeyEnv, ExtraParams: extraParams, ConcurrencyLimit: *value.ConcurrencyLimit, QueueCapacity: *value.QueueCapacity, QueueCapacitySet: true})
|
||||||
|
if err != nil {
|
||||||
|
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid backend definition", source.Name)
|
||||||
|
}
|
||||||
|
return normalized, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeExtraParams(data []byte) (map[string]any, error) {
|
||||||
|
if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||||
|
decoder.UseNumber()
|
||||||
|
var value map[string]any
|
||||||
|
if err := decoder.Decode(&value); err != nil || value == nil {
|
||||||
|
return nil, errors.New("extra parameters must be an object or null")
|
||||||
|
}
|
||||||
|
var trailing any
|
||||||
|
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||||
|
return nil, errors.New("extra parameters must contain exactly one JSON value")
|
||||||
|
}
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func containsField(fields []string, target string) bool {
|
||||||
|
for _, field := range fields {
|
||||||
|
if field == target {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func rejectSecretKeys(value map[string]any) error {
|
||||||
|
return rejectSecretValue(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func rejectSecretValue(value any) error {
|
||||||
|
switch value := value.(type) {
|
||||||
|
case map[string]any:
|
||||||
|
for key, child := range value {
|
||||||
|
switch strings.ToLower(key) {
|
||||||
|
case "api_key", "apikey", "authorization", "credential", "credentials", "password", "secret", "token", "access_token":
|
||||||
|
return errors.New("prohibited key")
|
||||||
|
}
|
||||||
|
if err := rejectSecretValue(child); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case []any:
|
||||||
|
for _, child := range value {
|
||||||
|
if err := rejectSecretValue(child); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
364
internal/catalog/catalog_test.go
Normal file
364
internal/catalog/catalog_test.go
Normal file
@@ -0,0 +1,364 @@
|
|||||||
|
package catalog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter"
|
||||||
|
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLoadPublishedCatalogsMatchCompatibilityFixture(t *testing.T) {
|
||||||
|
loaded, err := Load(
|
||||||
|
Source{Name: "OpenRouter", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
|
||||||
|
Source{Name: "Rakestrawhome", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load published catalogs: %v", err)
|
||||||
|
}
|
||||||
|
expected := loadCompatibilityFixture(t)
|
||||||
|
expectedIDs := fixtureProfileIDs(t, expected)
|
||||||
|
if !reflect.DeepEqual(loaded.profileIDs, expectedIDs) {
|
||||||
|
t.Fatalf("published profile IDs differ from compatibility fixture: got %q, want %q", loaded.profileIDs, expectedIDs)
|
||||||
|
}
|
||||||
|
actual := catalogValue(t, loaded)
|
||||||
|
actualJSON, err := json.Marshal(actual)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("encode loaded catalogs: %v", err)
|
||||||
|
}
|
||||||
|
expectedJSON, err := json.Marshal(expected)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("encode compatibility fixture: %v", err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(actualJSON, expectedJSON) {
|
||||||
|
t.Fatalf("published catalogs differ from compatibility fixture: got %#v, want %#v", actual, expected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadRejectsInvalidSources(t *testing.T) {
|
||||||
|
for name, sources := range map[string][]Source{
|
||||||
|
"none": nil,
|
||||||
|
"blank name": {{Name: " ", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root}},
|
||||||
|
"duplicate name": {
|
||||||
|
{Name: "same", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
|
||||||
|
{Name: "same", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
|
||||||
|
},
|
||||||
|
"nil filesystem": {{Name: "missing", ExpectedBackendID: "openrouter", Root: openrouter.Root}},
|
||||||
|
"invalid root": {{Name: "invalid-root", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: "."}},
|
||||||
|
"blank expected backend": {{Name: "blank-backend", ExpectedBackendID: " ", FS: openrouter.FS(), Root: openrouter.Root}},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
if _, err := Load(sources...); err == nil {
|
||||||
|
t.Fatal("expected error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadRejectsInvalidLayouts(t *testing.T) {
|
||||||
|
tests := map[string]func(fstest.MapFS){
|
||||||
|
"unexpected file": func(fsys fstest.MapFS) {
|
||||||
|
fsys["catalog/notes.txt"] = &fstest.MapFile{Data: []byte("unexpected")}
|
||||||
|
},
|
||||||
|
"unexpected directory": func(fsys fstest.MapFS) {
|
||||||
|
fsys["catalog/unexpected"] = &fstest.MapFile{Mode: fs.ModeDir}
|
||||||
|
},
|
||||||
|
"nonregular manifest": func(fsys fstest.MapFS) {
|
||||||
|
fsys["catalog/backend.json"].Mode = fs.ModeSymlink
|
||||||
|
},
|
||||||
|
"wrong profile extension": func(fsys fstest.MapFS) {
|
||||||
|
fsys["catalog/profiles/extra.yaml"] = &fstest.MapFile{Data: []byte(validProfile("extra", "one"))}
|
||||||
|
},
|
||||||
|
"nonregular profile": func(fsys fstest.MapFS) {
|
||||||
|
fsys["catalog/profiles/one-profile.yml"].Mode = fs.ModeSymlink
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for name, mutate := range tests {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
fsys := validCatalogFS("one")
|
||||||
|
mutate(fsys)
|
||||||
|
if _, err := Load(testSource("one", fsys)); err == nil {
|
||||||
|
t.Fatal("expected invalid layout error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadRejectsInvalidManifests(t *testing.T) {
|
||||||
|
tests := map[string]string{
|
||||||
|
"malformed": `{`,
|
||||||
|
"trailing value": validManifest("one", "TEST_API_KEY", "null") + `{}`,
|
||||||
|
"missing fields": `{"schema_version":1,"id":"one"}`,
|
||||||
|
"unsupported version": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"schema_version":1`, `"schema_version":2`, 1),
|
||||||
|
"unknown field": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"extra_params":null`, `"extra_params":null,"unknown":true`, 1),
|
||||||
|
"blank API key env": validManifest("one", " ", "null"),
|
||||||
|
"invalid API key env": validManifest("one", "LEAK-MARKER", "null"),
|
||||||
|
"invalid endpoint": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `https://one.example/v1`, `ftp://leak-marker.invalid/v1`, 1),
|
||||||
|
"zero concurrency": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"concurrency_limit":2`, `"concurrency_limit":0`, 1),
|
||||||
|
"non-object parameters": validManifest("one", "TEST_API_KEY", `[]`),
|
||||||
|
"secret parameter": validManifest("one", "TEST_API_KEY", `{"nested":{"token":"leak-marker"}}`),
|
||||||
|
}
|
||||||
|
for name, manifest := range tests {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
fsys := validCatalogFS("one")
|
||||||
|
fsys["catalog/backend.json"].Data = []byte(manifest)
|
||||||
|
_, err := Load(testSource("one", fsys))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid manifest error")
|
||||||
|
}
|
||||||
|
if strings.Contains(strings.ToLower(err.Error()), "leak-marker") {
|
||||||
|
t.Fatalf("catalog error exposed manifest content: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadPreservesManifestJSONNumbers(t *testing.T) {
|
||||||
|
fsys := validCatalogFS("one")
|
||||||
|
fsys["catalog/backend.json"].Data = []byte(validManifest(
|
||||||
|
"one",
|
||||||
|
"TEST_API_KEY",
|
||||||
|
`{"large":9007199254740993,"nested":[1.25]}`,
|
||||||
|
))
|
||||||
|
loaded, err := Load(testSource("one", fsys))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load catalog: %v", err)
|
||||||
|
}
|
||||||
|
if got := loaded.Backends[0].ExtraParams["large"]; got != json.Number("9007199254740993") {
|
||||||
|
t.Fatalf("large JSON integer = %#v, want preserved json.Number", got)
|
||||||
|
}
|
||||||
|
nested := loaded.Backends[0].ExtraParams["nested"].([]any)
|
||||||
|
if nested[0] != json.Number("1.25") {
|
||||||
|
t.Fatalf("nested JSON number = %#v, want preserved json.Number", nested[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadRejectsInvalidCatalogProfiles(t *testing.T) {
|
||||||
|
tests := map[string]func(fstest.MapFS){
|
||||||
|
"empty": func(fsys fstest.MapFS) {
|
||||||
|
delete(fsys, "catalog/profiles/one-profile.yml")
|
||||||
|
fsys["catalog/profiles"] = &fstest.MapFile{Mode: fs.ModeDir}
|
||||||
|
},
|
||||||
|
"malformed": func(fsys fstest.MapFS) {
|
||||||
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: [")
|
||||||
|
},
|
||||||
|
"raw API key": func(fsys fstest.MapFS) {
|
||||||
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key: leak-marker\n")
|
||||||
|
},
|
||||||
|
"endpoint field": func(fsys fstest.MapFS) {
|
||||||
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "endpoint: ''\n")
|
||||||
|
},
|
||||||
|
"API key environment field": func(fsys fstest.MapFS) {
|
||||||
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key_env: ''\n")
|
||||||
|
},
|
||||||
|
"owner mismatch": func(fsys fstest.MapFS) {
|
||||||
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "other"))
|
||||||
|
},
|
||||||
|
"missing base": func(fsys fstest.MapFS) {
|
||||||
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: leak-marker\n")
|
||||||
|
},
|
||||||
|
"cyclic base": func(fsys fstest.MapFS) {
|
||||||
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: second\n")
|
||||||
|
fsys["catalog/profiles/second.yml"] = &fstest.MapFile{Data: []byte("id: second\nbase_profile: one-profile\n")}
|
||||||
|
},
|
||||||
|
"secret profile parameter": func(fsys fstest.MapFS) {
|
||||||
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "extra_params:\n nested:\n password: leak-marker\n")
|
||||||
|
},
|
||||||
|
"unknown field is redacted": func(fsys fstest.MapFS) {
|
||||||
|
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "leak_marker: leak-marker\n")
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for name, mutate := range tests {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
fsys := validCatalogFS("one")
|
||||||
|
mutate(fsys)
|
||||||
|
_, err := Load(testSource("one", fsys))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid profile error")
|
||||||
|
}
|
||||||
|
if strings.Contains(strings.ToLower(err.Error()), "leak-marker") {
|
||||||
|
t.Fatalf("catalog error exposed profile content: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadRejectsCrossCatalogConflicts(t *testing.T) {
|
||||||
|
t.Run("duplicate backend", func(t *testing.T) {
|
||||||
|
second := catalogFS("one", map[string]string{
|
||||||
|
"catalog/profiles/second.yml": validProfile("second", "one"),
|
||||||
|
}, "null")
|
||||||
|
_, err := Load(
|
||||||
|
Source{Name: "first", ExpectedBackendID: "one", FS: validCatalogFS("one"), Root: "catalog"},
|
||||||
|
Source{Name: "second", ExpectedBackendID: "one", FS: second, Root: "catalog"},
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected duplicate backend error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("duplicate profile", func(t *testing.T) {
|
||||||
|
first := catalogFS("one", map[string]string{
|
||||||
|
"catalog/profiles/shared.yml": validProfile("shared", "one"),
|
||||||
|
}, "null")
|
||||||
|
second := catalogFS("two", map[string]string{
|
||||||
|
"catalog/profiles/shared.yml": validProfile("shared", "two"),
|
||||||
|
}, "null")
|
||||||
|
_, err := Load(testSource("one", first), testSource("two", second))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected duplicate profile error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("cross-catalog base", func(t *testing.T) {
|
||||||
|
first := catalogFS("one", map[string]string{
|
||||||
|
"catalog/profiles/base.yml": validProfile("base", "one"),
|
||||||
|
}, "null")
|
||||||
|
second := catalogFS("two", map[string]string{
|
||||||
|
"catalog/profiles/child.yml": "id: child\nbase_profile: base\n",
|
||||||
|
}, "null")
|
||||||
|
_, err := Load(testSource("one", first), testSource("two", second))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected cross-catalog base error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadReturnsDefensiveCatalogValues(t *testing.T) {
|
||||||
|
fsys := catalogFS("one", map[string]string{
|
||||||
|
"catalog/profiles/one-profile.yml": validProfile("one-profile", "one") + "extra_params:\n nested:\n value: profile\n",
|
||||||
|
}, `{"nested":{"value":"backend"}}`)
|
||||||
|
loaded, err := Load(testSource("one", fsys))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load catalog: %v", err)
|
||||||
|
}
|
||||||
|
loaded.Backends[0].ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||||
|
profileValue, err := loaded.Profiles.GetProfile(context.Background(), "one-profile")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load profile: %v", err)
|
||||||
|
}
|
||||||
|
profileValue.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||||
|
|
||||||
|
again, err := Load(testSource("one", fsys))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reload catalog: %v", err)
|
||||||
|
}
|
||||||
|
if got := again.Backends[0].ExtraParams["nested"].(map[string]any)["value"]; got != "backend" {
|
||||||
|
t.Fatalf("backend mutation escaped returned set: %#v", got)
|
||||||
|
}
|
||||||
|
againProfile, err := loaded.Profiles.GetProfile(context.Background(), "one-profile")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reload profile: %v", err)
|
||||||
|
}
|
||||||
|
if got := againProfile.ExtraParams["nested"].(map[string]any)["value"]; got != "profile" {
|
||||||
|
t.Fatalf("profile mutation escaped returned value: %#v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadCompatibilityFixture(t *testing.T) map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(filepath.Join("..", "..", "testdata", "builtin-catalog-v1.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read fixture: %v", err)
|
||||||
|
}
|
||||||
|
var value map[string]any
|
||||||
|
if err := json.Unmarshal(data, &value); err != nil {
|
||||||
|
t.Fatalf("decode fixture: %v", err)
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
func fixtureProfileIDs(t *testing.T, fixture map[string]any) []string {
|
||||||
|
t.Helper()
|
||||||
|
profiles, ok := fixture["profiles"].([]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("compatibility fixture profiles are malformed")
|
||||||
|
}
|
||||||
|
ids := make([]string, 0, len(profiles))
|
||||||
|
for _, entry := range profiles {
|
||||||
|
profileValue, ok := entry.(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("compatibility fixture profile is malformed")
|
||||||
|
}
|
||||||
|
id, ok := profileValue["id"].(string)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("compatibility fixture profile ID is malformed")
|
||||||
|
}
|
||||||
|
ids = append(ids, id)
|
||||||
|
}
|
||||||
|
sort.Strings(ids)
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
func catalogValue(t *testing.T, loaded Set) map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
backends := make([]any, 0, len(loaded.Backends))
|
||||||
|
for _, backend := range loaded.Backends {
|
||||||
|
backends = append(backends, map[string]any{"id": backend.ID, "endpoint": backend.Endpoint, "api_key_env": backend.APIKeyEnv, "extra_params": interfaceValue(backend.ExtraParams), "concurrency_limit": backend.ConcurrencyLimit, "queue_capacity": backend.QueueCapacity, "queue_capacity_set": backend.QueueCapacitySet})
|
||||||
|
}
|
||||||
|
sort.Slice(backends, func(left, right int) bool {
|
||||||
|
return backends[left].(map[string]any)["id"].(string) < backends[right].(map[string]any)["id"].(string)
|
||||||
|
})
|
||||||
|
actualProfiles := make([]any, 0, len(loaded.profileIDs))
|
||||||
|
for _, id := range loaded.profileIDs {
|
||||||
|
profile, err := loaded.Profiles.GetProfile(context.Background(), id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load profile %q: %v", id, err)
|
||||||
|
}
|
||||||
|
actualProfiles = append(actualProfiles, map[string]any{"id": profile.ID, "base_profile": profile.BaseProfileID, "backend": profile.BackendID, "endpoint": profile.Endpoint, "model": profile.Model, "temperature": profile.Temperature, "max_tokens": profile.MaxTokens, "top_p": profile.TopP, "timeout_seconds": profile.TimeoutSeconds, "service_tier": profile.ServiceTier, "reasoning_effort": profile.ReasoningEffort, "api_key_env": profile.APIKeyEnv, "api_key_required": profile.APIKeyRequired, "extra_params": interfaceValue(profile.ExtraParams)})
|
||||||
|
}
|
||||||
|
return map[string]any{"backends": backends, "profiles": actualProfiles}
|
||||||
|
}
|
||||||
|
|
||||||
|
func interfaceValue(value map[string]any) any {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
func testSource(id string, fsys fs.FS) Source {
|
||||||
|
return Source{Name: id, ExpectedBackendID: id, FS: fsys, Root: "catalog"}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validCatalogFS(id string) fstest.MapFS {
|
||||||
|
return catalogFS(id, map[string]string{
|
||||||
|
"catalog/profiles/" + id + "-profile.yml": validProfile(id+"-profile", id),
|
||||||
|
}, "null")
|
||||||
|
}
|
||||||
|
|
||||||
|
func catalogFS(id string, profiles map[string]string, extraParams string) fstest.MapFS {
|
||||||
|
fsys := fstest.MapFS{
|
||||||
|
"catalog/backend.json": &fstest.MapFile{Data: []byte(validManifest(id, "TEST_API_KEY", extraParams))},
|
||||||
|
}
|
||||||
|
for name, content := range profiles {
|
||||||
|
fsys[name] = &fstest.MapFile{Data: []byte(content)}
|
||||||
|
}
|
||||||
|
return fsys
|
||||||
|
}
|
||||||
|
|
||||||
|
func validManifest(id, apiKeyEnv, extraParams string) string {
|
||||||
|
return fmt.Sprintf(
|
||||||
|
`{"schema_version":1,"id":%q,"endpoint":%q,"api_key_env":%q,"concurrency_limit":2,"queue_capacity":3,"extra_params":%s}`,
|
||||||
|
id,
|
||||||
|
"https://"+id+".example/v1",
|
||||||
|
apiKeyEnv,
|
||||||
|
extraParams,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func validProfile(id, backendID string) string {
|
||||||
|
return fmt.Sprintf("id: %s\nbackend: %s\nmodel: test-model\n", id, backendID)
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ fs.FS = openrouter.FS()
|
||||||
@@ -15,10 +15,7 @@ const (
|
|||||||
OpenAIChatCompletionsPath = "/chat/completions"
|
OpenAIChatCompletionsPath = "/chat/completions"
|
||||||
|
|
||||||
ExecutionDefaultTimeoutSeconds = 600
|
ExecutionDefaultTimeoutSeconds = 600
|
||||||
)
|
LLMRequestTimeoutDefault = 10 * time.Minute
|
||||||
|
|
||||||
var (
|
|
||||||
LLMRequestTimeoutDefault = 10 * time.Minute
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func ExecutionTargetDefault() domain.ExecutionTarget {
|
func ExecutionTargetDefault() domain.ExecutionTarget {
|
||||||
|
|||||||
@@ -60,15 +60,16 @@ type CacheControl struct {
|
|||||||
|
|
||||||
// RunRequest represents a request to generate a single artifact.
|
// RunRequest represents a request to generate a single artifact.
|
||||||
type RunRequest struct {
|
type RunRequest struct {
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion string
|
||||||
ProfileID string
|
ProfileID string
|
||||||
SessionID string
|
SessionID string
|
||||||
APIKey string `json:"-" yaml:"-"`
|
APIKey string `json:"-" yaml:"-"`
|
||||||
Inputs map[string]ArtifactRef
|
Inputs map[string]ArtifactRef
|
||||||
Vars map[string]string
|
Vars map[string]string
|
||||||
Execution *ExecutionTargetOverride
|
Execution *ExecutionTargetOverride
|
||||||
Validation *OutputContract
|
Validation *OutputContract
|
||||||
|
AppendedMessages []RenderedMessage
|
||||||
}
|
}
|
||||||
|
|
||||||
// RunResult represents the complete result of a prompt execution run.
|
// RunResult represents the complete result of a prompt execution run.
|
||||||
@@ -97,22 +98,22 @@ type RunResult struct {
|
|||||||
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
|
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
|
||||||
// It must never include resolved API key values, model output, or validation data.
|
// It must never include resolved API key values, model output, or validation data.
|
||||||
type PreparedRun struct {
|
type PreparedRun struct {
|
||||||
PromptID string `json:"prompt_id"`
|
PromptID string
|
||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
PromptVersion string
|
||||||
PromptHash string `json:"prompt_hash,omitempty"`
|
PromptHash string
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
SelectedProfileID string
|
||||||
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
SelectedBackendID string
|
||||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
EffectiveModelParams ExecutionTarget
|
||||||
TargetPresence ExecutionTargetPresence `json:"-"`
|
TargetPresence ExecutionTargetPresence
|
||||||
OutputContract OutputContract `json:"output_contract"`
|
OutputContract OutputContract
|
||||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
StructuredOutput *StructuredOutputSpec
|
||||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
InputHashes map[string]string
|
||||||
SessionID string `json:"session_id,omitempty"`
|
SessionID string
|
||||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
RenderedPromptHash string
|
||||||
Messages []RenderedMessage `json:"messages"`
|
Messages []RenderedMessage
|
||||||
StartTime time.Time `json:"start_time,omitempty"`
|
StartTime time.Time
|
||||||
EndTime time.Time `json:"end_time,omitempty"`
|
EndTime time.Time
|
||||||
DurationMS int64 `json:"duration_ms,omitempty"`
|
DurationMS int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArtifactRef represents a reference to an input artifact.
|
// ArtifactRef represents a reference to an input artifact.
|
||||||
@@ -192,6 +193,7 @@ type BackendCapacityPolicy struct {
|
|||||||
// ExecutionProfile describes how and where to execute a model.
|
// ExecutionProfile describes how and where to execute a model.
|
||||||
type ExecutionProfile struct {
|
type ExecutionProfile struct {
|
||||||
ID string `yaml:"id"`
|
ID string `yaml:"id"`
|
||||||
|
BaseProfileID string `yaml:"base_profile"`
|
||||||
BackendID string `yaml:"backend"`
|
BackendID string `yaml:"backend"`
|
||||||
Endpoint string `yaml:"endpoint"`
|
Endpoint string `yaml:"endpoint"`
|
||||||
Model string `yaml:"model"`
|
Model string `yaml:"model"`
|
||||||
|
|||||||
39
internal/domain/endpoint.go
Normal file
39
internal/domain/endpoint.go
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NormalizeOpenAICompatibleBaseEndpoint trims and validates a source-neutral
|
||||||
|
// OpenAI-compatible provider base endpoint.
|
||||||
|
func NormalizeOpenAICompatibleBaseEndpoint(endpoint string) (string, error) {
|
||||||
|
endpoint = strings.TrimSpace(endpoint)
|
||||||
|
if endpoint == "" {
|
||||||
|
return "", errors.New("endpoint must not be blank")
|
||||||
|
}
|
||||||
|
if strings.Contains(endpoint, "#") {
|
||||||
|
return "", errors.New("endpoint must not contain a fragment")
|
||||||
|
}
|
||||||
|
|
||||||
|
parsed, err := url.Parse(endpoint)
|
||||||
|
if err != nil {
|
||||||
|
return "", errors.New("endpoint must be a valid URL")
|
||||||
|
}
|
||||||
|
parsed.Scheme = strings.ToLower(parsed.Scheme)
|
||||||
|
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||||
|
return "", errors.New("endpoint must use http or https")
|
||||||
|
}
|
||||||
|
if !parsed.IsAbs() || parsed.Hostname() == "" {
|
||||||
|
return "", errors.New("endpoint must be absolute and include a host")
|
||||||
|
}
|
||||||
|
if parsed.User != nil {
|
||||||
|
return "", errors.New("endpoint must not contain user information")
|
||||||
|
}
|
||||||
|
if parsed.RawQuery != "" || parsed.ForceQuery {
|
||||||
|
return "", errors.New("endpoint must not contain a query string")
|
||||||
|
}
|
||||||
|
|
||||||
|
return parsed.String(), nil
|
||||||
|
}
|
||||||
47
internal/domain/endpoint_test.go
Normal file
47
internal/domain/endpoint_test.go
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestNormalizeOpenAICompatibleBaseEndpoint(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
endpoint string
|
||||||
|
want string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "http host", endpoint: "http://provider.example", want: "http://provider.example"},
|
||||||
|
{name: "https nested path and whitespace", endpoint: " HTTPS://provider.example/api/openai/v1 ", want: "https://provider.example/api/openai/v1"},
|
||||||
|
{name: "IPv4 host and port", endpoint: "http://127.0.0.1:8080/v1", want: "http://127.0.0.1:8080/v1"},
|
||||||
|
{name: "IPv6 host and port", endpoint: "https://[::1]:8443/v1", want: "https://[::1]:8443/v1"},
|
||||||
|
{name: "repeated trailing slashes", endpoint: "https://provider.example/v1///", want: "https://provider.example/v1///"},
|
||||||
|
{name: "blank", endpoint: " \t\n ", wantErr: true},
|
||||||
|
{name: "relative path", endpoint: "/api/v1", wantErr: true},
|
||||||
|
{name: "scheme relative", endpoint: "//provider.example/v1", wantErr: true},
|
||||||
|
{name: "missing host", endpoint: "https:///v1", wantErr: true},
|
||||||
|
{name: "unsupported scheme", endpoint: "ftp://provider.example/v1", wantErr: true},
|
||||||
|
{name: "user information", endpoint: "https://user:secret@provider.example/v1", wantErr: true},
|
||||||
|
{name: "query", endpoint: "https://provider.example/v1?mode=chat", wantErr: true},
|
||||||
|
{name: "empty query", endpoint: "https://provider.example/v1?", wantErr: true},
|
||||||
|
{name: "fragment", endpoint: "https://provider.example/v1#chat", wantErr: true},
|
||||||
|
{name: "empty fragment", endpoint: "https://provider.example/v1#", wantErr: true},
|
||||||
|
{name: "malformed URL", endpoint: "https://provider.example/%zz", wantErr: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got, err := NormalizeOpenAICompatibleBaseEndpoint(tc.endpoint)
|
||||||
|
if tc.wantErr {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("expected endpoint error, got %q", got)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalize endpoint: %v", err)
|
||||||
|
}
|
||||||
|
if got != tc.want {
|
||||||
|
t.Fatalf("normalized endpoint = %q, want %q", got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
34
internal/domain/execution_settings.go
Normal file
34
internal/domain/execution_settings.go
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"math"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const maxExecutionTimeoutSeconds int64 = math.MaxInt64 / int64(time.Second)
|
||||||
|
|
||||||
|
// ValidateExecutionTargetSettings validates source-neutral execution-setting
|
||||||
|
// invariants on a resolved target.
|
||||||
|
func ValidateExecutionTargetSettings(target ExecutionTarget) error {
|
||||||
|
if !isFinite(target.Temperature) || target.Temperature < 0 || target.Temperature > 2 {
|
||||||
|
return errors.New("temperature must be finite and between 0 and 2")
|
||||||
|
}
|
||||||
|
if target.MaxTokens < 0 {
|
||||||
|
return errors.New("max_tokens must be greater than or equal to 0")
|
||||||
|
}
|
||||||
|
if !isFinite(target.TopP) || target.TopP < 0 || target.TopP > 1 {
|
||||||
|
return errors.New("top_p must be finite and between 0 and 1")
|
||||||
|
}
|
||||||
|
if target.TimeoutSeconds < 0 {
|
||||||
|
return errors.New("timeout_seconds must be greater than or equal to 0")
|
||||||
|
}
|
||||||
|
if int64(target.TimeoutSeconds) > maxExecutionTimeoutSeconds {
|
||||||
|
return errors.New("timeout_seconds exceeds the maximum supported duration")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isFinite(value float64) bool {
|
||||||
|
return !math.IsNaN(value) && !math.IsInf(value, 0)
|
||||||
|
}
|
||||||
73
internal/domain/execution_settings_test.go
Normal file
73
internal/domain/execution_settings_test.go
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidateExecutionTargetSettings(t *testing.T) {
|
||||||
|
valid := ExecutionTarget{
|
||||||
|
Temperature: 1,
|
||||||
|
MaxTokens: 1,
|
||||||
|
TopP: 0.5,
|
||||||
|
TimeoutSeconds: 1,
|
||||||
|
}
|
||||||
|
|
||||||
|
type testCase struct {
|
||||||
|
name string
|
||||||
|
change func(*ExecutionTarget)
|
||||||
|
wantErr string
|
||||||
|
}
|
||||||
|
tests := []testCase{
|
||||||
|
{name: "temperature lower boundary", change: func(v *ExecutionTarget) { v.Temperature = 0 }},
|
||||||
|
{name: "temperature finite lower neighbor", change: func(v *ExecutionTarget) { v.Temperature = math.Nextafter(0, 1) }},
|
||||||
|
{name: "temperature finite upper neighbor", change: func(v *ExecutionTarget) { v.Temperature = math.Nextafter(2, 0) }},
|
||||||
|
{name: "temperature upper boundary", change: func(v *ExecutionTarget) { v.Temperature = 2 }},
|
||||||
|
{name: "temperature below lower boundary", change: func(v *ExecutionTarget) { v.Temperature = math.Nextafter(0, math.Inf(-1)) }, wantErr: "temperature"},
|
||||||
|
{name: "temperature above upper boundary", change: func(v *ExecutionTarget) { v.Temperature = math.Nextafter(2, math.Inf(1)) }, wantErr: "temperature"},
|
||||||
|
{name: "temperature NaN", change: func(v *ExecutionTarget) { v.Temperature = math.NaN() }, wantErr: "temperature"},
|
||||||
|
{name: "temperature positive infinity", change: func(v *ExecutionTarget) { v.Temperature = math.Inf(1) }, wantErr: "temperature"},
|
||||||
|
{name: "temperature negative infinity", change: func(v *ExecutionTarget) { v.Temperature = math.Inf(-1) }, wantErr: "temperature"},
|
||||||
|
{name: "max tokens lower boundary", change: func(v *ExecutionTarget) { v.MaxTokens = 0 }},
|
||||||
|
{name: "max tokens finite neighbor", change: func(v *ExecutionTarget) { v.MaxTokens = 1 }},
|
||||||
|
{name: "max tokens below lower boundary", change: func(v *ExecutionTarget) { v.MaxTokens = -1 }, wantErr: "max_tokens"},
|
||||||
|
{name: "top p lower boundary", change: func(v *ExecutionTarget) { v.TopP = 0 }},
|
||||||
|
{name: "top p finite lower neighbor", change: func(v *ExecutionTarget) { v.TopP = math.Nextafter(0, 1) }},
|
||||||
|
{name: "top p finite upper neighbor", change: func(v *ExecutionTarget) { v.TopP = math.Nextafter(1, 0) }},
|
||||||
|
{name: "top p upper boundary", change: func(v *ExecutionTarget) { v.TopP = 1 }},
|
||||||
|
{name: "top p below lower boundary", change: func(v *ExecutionTarget) { v.TopP = math.Nextafter(0, math.Inf(-1)) }, wantErr: "top_p"},
|
||||||
|
{name: "top p above upper boundary", change: func(v *ExecutionTarget) { v.TopP = math.Nextafter(1, math.Inf(1)) }, wantErr: "top_p"},
|
||||||
|
{name: "top p NaN", change: func(v *ExecutionTarget) { v.TopP = math.NaN() }, wantErr: "top_p"},
|
||||||
|
{name: "top p positive infinity", change: func(v *ExecutionTarget) { v.TopP = math.Inf(1) }, wantErr: "top_p"},
|
||||||
|
{name: "top p negative infinity", change: func(v *ExecutionTarget) { v.TopP = math.Inf(-1) }, wantErr: "top_p"},
|
||||||
|
{name: "timeout lower boundary", change: func(v *ExecutionTarget) { v.TimeoutSeconds = 0 }},
|
||||||
|
{name: "timeout finite neighbor", change: func(v *ExecutionTarget) { v.TimeoutSeconds = 1 }},
|
||||||
|
{name: "timeout below lower boundary", change: func(v *ExecutionTarget) { v.TimeoutSeconds = -1 }, wantErr: "timeout_seconds"},
|
||||||
|
}
|
||||||
|
if strconv.IntSize == 64 {
|
||||||
|
durationLimit := maxExecutionTimeoutSeconds
|
||||||
|
tests = append(tests,
|
||||||
|
testCase{name: "timeout duration boundary", change: func(v *ExecutionTarget) { v.TimeoutSeconds = int(durationLimit) }},
|
||||||
|
testCase{name: "timeout above duration boundary", change: func(v *ExecutionTarget) { v.TimeoutSeconds = int(durationLimit) + 1 }, wantErr: "timeout_seconds"},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
target := valid
|
||||||
|
tt.change(&target)
|
||||||
|
err := ValidateExecutionTargetSettings(target)
|
||||||
|
if tt.wantErr == "" {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validate execution settings: %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||||
|
t.Fatalf("error = %v, want diagnostic containing %q", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
82
internal/domain/message.go
Normal file
82
internal/domain/message.go
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
RoleDeveloper = "developer"
|
||||||
|
RoleSystem = "system"
|
||||||
|
RoleUser = "user"
|
||||||
|
RoleAssistant = "assistant"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NormalizeMessageRole validates and canonicalizes a provider-bound chat role.
|
||||||
|
func NormalizeMessageRole(role string) (string, error) {
|
||||||
|
if !utf8.ValidString(role) {
|
||||||
|
return "", errors.New("message role must be valid UTF-8")
|
||||||
|
}
|
||||||
|
|
||||||
|
normalized := strings.ToLower(strings.TrimSpace(role))
|
||||||
|
switch normalized {
|
||||||
|
case RoleDeveloper, RoleSystem, RoleUser, RoleAssistant:
|
||||||
|
return normalized, nil
|
||||||
|
default:
|
||||||
|
return "", errors.New("message role must be developer, system, user, or assistant")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NormalizeCacheControl validates, canonicalizes, and copies cache metadata.
|
||||||
|
func NormalizeCacheControl(control *CacheControl) (*CacheControl, error) {
|
||||||
|
if control == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if !utf8.ValidString(string(control.Type)) {
|
||||||
|
return nil, errors.New("cache control type must be valid UTF-8")
|
||||||
|
}
|
||||||
|
if !utf8.ValidString(control.TTL) {
|
||||||
|
return nil, errors.New("cache control ttl must be valid UTF-8")
|
||||||
|
}
|
||||||
|
|
||||||
|
cacheType := strings.TrimSpace(string(control.Type))
|
||||||
|
if cacheType == "" {
|
||||||
|
return nil, errors.New("cache control type is required")
|
||||||
|
}
|
||||||
|
if CacheControlType(cacheType) != CacheControlEphemeral {
|
||||||
|
return nil, errors.New("unsupported type")
|
||||||
|
}
|
||||||
|
|
||||||
|
ttl := strings.TrimSpace(control.TTL)
|
||||||
|
if ttl != "" && ttl != "1h" {
|
||||||
|
return nil, errors.New("unsupported ttl")
|
||||||
|
}
|
||||||
|
|
||||||
|
return &CacheControl{Type: CacheControlType(cacheType), TTL: ttl}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CloneRenderedMessages returns a deep copy of rendered messages.
|
||||||
|
func CloneRenderedMessages(messages []RenderedMessage) []RenderedMessage {
|
||||||
|
cloned := make([]RenderedMessage, len(messages))
|
||||||
|
copyRenderedMessages(cloned, messages)
|
||||||
|
return cloned
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConcatRenderedMessages returns an independently owned concatenation of messages.
|
||||||
|
func ConcatRenderedMessages(prefix, suffix []RenderedMessage) []RenderedMessage {
|
||||||
|
messages := make([]RenderedMessage, len(prefix)+len(suffix))
|
||||||
|
copyRenderedMessages(messages, prefix)
|
||||||
|
copyRenderedMessages(messages[len(prefix):], suffix)
|
||||||
|
return messages
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyRenderedMessages(destination, source []RenderedMessage) {
|
||||||
|
for index, message := range source {
|
||||||
|
destination[index] = message
|
||||||
|
if message.CacheControl != nil {
|
||||||
|
cacheControl := *message.CacheControl
|
||||||
|
destination[index].CacheControl = &cacheControl
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
121
internal/domain/message_test.go
Normal file
121
internal/domain/message_test.go
Normal file
@@ -0,0 +1,121 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNormalizeMessageRole(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
input string
|
||||||
|
want string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "developer", input: RoleDeveloper, want: RoleDeveloper},
|
||||||
|
{name: "system", input: RoleSystem, want: RoleSystem},
|
||||||
|
{name: "user", input: RoleUser, want: RoleUser},
|
||||||
|
{name: "assistant", input: RoleAssistant, want: RoleAssistant},
|
||||||
|
{name: "surrounding whitespace and mixed case", input: " \u2003UsEr\u2003 ", want: RoleUser},
|
||||||
|
{name: "blank", input: " \t\n ", wantErr: true},
|
||||||
|
{name: "tool", input: "tool", wantErr: true},
|
||||||
|
{name: "function", input: "function", wantErr: true},
|
||||||
|
{name: "custom", input: "custom-role", wantErr: true},
|
||||||
|
{name: "invalid UTF-8", input: string([]byte{0xff}), wantErr: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
got, err := NormalizeMessageRole(test.input)
|
||||||
|
if test.wantErr {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected an error")
|
||||||
|
}
|
||||||
|
if strings.Contains(err.Error(), test.input) {
|
||||||
|
t.Fatalf("error exposed the input: %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NormalizeMessageRole() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != test.want {
|
||||||
|
t.Fatalf("NormalizeMessageRole() = %q, want %q", got, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeCacheControl(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
input *CacheControl
|
||||||
|
want *CacheControl
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "nil", input: nil, want: nil},
|
||||||
|
{
|
||||||
|
name: "canonical values",
|
||||||
|
input: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"},
|
||||||
|
want: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "trims values",
|
||||||
|
input: &CacheControl{Type: " ephemeral ", TTL: " 1h\t"},
|
||||||
|
want: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"},
|
||||||
|
},
|
||||||
|
{name: "empty type", input: &CacheControl{}, wantErr: true},
|
||||||
|
{name: "unsupported type", input: &CacheControl{Type: "persistent"}, wantErr: true},
|
||||||
|
{name: "unsupported ttl", input: &CacheControl{Type: CacheControlEphemeral, TTL: "5m"}, wantErr: true},
|
||||||
|
{name: "invalid type UTF-8", input: &CacheControl{Type: CacheControlType(string([]byte{0xff}))}, wantErr: true},
|
||||||
|
{name: "invalid ttl UTF-8", input: &CacheControl{Type: CacheControlEphemeral, TTL: string([]byte{0xff})}, wantErr: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
got, err := NormalizeCacheControl(test.input)
|
||||||
|
if test.wantErr {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected an error")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NormalizeCacheControl() error = %v", err)
|
||||||
|
}
|
||||||
|
if got == nil || test.want == nil {
|
||||||
|
if got != test.want {
|
||||||
|
t.Fatalf("NormalizeCacheControl() = %#v, want %#v", got, test.want)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if *got != *test.want {
|
||||||
|
t.Fatalf("NormalizeCacheControl() = %#v, want %#v", got, test.want)
|
||||||
|
}
|
||||||
|
if got == test.input {
|
||||||
|
t.Fatal("normalized cache control aliases its input")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderedMessageCloning(t *testing.T) {
|
||||||
|
prefix := []RenderedMessage{{Role: RoleSystem, Content: "prefix", CacheControl: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"}}}
|
||||||
|
suffix := []RenderedMessage{{Role: RoleUser, Content: " suffix "}, {Role: RoleAssistant, Content: "", CacheControl: &CacheControl{Type: CacheControlEphemeral}}}
|
||||||
|
|
||||||
|
cloned := CloneRenderedMessages(prefix)
|
||||||
|
combined := ConcatRenderedMessages(prefix, suffix)
|
||||||
|
if len(combined) != 3 || combined[0].Content != "prefix" || combined[1].Content != " suffix " || combined[2].Content != "" {
|
||||||
|
t.Fatalf("unexpected combined messages: %#v", combined)
|
||||||
|
}
|
||||||
|
if cloned[0].CacheControl == prefix[0].CacheControl || combined[0].CacheControl == prefix[0].CacheControl || combined[2].CacheControl == suffix[1].CacheControl {
|
||||||
|
t.Fatal("cloned cache controls alias their inputs")
|
||||||
|
}
|
||||||
|
|
||||||
|
prefix[0].Content = "changed"
|
||||||
|
prefix[0].CacheControl.TTL = ""
|
||||||
|
suffix[1].CacheControl.Type = "changed"
|
||||||
|
if cloned[0].Content != "prefix" || cloned[0].CacheControl.TTL != "1h" || combined[0].Content != "prefix" || combined[0].CacheControl.TTL != "1h" || combined[2].CacheControl.Type != CacheControlEphemeral {
|
||||||
|
t.Fatalf("cloned messages changed with their inputs: cloned=%#v combined=%#v", cloned, combined)
|
||||||
|
}
|
||||||
|
}
|
||||||
38
internal/domain/output_contract.go
Normal file
38
internal/domain/output_contract.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
const maxOutputRepairAttempts = 3
|
||||||
|
|
||||||
|
// ValidateOutputContract validates source-neutral output-contract invariants.
|
||||||
|
func ValidateOutputContract(contract OutputContract) error {
|
||||||
|
switch contract.Format {
|
||||||
|
case FormatText, FormatMarkdown, FormatJSON:
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("invalid output format: %q", contract.Format)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch contract.ValidationMode {
|
||||||
|
case ValidationNone, ValidationBasic, ValidationJSON, ValidationJSONSchema:
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("invalid validation mode: %q", contract.ValidationMode)
|
||||||
|
}
|
||||||
|
|
||||||
|
if contract.ValidationMode == ValidationJSONSchema && strings.TrimSpace(contract.SchemaPath) == "" {
|
||||||
|
return errors.New("schema_path is required when validation_mode is json_schema")
|
||||||
|
}
|
||||||
|
if contract.RepairAttempts < 0 {
|
||||||
|
return errors.New("repair_attempts must be greater than or equal to 0")
|
||||||
|
}
|
||||||
|
if contract.RepairAttempts > maxOutputRepairAttempts {
|
||||||
|
return fmt.Errorf("repair_attempts must be less than or equal to %d", maxOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
if contract.ValidationMode == ValidationNone && contract.RepairAttempts > 0 {
|
||||||
|
return errors.New("repair_attempts requires basic, json, or json_schema validation")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
87
internal/domain/output_contract_test.go
Normal file
87
internal/domain/output_contract_test.go
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidateOutputContract(t *testing.T) {
|
||||||
|
valid := OutputContract{
|
||||||
|
Format: FormatText,
|
||||||
|
ValidationMode: ValidationNone,
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
change func(*OutputContract)
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{name: "text format", change: func(c *OutputContract) { c.Format = FormatText }},
|
||||||
|
{name: "markdown format", change: func(c *OutputContract) { c.Format = FormatMarkdown }},
|
||||||
|
{name: "json format", change: func(c *OutputContract) { c.Format = FormatJSON }},
|
||||||
|
{name: "empty format", change: func(c *OutputContract) { c.Format = "" }, wantErr: "format"},
|
||||||
|
{name: "unsupported format", change: func(c *OutputContract) { c.Format = OutputFormat("binary") }, wantErr: "format"},
|
||||||
|
{name: "none validation", change: func(c *OutputContract) { c.ValidationMode = ValidationNone }},
|
||||||
|
{name: "basic validation", change: func(c *OutputContract) { c.ValidationMode = ValidationBasic }},
|
||||||
|
{name: "json validation", change: func(c *OutputContract) { c.ValidationMode = ValidationJSON }},
|
||||||
|
{name: "json schema validation", change: func(c *OutputContract) {
|
||||||
|
c.ValidationMode = ValidationJSONSchema
|
||||||
|
c.SchemaPath = "schema.json"
|
||||||
|
}},
|
||||||
|
{name: "empty validation mode", change: func(c *OutputContract) { c.ValidationMode = "" }, wantErr: "validation mode"},
|
||||||
|
{name: "unsupported validation mode", change: func(c *OutputContract) { c.ValidationMode = ValidationMode("unknown") }, wantErr: "validation mode"},
|
||||||
|
{name: "negative repair attempts", change: func(c *OutputContract) {
|
||||||
|
c.ValidationMode = ValidationBasic
|
||||||
|
c.RepairAttempts = -1
|
||||||
|
}, wantErr: "repair_attempts"},
|
||||||
|
{name: "zero repair attempts", change: func(c *OutputContract) { c.RepairAttempts = 0 }},
|
||||||
|
{name: "one repair attempt", change: func(c *OutputContract) {
|
||||||
|
c.ValidationMode = ValidationBasic
|
||||||
|
c.RepairAttempts = 1
|
||||||
|
}},
|
||||||
|
{name: "maximum repair attempts", change: func(c *OutputContract) {
|
||||||
|
c.ValidationMode = ValidationJSON
|
||||||
|
c.RepairAttempts = 3
|
||||||
|
}},
|
||||||
|
{name: "too many repair attempts", change: func(c *OutputContract) {
|
||||||
|
c.ValidationMode = ValidationJSONSchema
|
||||||
|
c.SchemaPath = "schema.json"
|
||||||
|
c.RepairAttempts = 4
|
||||||
|
}, wantErr: "repair_attempts"},
|
||||||
|
{name: "none validation with repair attempts", change: func(c *OutputContract) {
|
||||||
|
c.ValidationMode = ValidationNone
|
||||||
|
c.RepairAttempts = 1
|
||||||
|
}, wantErr: "repair_attempts"},
|
||||||
|
{name: "json schema empty path", change: func(c *OutputContract) {
|
||||||
|
c.ValidationMode = ValidationJSONSchema
|
||||||
|
c.SchemaPath = ""
|
||||||
|
}, wantErr: "schema_path"},
|
||||||
|
{name: "json schema whitespace path", change: func(c *OutputContract) {
|
||||||
|
c.ValidationMode = ValidationJSONSchema
|
||||||
|
c.SchemaPath = " \t "
|
||||||
|
}, wantErr: "schema_path"},
|
||||||
|
{name: "json schema nonblank path", change: func(c *OutputContract) {
|
||||||
|
c.ValidationMode = ValidationJSONSchema
|
||||||
|
c.SchemaPath = " schema.json "
|
||||||
|
}},
|
||||||
|
{name: "non-schema empty path", change: func(c *OutputContract) { c.SchemaPath = "" }},
|
||||||
|
{name: "non-schema populated path", change: func(c *OutputContract) { c.SchemaPath = "ignored.json" }},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
contract := valid
|
||||||
|
tt.change(&contract)
|
||||||
|
err := ValidateOutputContract(contract)
|
||||||
|
if tt.wantErr == "" {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validate output contract: %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||||
|
t.Fatalf("error = %v, want diagnostic containing %q", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,141 +0,0 @@
|
|||||||
package domain
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
|
|
||||||
const envName = "PROMPTKIT_TEST_API_KEY"
|
|
||||||
const secret = "super-secret-value"
|
|
||||||
t.Setenv(envName, secret)
|
|
||||||
|
|
||||||
prepared := PreparedRun{
|
|
||||||
PromptID: "prompt.id",
|
|
||||||
PromptVersion: "v1",
|
|
||||||
PromptHash: "prompt-hash",
|
|
||||||
SelectedProfileID: "local-fast",
|
|
||||||
EffectiveModelParams: ExecutionTarget{
|
|
||||||
Endpoint: "http://llm/v1",
|
|
||||||
Model: "gpt-test",
|
|
||||||
APIKeyEnv: envName,
|
|
||||||
APIKey: secret,
|
|
||||||
},
|
|
||||||
InputHashes: map[string]string{"transcript": "hash-1"},
|
|
||||||
RenderedPromptHash: "rendered-hash",
|
|
||||||
Messages: []RenderedMessage{
|
|
||||||
{Role: "system", Content: "You are helpful."},
|
|
||||||
{Role: "user", Content: "Summarize this."},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
b, err := json.Marshal(prepared)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
out := string(b)
|
|
||||||
if strings.Contains(out, secret) {
|
|
||||||
t.Fatalf("prepared run JSON unexpectedly contains secret value: %s", out)
|
|
||||||
}
|
|
||||||
if !strings.Contains(out, `"api_key_env":"`+envName+`"`) {
|
|
||||||
t.Fatalf("prepared run JSON should include api_key_env name: %s", out)
|
|
||||||
}
|
|
||||||
|
|
||||||
var top map[string]any
|
|
||||||
if err := json.Unmarshal(b, &top); err != nil {
|
|
||||||
t.Fatalf("unmarshal failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, forbidden := range []string{"raw_output", "validation", "artifact"} {
|
|
||||||
if _, ok := top[forbidden]; ok {
|
|
||||||
t.Fatalf("prepared run JSON should not include %q", forbidden)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPreparedRunJSONIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
|
||||||
prepared := PreparedRun{
|
|
||||||
PromptID: "prompt.id",
|
|
||||||
SelectedProfileID: "local-fast",
|
|
||||||
EffectiveModelParams: ExecutionTarget{
|
|
||||||
Endpoint: "http://llm/v1",
|
|
||||||
Model: "gpt-test",
|
|
||||||
},
|
|
||||||
RenderedPromptHash: "rendered-hash",
|
|
||||||
Messages: []RenderedMessage{
|
|
||||||
{
|
|
||||||
Role: "system",
|
|
||||||
Content: "You are helpful.",
|
|
||||||
CacheControl: &CacheControl{
|
|
||||||
Type: CacheControlEphemeral,
|
|
||||||
TTL: "1h",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{Role: "user", Content: "Summarize this."},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
b, err := json.Marshal(prepared)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var decoded struct {
|
|
||||||
Messages []map[string]any `json:"messages"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(b, &decoded); err != nil {
|
|
||||||
t.Fatalf("unmarshal failed: %v", err)
|
|
||||||
}
|
|
||||||
if len(decoded.Messages) != 2 {
|
|
||||||
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
|
|
||||||
}
|
|
||||||
|
|
||||||
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("expected cache_control on first message, got %#v", decoded.Messages[0])
|
|
||||||
}
|
|
||||||
if cacheControl["type"] != string(CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
|
||||||
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
|
||||||
}
|
|
||||||
if _, ok := decoded.Messages[1]["cache_control"]; ok {
|
|
||||||
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPreparedRunJSONIncludesSessionIDOnlyWhenPresent(t *testing.T) {
|
|
||||||
prepared := PreparedRun{
|
|
||||||
PromptID: "prompt.id",
|
|
||||||
SelectedProfileID: "local-fast",
|
|
||||||
EffectiveModelParams: ExecutionTarget{
|
|
||||||
Endpoint: "http://llm/v1",
|
|
||||||
Model: "gpt-test",
|
|
||||||
},
|
|
||||||
SessionID: "session-123",
|
|
||||||
RenderedPromptHash: "rendered-hash",
|
|
||||||
Messages: []RenderedMessage{{Role: "user", Content: "Summarize this."}},
|
|
||||||
}
|
|
||||||
|
|
||||||
b, err := json.Marshal(prepared)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var decoded map[string]any
|
|
||||||
if err := json.Unmarshal(b, &decoded); err != nil {
|
|
||||||
t.Fatalf("unmarshal failed: %v", err)
|
|
||||||
}
|
|
||||||
if decoded["session_id"] != "session-123" {
|
|
||||||
t.Fatalf("expected session_id in prepared run JSON, got %#v", decoded["session_id"])
|
|
||||||
}
|
|
||||||
|
|
||||||
prepared.SessionID = ""
|
|
||||||
b, err = json.Marshal(prepared)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal failed: %v", err)
|
|
||||||
}
|
|
||||||
if strings.Contains(string(b), "session_id") {
|
|
||||||
t.Fatalf("expected empty session_id to be omitted, got %s", b)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -8,6 +8,9 @@ import (
|
|||||||
|
|
||||||
// NormalizeSessionID applies the shared session identifier rule.
|
// NormalizeSessionID applies the shared session identifier rule.
|
||||||
func NormalizeSessionID(raw string) (string, error) {
|
func NormalizeSessionID(raw string) (string, error) {
|
||||||
|
if !utf8.ValidString(raw) {
|
||||||
|
return "", fmt.Errorf("session_id must contain valid UTF-8")
|
||||||
|
}
|
||||||
normalized := strings.TrimSpace(raw)
|
normalized := strings.TrimSpace(raw)
|
||||||
if normalized == "" {
|
if normalized == "" {
|
||||||
return "", nil
|
return "", nil
|
||||||
|
|||||||
@@ -7,10 +7,10 @@ import (
|
|||||||
|
|
||||||
func TestNormalizeSessionID(t *testing.T) {
|
func TestNormalizeSessionID(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
raw string
|
raw string
|
||||||
want string
|
want string
|
||||||
wantErr bool
|
wantErrContains string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "trims surrounding Unicode whitespace",
|
name: "trims surrounding Unicode whitespace",
|
||||||
@@ -28,21 +28,24 @@ func TestNormalizeSessionID(t *testing.T) {
|
|||||||
want: strings.Repeat("界", SessionIDMaxLength),
|
want: strings.Repeat("界", SessionIDMaxLength),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "one Unicode code point over maximum is rejected",
|
name: "one Unicode code point over maximum is rejected",
|
||||||
raw: strings.Repeat("界", SessionIDMaxLength+1),
|
raw: strings.Repeat("界", SessionIDMaxLength+1),
|
||||||
wantErr: true,
|
wantErrContains: "exceeds maximum",
|
||||||
},
|
},
|
||||||
|
{name: "invalid UTF-8 before valid content", raw: string([]byte{0xff}) + "session", wantErrContains: "valid UTF-8"},
|
||||||
|
{name: "invalid UTF-8 within valid content", raw: "ses" + string([]byte{0xff}) + "sion", wantErrContains: "valid UTF-8"},
|
||||||
|
{name: "invalid UTF-8 after valid content", raw: "session" + string([]byte{0xff}), wantErrContains: "valid UTF-8"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
got, err := NormalizeSessionID(tt.raw)
|
got, err := NormalizeSessionID(tt.raw)
|
||||||
if tt.wantErr {
|
if tt.wantErrContains != "" {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected normalization error")
|
t.Fatal("expected normalization error")
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "exceeds maximum") {
|
if !strings.Contains(err.Error(), tt.wantErrContains) {
|
||||||
t.Fatalf("expected useful length diagnostic, got %v", err)
|
t.Fatalf("expected diagnostic containing %q, got %v", tt.wantErrContains, err)
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -36,7 +36,8 @@ func FindYAMLFiles(ctx context.Context, root string) ([]string, error) {
|
|||||||
return files, err
|
return files, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// FindFSYAMLFiles returns sorted paths for .yaml and .yml files under root in fsys.
|
// FindFSYAMLFiles returns root itself when it names a file. For a directory
|
||||||
|
// root, it returns sorted paths for .yaml and .yml files beneath that root.
|
||||||
func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, error) {
|
func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, error) {
|
||||||
cleanRoot := CleanFSRoot(root)
|
cleanRoot := CleanFSRoot(root)
|
||||||
var files []string
|
var files []string
|
||||||
@@ -52,6 +53,10 @@ func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, er
|
|||||||
if d.IsDir() {
|
if d.IsDir() {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if name == cleanRoot {
|
||||||
|
files = append(files, name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
if !IsYAMLFile(d.Name()) {
|
if !IsYAMLFile(d.Name()) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -71,10 +76,10 @@ func RelativePath(root string, filePath string) string {
|
|||||||
return filepath.Clean(rel)
|
return filepath.Clean(rel)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CleanFSRoot normalizes a root path for use with fs.FS.
|
// CleanFSRoot normalizes a root path for use with fs.FS while preserving
|
||||||
|
// nonblank leading and trailing whitespace.
|
||||||
func CleanFSRoot(root string) string {
|
func CleanFSRoot(root string) string {
|
||||||
root = strings.TrimSpace(root)
|
if strings.TrimSpace(root) == "" || root == "." {
|
||||||
if root == "" || root == "." {
|
|
||||||
return "."
|
return "."
|
||||||
}
|
}
|
||||||
return path.Clean(root)
|
return path.Clean(root)
|
||||||
@@ -97,22 +102,21 @@ func DisplayPath(root string, name string) string {
|
|||||||
// ResolveFSPath resolves userPath from baseDir and keeps it inside root.
|
// ResolveFSPath resolves userPath from baseDir and keeps it inside root.
|
||||||
func ResolveFSPath(root string, baseDir string, userPath string) (string, string, error) {
|
func ResolveFSPath(root string, baseDir string, userPath string) (string, string, error) {
|
||||||
cleanRoot := CleanFSRoot(root)
|
cleanRoot := CleanFSRoot(root)
|
||||||
cleanBase := path.Clean(strings.TrimSpace(baseDir))
|
cleanBase := path.Clean(baseDir)
|
||||||
if cleanBase == "" {
|
if strings.TrimSpace(baseDir) == "" {
|
||||||
cleanBase = cleanRoot
|
cleanBase = cleanRoot
|
||||||
}
|
}
|
||||||
if !containsFSPath(cleanRoot, cleanBase) {
|
if !containsFSPath(cleanRoot, cleanBase) {
|
||||||
return "", "", fmt.Errorf("base path %q is outside source root %q", cleanBase, cleanRoot)
|
return "", "", fmt.Errorf("base path %q is outside source root %q", cleanBase, cleanRoot)
|
||||||
}
|
}
|
||||||
|
|
||||||
cleanUserPath := strings.TrimSpace(userPath)
|
if strings.TrimSpace(userPath) == "" {
|
||||||
if cleanUserPath == "" {
|
|
||||||
return "", "", fmt.Errorf("path is required")
|
return "", "", fmt.Errorf("path is required")
|
||||||
}
|
}
|
||||||
cleanUserPath = path.Clean(cleanUserPath)
|
if path.IsAbs(userPath) {
|
||||||
if path.IsAbs(cleanUserPath) {
|
|
||||||
return "", "", fmt.Errorf("path %q must be relative", userPath)
|
return "", "", fmt.Errorf("path %q must be relative", userPath)
|
||||||
}
|
}
|
||||||
|
cleanUserPath := path.Clean(userPath)
|
||||||
|
|
||||||
resolved := path.Clean(path.Join(cleanBase, cleanUserPath))
|
resolved := path.Clean(path.Join(cleanBase, cleanUserPath))
|
||||||
if !containsFSPath(cleanRoot, resolved) {
|
if !containsFSPath(cleanRoot, resolved) {
|
||||||
@@ -130,13 +134,6 @@ func containsFSPath(root string, name string) bool {
|
|||||||
return name == root || strings.HasPrefix(name, strings.TrimSuffix(root, "/")+"/")
|
return name == root || strings.HasPrefix(name, strings.TrimSuffix(root, "/")+"/")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stem strips .yaml or .yml from a file name.
|
|
||||||
func Stem(name string) string {
|
|
||||||
name = strings.TrimSuffix(name, ".yaml")
|
|
||||||
name = strings.TrimSuffix(name, ".yml")
|
|
||||||
return name
|
|
||||||
}
|
|
||||||
|
|
||||||
func IsYAMLFile(name string) bool {
|
func IsYAMLFile(name string) bool {
|
||||||
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
|
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ func TestFindFSYAMLFilesNestedSortedAndFiltered(t *testing.T) {
|
|||||||
"other/ignored.yaml": &fstest.MapFile{Data: []byte("id: ignored")},
|
"other/ignored.yaml": &fstest.MapFile{Data: []byte("id: ignored")},
|
||||||
}
|
}
|
||||||
|
|
||||||
got, err := FindFSYAMLFiles(context.Background(), fsys, " prompts ")
|
got, err := FindFSYAMLFiles(context.Background(), fsys, "prompts")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -98,8 +98,11 @@ func TestCleanFSRoot(t *testing.T) {
|
|||||||
want string
|
want string
|
||||||
}{
|
}{
|
||||||
{name: "empty", root: "", want: "."},
|
{name: "empty", root: "", want: "."},
|
||||||
|
{name: "whitespace only", root: " \t ", want: "."},
|
||||||
{name: "dot", root: ".", want: "."},
|
{name: "dot", root: ".", want: "."},
|
||||||
{name: "trimmed", root: " prompts/../profiles ", want: "profiles"},
|
{name: "cleaned", root: "prompts/../profiles", want: "profiles"},
|
||||||
|
{name: "leading whitespace preserved", root: " profiles", want: " profiles"},
|
||||||
|
{name: "trailing whitespace preserved", root: "profiles ", want: "profiles "},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
@@ -158,6 +161,22 @@ func TestResolveFSPath(t *testing.T) {
|
|||||||
wantPath: "prompts/shared/user.tmpl",
|
wantPath: "prompts/shared/user.tmpl",
|
||||||
wantDisplay: "shared/user.tmpl",
|
wantDisplay: "shared/user.tmpl",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "leading whitespace preserved",
|
||||||
|
root: "prompts",
|
||||||
|
baseDir: "prompts/nested",
|
||||||
|
userPath: " user.tmpl",
|
||||||
|
wantPath: "prompts/nested/ user.tmpl",
|
||||||
|
wantDisplay: "nested/ user.tmpl",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "trailing whitespace preserved",
|
||||||
|
root: "prompts",
|
||||||
|
baseDir: "prompts/nested",
|
||||||
|
userPath: "user.tmpl ",
|
||||||
|
wantPath: "prompts/nested/user.tmpl ",
|
||||||
|
wantDisplay: "nested/user.tmpl ",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "escape rejected",
|
name: "escape rejected",
|
||||||
root: "prompts",
|
root: "prompts",
|
||||||
@@ -218,26 +237,6 @@ func TestResolveFSPath(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStemStripsYAMLExtensions(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
in string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{name: "yaml", in: "prompt.yaml", want: "prompt"},
|
|
||||||
{name: "yml", in: "profile.yml", want: "profile"},
|
|
||||||
{name: "other", in: "file.txt", want: "file.txt"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
if got := Stem(tc.in); got != tc.want {
|
|
||||||
t.Fatalf("expected %q, got %q", tc.want, got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIsYAMLFile(t *testing.T) {
|
func TestIsYAMLFile(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
// Package jsonvalue validates and defensively copies JSON-compatible value
|
// Package jsonvalue validates and defensively copies bounded JSON-compatible
|
||||||
// trees used by configuration, request, and prepared-state boundaries.
|
// value trees used by configuration, request, and prepared-state boundaries.
|
||||||
package jsonvalue
|
package jsonvalue
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -8,29 +8,39 @@ import (
|
|||||||
"math"
|
"math"
|
||||||
"reflect"
|
"reflect"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const maxSafeJSONInteger = 1<<53 - 1
|
const (
|
||||||
|
maxContainerDepth = 100
|
||||||
|
maxProducedNodes = 100_000
|
||||||
|
)
|
||||||
|
|
||||||
type visit struct {
|
type visit struct {
|
||||||
typ reflect.Type
|
typ reflect.Type
|
||||||
ptr uintptr
|
ptr uintptr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type traversalState struct {
|
||||||
|
active map[visit]struct{}
|
||||||
|
producedNodes int
|
||||||
|
}
|
||||||
|
|
||||||
// Copy validates and deeply copies a JSON-compatible value while preserving
|
// Copy validates and deeply copies a JSON-compatible value while preserving
|
||||||
// compatible concrete map, slice, array, scalar, and number types.
|
// compatible concrete map, slice, array, scalar, and number types. It rejects
|
||||||
|
// cycles and values that exceed the package's traversal limits.
|
||||||
func Copy(src any) (any, error) {
|
func Copy(src any) (any, error) {
|
||||||
return copyValue(reflect.ValueOf(src), "value", make(map[visit]struct{}), true)
|
return copyValue(reflect.ValueOf(src), "value", newTraversalState(), true, 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CopyMap validates and deeply copies an extra-parameter map while preserving
|
// CopyMap validates and deeply copies an extra-parameter map while preserving
|
||||||
// compatible concrete map, slice, array, scalar, and number types.
|
// compatible concrete map, slice, array, scalar, and number types. It rejects
|
||||||
|
// empty object keys, cycles, and values that exceed the package's traversal
|
||||||
|
// limits.
|
||||||
func CopyMap(src map[string]any) (map[string]any, error) {
|
func CopyMap(src map[string]any) (map[string]any, error) {
|
||||||
if src == nil {
|
if src == nil {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
copied, err := copyValue(reflect.ValueOf(src), "extra_params", make(map[visit]struct{}), false)
|
copied, err := copyValue(reflect.ValueOf(src), "extra_params", newTraversalState(), false, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -44,71 +54,90 @@ func CopyMap(src map[string]any) (map[string]any, error) {
|
|||||||
func copyValue(
|
func copyValue(
|
||||||
value reflect.Value,
|
value reflect.Value,
|
||||||
path string,
|
path string,
|
||||||
seen map[visit]struct{},
|
state *traversalState,
|
||||||
allowEmptyMapKeys bool,
|
allowEmptyMapKeys bool,
|
||||||
|
containerDepth int,
|
||||||
) (any, error) {
|
) (any, error) {
|
||||||
if !value.IsValid() {
|
resolved, cleanup, isNull, err := state.resolveIndirection(value, path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer cleanup()
|
||||||
|
if isNull {
|
||||||
|
if err := state.produceNode(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
if value.Kind() == reflect.Interface {
|
value = resolved
|
||||||
if value.IsNil() {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return copyValue(value.Elem(), path, seen, allowEmptyMapKeys)
|
|
||||||
}
|
|
||||||
if !value.CanInterface() {
|
if !value.CanInterface() {
|
||||||
return nil, fmt.Errorf("%s: value cannot be copied", path)
|
return nil, fmt.Errorf("%s: value cannot be copied", path)
|
||||||
}
|
}
|
||||||
if number, ok := value.Interface().(json.Number); ok {
|
if number, ok := value.Interface().(json.Number); ok {
|
||||||
if _, err := json.Marshal(number); err != nil {
|
if !validJSONNumber(number) {
|
||||||
return nil, fmt.Errorf("%s: invalid JSON number", path)
|
return nil, fmt.Errorf("%s: invalid JSON number", path)
|
||||||
}
|
}
|
||||||
f, err := strconv.ParseFloat(number.String(), 64)
|
if err := state.produceNode(path); err != nil {
|
||||||
if err != nil || math.IsNaN(f) || math.IsInf(f, 0) {
|
return nil, err
|
||||||
return nil, fmt.Errorf("%s: invalid JSON number", path)
|
|
||||||
}
|
}
|
||||||
return number, nil
|
return number, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
switch value.Kind() {
|
switch value.Kind() {
|
||||||
case reflect.Bool, reflect.String:
|
case reflect.Bool, reflect.String:
|
||||||
|
if err := state.produceNode(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
return value.Interface(), nil
|
return value.Interface(), nil
|
||||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||||
if value.Int() < -maxSafeJSONInteger || value.Int() > maxSafeJSONInteger {
|
if err := state.produceNode(path); err != nil {
|
||||||
return nil, fmt.Errorf("%s: integer is outside the JSON-safe range", path)
|
return nil, err
|
||||||
}
|
}
|
||||||
return value.Interface(), nil
|
return value.Interface(), nil
|
||||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
|
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
|
||||||
if value.Uint() > maxSafeJSONInteger {
|
if err := state.produceNode(path); err != nil {
|
||||||
return nil, fmt.Errorf("%s: integer is outside the JSON-safe range", path)
|
return nil, err
|
||||||
}
|
}
|
||||||
return value.Interface(), nil
|
return value.Interface(), nil
|
||||||
case reflect.Float32, reflect.Float64:
|
case reflect.Float32, reflect.Float64:
|
||||||
number := value.Convert(reflect.TypeOf(float64(0))).Float()
|
number := value.Float()
|
||||||
if math.IsNaN(number) || math.IsInf(number, 0) {
|
if math.IsNaN(number) || math.IsInf(number, 0) {
|
||||||
return nil, fmt.Errorf("%s: floating-point value must be finite", path)
|
return nil, fmt.Errorf("%s: floating-point value must be finite", path)
|
||||||
}
|
}
|
||||||
|
if err := state.produceNode(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
return value.Interface(), nil
|
return value.Interface(), nil
|
||||||
case reflect.Pointer:
|
case reflect.Map:
|
||||||
if value.IsNil() {
|
if value.IsNil() {
|
||||||
|
if err := state.produceNode(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
current := visit{typ: value.Type(), ptr: value.Pointer()}
|
nextDepth, err := state.enterContainer(path, containerDepth)
|
||||||
if _, ok := seen[current]; ok {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
return nil, err
|
||||||
}
|
}
|
||||||
seen[current] = struct{}{}
|
return copyMapValue(value, path, state, allowEmptyMapKeys, nextDepth)
|
||||||
defer delete(seen, current)
|
|
||||||
return copyValue(value.Elem(), path, seen, allowEmptyMapKeys)
|
|
||||||
case reflect.Map:
|
|
||||||
return copyMapValue(value, path, seen, allowEmptyMapKeys)
|
|
||||||
case reflect.Slice:
|
case reflect.Slice:
|
||||||
if value.IsNil() {
|
if value.IsNil() {
|
||||||
|
if err := state.produceNode(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
return copySequenceValue(value, path, seen, allowEmptyMapKeys)
|
nextDepth, err := state.enterContainer(path, containerDepth)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return copySequenceValue(value, path, state, allowEmptyMapKeys, nextDepth)
|
||||||
case reflect.Array:
|
case reflect.Array:
|
||||||
return copySequenceValue(value, path, seen, allowEmptyMapKeys)
|
nextDepth, err := state.enterContainer(path, containerDepth)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return copySequenceValue(value, path, state, allowEmptyMapKeys, nextDepth)
|
||||||
default:
|
default:
|
||||||
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
|
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
|
||||||
}
|
}
|
||||||
@@ -117,22 +146,26 @@ func copyValue(
|
|||||||
func copyMapValue(
|
func copyMapValue(
|
||||||
value reflect.Value,
|
value reflect.Value,
|
||||||
path string,
|
path string,
|
||||||
seen map[visit]struct{},
|
state *traversalState,
|
||||||
allowEmptyMapKeys bool,
|
allowEmptyMapKeys bool,
|
||||||
|
containerDepth int,
|
||||||
) (any, error) {
|
) (any, error) {
|
||||||
if value.IsNil() {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if value.Type().Key().Kind() != reflect.String {
|
if value.Type().Key().Kind() != reflect.String {
|
||||||
return nil, fmt.Errorf("%s: map key type %s is not supported", path, value.Type().Key())
|
return nil, fmt.Errorf("%s: map key type %s is not supported", path, value.Type().Key())
|
||||||
}
|
}
|
||||||
|
if err := state.produceNode(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := state.ensureChildCapacity(path, value.Len()); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
current := visit{typ: value.Type(), ptr: value.Pointer()}
|
current := visit{typ: value.Type(), ptr: value.Pointer()}
|
||||||
if _, ok := seen[current]; ok {
|
if _, ok := state.active[current]; ok {
|
||||||
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||||
}
|
}
|
||||||
seen[current] = struct{}{}
|
state.active[current] = struct{}{}
|
||||||
defer delete(seen, current)
|
defer delete(state.active, current)
|
||||||
|
|
||||||
keys := value.MapKeys()
|
keys := value.MapKeys()
|
||||||
sort.Slice(keys, func(i, j int) bool {
|
sort.Slice(keys, func(i, j int) bool {
|
||||||
@@ -152,7 +185,13 @@ func copyMapValue(
|
|||||||
if name == "" && !allowEmptyMapKeys {
|
if name == "" && !allowEmptyMapKeys {
|
||||||
return nil, fmt.Errorf("%s: map key must not be empty", path)
|
return nil, fmt.Errorf("%s: map key must not be empty", path)
|
||||||
}
|
}
|
||||||
copied, err := copyValue(value.MapIndex(key), path+"."+name, seen, allowEmptyMapKeys)
|
copied, err := copyValue(
|
||||||
|
value.MapIndex(key),
|
||||||
|
path+"."+name,
|
||||||
|
state,
|
||||||
|
allowEmptyMapKeys,
|
||||||
|
containerDepth,
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -190,17 +229,25 @@ func copyMapValue(
|
|||||||
func copySequenceValue(
|
func copySequenceValue(
|
||||||
value reflect.Value,
|
value reflect.Value,
|
||||||
path string,
|
path string,
|
||||||
seen map[visit]struct{},
|
state *traversalState,
|
||||||
allowEmptyMapKeys bool,
|
allowEmptyMapKeys bool,
|
||||||
|
containerDepth int,
|
||||||
) (any, error) {
|
) (any, error) {
|
||||||
|
if err := state.produceNode(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := state.ensureChildCapacity(path, value.Len()); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
var current visit
|
var current visit
|
||||||
if value.Kind() == reflect.Slice {
|
if value.Kind() == reflect.Slice {
|
||||||
current = visit{typ: value.Type(), ptr: value.Pointer()}
|
current = visit{typ: value.Type(), ptr: value.Pointer()}
|
||||||
if _, ok := seen[current]; ok {
|
if _, ok := state.active[current]; ok {
|
||||||
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||||
}
|
}
|
||||||
seen[current] = struct{}{}
|
state.active[current] = struct{}{}
|
||||||
defer delete(seen, current)
|
defer delete(state.active, current)
|
||||||
}
|
}
|
||||||
|
|
||||||
values := make([]any, value.Len())
|
values := make([]any, value.Len())
|
||||||
@@ -210,8 +257,9 @@ func copySequenceValue(
|
|||||||
copied, err := copyValue(
|
copied, err := copyValue(
|
||||||
value.Index(i),
|
value.Index(i),
|
||||||
fmt.Sprintf("%s[%d]", path, i),
|
fmt.Sprintf("%s[%d]", path, i),
|
||||||
seen,
|
state,
|
||||||
allowEmptyMapKeys,
|
allowEmptyMapKeys,
|
||||||
|
containerDepth,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -248,6 +296,73 @@ func copySequenceValue(
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validJSONNumber(number json.Number) bool {
|
||||||
|
var parsed json.Number
|
||||||
|
if err := json.Unmarshal([]byte(number.String()), &parsed); err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return parsed.String() == number.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTraversalState() *traversalState {
|
||||||
|
return &traversalState{active: make(map[visit]struct{})}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *traversalState) produceNode(path string) error {
|
||||||
|
if state.producedNodes >= maxProducedNodes {
|
||||||
|
return fmt.Errorf("%s: JSON value work limit exceeded", path)
|
||||||
|
}
|
||||||
|
state.producedNodes++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *traversalState) enterContainer(path string, depth int) (int, error) {
|
||||||
|
depth++
|
||||||
|
if depth > maxContainerDepth {
|
||||||
|
return 0, fmt.Errorf("%s: JSON container depth limit exceeded", path)
|
||||||
|
}
|
||||||
|
return depth, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *traversalState) ensureChildCapacity(path string, count int) error {
|
||||||
|
if count > maxProducedNodes-state.producedNodes {
|
||||||
|
return fmt.Errorf("%s: JSON value work limit exceeded", path)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *traversalState) resolveIndirection(
|
||||||
|
value reflect.Value,
|
||||||
|
path string,
|
||||||
|
) (reflect.Value, func(), bool, error) {
|
||||||
|
var visits []visit
|
||||||
|
cleanup := func() {
|
||||||
|
for _, current := range visits {
|
||||||
|
delete(state.active, current)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for value.IsValid() && (value.Kind() == reflect.Interface || value.Kind() == reflect.Pointer) {
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Value{}, cleanup, true, nil
|
||||||
|
}
|
||||||
|
if value.Kind() == reflect.Pointer {
|
||||||
|
current := visit{typ: value.Type(), ptr: value.Pointer()}
|
||||||
|
if _, ok := state.active[current]; ok {
|
||||||
|
cleanup()
|
||||||
|
return reflect.Value{}, nil, false, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||||
|
}
|
||||||
|
state.active[current] = struct{}{}
|
||||||
|
visits = append(visits, current)
|
||||||
|
}
|
||||||
|
value = value.Elem()
|
||||||
|
}
|
||||||
|
if !value.IsValid() {
|
||||||
|
return reflect.Value{}, cleanup, true, nil
|
||||||
|
}
|
||||||
|
return value, cleanup, false, nil
|
||||||
|
}
|
||||||
|
|
||||||
func canAssignNil(typ reflect.Type) bool {
|
func canAssignNil(typ reflect.Type) bool {
|
||||||
switch typ.Kind() {
|
switch typ.Kind() {
|
||||||
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
||||||
|
|||||||
@@ -1,112 +1,332 @@
|
|||||||
package jsonvalue_test
|
package jsonvalue
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"math"
|
"math"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCopyMapPreservesTypesAndIsolatesMutations(t *testing.T) {
|
type (
|
||||||
nested := map[string]int{"limit": 2}
|
namedBool bool
|
||||||
sequence := []string{"one", "two"}
|
namedString string
|
||||||
input := map[string]any{
|
namedInt64 int64
|
||||||
"count": int64(7),
|
namedUint64 uint64
|
||||||
"number": json.Number("-1.25e+2"),
|
namedFloat32 float32
|
||||||
"nested": nested,
|
namedFloat64 float64
|
||||||
"sequence": sequence,
|
namedKey string
|
||||||
}
|
namedMap map[namedKey]namedInt64
|
||||||
|
namedSlice []namedString
|
||||||
copied, err := jsonvalue.CopyMap(input)
|
namedArray [1]map[string]int
|
||||||
if err != nil {
|
)
|
||||||
t.Fatalf("copy map: %v", err)
|
|
||||||
}
|
|
||||||
nested["limit"] = 99
|
|
||||||
sequence[0] = "changed"
|
|
||||||
input["added"] = true
|
|
||||||
|
|
||||||
if got, ok := copied["count"].(int64); !ok || got != 7 {
|
|
||||||
t.Fatalf("integer type or value changed: %#v", copied["count"])
|
|
||||||
}
|
|
||||||
if got, ok := copied["number"].(json.Number); !ok || got != "-1.25e+2" {
|
|
||||||
t.Fatalf("JSON number type or value changed: %#v", copied["number"])
|
|
||||||
}
|
|
||||||
if got := copied["nested"].(map[string]int)["limit"]; got != 2 {
|
|
||||||
t.Fatalf("nested map was not isolated: %d", got)
|
|
||||||
}
|
|
||||||
if got := copied["sequence"].([]string)[0]; got != "one" {
|
|
||||||
t.Fatalf("sequence was not isolated: %q", got)
|
|
||||||
}
|
|
||||||
if _, ok := copied["added"]; ok {
|
|
||||||
t.Fatalf("top-level map was not isolated: %#v", copied)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCopyAllowsEmptyObjectKeysAndIsolatesMutations(t *testing.T) {
|
|
||||||
nested := map[string]any{"": []any{"original"}}
|
|
||||||
|
|
||||||
copiedValue, err := jsonvalue.Copy(nested)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("copy value: %v", err)
|
|
||||||
}
|
|
||||||
nested[""].([]any)[0] = "changed"
|
|
||||||
|
|
||||||
copied := copiedValue.(map[string]any)
|
|
||||||
if got := copied[""].([]any)[0]; got != "original" {
|
|
||||||
t.Fatalf("copied value was not isolated: %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCopyMapRejectsInvalidValues(t *testing.T) {
|
|
||||||
cyclicMap := map[string]any{}
|
|
||||||
cyclicMap["self"] = cyclicMap
|
|
||||||
cyclicSlice := []any{nil}
|
|
||||||
cyclicSlice[0] = cyclicSlice
|
|
||||||
|
|
||||||
|
func TestCopyPreservesSupportedScalarAndNumberTypes(t *testing.T) {
|
||||||
|
maxInt := int(^uint(0) >> 1)
|
||||||
|
minInt := -maxInt - 1
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
value any
|
value any
|
||||||
}{
|
}{
|
||||||
{name: "empty nested key", value: map[string]int{"": 1}},
|
{name: "bool", value: true},
|
||||||
{name: "non-string map key", value: map[int]string{1: "one"}},
|
{name: "named bool", value: namedBool(true)},
|
||||||
{name: "unsupported value", value: make(chan int)},
|
{name: "string", value: "value"},
|
||||||
{name: "cyclic map", value: cyclicMap},
|
{name: "named string", value: namedString("value")},
|
||||||
{name: "cyclic slice", value: cyclicSlice},
|
{name: "int", value: minInt},
|
||||||
{name: "NaN", value: math.NaN()},
|
{name: "int8", value: int8(-1 << 7)},
|
||||||
{name: "positive infinity", value: math.Inf(1)},
|
{name: "int16", value: int16(-1 << 15)},
|
||||||
{name: "unsafe signed integer", value: int64(1 << 53)},
|
{name: "int32", value: int32(-1 << 31)},
|
||||||
{name: "unsafe unsigned integer", value: uint64(1 << 53)},
|
{name: "int64", value: int64(-1 << 63)},
|
||||||
|
{name: "named int64", value: namedInt64(1<<63 - 1)},
|
||||||
|
{name: "uint", value: ^uint(0)},
|
||||||
|
{name: "uint8", value: ^uint8(0)},
|
||||||
|
{name: "uint16", value: ^uint16(0)},
|
||||||
|
{name: "uint32", value: ^uint32(0)},
|
||||||
|
{name: "uint64", value: ^uint64(0)},
|
||||||
|
{name: "uintptr", value: ^uintptr(0)},
|
||||||
|
{name: "named uint64", value: namedUint64(^uint64(0))},
|
||||||
|
{name: "float32", value: float32(1.25)},
|
||||||
|
{name: "float64", value: float64(-2.5e100)},
|
||||||
|
{name: "named float32", value: namedFloat32(3.5)},
|
||||||
|
{name: "named float64", value: namedFloat64(-4.5e200)},
|
||||||
|
{name: "JSON number integer", value: json.Number("18446744073709551615")},
|
||||||
|
{name: "JSON number fraction", value: json.Number("-1.25e+2")},
|
||||||
|
{name: "JSON number beyond float64", value: json.Number("1e9999")},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
if _, err := jsonvalue.CopyMap(map[string]any{"value": tc.value}); err == nil {
|
got, err := Copy(tc.value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("copy value: %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, tc.value) {
|
||||||
|
t.Fatalf("value or concrete type changed: got %#v (%T), want %#v (%T)", got, got, tc.value, tc.value)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyRejectsInvalidNumbers(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
value any
|
||||||
|
}{
|
||||||
|
{name: "float32 NaN", value: float32(math.NaN())},
|
||||||
|
{name: "float64 NaN", value: math.NaN()},
|
||||||
|
{name: "named float NaN", value: namedFloat64(math.NaN())},
|
||||||
|
{name: "positive infinity", value: math.Inf(1)},
|
||||||
|
{name: "negative infinity", value: math.Inf(-1)},
|
||||||
|
{name: "empty JSON number", value: json.Number("")},
|
||||||
|
{name: "leading zero JSON number", value: json.Number("01")},
|
||||||
|
{name: "leading plus JSON number", value: json.Number("+1")},
|
||||||
|
{name: "trailing decimal JSON number", value: json.Number("1.")},
|
||||||
|
{name: "leading decimal JSON number", value: json.Number(".1")},
|
||||||
|
{name: "non-number JSON number", value: json.Number("NaN")},
|
||||||
|
{name: "spaced JSON number", value: json.Number(" 1")},
|
||||||
|
{name: "quoted JSON number", value: json.Number(`"1"`)},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if _, err := Copy(tc.value); err == nil {
|
||||||
t.Fatal("expected validation error")
|
t.Fatal("expected validation error")
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCopyMapValidatesJSONNumberSyntaxAndRange(t *testing.T) {
|
func TestCopyPreservesCompatibleCollectionsAndNilEmptyDistinctions(t *testing.T) {
|
||||||
for _, number := range []json.Number{"0", "-1", "1.25", "-1.25e+2"} {
|
collections := []struct {
|
||||||
t.Run("valid "+number.String(), func(t *testing.T) {
|
name string
|
||||||
got, err := jsonvalue.CopyMap(map[string]any{"value": number})
|
value any
|
||||||
|
}{
|
||||||
|
{name: "unnamed map", value: map[string]int{"limit": 2}},
|
||||||
|
{name: "named map", value: namedMap{"limit": 2}},
|
||||||
|
{name: "unnamed slice", value: []string{"one", "two"}},
|
||||||
|
{name: "named slice", value: namedSlice{"one", "two"}},
|
||||||
|
{name: "unnamed array", value: [2]int{1, 2}},
|
||||||
|
{name: "named array", value: namedArray{{"limit": 2}}},
|
||||||
|
{name: "empty map", value: map[string]int{}},
|
||||||
|
{name: "empty named map", value: namedMap{}},
|
||||||
|
{name: "empty slice", value: []string{}},
|
||||||
|
{name: "empty named slice", value: namedSlice{}},
|
||||||
|
{name: "empty array", value: [0]string{}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range collections {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got, err := Copy(tc.value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("copy valid JSON number: %v", err)
|
t.Fatalf("copy collection: %v", err)
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(got["value"], number) {
|
if !reflect.DeepEqual(got, tc.value) || reflect.TypeOf(got) != reflect.TypeOf(tc.value) {
|
||||||
t.Fatalf("JSON number changed: got %#v want %#v", got["value"], number)
|
t.Fatalf("collection changed: got %#v (%T), want %#v (%T)", got, got, tc.value, tc.value)
|
||||||
|
}
|
||||||
|
kind := reflect.ValueOf(got).Kind()
|
||||||
|
if (kind == reflect.Map || kind == reflect.Slice) && reflect.ValueOf(got).IsNil() {
|
||||||
|
t.Fatal("non-nil collection became nil")
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, number := range []json.Number{"", "01", "+1", "1.", ".1", "1e9999", "not-a-number"} {
|
var nilMap map[string]int
|
||||||
t.Run("invalid "+number.String(), func(t *testing.T) {
|
var nilSlice []string
|
||||||
if _, err := jsonvalue.CopyMap(map[string]any{"value": number}); err == nil {
|
var nilPointer *namedInt64
|
||||||
t.Fatal("expected invalid JSON number error")
|
for _, value := range []any{nil, nilMap, nilSlice, nilPointer} {
|
||||||
|
got, err := Copy(value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("copy null value: %v", err)
|
||||||
|
}
|
||||||
|
if got != nil {
|
||||||
|
t.Fatalf("null value became %#v (%T)", got, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
gotNil, err := CopyMap(nil)
|
||||||
|
if err != nil || gotNil != nil {
|
||||||
|
t.Fatalf("nil CopyMap result = %#v, %v", gotNil, err)
|
||||||
|
}
|
||||||
|
gotEmpty, err := CopyMap(map[string]any{})
|
||||||
|
if err != nil || gotEmpty == nil || len(gotEmpty) != 0 {
|
||||||
|
t.Fatalf("empty CopyMap result = %#v, %v", gotEmpty, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyHandlesIndirectionAndIsolatesNestedMutations(t *testing.T) {
|
||||||
|
integer := namedInt64(7)
|
||||||
|
nestedMap := namedMap{"limit": 2}
|
||||||
|
nestedSlice := namedSlice{"original"}
|
||||||
|
nestedArray := namedArray{{"limit": 3}}
|
||||||
|
shared := []any{map[string]int{"value": 4}}
|
||||||
|
input := map[string]any{
|
||||||
|
"integer": &integer,
|
||||||
|
"map": nestedMap,
|
||||||
|
"slice": nestedSlice,
|
||||||
|
"array": nestedArray,
|
||||||
|
"first": shared,
|
||||||
|
"second": shared,
|
||||||
|
}
|
||||||
|
|
||||||
|
copiedValue, err := Copy(input)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("copy mixed tree: %v", err)
|
||||||
|
}
|
||||||
|
copied := copiedValue.(map[string]any)
|
||||||
|
nestedMap["limit"] = 20
|
||||||
|
nestedSlice[0] = "changed"
|
||||||
|
nestedArray[0]["limit"] = 30
|
||||||
|
shared[0].(map[string]int)["value"] = 40
|
||||||
|
|
||||||
|
if got, ok := copied["integer"].(namedInt64); !ok || got != 7 {
|
||||||
|
t.Fatalf("pointer target changed: %#v", copied["integer"])
|
||||||
|
}
|
||||||
|
if got := copied["map"].(namedMap)["limit"]; got != 2 {
|
||||||
|
t.Fatalf("nested map aliased input: %d", got)
|
||||||
|
}
|
||||||
|
if got := copied["slice"].(namedSlice)[0]; got != "original" {
|
||||||
|
t.Fatalf("nested slice aliased input: %q", got)
|
||||||
|
}
|
||||||
|
if got := copied["array"].(namedArray)[0]["limit"]; got != 3 {
|
||||||
|
t.Fatalf("nested array aliased input: %d", got)
|
||||||
|
}
|
||||||
|
first := copied["first"].([]any)
|
||||||
|
second := copied["second"].([]any)
|
||||||
|
if got := first[0].(map[string]int)["value"]; got != 4 {
|
||||||
|
t.Fatalf("shared child aliased input: %d", got)
|
||||||
|
}
|
||||||
|
first[0].(map[string]int)["value"] = 99
|
||||||
|
if got := second[0].(map[string]int)["value"]; got != 4 {
|
||||||
|
t.Fatalf("repeated acyclic value shared copied output: %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyAndCopyMapApplyDistinctEmptyKeyRules(t *testing.T) {
|
||||||
|
nested := map[string]any{"": []any{"original"}}
|
||||||
|
copiedValue, err := Copy(nested)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Copy rejected empty schema key: %v", err)
|
||||||
|
}
|
||||||
|
nested[""].([]any)[0] = "changed"
|
||||||
|
if got := copiedValue.(map[string]any)[""].([]any)[0]; got != "original" {
|
||||||
|
t.Fatalf("copied schema value was not isolated: %v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = CopyMap(map[string]any{"nested": map[string]any{"": true}})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "extra_params.nested") {
|
||||||
|
t.Fatalf("CopyMap empty-key error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyRejectsUnsupportedValuesAndActiveCycles(t *testing.T) {
|
||||||
|
cyclicMap := map[string]any{}
|
||||||
|
cyclicMap["self"] = cyclicMap
|
||||||
|
cyclicSlice := []any{nil}
|
||||||
|
cyclicSlice[0] = cyclicSlice
|
||||||
|
var cyclicPointer any
|
||||||
|
cyclicPointer = &cyclicPointer
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
value any
|
||||||
|
wantPath string
|
||||||
|
}{
|
||||||
|
{name: "non-string map key", value: map[int]string{1: "one"}, wantPath: "value"},
|
||||||
|
{name: "unsupported channel", value: make(chan int), wantPath: "value"},
|
||||||
|
{name: "deterministic map path", value: map[string]any{"z": make(chan int), "a": make(chan int)}, wantPath: "value.a"},
|
||||||
|
{name: "cyclic map", value: cyclicMap, wantPath: "value.self"},
|
||||||
|
{name: "cyclic slice", value: cyclicSlice, wantPath: "value[0]"},
|
||||||
|
{name: "cyclic pointer", value: cyclicPointer, wantPath: "value"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := Copy(tc.value)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tc.wantPath) {
|
||||||
|
t.Fatalf("error = %v, want structural path %q", err, tc.wantPath)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCopyEnforcesContainerDepth(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
depth int
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "just below", depth: maxContainerDepth - 1},
|
||||||
|
{name: "at limit", depth: maxContainerDepth},
|
||||||
|
{name: "over limit", depth: maxContainerDepth + 1, wantErr: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := Copy(alternatingContainers(tc.depth))
|
||||||
|
if tc.wantErr {
|
||||||
|
if err == nil || !strings.HasPrefix(err.Error(), "value") || !strings.Contains(err.Error(), "container depth limit") {
|
||||||
|
t.Fatalf("depth error = %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("copy depth %d: %v", tc.depth, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyEnforcesProducedNodeBudgetForRepeatedAcyclicValues(t *testing.T) {
|
||||||
|
shared := []any{true}
|
||||||
|
sharedOccurrences := (maxProducedNodes - 2) / 2
|
||||||
|
justBelow := repeatedValues(shared, sharedOccurrences, 0)
|
||||||
|
atLimit := repeatedValues(shared, sharedOccurrences, 1)
|
||||||
|
overLimit := repeatedValues(shared, sharedOccurrences, 2)
|
||||||
|
|
||||||
|
for name, value := range map[string]any{
|
||||||
|
"just below": justBelow,
|
||||||
|
"at limit": atLimit,
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
if _, err := Copy(value); err != nil {
|
||||||
|
t.Fatalf("copy value within work budget: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := Copy(overLimit)
|
||||||
|
if err == nil || !strings.HasPrefix(err.Error(), "value[") || !strings.Contains(err.Error(), "value work limit") {
|
||||||
|
t.Fatalf("work-budget error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = Copy(make([]any, maxProducedNodes))
|
||||||
|
if err == nil || !strings.HasPrefix(err.Error(), "value:") || !strings.Contains(err.Error(), "value work limit") {
|
||||||
|
t.Fatalf("flat work-budget error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func alternatingContainers(depth int) any {
|
||||||
|
var value any = true
|
||||||
|
for level := 0; level < depth; level++ {
|
||||||
|
switch level % 3 {
|
||||||
|
case 0:
|
||||||
|
value = map[string]any{"child": value}
|
||||||
|
case 1:
|
||||||
|
value = []any{value}
|
||||||
|
default:
|
||||||
|
value = [1]any{value}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
func repeatedValues(shared []any, occurrences, leadingScalars int) []any {
|
||||||
|
values := make([]any, 0, leadingScalars+occurrences)
|
||||||
|
for i := 0; i < leadingScalars; i++ {
|
||||||
|
values = append(values, false)
|
||||||
|
}
|
||||||
|
for i := 0; i < occurrences; i++ {
|
||||||
|
values = append(values, shared)
|
||||||
|
}
|
||||||
|
return values
|
||||||
|
}
|
||||||
|
|||||||
@@ -25,6 +25,20 @@ var (
|
|||||||
ErrMalformedResponse = errors.New("malformed llm response")
|
ErrMalformedResponse = errors.New("malformed llm response")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const maxOpenAIChatResponseBytes int64 = 16 << 20
|
||||||
|
|
||||||
|
type requestFailedError struct {
|
||||||
|
cause error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *requestFailedError) Error() string {
|
||||||
|
return ErrRequestFailed.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *requestFailedError) Unwrap() []error {
|
||||||
|
return []error{ErrRequestFailed, e.cause}
|
||||||
|
}
|
||||||
|
|
||||||
type OpenAICompatibleConfig struct {
|
type OpenAICompatibleConfig struct {
|
||||||
BaseURL string
|
BaseURL string
|
||||||
Model string
|
Model string
|
||||||
@@ -39,9 +53,11 @@ type OpenAICompatibleClient struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) {
|
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) {
|
||||||
baseURL := strings.TrimSpace(cfg.BaseURL)
|
baseURL := ""
|
||||||
if baseURL != "" {
|
if strings.TrimSpace(cfg.BaseURL) != "" {
|
||||||
if _, err := url.ParseRequestURI(baseURL); err != nil {
|
var err error
|
||||||
|
baseURL, err = domain.NormalizeOpenAICompatibleBaseEndpoint(cfg.BaseURL)
|
||||||
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err)
|
return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -63,25 +79,29 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &OpenAICompatibleClient{
|
return &OpenAICompatibleClient{
|
||||||
baseURL: strings.TrimRight(baseURL, "/"),
|
baseURL: baseURL,
|
||||||
defaultModel: cfg.Model,
|
defaultModel: cfg.Model,
|
||||||
httpClient: client,
|
httpClient: client,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
if req.Target.TimeoutSeconds < 0 {
|
if err := domain.ValidateExecutionTargetSettings(req.Target); err != nil {
|
||||||
return nil, fmt.Errorf("%w: timeout_seconds must be greater than or equal to 0", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
endpoint := strings.TrimSpace(req.Target.Endpoint)
|
selectedEndpoint := req.Target.Endpoint
|
||||||
if endpoint == "" {
|
if strings.TrimSpace(selectedEndpoint) == "" {
|
||||||
endpoint = c.baseURL
|
selectedEndpoint = c.baseURL
|
||||||
}
|
}
|
||||||
if endpoint == "" {
|
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(selectedEndpoint)
|
||||||
return nil, fmt.Errorf("%w: endpoint is required", ErrInvalidRequest)
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: invalid endpoint: %v", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
endpoint, err = url.JoinPath(endpoint, defaults.OpenAIChatCompletionsPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: invalid endpoint path: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
endpoint = strings.TrimRight(endpoint, "/") + defaults.OpenAIChatCompletionsPath
|
|
||||||
|
|
||||||
wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel)
|
wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -113,13 +133,18 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
|
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
|
||||||
}
|
}
|
||||||
httpReq.Header.Set("Content-Type", "application/json")
|
httpReq.Header.Set("Content-Type", "application/json")
|
||||||
if apiKey := strings.TrimSpace(req.Target.APIKey); apiKey != "" {
|
apiKey := strings.TrimSpace(req.Target.APIKey)
|
||||||
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
envName := strings.TrimSpace(req.Target.APIKeyEnv)
|
||||||
} else if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" {
|
if apiKey == "" && envName != "" {
|
||||||
apiKey := strings.TrimSpace(os.Getenv(envName))
|
apiKey = strings.TrimSpace(os.Getenv(envName))
|
||||||
if apiKey == "" {
|
}
|
||||||
|
if apiKey == "" && req.Target.APIKeyRequired {
|
||||||
|
if envName != "" {
|
||||||
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
|
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
|
||||||
}
|
}
|
||||||
|
return nil, fmt.Errorf("%w: api key is required", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
if apiKey != "" {
|
||||||
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -130,30 +155,36 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
|
|
||||||
httpResp, err := httpClient.Do(httpReq)
|
httpResp, err := httpClient.Do(httpReq)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %v", ErrRequestFailed, err)
|
return nil, &requestFailedError{cause: err}
|
||||||
}
|
}
|
||||||
defer httpResp.Body.Close()
|
defer httpResp.Body.Close()
|
||||||
|
|
||||||
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
|
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
|
||||||
_, _ = io.Copy(io.Discard, io.LimitReader(httpResp.Body, 4096))
|
return nil, providerHTTPErrorFromBody(
|
||||||
return nil, fmt.Errorf("%w: status=%d", ErrUnexpectedStatus, httpResp.StatusCode)
|
httpResp.StatusCode,
|
||||||
|
httpResp.ContentLength,
|
||||||
|
httpResp.Body,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if httpResp.ContentLength > maxOpenAIChatResponseBytes {
|
||||||
|
return nil, openAIChatResponseTooLargeError()
|
||||||
}
|
}
|
||||||
|
|
||||||
var wireResp openAIChatResponse
|
wireResp, err := decodeOpenAIChatResponse(httpResp.Body)
|
||||||
if err := json.NewDecoder(httpResp.Body).Decode(&wireResp); err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: failed to decode response: %v", ErrMalformedResponse, err)
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(wireResp.Choices) == 0 {
|
if len(wireResp.Choices) == 0 {
|
||||||
return nil, fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
|
return nil, fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
|
||||||
}
|
}
|
||||||
content := wireResp.Choices[0].Message.Content
|
content := wireResp.Choices[0].Message.Content
|
||||||
if content == "" {
|
if content == nil {
|
||||||
return nil, fmt.Errorf("%w: first choice has empty message content", ErrMalformedResponse)
|
return nil, fmt.Errorf("%w: first choice has missing message content", ErrMalformedResponse)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &domain.GenerateResponse{
|
return &domain.GenerateResponse{
|
||||||
Content: content,
|
Content: *content,
|
||||||
Usage: domain.TokenUsage{
|
Usage: domain.TokenUsage{
|
||||||
PromptTokens: wireResp.Usage.PromptTokens,
|
PromptTokens: wireResp.Usage.PromptTokens,
|
||||||
CompletionTokens: wireResp.Usage.CompletionTokens,
|
CompletionTokens: wireResp.Usage.CompletionTokens,
|
||||||
@@ -164,6 +195,46 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func decodeOpenAIChatResponse(body io.Reader) (openAIChatResponse, error) {
|
||||||
|
limited := &io.LimitedReader{
|
||||||
|
R: body,
|
||||||
|
N: maxOpenAIChatResponseBytes + 1,
|
||||||
|
}
|
||||||
|
decoder := json.NewDecoder(limited)
|
||||||
|
|
||||||
|
var response openAIChatResponse
|
||||||
|
if err := decoder.Decode(&response); err != nil {
|
||||||
|
if limited.N == 0 {
|
||||||
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
|
}
|
||||||
|
return openAIChatResponse{}, fmt.Errorf("%w: failed to decode response", ErrMalformedResponse)
|
||||||
|
}
|
||||||
|
if limited.N == 0 {
|
||||||
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
|
}
|
||||||
|
|
||||||
|
var trailing any
|
||||||
|
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||||
|
if limited.N == 0 {
|
||||||
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
|
}
|
||||||
|
return openAIChatResponse{}, fmt.Errorf("%w: response contains trailing data", ErrMalformedResponse)
|
||||||
|
}
|
||||||
|
if limited.N == 0 {
|
||||||
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
|
}
|
||||||
|
|
||||||
|
return response, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func openAIChatResponseTooLargeError() error {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%w: response exceeds %d-byte limit",
|
||||||
|
ErrMalformedResponse,
|
||||||
|
maxOpenAIChatResponseBytes,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) {
|
func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) {
|
||||||
model := strings.TrimSpace(req.Target.Model)
|
model := strings.TrimSpace(req.Target.Model)
|
||||||
if model == "" {
|
if model == "" {
|
||||||
@@ -308,8 +379,8 @@ type openAICacheControl struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type openAIChatResponseMessage struct {
|
type openAIChatResponseMessage struct {
|
||||||
Role string `json:"role"`
|
Role string `json:"role"`
|
||||||
Content string `json:"content"`
|
Content *string `json:"content"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type openAIChatResponse struct {
|
type openAIChatResponse struct {
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
189
internal/llm/provider_http_error.go
Normal file
189
internal/llm/provider_http_error.go
Normal file
@@ -0,0 +1,189 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
maxProviderErrorResponseBytes int64 = 64 << 10
|
||||||
|
maxProviderErrorIdentifierRunes = 256
|
||||||
|
maxProviderErrorMessageRunes = 4096
|
||||||
|
)
|
||||||
|
|
||||||
|
// ProviderHTTPError describes a non-success response from an LLM provider.
|
||||||
|
type ProviderHTTPError struct {
|
||||||
|
statusCode int
|
||||||
|
providerCode string
|
||||||
|
providerType string
|
||||||
|
providerMessage string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *ProviderHTTPError) StatusCode() int {
|
||||||
|
if e == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return e.statusCode
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *ProviderHTTPError) ProviderCode() string {
|
||||||
|
if e == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return e.providerCode
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *ProviderHTTPError) ProviderType() string {
|
||||||
|
if e == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return e.providerType
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *ProviderHTTPError) ProviderMessage() string {
|
||||||
|
if e == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return e.providerMessage
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *ProviderHTTPError) Error() string {
|
||||||
|
if e == nil || e.statusCode == 0 {
|
||||||
|
return ErrUnexpectedStatus.Error()
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s: status=%d", ErrUnexpectedStatus, e.statusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *ProviderHTTPError) GoString() string {
|
||||||
|
return e.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *ProviderHTTPError) Unwrap() error {
|
||||||
|
return ErrUnexpectedStatus
|
||||||
|
}
|
||||||
|
|
||||||
|
type providerErrorDetails struct {
|
||||||
|
providerCode string
|
||||||
|
providerType string
|
||||||
|
providerMessage string
|
||||||
|
}
|
||||||
|
|
||||||
|
func newProviderHTTPError(statusCode int, details providerErrorDetails) *ProviderHTTPError {
|
||||||
|
return &ProviderHTTPError{
|
||||||
|
statusCode: statusCode,
|
||||||
|
providerCode: details.providerCode,
|
||||||
|
providerType: details.providerType,
|
||||||
|
providerMessage: details.providerMessage,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func providerHTTPErrorFromBody(statusCode int, contentLength int64, body io.Reader) *ProviderHTTPError {
|
||||||
|
if contentLength > maxProviderErrorResponseBytes {
|
||||||
|
return newProviderHTTPError(statusCode, providerErrorDetails{})
|
||||||
|
}
|
||||||
|
|
||||||
|
limited := &io.LimitedReader{
|
||||||
|
R: body,
|
||||||
|
N: maxProviderErrorResponseBytes + 1,
|
||||||
|
}
|
||||||
|
contents, err := io.ReadAll(limited)
|
||||||
|
if err != nil || limited.N == 0 {
|
||||||
|
return newProviderHTTPError(statusCode, providerErrorDetails{})
|
||||||
|
}
|
||||||
|
return newProviderHTTPError(statusCode, parseProviderErrorEnvelope(contents))
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseProviderErrorEnvelope(body []byte) providerErrorDetails {
|
||||||
|
decoder := json.NewDecoder(strings.NewReader(string(body)))
|
||||||
|
decoder.UseNumber()
|
||||||
|
|
||||||
|
var envelope map[string]json.RawMessage
|
||||||
|
if err := decoder.Decode(&envelope); err != nil {
|
||||||
|
return providerErrorDetails{}
|
||||||
|
}
|
||||||
|
var trailing any
|
||||||
|
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||||
|
return providerErrorDetails{}
|
||||||
|
}
|
||||||
|
|
||||||
|
rawError, ok := envelope["error"]
|
||||||
|
if !ok {
|
||||||
|
return providerErrorDetails{}
|
||||||
|
}
|
||||||
|
var providerError map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(rawError, &providerError); err != nil || providerError == nil {
|
||||||
|
return providerErrorDetails{}
|
||||||
|
}
|
||||||
|
|
||||||
|
var details providerErrorDetails
|
||||||
|
if raw, ok := providerError["message"]; ok {
|
||||||
|
var value string
|
||||||
|
if json.Unmarshal(raw, &value) == nil {
|
||||||
|
details.providerMessage = normalizeProviderErrorMessage(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if raw, ok := providerError["type"]; ok {
|
||||||
|
var value string
|
||||||
|
if json.Unmarshal(raw, &value) == nil {
|
||||||
|
details.providerType = normalizeProviderErrorIdentifier(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if raw, ok := providerError["code"]; ok {
|
||||||
|
var value any
|
||||||
|
fieldDecoder := json.NewDecoder(strings.NewReader(string(raw)))
|
||||||
|
fieldDecoder.UseNumber()
|
||||||
|
if fieldDecoder.Decode(&value) == nil {
|
||||||
|
switch value := value.(type) {
|
||||||
|
case string:
|
||||||
|
details.providerCode = normalizeProviderErrorIdentifier(value)
|
||||||
|
case json.Number:
|
||||||
|
details.providerCode = normalizeProviderErrorIdentifier(value.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return details
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeProviderErrorIdentifier(value string) string {
|
||||||
|
normalized := normalizeProviderErrorText(value)
|
||||||
|
if len([]rune(normalized)) > maxProviderErrorIdentifierRunes {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return normalized
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeProviderErrorMessage(value string) string {
|
||||||
|
normalized := normalizeProviderErrorText(value)
|
||||||
|
runes := []rune(normalized)
|
||||||
|
if len(runes) <= maxProviderErrorMessageRunes {
|
||||||
|
return normalized
|
||||||
|
}
|
||||||
|
return string(runes[:maxProviderErrorMessageRunes-1]) + "…"
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeProviderErrorText(value string) string {
|
||||||
|
value = strings.ToValidUTF8(value, "<22>")
|
||||||
|
|
||||||
|
var result strings.Builder
|
||||||
|
result.Grow(len(value))
|
||||||
|
separatorPending := false
|
||||||
|
for _, r := range value {
|
||||||
|
if unicode.IsSpace(r) || unicode.IsControl(r) || unicode.In(r, unicode.Cf) {
|
||||||
|
if result.Len() > 0 {
|
||||||
|
separatorPending = true
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if separatorPending {
|
||||||
|
result.WriteByte(' ')
|
||||||
|
separatorPending = false
|
||||||
|
}
|
||||||
|
result.WriteRune(r)
|
||||||
|
}
|
||||||
|
return result.String()
|
||||||
|
}
|
||||||
255
internal/llm/provider_http_error_test.go
Normal file
255
internal/llm/provider_http_error_test.go
Normal file
@@ -0,0 +1,255 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
type guardedReader struct {
|
||||||
|
reader io.Reader
|
||||||
|
remaining int64
|
||||||
|
bytes int64
|
||||||
|
violated bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *guardedReader) Read(buffer []byte) (int, error) {
|
||||||
|
if int64(len(buffer)) > r.remaining {
|
||||||
|
r.violated = true
|
||||||
|
return 0, errors.New("reader was read past its allowed boundary")
|
||||||
|
}
|
||||||
|
n, err := r.reader.Read(buffer)
|
||||||
|
r.bytes += int64(n)
|
||||||
|
r.remaining -= int64(n)
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
type failingReader struct {
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r failingReader) Read([]byte) (int, error) {
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProviderHTTPErrorEnvelopeParsing(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
body string
|
||||||
|
want providerErrorDetails
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "all supported string fields",
|
||||||
|
body: `{"error":{"message":"diagnostic","type":"invalid_request_error","code":"unsupported_parameter"}}`,
|
||||||
|
want: providerErrorDetails{providerMessage: "diagnostic", providerType: "invalid_request_error", providerCode: "unsupported_parameter"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "integer code",
|
||||||
|
body: `{"error":{"code":17}}`,
|
||||||
|
want: providerErrorDetails{providerCode: "17"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fractional code",
|
||||||
|
body: `{"error":{"code":1.25}}`,
|
||||||
|
want: providerErrorDetails{providerCode: "1.25"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "exponent code",
|
||||||
|
body: `{"error":{"code":6.02e+23}}`,
|
||||||
|
want: providerErrorDetails{providerCode: "6.02e+23"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid fields do not discard valid fields",
|
||||||
|
body: `{"error":{"message":null,"type":"invalid_request_error","code":false}}`,
|
||||||
|
want: providerErrorDetails{providerType: "invalid_request_error"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unknown fields are ignored",
|
||||||
|
body: `{"trace":"do not retain","error":{"param":"temperature","metadata":{"secret":"x"}}}`,
|
||||||
|
want: providerErrorDetails{},
|
||||||
|
},
|
||||||
|
{name: "missing error", body: `{}`, want: providerErrorDetails{}},
|
||||||
|
{name: "null error", body: `{"error":null}`, want: providerErrorDetails{}},
|
||||||
|
{name: "scalar error", body: `{"error":"nope"}`, want: providerErrorDetails{}},
|
||||||
|
{name: "empty error", body: `{"error":{}}`, want: providerErrorDetails{}},
|
||||||
|
{name: "malformed", body: `{"error":`, want: providerErrorDetails{}},
|
||||||
|
{name: "truncated", body: `{"error":{"message":"x"`, want: providerErrorDetails{}},
|
||||||
|
{name: "trailing garbage", body: `{"error":{"message":"x"}} garbage`, want: providerErrorDetails{}},
|
||||||
|
{name: "second document", body: `{"error":{"message":"x"}} {}`, want: providerErrorDetails{}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if got := parseProviderErrorEnvelope([]byte(tc.body)); !reflect.DeepEqual(got, tc.want) {
|
||||||
|
t.Fatalf("parseProviderErrorEnvelope() = %#v, want %#v", got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProviderErrorTextNormalizationAndLimits(t *testing.T) {
|
||||||
|
validIdentifier := strings.Repeat("界", maxProviderErrorIdentifierRunes)
|
||||||
|
validMessage := strings.Repeat("界", maxProviderErrorMessageRunes)
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
got string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "multibyte text", got: "Grüße 世界", want: "Grüße 世界"},
|
||||||
|
{name: "invalid UTF-8", got: string([]byte{'a', 0xff, 'b'}), want: "a<>b"},
|
||||||
|
{name: "whitespace control and format runs", got: " \n\talpha\x00\u200b\u200bbeta \r ", want: "alpha beta"},
|
||||||
|
{name: "blank normalization", got: "\t\u200b\n", want: ""},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if got := normalizeProviderErrorText(tc.got); got != tc.want {
|
||||||
|
t.Fatalf("normalizeProviderErrorText() = %q, want %q", got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := normalizeProviderErrorIdentifier(validIdentifier); got != validIdentifier {
|
||||||
|
t.Fatalf("exact identifier boundary = %q, want retained value", got)
|
||||||
|
}
|
||||||
|
if got := normalizeProviderErrorIdentifier(validIdentifier + "界"); got != "" {
|
||||||
|
t.Fatalf("overlong identifier = %q, want empty", got)
|
||||||
|
}
|
||||||
|
if got := normalizeProviderErrorMessage(validMessage); got != validMessage {
|
||||||
|
t.Fatalf("exact message boundary = %q, want retained value", got)
|
||||||
|
}
|
||||||
|
wantTruncatedMessage := strings.Repeat("界", maxProviderErrorMessageRunes-1) + "…"
|
||||||
|
if got := normalizeProviderErrorMessage(validMessage + "界"); got != wantTruncatedMessage {
|
||||||
|
t.Fatalf("overlong message length = %d, want %d", utf8.RuneCountInString(got), maxProviderErrorMessageRunes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProviderHTTPErrorIdentityAndFormatting(t *testing.T) {
|
||||||
|
const marker = "provider-secret-marker"
|
||||||
|
err := newProviderHTTPError(429, providerErrorDetails{
|
||||||
|
providerCode: marker + "-code",
|
||||||
|
providerType: marker + "-type",
|
||||||
|
providerMessage: marker + "-message",
|
||||||
|
})
|
||||||
|
|
||||||
|
if err.StatusCode() != 429 || err.ProviderCode() != marker+"-code" || err.ProviderType() != marker+"-type" || err.ProviderMessage() != marker+"-message" {
|
||||||
|
t.Fatalf("accessors returned unexpected values: %#v", err)
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrUnexpectedStatus) {
|
||||||
|
t.Fatalf("errors.Is(%v, ErrUnexpectedStatus) = false", err)
|
||||||
|
}
|
||||||
|
for _, rendered := range []string{fmt.Sprintf("%v", err), fmt.Sprintf("%+v", err), fmt.Sprintf("%#v", err)} {
|
||||||
|
if rendered != "llm returned non-success status: status=429" {
|
||||||
|
t.Fatalf("formatted error = %q", rendered)
|
||||||
|
}
|
||||||
|
if strings.Contains(rendered, marker) {
|
||||||
|
t.Fatalf("formatted error exposed provider marker: %q", rendered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var nilError *ProviderHTTPError
|
||||||
|
if nilError.StatusCode() != 0 || nilError.ProviderCode() != "" || nilError.ProviderType() != "" || nilError.ProviderMessage() != "" {
|
||||||
|
t.Fatal("nil accessors returned provider values")
|
||||||
|
}
|
||||||
|
if nilError.Error() != "llm returned non-success status" || nilError.GoString() != "llm returned non-success status" || !errors.Is(nilError, ErrUnexpectedStatus) {
|
||||||
|
t.Fatalf("nil error behavior is not safe: %v", nilError)
|
||||||
|
}
|
||||||
|
|
||||||
|
zero := &ProviderHTTPError{}
|
||||||
|
if zero.Error() != "llm returned non-success status" || zero.GoString() != "llm returned non-success status" || !errors.Is(zero, ErrUnexpectedStatus) {
|
||||||
|
t.Fatalf("zero error behavior is not safe: %v", zero)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProviderHTTPErrorBodyBounds(t *testing.T) {
|
||||||
|
const (
|
||||||
|
statusCode = 502
|
||||||
|
marker = "provider-body-marker"
|
||||||
|
)
|
||||||
|
|
||||||
|
ordinaryBody := `{"error":{"message":"` + marker + `"}}`
|
||||||
|
exactLimitBody := ordinaryBody + strings.Repeat(" ", int(maxProviderErrorResponseBytes)-len(ordinaryBody))
|
||||||
|
overLimitBody := ordinaryBody + strings.Repeat(" ", int(maxProviderErrorResponseBytes)+1-len(ordinaryBody))
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
contentLength int64
|
||||||
|
reader io.Reader
|
||||||
|
wantRead int64
|
||||||
|
wantMessage string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "recognized envelope",
|
||||||
|
contentLength: int64(len(ordinaryBody)),
|
||||||
|
reader: strings.NewReader(ordinaryBody),
|
||||||
|
wantRead: int64(len(ordinaryBody)),
|
||||||
|
wantMessage: marker,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "exact limit",
|
||||||
|
contentLength: maxProviderErrorResponseBytes,
|
||||||
|
reader: strings.NewReader(exactLimitBody),
|
||||||
|
wantRead: maxProviderErrorResponseBytes,
|
||||||
|
wantMessage: marker,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "declared oversize does not read",
|
||||||
|
contentLength: maxProviderErrorResponseBytes + 1,
|
||||||
|
reader: strings.NewReader(ordinaryBody),
|
||||||
|
wantRead: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unknown length oversize",
|
||||||
|
contentLength: -1,
|
||||||
|
reader: strings.NewReader(overLimitBody),
|
||||||
|
wantRead: maxProviderErrorResponseBytes + 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "underreported oversize",
|
||||||
|
contentLength: maxProviderErrorResponseBytes,
|
||||||
|
reader: strings.NewReader(overLimitBody),
|
||||||
|
wantRead: maxProviderErrorResponseBytes + 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "read failure",
|
||||||
|
contentLength: -1,
|
||||||
|
reader: failingReader{err: errors.New("read failure")},
|
||||||
|
wantRead: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty body",
|
||||||
|
contentLength: 0,
|
||||||
|
reader: strings.NewReader(""),
|
||||||
|
wantRead: 0,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
reader := &guardedReader{
|
||||||
|
reader: tc.reader,
|
||||||
|
remaining: maxProviderErrorResponseBytes + 1,
|
||||||
|
}
|
||||||
|
err := providerHTTPErrorFromBody(statusCode, tc.contentLength, reader)
|
||||||
|
if err == nil || err.StatusCode() != statusCode {
|
||||||
|
t.Fatalf("error status = %v, want %d", err, statusCode)
|
||||||
|
}
|
||||||
|
if reader.bytes != tc.wantRead {
|
||||||
|
t.Fatalf("body bytes read = %d, want %d", reader.bytes, tc.wantRead)
|
||||||
|
}
|
||||||
|
if reader.violated {
|
||||||
|
t.Fatal("body reader was asked to read beyond the overflow probe")
|
||||||
|
}
|
||||||
|
if got := err.ProviderMessage(); got != tc.wantMessage {
|
||||||
|
t.Fatalf("provider message = %q, want %q", got, tc.wantMessage)
|
||||||
|
}
|
||||||
|
if tc.wantMessage == "" {
|
||||||
|
if err.ProviderCode() != "" || err.ProviderType() != "" || strings.Contains(err.Error(), marker) {
|
||||||
|
t.Fatalf("discarded details were retained: %#v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
145
internal/llm/provider_http_error_transport_test.go
Normal file
145
internal/llm/provider_http_error_transport_test.go
Normal file
@@ -0,0 +1,145 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientStructuredNonSuccessResponse(t *testing.T) {
|
||||||
|
body := `{"error":{"message":" provider\nmessage\u200b","type":"invalid\ttype","code":1.5e+4}}`
|
||||||
|
responseBody := &countingReadCloser{reader: strings.NewReader(body)}
|
||||||
|
client := newNonSuccessResponseClient(t, http.StatusBadRequest, int64(len(body)), responseBody)
|
||||||
|
|
||||||
|
response, err := client.Generate(context.Background(), ordinaryGenerateRequest())
|
||||||
|
if response != nil {
|
||||||
|
t.Fatalf("response = %#v, want nil", response)
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrUnexpectedStatus) {
|
||||||
|
t.Fatalf("errors.Is(%v, ErrUnexpectedStatus) = false", err)
|
||||||
|
}
|
||||||
|
var providerHTTPError *ProviderHTTPError
|
||||||
|
if !errors.As(err, &providerHTTPError) {
|
||||||
|
t.Fatalf("error = %T, want *ProviderHTTPError", err)
|
||||||
|
}
|
||||||
|
if providerHTTPError.StatusCode() != http.StatusBadRequest || providerHTTPError.ProviderCode() != "1.5e+4" || providerHTTPError.ProviderType() != "invalid type" || providerHTTPError.ProviderMessage() != "provider message" {
|
||||||
|
t.Fatalf("provider error = %#v", providerHTTPError)
|
||||||
|
}
|
||||||
|
if !responseBody.closed {
|
||||||
|
t.Fatal("non-success response body was not closed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientNonSuccessBodyOwnership(t *testing.T) {
|
||||||
|
const marker = "provider-body-marker"
|
||||||
|
normalBody := `{"error":{"message":"` + marker + `"}}`
|
||||||
|
overLimitBody := normalBody + strings.Repeat(" ", int(maxProviderErrorResponseBytes)+1-len(normalBody))
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
contentLength int64
|
||||||
|
reader io.Reader
|
||||||
|
wantRead int64
|
||||||
|
wantMessage string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "normal",
|
||||||
|
contentLength: int64(len(normalBody)),
|
||||||
|
reader: strings.NewReader(normalBody),
|
||||||
|
wantRead: int64(len(normalBody)),
|
||||||
|
wantMessage: marker,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "declared oversize",
|
||||||
|
contentLength: maxProviderErrorResponseBytes + 1,
|
||||||
|
reader: strings.NewReader(normalBody),
|
||||||
|
wantRead: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "streamed oversize",
|
||||||
|
contentLength: -1,
|
||||||
|
reader: &guardedReader{
|
||||||
|
reader: strings.NewReader(overLimitBody),
|
||||||
|
remaining: maxProviderErrorResponseBytes + 1,
|
||||||
|
},
|
||||||
|
wantRead: maxProviderErrorResponseBytes + 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "underreported oversize",
|
||||||
|
contentLength: maxProviderErrorResponseBytes,
|
||||||
|
reader: &guardedReader{
|
||||||
|
reader: strings.NewReader(overLimitBody),
|
||||||
|
remaining: maxProviderErrorResponseBytes + 1,
|
||||||
|
},
|
||||||
|
wantRead: maxProviderErrorResponseBytes + 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "malformed",
|
||||||
|
contentLength: 1,
|
||||||
|
reader: strings.NewReader("{"),
|
||||||
|
wantRead: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "read failure",
|
||||||
|
contentLength: -1,
|
||||||
|
reader: failingReader{err: errors.New("response read failed")},
|
||||||
|
wantRead: 0,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
body := &countingReadCloser{reader: tc.reader}
|
||||||
|
client := newNonSuccessResponseClient(t, http.StatusBadGateway, tc.contentLength, body)
|
||||||
|
|
||||||
|
response, err := client.Generate(context.Background(), ordinaryGenerateRequest())
|
||||||
|
if response != nil {
|
||||||
|
t.Fatalf("response = %#v, want nil", response)
|
||||||
|
}
|
||||||
|
var providerHTTPError *ProviderHTTPError
|
||||||
|
if !errors.As(err, &providerHTTPError) {
|
||||||
|
t.Fatalf("error = %T, want *ProviderHTTPError", err)
|
||||||
|
}
|
||||||
|
if !body.closed {
|
||||||
|
t.Fatal("response body was not closed")
|
||||||
|
}
|
||||||
|
if body.bytesRead != tc.wantRead {
|
||||||
|
t.Fatalf("body bytes read = %d, want %d", body.bytesRead, tc.wantRead)
|
||||||
|
}
|
||||||
|
if body.bytesRead > maxProviderErrorResponseBytes+1 {
|
||||||
|
t.Fatalf("body bytes read = %d, exceeds overflow probe", body.bytesRead)
|
||||||
|
}
|
||||||
|
if guarded, ok := tc.reader.(*guardedReader); ok && guarded.violated {
|
||||||
|
t.Fatal("body reader was asked to read beyond the overflow probe")
|
||||||
|
}
|
||||||
|
if got := providerHTTPError.ProviderMessage(); got != tc.wantMessage {
|
||||||
|
t.Fatalf("provider message = %q, want %q", got, tc.wantMessage)
|
||||||
|
}
|
||||||
|
if tc.wantMessage == "" && (providerHTTPError.ProviderCode() != "" || providerHTTPError.ProviderType() != "" || strings.Contains(providerHTTPError.Error(), marker)) {
|
||||||
|
t.Fatalf("discarded details were retained: %#v", providerHTTPError)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newNonSuccessResponseClient(t *testing.T, statusCode int, contentLength int64, body io.ReadCloser) *OpenAICompatibleClient {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
|
BaseURL: "https://provider.example/v1",
|
||||||
|
Model: "m",
|
||||||
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{
|
||||||
|
StatusCode: statusCode,
|
||||||
|
ContentLength: contentLength,
|
||||||
|
Body: body,
|
||||||
|
}, nil
|
||||||
|
})},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct client: %v", err)
|
||||||
|
}
|
||||||
|
return client
|
||||||
|
}
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
id: aion-2
|
|
||||||
backend: openrouter
|
|
||||||
model: aion-labs/aion-2.0
|
|
||||||
temperature: 0.72
|
|
||||||
reasoning_effort: high
|
|
||||||
top_p: 0.95
|
|
||||||
timeout_seconds: 180
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
id: claude-fable-latest
|
|
||||||
backend: openrouter
|
|
||||||
model: "~anthropic/claude-fable-latest"
|
|
||||||
reasoning_effort: high
|
|
||||||
timeout_seconds: 600
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
id: claude-haiku-latest
|
|
||||||
backend: openrouter
|
|
||||||
model: "~anthropic/claude-haiku-latest"
|
|
||||||
reasoning_effort: medium
|
|
||||||
timeout_seconds: 240
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
id: claude-opus-latest
|
|
||||||
backend: openrouter
|
|
||||||
model: "~anthropic/claude-opus-latest"
|
|
||||||
reasoning_effort: high
|
|
||||||
timeout_seconds: 240
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
id: claude-sonnet-latest
|
|
||||||
backend: openrouter
|
|
||||||
model: "~anthropic/claude-sonnet-latest"
|
|
||||||
reasoning_effort: high
|
|
||||||
timeout_seconds: 240
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
id: deepseek-3-2
|
|
||||||
backend: openrouter
|
|
||||||
model: deepseek/deepseek-v3.2
|
|
||||||
reasoning_effort: high
|
|
||||||
timeout_seconds: 180
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
id: deepseek-4-flash
|
|
||||||
backend: openrouter
|
|
||||||
model: deepseek/deepseek-v4-flash
|
|
||||||
#reasoning_effort: medium
|
|
||||||
timeout_seconds: 180
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
id: deepseek-4-pro
|
|
||||||
backend: openrouter
|
|
||||||
model: deepseek/deepseek-v4-pro
|
|
||||||
reasoning_effort: high
|
|
||||||
timeout_seconds: 180
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
id: gemini-2-flash-lite
|
|
||||||
backend: openrouter
|
|
||||||
model: "google/gemini-2.5-flash-lite"
|
|
||||||
#temperature: 0.15
|
|
||||||
reasoning_effort: high
|
|
||||||
#top_p: 0.98
|
|
||||||
timeout_seconds: 240
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
id: gemini-2-flash
|
|
||||||
backend: openrouter
|
|
||||||
model: "google/gemini-2.5-flash"
|
|
||||||
#temperature: 0.15
|
|
||||||
reasoning_effort: high
|
|
||||||
#top_p: 0.98
|
|
||||||
timeout_seconds: 240
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
id: gemini-2-pro
|
|
||||||
backend: openrouter
|
|
||||||
model: "google/gemini-2.5-pro"
|
|
||||||
#temperature: 0.15
|
|
||||||
reasoning_effort: high
|
|
||||||
#top_p: 0.98
|
|
||||||
timeout_seconds: 240
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
id: gemini-3-flash-lite
|
|
||||||
backend: openrouter
|
|
||||||
model: "google/gemini-3.1-flash-lite"
|
|
||||||
#temperature: 0.15
|
|
||||||
reasoning_effort: high
|
|
||||||
#top_p: 0.98
|
|
||||||
timeout_seconds: 240
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
id: gemini-flash-latest
|
|
||||||
backend: openrouter
|
|
||||||
model: "~google/gemini-flash-latest"
|
|
||||||
#temperature: 0.15
|
|
||||||
reasoning_effort: high
|
|
||||||
#top_p: 0.98
|
|
||||||
timeout_seconds: 240
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
id: gemini-pro-latest
|
|
||||||
backend: openrouter
|
|
||||||
model: "~google/gemini-pro-latest"
|
|
||||||
#temperature: 0.15
|
|
||||||
reasoning_effort: high
|
|
||||||
#top_p: 0.98
|
|
||||||
timeout_seconds: 240
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
id: gemma-4-31b
|
|
||||||
backend: openrouter
|
|
||||||
model: google/gemma-4-31b-it:exacto
|
|
||||||
temperature: 0.15
|
|
||||||
reasoning_effort: high
|
|
||||||
top_p: 0.98
|
|
||||||
timeout_seconds: 240
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
id: minimax-m2
|
|
||||||
backend: openrouter
|
|
||||||
model: minimax/minimax-m2.5
|
|
||||||
temperature: 0.5
|
|
||||||
reasoning_effort: high
|
|
||||||
top_p: 0.95
|
|
||||||
timeout_seconds: 180
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
id: minimax-m3
|
|
||||||
backend: openrouter
|
|
||||||
model: minimax/minimax-m3
|
|
||||||
#temperature: 0.5
|
|
||||||
reasoning_effort: high
|
|
||||||
#top_p: 0.95
|
|
||||||
timeout_seconds: 180
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
id: mistral-large-2512
|
|
||||||
backend: openrouter
|
|
||||||
model: mistralai/mistral-large-2512
|
|
||||||
temperature: 0.15
|
|
||||||
top_p: 0.98
|
|
||||||
timeout_seconds: 180
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: mistral-medium-3-5
|
|
||||||
backend: openrouter
|
|
||||||
model: mistralai/mistral-medium-3-5
|
|
||||||
temperature: 0.15
|
|
||||||
reasoning_effort: high
|
|
||||||
top_p: 0.98
|
|
||||||
timeout_seconds: 180
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
id: mistral-small-3
|
|
||||||
backend: openrouter
|
|
||||||
model: mistralai/mistral-small-3.2-24b-instruct
|
|
||||||
temperature: 0.05
|
|
||||||
top_p: 1.0
|
|
||||||
timeout_seconds: 180
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: mistral-small-4
|
|
||||||
backend: openrouter
|
|
||||||
model: mistralai/mistral-small-2603
|
|
||||||
temperature: 0.1
|
|
||||||
reasoning_effort: high
|
|
||||||
top_p: 0.98
|
|
||||||
timeout_seconds: 180
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
id: nemotron-3-ultra
|
|
||||||
backend: openrouter
|
|
||||||
model: nvidia/nemotron-3-ultra-550b-a55b
|
|
||||||
reasoning_effort: high
|
|
||||||
timeout_seconds: 180
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
id: gpt-5-mini
|
|
||||||
backend: openrouter
|
|
||||||
model: "openai/gpt-5.4-mini"
|
|
||||||
reasoning_effort: high
|
|
||||||
timeout_seconds: 240
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
id: gpt-5-nano
|
|
||||||
backend: openrouter
|
|
||||||
model: "openai/gpt-5.4-nano"
|
|
||||||
reasoning_effort: high
|
|
||||||
timeout_seconds: 240
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
package builtin
|
|
||||||
|
|
||||||
import (
|
|
||||||
"embed"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
|
||||||
)
|
|
||||||
|
|
||||||
const assetRoot = "assets"
|
|
||||||
|
|
||||||
//go:embed assets/**/*.yml
|
|
||||||
var assets embed.FS
|
|
||||||
|
|
||||||
func NewRepository() profile.Repository {
|
|
||||||
return profile.NewFSRepository(assets, assetRoot)
|
|
||||||
}
|
|
||||||
@@ -1,90 +0,0 @@
|
|||||||
package builtin
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"io/fs"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
|
||||||
"gopkg.in/yaml.v3"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
|
|
||||||
repo := NewRepository()
|
|
||||||
ids := loadBuiltInProfileIDs(t)
|
|
||||||
if len(ids) == 0 {
|
|
||||||
t.Fatal("expected built-in profiles")
|
|
||||||
}
|
|
||||||
|
|
||||||
for id := range ids {
|
|
||||||
t.Run(id, func(t *testing.T) {
|
|
||||||
p, err := repo.GetProfile(context.Background(), id)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected built-in profile %q to load, got %v", id, err)
|
|
||||||
}
|
|
||||||
if p.ID != id {
|
|
||||||
t.Fatalf("expected profile id %q, got %q", id, p.ID)
|
|
||||||
}
|
|
||||||
if p.BackendID != backend.OpenRouterID {
|
|
||||||
t.Fatalf("expected profile %q to select %q, got %q", id, backend.OpenRouterID, p.BackendID)
|
|
||||||
}
|
|
||||||
if p.Endpoint != "" || p.APIKeyEnv != "" {
|
|
||||||
t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuiltInProfilesDoNotContainDuplicateIDsOrRawAPIKeys(t *testing.T) {
|
|
||||||
loadBuiltInProfileIDs(t)
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadBuiltInProfileIDs(t *testing.T) map[string]string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
ids := map[string]string{}
|
|
||||||
err := fs.WalkDir(assets, assetRoot, func(name string, d fs.DirEntry, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if d.IsDir() || !strings.HasSuffix(name, ".yml") {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := assets.ReadFile(name)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read built-in profile %s: %v", name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var raw map[string]any
|
|
||||||
if err := yaml.Unmarshal(data, &raw); err != nil {
|
|
||||||
t.Fatalf("failed to decode built-in profile %s: %v", name, err)
|
|
||||||
}
|
|
||||||
if _, ok := raw["api_key"]; ok {
|
|
||||||
t.Fatalf("built-in profile %s contains raw api_key", name)
|
|
||||||
}
|
|
||||||
if raw["backend"] != backend.OpenRouterID {
|
|
||||||
t.Fatalf("built-in profile %s does not select %q", name, backend.OpenRouterID)
|
|
||||||
}
|
|
||||||
if _, ok := raw["endpoint"]; ok {
|
|
||||||
t.Fatalf("built-in profile %s repeats endpoint", name)
|
|
||||||
}
|
|
||||||
if _, ok := raw["api_key_env"]; ok {
|
|
||||||
t.Fatalf("built-in profile %s repeats api_key_env", name)
|
|
||||||
}
|
|
||||||
id, ok := raw["id"].(string)
|
|
||||||
if !ok || strings.TrimSpace(id) == "" {
|
|
||||||
t.Fatalf("built-in profile %s has missing id", name)
|
|
||||||
}
|
|
||||||
if previous, ok := ids[id]; ok {
|
|
||||||
t.Fatalf("duplicate built-in profile id %q in %s and %s", id, previous, name)
|
|
||||||
}
|
|
||||||
ids[id] = name
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to walk built-in profiles: %v", err)
|
|
||||||
}
|
|
||||||
return ids
|
|
||||||
}
|
|
||||||
47
internal/profile/definition.go
Normal file
47
internal/profile/definition.go
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
package profile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NormalizeAndValidateDefinition normalizes and validates one source-local
|
||||||
|
// profile definition without resolving a base profile.
|
||||||
|
func NormalizeAndValidateDefinition(profile *domain.ExecutionProfile) error {
|
||||||
|
if profile == nil {
|
||||||
|
return errors.New("profile is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
profile.ID = strings.TrimSpace(profile.ID)
|
||||||
|
profile.BaseProfileID = strings.TrimSpace(profile.BaseProfileID)
|
||||||
|
profile.BackendID = strings.TrimSpace(profile.BackendID)
|
||||||
|
profile.Endpoint = strings.TrimSpace(profile.Endpoint)
|
||||||
|
|
||||||
|
if profile.ID == "" {
|
||||||
|
return errors.New("id is required")
|
||||||
|
}
|
||||||
|
if profile.Endpoint != "" {
|
||||||
|
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(profile.Endpoint)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
profile.Endpoint = endpoint
|
||||||
|
}
|
||||||
|
if profile.BaseProfileID == "" {
|
||||||
|
if profile.BackendID == "" && profile.Endpoint == "" {
|
||||||
|
return errors.New("backend or endpoint is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(profile.Model) == "" {
|
||||||
|
return errors.New("model is required")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{
|
||||||
|
Temperature: profile.Temperature,
|
||||||
|
MaxTokens: profile.MaxTokens,
|
||||||
|
TopP: profile.TopP,
|
||||||
|
TimeoutSeconds: profile.TimeoutSeconds,
|
||||||
|
})
|
||||||
|
}
|
||||||
98
internal/profile/eager_repository.go
Normal file
98
internal/profile/eager_repository.go
Normal file
@@ -0,0 +1,98 @@
|
|||||||
|
package profile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// LoadedProfileMetadata identifies one profile accepted by LoadFSRepository.
|
||||||
|
type LoadedProfileMetadata struct {
|
||||||
|
// ID is the normalized profile ID.
|
||||||
|
ID string
|
||||||
|
// Path is the safe root-relative source path.
|
||||||
|
Path string
|
||||||
|
// ExplicitFields lists the sorted top-level YAML fields present in source.
|
||||||
|
ExplicitFields []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadFSRepository eagerly validates every profile under root and returns an
|
||||||
|
// immutable raw repository and independently owned source metadata.
|
||||||
|
func LoadFSRepository(ctx context.Context, fsys fs.FS, root string) (Repository, []LoadedProfileMetadata, error) {
|
||||||
|
if fsys == nil {
|
||||||
|
return nil, nil, fmt.Errorf("failed to read profile directory: filesystem is nil")
|
||||||
|
}
|
||||||
|
paths, err := filecatalog.FindFSYAMLFiles(ctx, fsys, root)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("failed to read profile directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
repository := &loadedRepository{profiles: make(map[string]domain.ExecutionProfile, len(paths))}
|
||||||
|
metadata := make([]LoadedProfileMetadata, 0, len(paths))
|
||||||
|
for _, path := range paths {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
data, err := fs.ReadFile(fsys, path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("failed to read profile file %s: %w", filecatalog.DisplayPath(root, path), err)
|
||||||
|
}
|
||||||
|
fileMetadata, err := readProfileFileMetadata(data)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidYAML, filecatalog.DisplayPath(root, path))
|
||||||
|
}
|
||||||
|
if fileMetadata.hasRawAPIKey {
|
||||||
|
return nil, nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, filecatalog.DisplayPath(root, path))
|
||||||
|
}
|
||||||
|
definition, err := decodeProfile(data)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidYAML, filecatalog.DisplayPath(root, path))
|
||||||
|
}
|
||||||
|
definition.ExtraParams, err = jsonvalue.CopyMap(definition.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidProfile, filecatalog.DisplayPath(root, path))
|
||||||
|
}
|
||||||
|
if err := NormalizeAndValidateDefinition(definition); err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidProfile, filecatalog.DisplayPath(root, path))
|
||||||
|
}
|
||||||
|
if _, exists := repository.profiles[definition.ID]; exists {
|
||||||
|
return nil, nil, fmt.Errorf("%w: %s: duplicate profile ID", ErrInvalidProfile, filecatalog.DisplayPath(root, path))
|
||||||
|
}
|
||||||
|
repository.profiles[definition.ID] = *definition
|
||||||
|
fields := append([]string(nil), fileMetadata.explicitFields...)
|
||||||
|
sort.Strings(fields)
|
||||||
|
metadata = append(metadata, LoadedProfileMetadata{
|
||||||
|
ID: definition.ID,
|
||||||
|
Path: filecatalog.DisplayPath(root, path),
|
||||||
|
ExplicitFields: fields,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
sort.Slice(metadata, func(left, right int) bool { return metadata[left].ID < metadata[right].ID })
|
||||||
|
return repository, metadata, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type loadedRepository struct {
|
||||||
|
profiles map[string]domain.ExecutionProfile
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *loadedRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
definition, found := r.profiles[strings.TrimSpace(id)]
|
||||||
|
if !found {
|
||||||
|
return nil, ErrProfileNotFound
|
||||||
|
}
|
||||||
|
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("copy loaded profile %q: %w", definition.ID, err)
|
||||||
|
}
|
||||||
|
definition.ExtraParams = extraParams
|
||||||
|
return &definition, nil
|
||||||
|
}
|
||||||
110
internal/profile/eager_repository_test.go
Normal file
110
internal/profile/eager_repository_test.go
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
package profile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io/fs"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLoadFSRepositoryLoadsSortedIndependentProfiles(t *testing.T) {
|
||||||
|
fsys := fstest.MapFS{
|
||||||
|
"profiles/z.yml": {Data: []byte("id: z\nbackend: local\nmodel: z-model\nextra_params:\n nested:\n value: one\n")},
|
||||||
|
"profiles/a.yml": {Data: []byte("id: a\nbackend: local\nmodel: a-model\nendpoint: ''\n")},
|
||||||
|
}
|
||||||
|
|
||||||
|
repository, metadata, err := LoadFSRepository(context.Background(), fsys, "profiles")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load repository: %v", err)
|
||||||
|
}
|
||||||
|
if len(metadata) != 2 || metadata[0].ID != "a" || metadata[1].ID != "z" || metadata[0].Path != "a.yml" {
|
||||||
|
t.Fatalf("unexpected metadata: %#v", metadata)
|
||||||
|
}
|
||||||
|
if len(metadata[0].ExplicitFields) != 4 || metadata[0].ExplicitFields[0] != "backend" || metadata[0].ExplicitFields[1] != "endpoint" {
|
||||||
|
t.Fatalf("expected explicitly empty endpoint metadata, got %#v", metadata[0].ExplicitFields)
|
||||||
|
}
|
||||||
|
metadata[1].ExplicitFields[0] = "changed"
|
||||||
|
first, err := repository.GetProfile(context.Background(), "z")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load profile: %v", err)
|
||||||
|
}
|
||||||
|
first.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||||
|
second, err := repository.GetProfile(context.Background(), "z")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reload profile: %v", err)
|
||||||
|
}
|
||||||
|
if second.ExtraParams["nested"].(map[string]any)["value"] != "one" {
|
||||||
|
t.Fatalf("profile value was not defensively copied: %#v", second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadFSRepositoryRejectsInvalidProfiles(t *testing.T) {
|
||||||
|
tests := map[string]fstest.MapFS{
|
||||||
|
"duplicate IDs": {
|
||||||
|
"profiles/one.yml": {Data: []byte("id: duplicate\nbackend: local\nmodel: one\n")},
|
||||||
|
"profiles/two.yml": {Data: []byte("id: duplicate\nbackend: local\nmodel: two\n")},
|
||||||
|
},
|
||||||
|
"raw API key": {
|
||||||
|
"profiles/one.yml": {Data: []byte("id: one\nbackend: local\nmodel: one\napi_key: forbidden\n")},
|
||||||
|
},
|
||||||
|
"multiple documents": {
|
||||||
|
"profiles/one.yml": {Data: []byte("id: one\nbackend: local\nmodel: one\n---\nid: two\n")},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for name, fsys := range tests {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
_, _, err := LoadFSRepository(context.Background(), fsys, "profiles")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected load failure")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadedRepositoryHonorsCancellationAndMissingProfiles(t *testing.T) {
|
||||||
|
repository, _, err := LoadFSRepository(context.Background(), fstest.MapFS{
|
||||||
|
"profiles/one.yml": {Data: []byte("id: one\nbackend: local\nmodel: one\n")},
|
||||||
|
}, "profiles")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load repository: %v", err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if _, err := repository.GetProfile(ctx, "one"); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("expected cancellation, got %v", err)
|
||||||
|
}
|
||||||
|
if _, err := repository.GetProfile(context.Background(), "missing"); !errors.Is(err, ErrProfileNotFound) {
|
||||||
|
t.Fatalf("expected missing profile, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadFSRepositoryMatchesPointLookupForRawProfiles(t *testing.T) {
|
||||||
|
fsys := fstest.MapFS{
|
||||||
|
"profiles/base.yml": {Data: []byte("id: base\nbackend: local\nmodel: base-model\n")},
|
||||||
|
"profiles/derived.yml": {Data: []byte("id: derived\nbase_profile: base\nreasoning_effort: high\n")},
|
||||||
|
}
|
||||||
|
eager, _, err := LoadFSRepository(context.Background(), fsys, "profiles")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load eager repository: %v", err)
|
||||||
|
}
|
||||||
|
pointLookup := NewFSRepository(fsys, "profiles")
|
||||||
|
for _, id := range []string{"base", "derived"} {
|
||||||
|
t.Run(id, func(t *testing.T) {
|
||||||
|
got, err := eager.GetProfile(context.Background(), id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load eager profile: %v", err)
|
||||||
|
}
|
||||||
|
want, err := pointLookup.GetProfile(context.Background(), id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load point-in-time profile: %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("raw profiles differ: got %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ fs.FS = fstest.MapFS{}
|
||||||
@@ -5,13 +5,14 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -73,7 +74,8 @@ func (r *overlayRepository) GetProfile(ctx context.Context, id string) (*domain.
|
|||||||
}
|
}
|
||||||
|
|
||||||
func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*domain.ExecutionProfile, error) {
|
func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*domain.ExecutionProfile, error) {
|
||||||
if strings.TrimSpace(id) == "" {
|
id = strings.TrimSpace(id)
|
||||||
|
if id == "" {
|
||||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
|
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
|
||||||
}
|
}
|
||||||
if fsys == nil {
|
if fsys == nil {
|
||||||
@@ -94,42 +96,49 @@ func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*doma
|
|||||||
}
|
}
|
||||||
|
|
||||||
relPath := filecatalog.DisplayPath(root, fullPath)
|
relPath := filecatalog.DisplayPath(root, fullPath)
|
||||||
fileMatch := filecatalog.Stem(path.Base(fullPath)) == id
|
|
||||||
data, err := fs.ReadFile(fsys, fullPath)
|
data, err := fs.ReadFile(fsys, fullPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
|
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
|
||||||
}
|
}
|
||||||
metadata := readProfileFileMetadata(data)
|
metadata, metadataErr := readProfileFileMetadata(data)
|
||||||
idMatch := fileMatch || metadata.id == id
|
idMatch := metadata.matchesID(id)
|
||||||
|
if metadataErr != nil {
|
||||||
|
if idMatch {
|
||||||
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, metadataErr)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
if metadata.hasRawAPIKey {
|
if metadata.hasRawAPIKey {
|
||||||
if idMatch {
|
if idMatch {
|
||||||
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath)
|
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath)
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if !idMatch {
|
||||||
var prof domain.ExecutionProfile
|
|
||||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
|
||||||
decoder.KnownFields(true)
|
|
||||||
if err := decoder.Decode(&prof); err != nil {
|
|
||||||
if idMatch {
|
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
|
|
||||||
}
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
prof, err := decodeProfile(data)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prof.ID = strings.TrimSpace(prof.ID)
|
||||||
if prof.ID != id {
|
if prof.ID != id {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
prof.BackendID = strings.TrimSpace(prof.BackendID)
|
prof.ExtraParams, err = jsonvalue.CopyMap(prof.ExtraParams)
|
||||||
if err := validateProfile(&prof); err != nil {
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
|
||||||
|
}
|
||||||
|
if err := NormalizeAndValidateDefinition(prof); err != nil {
|
||||||
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||||
return nil, fmt.Errorf("%w: %s", err, relPath)
|
return nil, fmt.Errorf("%w: %s", err, relPath)
|
||||||
}
|
}
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
|
||||||
}
|
}
|
||||||
matches = append(matches, profileMatch{
|
matches = append(matches, profileMatch{
|
||||||
profile: &prof,
|
profile: prof,
|
||||||
path: relPath,
|
path: relPath,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -155,15 +164,37 @@ type profileMatch struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type profileFileMetadata struct {
|
type profileFileMetadata struct {
|
||||||
id string
|
ids []string
|
||||||
hasRawAPIKey bool
|
hasRawAPIKey bool
|
||||||
|
explicitFields []string
|
||||||
}
|
}
|
||||||
|
|
||||||
func readProfileFileMetadata(data []byte) profileFileMetadata {
|
func readProfileFileMetadata(data []byte) (profileFileMetadata, error) {
|
||||||
|
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||||
var node yaml.Node
|
var node yaml.Node
|
||||||
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&node); err != nil {
|
if err := decoder.Decode(&node); err != nil {
|
||||||
return profileFileMetadata{}
|
return profileFileMetadata{}, err
|
||||||
}
|
}
|
||||||
|
metadata := profileMetadataFromNode(&node)
|
||||||
|
documentCount := 1
|
||||||
|
for {
|
||||||
|
var trailing yaml.Node
|
||||||
|
err := decoder.Decode(&trailing)
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
if documentCount == 1 {
|
||||||
|
return metadata, nil
|
||||||
|
}
|
||||||
|
return metadata, errors.New("profile file must contain exactly one YAML document")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return metadata, err
|
||||||
|
}
|
||||||
|
documentCount++
|
||||||
|
metadata.merge(profileMetadataFromNode(&trailing))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func profileMetadataFromNode(node *yaml.Node) profileFileMetadata {
|
||||||
if node.Kind != yaml.DocumentNode || len(node.Content) == 0 {
|
if node.Kind != yaml.DocumentNode || len(node.Content) == 0 {
|
||||||
return profileFileMetadata{}
|
return profileFileMetadata{}
|
||||||
}
|
}
|
||||||
@@ -176,9 +207,10 @@ func readProfileFileMetadata(data []byte) profileFileMetadata {
|
|||||||
for i := 0; i+1 < len(mapping.Content); i += 2 {
|
for i := 0; i+1 < len(mapping.Content); i += 2 {
|
||||||
key := mapping.Content[i]
|
key := mapping.Content[i]
|
||||||
value := mapping.Content[i+1]
|
value := mapping.Content[i+1]
|
||||||
|
metadata.explicitFields = append(metadata.explicitFields, key.Value)
|
||||||
switch key.Value {
|
switch key.Value {
|
||||||
case "id":
|
case "id":
|
||||||
metadata.id = strings.TrimSpace(value.Value)
|
metadata.ids = append(metadata.ids, strings.TrimSpace(value.Value))
|
||||||
case "api_key":
|
case "api_key":
|
||||||
metadata.hasRawAPIKey = true
|
metadata.hasRawAPIKey = true
|
||||||
}
|
}
|
||||||
@@ -186,29 +218,42 @@ func readProfileFileMetadata(data []byte) profileFileMetadata {
|
|||||||
return metadata
|
return metadata
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateProfile(p *domain.ExecutionProfile) error {
|
func (m profileFileMetadata) matchesID(id string) bool {
|
||||||
if strings.TrimSpace(p.ID) == "" {
|
for _, candidate := range m.ids {
|
||||||
return errors.New("id is required")
|
if candidate == id {
|
||||||
|
return true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(p.BackendID) == "" && strings.TrimSpace(p.Endpoint) == "" {
|
return false
|
||||||
return errors.New("backend or endpoint is required")
|
}
|
||||||
}
|
|
||||||
if strings.TrimSpace(p.Model) == "" {
|
func (m *profileFileMetadata) merge(other profileFileMetadata) {
|
||||||
return errors.New("model is required")
|
m.ids = append(m.ids, other.ids...)
|
||||||
}
|
m.hasRawAPIKey = m.hasRawAPIKey || other.hasRawAPIKey
|
||||||
|
m.explicitFields = append(m.explicitFields, other.explicitFields...)
|
||||||
if p.Temperature < 0 || p.Temperature > 2 {
|
}
|
||||||
return errors.New("temperature must be between 0 and 2")
|
|
||||||
}
|
func decodeProfile(data []byte) (*domain.ExecutionProfile, error) {
|
||||||
if p.MaxTokens < 0 {
|
var prof domain.ExecutionProfile
|
||||||
return errors.New("max_tokens must be greater than or equal to 0")
|
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||||
}
|
decoder.KnownFields(true)
|
||||||
if p.TopP < 0 || p.TopP > 1 {
|
if err := decoder.Decode(&prof); err != nil {
|
||||||
return errors.New("top_p must be between 0 and 1")
|
return nil, err
|
||||||
}
|
}
|
||||||
if p.TimeoutSeconds < 0 {
|
if err := requireYAMLStreamEnd(decoder); err != nil {
|
||||||
return errors.New("timeout_seconds must be greater than or equal to 0")
|
return nil, err
|
||||||
}
|
}
|
||||||
|
return &prof, nil
|
||||||
return nil
|
}
|
||||||
|
|
||||||
|
func requireYAMLStreamEnd(decoder *yaml.Decoder) error {
|
||||||
|
var trailing yaml.Node
|
||||||
|
err := decoder.Decode(&trailing)
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return errors.New("profile file must contain exactly one YAML document")
|
||||||
}
|
}
|
||||||
|
|||||||
58
internal/profile/repository_benchmark_test.go
Normal file
58
internal/profile/repository_benchmark_test.go
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
package profile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
)
|
||||||
|
|
||||||
|
func BenchmarkProfileRepositoryLookup(b *testing.B) {
|
||||||
|
for _, size := range []int{10, 1000} {
|
||||||
|
b.Run(fmt.Sprintf("catalog-%d", size), func(b *testing.B) {
|
||||||
|
files := fstest.MapFS{
|
||||||
|
"target.yaml": profileMapFile(`
|
||||||
|
id: target
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: target-model
|
||||||
|
extra_params:
|
||||||
|
selected: true
|
||||||
|
`),
|
||||||
|
}
|
||||||
|
metadataNames := []string{"target.yaml"}
|
||||||
|
for i := 1; i < size; i++ {
|
||||||
|
name := fmt.Sprintf("profile-%04d.yaml", i)
|
||||||
|
files[name] = profileMapFile(fmt.Sprintf(`
|
||||||
|
id: profile-%04d
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: unrelated-model
|
||||||
|
temperature: 0.5
|
||||||
|
max_tokens: 500
|
||||||
|
extra_params:
|
||||||
|
provider:
|
||||||
|
order:
|
||||||
|
- first
|
||||||
|
- second
|
||||||
|
`, i))
|
||||||
|
metadataNames = append(metadataNames, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
fsys := &recordingProfileFS{FS: files}
|
||||||
|
repo := NewFSRepository(fsys, ".")
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
if _, err := repo.GetProfile(context.Background(), "target"); err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
|
||||||
|
for _, name := range metadataNames {
|
||||||
|
if got := fsys.openCount(name); got != b.N {
|
||||||
|
b.Fatalf("metadata %q opens = %d, want %d", name, got, b.N)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,11 +2,13 @@ package profile
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
|
|
||||||
@@ -61,7 +63,7 @@ func TestFilesystemRepository_GetProfile(t *testing.T) {
|
|||||||
wantErr bool
|
wantErr bool
|
||||||
}{
|
}{
|
||||||
{name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"},
|
{name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"},
|
||||||
{name: "endpoint only", connection: "endpoint: http://localhost:8000/v1", wantEndpoint: "http://localhost:8000/v1"},
|
{name: "endpoint only", connection: "endpoint: ' https://localhost:8000/nested/v1 '", wantEndpoint: "https://localhost:8000/nested/v1"},
|
||||||
{name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"},
|
{name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"},
|
||||||
{name: "neither", wantErr: true},
|
{name: "neither", wantErr: true},
|
||||||
{name: "blank backend", connection: "backend: ' '", wantErr: true},
|
{name: "blank backend", connection: "backend: ' '", wantErr: true},
|
||||||
@@ -126,63 +128,6 @@ temperature: 0.1
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("valid profile with JSON-compatible extra params", func(t *testing.T) {
|
|
||||||
writeProfileTestFile(t, filepath.Join(tmpDir, "json-extra-params.yaml"), `
|
|
||||||
id: json-extra-params
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: nested-model
|
|
||||||
extra_params:
|
|
||||||
string_value: enabled
|
|
||||||
number_value: 42
|
|
||||||
boolean_value: true
|
|
||||||
object_value:
|
|
||||||
nested: value
|
|
||||||
count: 2
|
|
||||||
array_value:
|
|
||||||
- first
|
|
||||||
- 3
|
|
||||||
- false
|
|
||||||
`)
|
|
||||||
|
|
||||||
p, err := repo.GetProfile(ctx, "json-extra-params")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var got map[string]any
|
|
||||||
encoded, err := json.Marshal(p.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected extra_params to marshal as JSON, got %v", err)
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(encoded, &got); err != nil {
|
|
||||||
t.Fatalf("expected extra_params JSON to decode, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got["string_value"] != "enabled" {
|
|
||||||
t.Fatalf("unexpected string extra param: %#v", got["string_value"])
|
|
||||||
}
|
|
||||||
if got["number_value"] != float64(42) {
|
|
||||||
t.Fatalf("unexpected number extra param: %#v", got["number_value"])
|
|
||||||
}
|
|
||||||
if got["boolean_value"] != true {
|
|
||||||
t.Fatalf("unexpected boolean extra param: %#v", got["boolean_value"])
|
|
||||||
}
|
|
||||||
objectValue, ok := got["object_value"].(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("expected object extra param, got %#v", got["object_value"])
|
|
||||||
}
|
|
||||||
if objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
|
|
||||||
t.Fatalf("unexpected object extra param: %#v", objectValue)
|
|
||||||
}
|
|
||||||
arrayValue, ok := got["array_value"].([]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("expected array extra param, got %#v", got["array_value"])
|
|
||||||
}
|
|
||||||
if len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
|
|
||||||
t.Fatalf("unexpected array extra param: %#v", arrayValue)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
|
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
|
||||||
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
|
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
|
||||||
id: duplicate-profile
|
id: duplicate-profile
|
||||||
@@ -245,10 +190,10 @@ api_key: secret
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("invalid yaml", func(t *testing.T) {
|
t.Run("unidentifiable invalid yaml is unrelated", func(t *testing.T) {
|
||||||
_, err := repo.GetProfile(ctx, "invalid_yaml")
|
_, err := repo.GetProfile(ctx, "invalid_yaml")
|
||||||
if !errors.Is(err, ErrInvalidYAML) {
|
if !errors.Is(err, ErrProfileNotFound) {
|
||||||
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -274,14 +219,14 @@ api_key: secret
|
|||||||
})
|
})
|
||||||
|
|
||||||
t.Run("unknown field", func(t *testing.T) {
|
t.Run("unknown field", func(t *testing.T) {
|
||||||
_, err := repo.GetProfile(ctx, "unknown_field")
|
_, err := repo.GetProfile(ctx, "unknown-field")
|
||||||
if !errors.Is(err, ErrInvalidYAML) {
|
if !errors.Is(err, ErrInvalidYAML) {
|
||||||
t.Fatalf("expected ErrInvalidYAML for strict decode unknown field, got %v", err)
|
t.Fatalf("expected ErrInvalidYAML for strict decode unknown field, got %v", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("raw api_key rejected", func(t *testing.T) {
|
t.Run("raw api_key rejected", func(t *testing.T) {
|
||||||
_, err := repo.GetProfile(ctx, "raw_api_key")
|
_, err := repo.GetProfile(ctx, "raw-api-key")
|
||||||
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||||
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -406,6 +351,614 @@ model: second
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestProfileRepositoriesRejectInvalidEndpoints(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
endpoint string
|
||||||
|
withBackend bool
|
||||||
|
}{
|
||||||
|
{name: "relative", endpoint: "/v1"},
|
||||||
|
{name: "missing host", endpoint: "https:///v1"},
|
||||||
|
{name: "unsupported scheme", endpoint: "ftp://provider.example/v1"},
|
||||||
|
{name: "user information", endpoint: "https://user@provider.example/v1"},
|
||||||
|
{name: "query", endpoint: "https://provider.example/v1?mode=chat"},
|
||||||
|
{name: "fragment", endpoint: "https://provider.example/v1#chat"},
|
||||||
|
{name: "backend with invalid override", endpoint: "/v1", withBackend: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
backend := ""
|
||||||
|
if tc.withBackend {
|
||||||
|
backend = "backend: openrouter\n"
|
||||||
|
}
|
||||||
|
repo := NewFSRepository(fstest.MapFS{
|
||||||
|
"profiles/invalid.yaml": profileMapFile(fmt.Sprintf(
|
||||||
|
"id: invalid-endpoint\nmodel: model\n%sendpoint: %q\n",
|
||||||
|
backend,
|
||||||
|
tc.endpoint,
|
||||||
|
)),
|
||||||
|
}, "profiles")
|
||||||
|
|
||||||
|
_, err := repo.GetProfile(context.Background(), "invalid-endpoint")
|
||||||
|
if !errors.Is(err, ErrInvalidProfile) {
|
||||||
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileRepositoriesValidateExtraParams(t *testing.T) {
|
||||||
|
const validProfile = `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
extra_params:
|
||||||
|
string_value: enabled
|
||||||
|
object_value:
|
||||||
|
nested: true
|
||||||
|
array_value:
|
||||||
|
- first
|
||||||
|
- 3
|
||||||
|
`
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
definition string
|
||||||
|
wantErr bool
|
||||||
|
diagnostics []string
|
||||||
|
}{
|
||||||
|
{name: "valid nested values", definition: validProfile},
|
||||||
|
{
|
||||||
|
name: "empty key",
|
||||||
|
definition: `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
extra_params:
|
||||||
|
"": value
|
||||||
|
`,
|
||||||
|
wantErr: true,
|
||||||
|
diagnostics: []string{"extra_params", "key must not be empty"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non-finite value",
|
||||||
|
definition: `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
extra_params:
|
||||||
|
invalid: .nan
|
||||||
|
`,
|
||||||
|
wantErr: true,
|
||||||
|
diagnostics: []string{"extra_params.invalid", "must be finite"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nested non-finite value",
|
||||||
|
definition: `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
extra_params:
|
||||||
|
outer:
|
||||||
|
invalid: .inf
|
||||||
|
`,
|
||||||
|
wantErr: true,
|
||||||
|
diagnostics: []string{"extra_params.outer.invalid", "must be finite"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unsupported decoded value",
|
||||||
|
definition: `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
extra_params:
|
||||||
|
timestamp: 2026-08-11T12:34:56Z
|
||||||
|
`,
|
||||||
|
wantErr: true,
|
||||||
|
diagnostics: []string{"extra_params.timestamp", "unsupported JSON value type"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "excessive nesting",
|
||||||
|
definition: deeplyNestedExtraParamsProfile(101),
|
||||||
|
wantErr: true,
|
||||||
|
diagnostics: []string{"extra_params", "JSON container depth limit exceeded"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range profileRepositorySources() {
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
||||||
|
repo := source.newRepository(t, map[string]string{"selected.yaml": tc.definition})
|
||||||
|
got, err := repo.GetProfile(context.Background(), "selected-profile")
|
||||||
|
if tc.wantErr {
|
||||||
|
if !errors.Is(err, ErrInvalidProfile) {
|
||||||
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "selected.yaml") {
|
||||||
|
t.Fatalf("expected source path in error, got %v", err)
|
||||||
|
}
|
||||||
|
for _, diagnostic := range tc.diagnostics {
|
||||||
|
if !strings.Contains(err.Error(), diagnostic) {
|
||||||
|
t.Fatalf("expected error to contain %q, got %v", diagnostic, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load valid profile: %v", err)
|
||||||
|
}
|
||||||
|
if got.ExtraParams["string_value"] != "enabled" {
|
||||||
|
t.Fatalf("unexpected copied extra params: %#v", got.ExtraParams)
|
||||||
|
}
|
||||||
|
objectValue, objectOK := got.ExtraParams["object_value"].(map[string]any)
|
||||||
|
arrayValue, arrayOK := got.ExtraParams["array_value"].([]any)
|
||||||
|
if !objectOK || objectValue["nested"] != true ||
|
||||||
|
!arrayOK || len(arrayValue) != 2 || arrayValue[0] != "first" || arrayValue[1] != 3 {
|
||||||
|
t.Fatalf("unexpected copied nested extra params: %#v", got.ExtraParams)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileRepositoriesSelectCanonicalYAMLID(t *testing.T) {
|
||||||
|
const validProfile = `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: selected-model
|
||||||
|
`
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
files map[string]string
|
||||||
|
wantErr error
|
||||||
|
diagnostics []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "same stem unknown field with different id is unrelated",
|
||||||
|
files: map[string]string{
|
||||||
|
"selected-profile.yaml": `
|
||||||
|
id: unrelated-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: unrelated
|
||||||
|
unknown: true
|
||||||
|
`,
|
||||||
|
"valid.yaml": validProfile,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "same stem unidentifiable yaml is unrelated",
|
||||||
|
files: map[string]string{
|
||||||
|
"selected-profile.yaml": "id: [",
|
||||||
|
"valid.yaml": validProfile,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "same stem raw key with different id is unrelated",
|
||||||
|
files: map[string]string{
|
||||||
|
"selected-profile.yaml": `
|
||||||
|
id: unrelated-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: unrelated
|
||||||
|
api_key: secret
|
||||||
|
`,
|
||||||
|
"valid.yaml": validProfile,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "leading and trailing whitespace is normalized",
|
||||||
|
files: map[string]string{
|
||||||
|
"padded.yaml": `
|
||||||
|
id: " selected-profile "
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: selected-model
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "blank id is unrelated",
|
||||||
|
files: map[string]string{
|
||||||
|
"selected-profile.yaml": `
|
||||||
|
id: " "
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: unrelated
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantErr: ErrProfileNotFound,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "normalized duplicates are ambiguous",
|
||||||
|
files: map[string]string{
|
||||||
|
"first.yaml": validProfile,
|
||||||
|
"nested/second.yaml": `
|
||||||
|
id: " selected-profile "
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: duplicate
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantErr: ErrInvalidProfile,
|
||||||
|
diagnostics: []string{"duplicate execution profile id", "first.yaml", "nested/second.yaml"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "selected unknown field is authoritative",
|
||||||
|
files: map[string]string{
|
||||||
|
"malformed.yaml": `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: selected-model
|
||||||
|
unknown: true
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantErr: ErrInvalidYAML,
|
||||||
|
diagnostics: []string{"malformed.yaml"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "selected raw key is authoritative",
|
||||||
|
files: map[string]string{
|
||||||
|
"insecure.yaml": `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: selected-model
|
||||||
|
api_key: secret
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantErr: ErrRawAPIKeyNotAllowed,
|
||||||
|
diagnostics: []string{"insecure.yaml"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "selected identity in an additional document is authoritative",
|
||||||
|
files: map[string]string{
|
||||||
|
"additional-document.yaml": `
|
||||||
|
---
|
||||||
|
---
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: selected-model
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantErr: ErrInvalidYAML,
|
||||||
|
diagnostics: []string{"additional-document.yaml", "exactly one YAML document"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range profileRepositorySources() {
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
||||||
|
repo := source.newRepository(t, tc.files)
|
||||||
|
got, err := repo.GetProfile(context.Background(), " selected-profile ")
|
||||||
|
if tc.wantErr != nil {
|
||||||
|
if !errors.Is(err, tc.wantErr) {
|
||||||
|
t.Fatalf("expected %v, got %v", tc.wantErr, err)
|
||||||
|
}
|
||||||
|
for _, diagnostic := range tc.diagnostics {
|
||||||
|
if !strings.Contains(err.Error(), diagnostic) {
|
||||||
|
t.Fatalf("expected error to contain %q, got %v", diagnostic, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load selected profile: %v", err)
|
||||||
|
}
|
||||||
|
if got.ID != "selected-profile" || got.Model != "selected-model" {
|
||||||
|
t.Fatalf("unexpected selected profile: %+v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileRepositoriesRequireOneYAMLDocument(t *testing.T) {
|
||||||
|
const profile = `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: selected-model
|
||||||
|
`
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
suffix string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "comments and trailing whitespace", suffix: "\n# trailing comment\n\n"},
|
||||||
|
{name: "second populated document", suffix: "\n---\nid: another\n", wantErr: true},
|
||||||
|
{name: "second empty document", suffix: "\n---\n", wantErr: true},
|
||||||
|
{name: "malformed trailing yaml", suffix: "\n---\n[", wantErr: true},
|
||||||
|
{name: "raw key in trailing document", suffix: "\n---\napi_key: secret\n", wantErr: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range profileRepositorySources() {
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
||||||
|
repo := source.newRepository(t, map[string]string{"definition.yaml": profile + tc.suffix})
|
||||||
|
got, err := repo.GetProfile(context.Background(), "selected-profile")
|
||||||
|
if tc.wantErr {
|
||||||
|
if !errors.Is(err, ErrInvalidYAML) {
|
||||||
|
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "definition.yaml") {
|
||||||
|
t.Fatalf("expected source path in error, got %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load one-document profile: %v", err)
|
||||||
|
}
|
||||||
|
if got.ID != "selected-profile" {
|
||||||
|
t.Fatalf("unexpected profile: %+v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileRepositoriesPreserveOverlayFallbackRules(t *testing.T) {
|
||||||
|
fallback := staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"selected-profile": {ID: "selected-profile", Endpoint: "http://fallback", Model: "fallback-model"},
|
||||||
|
}}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
files map[string]string
|
||||||
|
wantModel string
|
||||||
|
wantErr error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "same stem malformed different id falls back",
|
||||||
|
files: map[string]string{
|
||||||
|
"selected-profile.yaml": `
|
||||||
|
id: unrelated-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: unrelated
|
||||||
|
unknown: true
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantModel: "fallback-model",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "blank id falls back",
|
||||||
|
files: map[string]string{
|
||||||
|
"selected-profile.yaml": `
|
||||||
|
id: " "
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: unrelated
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantModel: "fallback-model",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "selected malformed profile stops fallback",
|
||||||
|
files: map[string]string{
|
||||||
|
"other-name.yaml": `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: selected
|
||||||
|
unknown: true
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantErr: ErrInvalidYAML,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range profileRepositorySources() {
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
||||||
|
primary := source.newRepository(t, tc.files)
|
||||||
|
got, err := NewOverlayRepository(primary, fallback).GetProfile(context.Background(), "selected-profile")
|
||||||
|
if tc.wantErr != nil {
|
||||||
|
if !errors.Is(err, tc.wantErr) {
|
||||||
|
t.Fatalf("expected %v, got %v", tc.wantErr, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load fallback profile: %v", err)
|
||||||
|
}
|
||||||
|
if got.Model != tc.wantModel {
|
||||||
|
t.Fatalf("model = %q, want %q", got.Model, tc.wantModel)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileRepositoryReadsSourcesFreshOnEveryLookup(t *testing.T) {
|
||||||
|
newSource := func() (*recordingProfileFS, Repository) {
|
||||||
|
fsys := &recordingProfileFS{FS: fstest.MapFS{
|
||||||
|
"target.yaml": profileMapFile(`
|
||||||
|
id: target
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: target-model
|
||||||
|
`),
|
||||||
|
"unrelated.yaml": profileMapFile(`
|
||||||
|
id: unrelated
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: unrelated-model
|
||||||
|
`),
|
||||||
|
}}
|
||||||
|
return fsys, NewFSRepository(fsys, ".")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("selected source", func(t *testing.T) {
|
||||||
|
fsys, repo := newSource()
|
||||||
|
for lookup := 1; lookup <= 2; lookup++ {
|
||||||
|
got, err := repo.GetProfile(context.Background(), "target")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("lookup %d: %v", lookup, err)
|
||||||
|
}
|
||||||
|
if got.Model != "target-model" {
|
||||||
|
t.Fatalf("lookup %d model = %q", lookup, got.Model)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"target.yaml", "unrelated.yaml"} {
|
||||||
|
if count := fsys.openCount(name); count != lookup {
|
||||||
|
t.Fatalf("%s opens after lookup %d = %d, want %d", name, lookup, count, lookup)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("overlay fallthrough", func(t *testing.T) {
|
||||||
|
primaryFS := &recordingProfileFS{FS: fstest.MapFS{
|
||||||
|
"unrelated.yaml": profileMapFile(`
|
||||||
|
id: unrelated
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: unrelated-model
|
||||||
|
`),
|
||||||
|
}}
|
||||||
|
fallbackFS, fallback := newSource()
|
||||||
|
repo := NewOverlayRepository(NewFSRepository(primaryFS, "."), fallback)
|
||||||
|
|
||||||
|
for lookup := 1; lookup <= 2; lookup++ {
|
||||||
|
got, err := repo.GetProfile(context.Background(), "target")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("lookup %d: %v", lookup, err)
|
||||||
|
}
|
||||||
|
if got.Model != "target-model" {
|
||||||
|
t.Fatalf("lookup %d model = %q", lookup, got.Model)
|
||||||
|
}
|
||||||
|
if count := primaryFS.openCount("unrelated.yaml"); count != lookup {
|
||||||
|
t.Fatalf("primary opens after lookup %d = %d, want %d", lookup, count, lookup)
|
||||||
|
}
|
||||||
|
if count := fallbackFS.openCount("target.yaml"); count != lookup {
|
||||||
|
t.Fatalf("fallback opens after lookup %d = %d, want %d", lookup, count, lookup)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileRepositoriesRejectInvalidExecutionSettings(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
t.Run("operating-system filesystem", func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
writeProfileTestFile(t, filepath.Join(dir, "invalid.yaml"), `
|
||||||
|
id: invalid
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
temperature: .nan
|
||||||
|
`)
|
||||||
|
|
||||||
|
_, err := NewFilesystemRepository(dir).GetProfile(ctx, "invalid")
|
||||||
|
if !errors.Is(err, ErrInvalidProfile) {
|
||||||
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("fs.FS", func(t *testing.T) {
|
||||||
|
repo := NewFSRepository(fstest.MapFS{
|
||||||
|
"profiles/invalid.yaml": profileMapFile(`
|
||||||
|
id: invalid
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
top_p: .inf
|
||||||
|
`),
|
||||||
|
}, "profiles")
|
||||||
|
|
||||||
|
_, err := repo.GetProfile(ctx, "invalid")
|
||||||
|
if !errors.Is(err, ErrInvalidProfile) {
|
||||||
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileRepositoriesValidateDerivedDefinitions(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
files map[string]string
|
||||||
|
wantError error
|
||||||
|
wantBaseID string
|
||||||
|
wantProfile bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "alias is locally valid and normalizes base id",
|
||||||
|
files: map[string]string{"alias.yaml": `
|
||||||
|
id: selected-profile
|
||||||
|
base_profile: " base-profile "
|
||||||
|
`},
|
||||||
|
wantBaseID: "base-profile",
|
||||||
|
wantProfile: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "derived endpoint remains valid",
|
||||||
|
files: map[string]string{"invalid.yaml": `
|
||||||
|
id: selected-profile
|
||||||
|
base_profile: base-profile
|
||||||
|
endpoint: /v1
|
||||||
|
`},
|
||||||
|
wantError: ErrInvalidProfile,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "derived settings remain valid",
|
||||||
|
files: map[string]string{"invalid.yaml": `
|
||||||
|
id: selected-profile
|
||||||
|
base_profile: base-profile
|
||||||
|
top_p: 1.1
|
||||||
|
`},
|
||||||
|
wantError: ErrInvalidProfile,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "derived extra params remain valid",
|
||||||
|
files: map[string]string{"invalid.yaml": `
|
||||||
|
id: selected-profile
|
||||||
|
base_profile: base-profile
|
||||||
|
extra_params:
|
||||||
|
timestamp: 2026-08-11T12:34:56Z
|
||||||
|
`},
|
||||||
|
wantError: ErrInvalidProfile,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "derived raw key remains prohibited",
|
||||||
|
files: map[string]string{"invalid.yaml": `
|
||||||
|
id: selected-profile
|
||||||
|
base_profile: base-profile
|
||||||
|
api_key: secret
|
||||||
|
`},
|
||||||
|
wantError: ErrRawAPIKeyNotAllowed,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "derived duplicate id remains invalid",
|
||||||
|
files: map[string]string{
|
||||||
|
"first.yaml": "id: selected-profile\nbase_profile: first-base\n",
|
||||||
|
"second.yaml": "id: selected-profile\nbase_profile: second-base\n",
|
||||||
|
},
|
||||||
|
wantError: ErrInvalidProfile,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "derived extra document remains invalid",
|
||||||
|
files: map[string]string{"invalid.yaml": `
|
||||||
|
id: selected-profile
|
||||||
|
base_profile: base-profile
|
||||||
|
---
|
||||||
|
id: other
|
||||||
|
`},
|
||||||
|
wantError: ErrInvalidYAML,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "standalone profile remains complete",
|
||||||
|
files: map[string]string{"invalid.yaml": "id: selected-profile\n"},
|
||||||
|
wantError: ErrInvalidProfile,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range profileRepositorySources() {
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
||||||
|
repo := source.newRepository(t, tc.files)
|
||||||
|
got, err := repo.GetProfile(context.Background(), "selected-profile")
|
||||||
|
if tc.wantError != nil {
|
||||||
|
if !errors.Is(err, tc.wantError) {
|
||||||
|
t.Fatalf("error = %v, want %v", err, tc.wantError)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil || !tc.wantProfile {
|
||||||
|
t.Fatalf("profile = %+v, error = %v, want valid derived definition", got, err)
|
||||||
|
}
|
||||||
|
if got.BaseProfileID != tc.wantBaseID {
|
||||||
|
t.Fatalf("BaseProfileID = %q, want %q", got.BaseProfileID, tc.wantBaseID)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestOverlayRepository(t *testing.T) {
|
func TestOverlayRepository(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"}
|
primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"}
|
||||||
@@ -495,6 +1048,77 @@ func TestOverlayRepository(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type profileRepositorySource struct {
|
||||||
|
name string
|
||||||
|
newRepository func(t *testing.T, files map[string]string) Repository
|
||||||
|
}
|
||||||
|
|
||||||
|
type recordingProfileFS struct {
|
||||||
|
fs.FS
|
||||||
|
mu sync.Mutex
|
||||||
|
opened []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *recordingProfileFS) Open(name string) (fs.File, error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
f.opened = append(f.opened, name)
|
||||||
|
f.mu.Unlock()
|
||||||
|
return f.FS.Open(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *recordingProfileFS) openCount(name string) int {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
count := 0
|
||||||
|
for _, opened := range f.opened {
|
||||||
|
if opened == name {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
func profileRepositorySources() []profileRepositorySource {
|
||||||
|
return []profileRepositorySource{
|
||||||
|
{
|
||||||
|
name: "operating system",
|
||||||
|
newRepository: func(t *testing.T, files map[string]string) Repository {
|
||||||
|
t.Helper()
|
||||||
|
root := t.TempDir()
|
||||||
|
for name, content := range files {
|
||||||
|
filePath := filepath.Join(root, filepath.FromSlash(name))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil {
|
||||||
|
t.Fatalf("create profile directory: %v", err)
|
||||||
|
}
|
||||||
|
writeProfileTestFile(t, filePath, content)
|
||||||
|
}
|
||||||
|
return NewFilesystemRepository(root)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "filesystem",
|
||||||
|
newRepository: func(t *testing.T, files map[string]string) Repository {
|
||||||
|
t.Helper()
|
||||||
|
fsys := make(fstest.MapFS, len(files))
|
||||||
|
for name, content := range files {
|
||||||
|
fsys[name] = profileMapFile(content)
|
||||||
|
}
|
||||||
|
return NewFSRepository(fsys, ".")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func deeplyNestedExtraParamsProfile(depth int) string {
|
||||||
|
var definition strings.Builder
|
||||||
|
definition.WriteString("id: selected-profile\nendpoint: http://localhost:8000/v1\nmodel: model\nextra_params:\n")
|
||||||
|
for level := 0; level < depth; level++ {
|
||||||
|
fmt.Fprintf(&definition, "%slevel_%d:\n", strings.Repeat(" ", level+1), level)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&definition, "%svalue: true\n", strings.Repeat(" ", depth+1))
|
||||||
|
return definition.String()
|
||||||
|
}
|
||||||
|
|
||||||
func profileMapFile(content string) *fstest.MapFile {
|
func profileMapFile(content string) *fstest.MapFile {
|
||||||
return &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))}
|
return &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))}
|
||||||
}
|
}
|
||||||
|
|||||||
160
internal/profile/resolving_repository.go
Normal file
160
internal/profile/resolving_repository.go
Normal file
@@ -0,0 +1,160 @@
|
|||||||
|
package profile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||||
|
)
|
||||||
|
|
||||||
|
const maximumProfileChainLength = 32
|
||||||
|
|
||||||
|
type resolvingRepository struct {
|
||||||
|
source Repository
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewResolvingRepository resolves inherited profile definitions from source.
|
||||||
|
func NewResolvingRepository(source Repository) Repository {
|
||||||
|
return &resolvingRepository{source: source}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *resolvingRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
if r == nil || r.source == nil {
|
||||||
|
return nil, fmt.Errorf("%w: profile repository is required", ErrInvalidProfile)
|
||||||
|
}
|
||||||
|
requestedID := strings.TrimSpace(id)
|
||||||
|
if requestedID == "" {
|
||||||
|
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
profile, err := r.getRawProfile(ctx, requestedID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if profile == nil {
|
||||||
|
return nil, fmt.Errorf("%w: selected profile %q is nil", ErrInvalidProfile, requestedID)
|
||||||
|
}
|
||||||
|
|
||||||
|
chain := []*domain.ExecutionProfile{profile}
|
||||||
|
chainIDs := []string{requestedID}
|
||||||
|
visited := map[string]struct{}{requestedID: {}}
|
||||||
|
current := profile
|
||||||
|
|
||||||
|
for {
|
||||||
|
baseID := strings.TrimSpace(current.BaseProfileID)
|
||||||
|
if baseID == "" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if _, seen := visited[baseID]; seen {
|
||||||
|
return nil, fmt.Errorf("%w: profile inheritance cycle %s", ErrInvalidProfile, joinProfileChain(chainIDs, baseID))
|
||||||
|
}
|
||||||
|
if len(chain) >= maximumProfileChainLength {
|
||||||
|
return nil, fmt.Errorf("%w: profile inheritance chain exceeds %d profiles: %s", ErrInvalidProfile, maximumProfileChainLength, joinProfileChain(chainIDs, baseID))
|
||||||
|
}
|
||||||
|
|
||||||
|
base, err := r.getRawProfile(ctx, baseID)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrProfileNotFound) {
|
||||||
|
return nil, fmt.Errorf("%w: base profile %q is missing in chain %s", ErrInvalidProfile, baseID, joinProfileChain(chainIDs, baseID))
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("%w: failed to load base profile %q in chain %s: %w", ErrInvalidProfile, baseID, joinProfileChain(chainIDs, baseID), err)
|
||||||
|
}
|
||||||
|
if base == nil {
|
||||||
|
return nil, fmt.Errorf("%w: base profile %q is nil in chain %s", ErrInvalidProfile, baseID, joinProfileChain(chainIDs, baseID))
|
||||||
|
}
|
||||||
|
|
||||||
|
chain = append(chain, base)
|
||||||
|
chainIDs = append(chainIDs, baseID)
|
||||||
|
visited[baseID] = struct{}{}
|
||||||
|
current = base
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved, err := mergeProfileChain(chain)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: resolved profile chain %s: %w", ErrInvalidProfile, strings.Join(chainIDs, " -> "), err)
|
||||||
|
}
|
||||||
|
if err := validateResolvedProfile(resolved); err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: resolved profile chain %s: %w", ErrInvalidProfile, strings.Join(chainIDs, " -> "), err)
|
||||||
|
}
|
||||||
|
return resolved, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *resolvingRepository) getRawProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
profile, err := r.source.GetProfile(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return profile, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func joinProfileChain(chain []string, next string) string {
|
||||||
|
return strings.Join(append(append([]string(nil), chain...), next), " -> ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeProfileChain(chain []*domain.ExecutionProfile) (*domain.ExecutionProfile, error) {
|
||||||
|
resolved := &domain.ExecutionProfile{ID: chain[0].ID}
|
||||||
|
for index := len(chain) - 1; index >= 0; index-- {
|
||||||
|
definition := chain[index]
|
||||||
|
if strings.TrimSpace(definition.BackendID) != "" {
|
||||||
|
resolved.BackendID = definition.BackendID
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(definition.Endpoint) != "" {
|
||||||
|
resolved.Endpoint = definition.Endpoint
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(definition.Model) != "" {
|
||||||
|
resolved.Model = definition.Model
|
||||||
|
}
|
||||||
|
if definition.Temperature != 0 {
|
||||||
|
resolved.Temperature = definition.Temperature
|
||||||
|
}
|
||||||
|
if definition.MaxTokens != 0 {
|
||||||
|
resolved.MaxTokens = definition.MaxTokens
|
||||||
|
}
|
||||||
|
if definition.TopP != 0 {
|
||||||
|
resolved.TopP = definition.TopP
|
||||||
|
}
|
||||||
|
if definition.TimeoutSeconds != 0 {
|
||||||
|
resolved.TimeoutSeconds = definition.TimeoutSeconds
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(definition.ServiceTier) != "" {
|
||||||
|
resolved.ServiceTier = definition.ServiceTier
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(definition.ReasoningEffort) != "" {
|
||||||
|
resolved.ReasoningEffort = definition.ReasoningEffort
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(definition.APIKeyEnv) != "" {
|
||||||
|
resolved.APIKeyEnv = definition.APIKeyEnv
|
||||||
|
}
|
||||||
|
resolved.APIKeyRequired = resolved.APIKeyRequired || definition.APIKeyRequired
|
||||||
|
if len(definition.ExtraParams) != 0 {
|
||||||
|
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resolved.ExtraParams = extraParams
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resolved.ID = chain[0].ID
|
||||||
|
resolved.BaseProfileID = ""
|
||||||
|
return resolved, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateResolvedProfile(profile *domain.ExecutionProfile) error {
|
||||||
|
if profile == nil {
|
||||||
|
return errors.New("resolved profile is required")
|
||||||
|
}
|
||||||
|
profile.BaseProfileID = ""
|
||||||
|
return NormalizeAndValidateDefinition(profile)
|
||||||
|
}
|
||||||
418
internal/profile/resolving_repository_test.go
Normal file
418
internal/profile/resolving_repository_test.go
Normal file
@@ -0,0 +1,418 @@
|
|||||||
|
package profile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResolvingRepositoryMergesProfileChain(t *testing.T) {
|
||||||
|
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"leaf": {
|
||||||
|
ID: "leaf",
|
||||||
|
BaseProfileID: "middle",
|
||||||
|
BackendID: "leaf-backend",
|
||||||
|
TopP: 0.8,
|
||||||
|
TimeoutSeconds: 45,
|
||||||
|
ReasoningEffort: "high",
|
||||||
|
},
|
||||||
|
"middle": {
|
||||||
|
ID: "middle",
|
||||||
|
BaseProfileID: "root",
|
||||||
|
Endpoint: "https://middle.example/v1",
|
||||||
|
Model: "middle-model",
|
||||||
|
MaxTokens: 256,
|
||||||
|
APIKeyEnv: "MIDDLE_API_KEY",
|
||||||
|
APIKeyRequired: true,
|
||||||
|
ExtraParams: map[string]any{"middle": map[string]any{"value": "middle"}},
|
||||||
|
},
|
||||||
|
"root": {
|
||||||
|
ID: "root",
|
||||||
|
BackendID: "root-backend",
|
||||||
|
Endpoint: "https://root.example/v1",
|
||||||
|
Model: "root-model",
|
||||||
|
Temperature: 0.3,
|
||||||
|
ServiceTier: "priority",
|
||||||
|
ExtraParams: map[string]any{"root": "value"},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
|
||||||
|
got, err := NewResolvingRepository(repo).GetProfile(context.Background(), "leaf")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve profile: %v", err)
|
||||||
|
}
|
||||||
|
want := &domain.ExecutionProfile{
|
||||||
|
ID: "leaf",
|
||||||
|
BackendID: "leaf-backend",
|
||||||
|
Endpoint: "https://middle.example/v1",
|
||||||
|
Model: "middle-model",
|
||||||
|
Temperature: 0.3,
|
||||||
|
MaxTokens: 256,
|
||||||
|
TopP: 0.8,
|
||||||
|
TimeoutSeconds: 45,
|
||||||
|
ServiceTier: "priority",
|
||||||
|
ReasoningEffort: "high",
|
||||||
|
APIKeyEnv: "MIDDLE_API_KEY",
|
||||||
|
APIKeyRequired: true,
|
||||||
|
ExtraParams: map[string]any{"middle": map[string]any{"value": "middle"}},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("resolved profile:\n got %#v\nwant %#v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvingRepositoryRejectsMissingSourceAndProfileID(t *testing.T) {
|
||||||
|
if _, err := NewResolvingRepository(nil).GetProfile(context.Background(), "profile"); !errors.Is(err, ErrInvalidProfile) {
|
||||||
|
t.Fatalf("nil source error = %v, want ErrInvalidProfile", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{}}
|
||||||
|
if _, err := NewResolvingRepository(repo).GetProfile(context.Background(), " \t "); !errors.Is(err, ErrInvalidProfile) {
|
||||||
|
t.Fatalf("blank id error = %v, want ErrInvalidProfile", err)
|
||||||
|
}
|
||||||
|
if got := repo.callCount(" "); got != 0 {
|
||||||
|
t.Fatalf("blank id looked up source %d times", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvingRepositoryCopiesExtraParams(t *testing.T) {
|
||||||
|
baseParams := map[string]any{"nested": map[string]any{"value": "base"}}
|
||||||
|
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"child": {ID: "child", BaseProfileID: "base"},
|
||||||
|
"base": {
|
||||||
|
ID: "base",
|
||||||
|
Endpoint: "https://base.example/v1",
|
||||||
|
Model: "model",
|
||||||
|
ExtraParams: baseParams,
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
resolver := NewResolvingRepository(repo)
|
||||||
|
|
||||||
|
first, err := resolver.GetProfile(context.Background(), "child")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve inherited map: %v", err)
|
||||||
|
}
|
||||||
|
first.ExtraParams["nested"].(map[string]any)["value"] = "mutated"
|
||||||
|
second, err := resolver.GetProfile(context.Background(), "child")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve inherited map again: %v", err)
|
||||||
|
}
|
||||||
|
if got := second.ExtraParams["nested"].(map[string]any)["value"]; got != "base" {
|
||||||
|
t.Fatalf("later result retained mutation: %v", got)
|
||||||
|
}
|
||||||
|
if got := baseParams["nested"].(map[string]any)["value"]; got != "base" {
|
||||||
|
t.Fatalf("source map retained mutation: %v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
repo.set("child", &domain.ExecutionProfile{
|
||||||
|
ID: "child",
|
||||||
|
BaseProfileID: "base",
|
||||||
|
ExtraParams: map[string]any{"child": "replacement"},
|
||||||
|
})
|
||||||
|
replaced, err := resolver.GetProfile(context.Background(), "child")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve replacement map: %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(replaced.ExtraParams, map[string]any{"child": "replacement"}) {
|
||||||
|
t.Fatalf("extra params = %#v, want complete child replacement", replaced.ExtraParams)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvingRepositoryUsesRawOverlayForEachLookup(t *testing.T) {
|
||||||
|
leafSource := NewFSRepository(profileTestFS(map[string]string{
|
||||||
|
"leaf.yaml": "id: leaf\nbase_profile: base\n",
|
||||||
|
}), ".")
|
||||||
|
fallback := NewFSRepository(profileTestFS(map[string]string{
|
||||||
|
"base.yaml": "id: base\nendpoint: https://fallback.example/v1\nmodel: fallback-model\n",
|
||||||
|
}), ".")
|
||||||
|
overlay := NewOverlayRepository(leafSource, fallback)
|
||||||
|
resolver := NewResolvingRepository(overlay)
|
||||||
|
|
||||||
|
got, err := resolver.GetProfile(context.Background(), "leaf")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve fallback base: %v", err)
|
||||||
|
}
|
||||||
|
if got.Model != "fallback-model" {
|
||||||
|
t.Fatalf("fallback base model = %q", got.Model)
|
||||||
|
}
|
||||||
|
|
||||||
|
shadowing := NewOverlayRepository(NewFSRepository(profileTestFS(map[string]string{
|
||||||
|
"leaf.yaml": "id: leaf\nbase_profile: base\n",
|
||||||
|
"base.yaml": "id: base\nendpoint: https://primary.example/v1\nmodel: primary-model\n",
|
||||||
|
}), "."), fallback)
|
||||||
|
got, err = NewResolvingRepository(shadowing).GetProfile(context.Background(), "leaf")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve shadowed base: %v", err)
|
||||||
|
}
|
||||||
|
if got.Model != "primary-model" || got.Endpoint != "https://primary.example/v1" {
|
||||||
|
t.Fatalf("shadowed base = %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvingRepositoryReportsSafetyAndSourceErrors(t *testing.T) {
|
||||||
|
sourceErr := errors.New("source failure")
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
repo *resolvingTestRepository
|
||||||
|
id string
|
||||||
|
want []error
|
||||||
|
wantNot error
|
||||||
|
contains []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "missing selected profile preserves not found",
|
||||||
|
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{}},
|
||||||
|
id: "missing",
|
||||||
|
want: []error{ErrProfileNotFound},
|
||||||
|
wantNot: ErrInvalidProfile,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing base is invalid but not not found",
|
||||||
|
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"leaf": {ID: "leaf", BaseProfileID: "missing"},
|
||||||
|
}},
|
||||||
|
id: "leaf",
|
||||||
|
want: []error{ErrInvalidProfile},
|
||||||
|
wantNot: ErrProfileNotFound,
|
||||||
|
contains: []string{"missing", "leaf -> missing"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "direct cycle",
|
||||||
|
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"a": {ID: "a", BaseProfileID: "a"},
|
||||||
|
}},
|
||||||
|
id: "a",
|
||||||
|
want: []error{ErrInvalidProfile},
|
||||||
|
contains: []string{"a -> a"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "indirect cycle",
|
||||||
|
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"a": {ID: "a", BaseProfileID: "b"},
|
||||||
|
"b": {ID: "b", BaseProfileID: "c"},
|
||||||
|
"c": {ID: "c", BaseProfileID: "a"},
|
||||||
|
}},
|
||||||
|
id: "a",
|
||||||
|
want: []error{ErrInvalidProfile},
|
||||||
|
contains: []string{"a -> b -> c -> a"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nil result",
|
||||||
|
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"leaf": nil,
|
||||||
|
}},
|
||||||
|
id: "leaf",
|
||||||
|
want: []error{ErrInvalidProfile},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "incomplete resolved profile",
|
||||||
|
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"leaf": {ID: "leaf", BaseProfileID: "base"},
|
||||||
|
"base": {ID: "base", Model: "model"},
|
||||||
|
}},
|
||||||
|
id: "leaf",
|
||||||
|
want: []error{ErrInvalidProfile},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "base source error is retained",
|
||||||
|
repo: &resolvingTestRepository{
|
||||||
|
profiles: map[string]*domain.ExecutionProfile{"leaf": {ID: "leaf", BaseProfileID: "base"}},
|
||||||
|
errors: map[string]error{"base": sourceErr},
|
||||||
|
},
|
||||||
|
id: "leaf",
|
||||||
|
want: []error{ErrInvalidProfile, sourceErr},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := NewResolvingRepository(tc.repo).GetProfile(context.Background(), tc.id)
|
||||||
|
for _, want := range tc.want {
|
||||||
|
if !errors.Is(err, want) {
|
||||||
|
t.Fatalf("error = %v, want %v", err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tc.wantNot != nil && errors.Is(err, tc.wantNot) {
|
||||||
|
t.Fatalf("error = %v, must not match %v", err, tc.wantNot)
|
||||||
|
}
|
||||||
|
for _, fragment := range tc.contains {
|
||||||
|
if !strings.Contains(err.Error(), fragment) {
|
||||||
|
t.Fatalf("error = %v, want %q", err, fragment)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvingRepositoryEnforcesChainLength(t *testing.T) {
|
||||||
|
for _, count := range []int{maximumProfileChainLength, maximumProfileChainLength + 1} {
|
||||||
|
t.Run(fmt.Sprintf("%d profiles", count), func(t *testing.T) {
|
||||||
|
profiles := make(map[string]*domain.ExecutionProfile, count)
|
||||||
|
for index := 1; index <= count; index++ {
|
||||||
|
id := fmt.Sprintf("profile-%d", index)
|
||||||
|
definition := &domain.ExecutionProfile{ID: id}
|
||||||
|
if index == count {
|
||||||
|
definition.Endpoint = "https://root.example/v1"
|
||||||
|
definition.Model = "model"
|
||||||
|
} else {
|
||||||
|
definition.BaseProfileID = fmt.Sprintf("profile-%d", index+1)
|
||||||
|
}
|
||||||
|
profiles[id] = definition
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := NewResolvingRepository(&resolvingTestRepository{profiles: profiles}).GetProfile(context.Background(), "profile-1")
|
||||||
|
if count == maximumProfileChainLength {
|
||||||
|
if err != nil || got == nil {
|
||||||
|
t.Fatalf("profile = %+v, error = %v, want accepted chain", got, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrInvalidProfile) {
|
||||||
|
t.Fatalf("error = %v, want ErrInvalidProfile", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvingRepositoryIsFreshAndCancellationAware(t *testing.T) {
|
||||||
|
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"leaf": {ID: "leaf", BaseProfileID: "base"},
|
||||||
|
"base": {ID: "base", Endpoint: "https://base.example/v1", Model: "first", ExtraParams: map[string]any{"nested": map[string]any{"value": "first"}}},
|
||||||
|
}}
|
||||||
|
resolver := NewResolvingRepository(repo)
|
||||||
|
|
||||||
|
first, err := resolver.GetProfile(context.Background(), "leaf")
|
||||||
|
if err != nil || first.Model != "first" {
|
||||||
|
t.Fatalf("first result=(%+v, %v)", first, err)
|
||||||
|
}
|
||||||
|
repo.set("base", &domain.ExecutionProfile{ID: "base", Endpoint: "https://base.example/v1", Model: "second", ExtraParams: map[string]any{"nested": map[string]any{"value": "second"}}})
|
||||||
|
second, err := resolver.GetProfile(context.Background(), "leaf")
|
||||||
|
if err != nil || second.Model != "second" {
|
||||||
|
t.Fatalf("second result=(%+v, %v)", second, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
canceled, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if _, err := resolver.GetProfile(canceled, "leaf"); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("canceled lookup error = %v", err)
|
||||||
|
}
|
||||||
|
if got := repo.callCount("leaf"); got != 2 {
|
||||||
|
t.Fatalf("calls after canceled lookup = %d, want 2", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
duringTraversal, cancelDuringTraversal := context.WithCancel(context.Background())
|
||||||
|
repo.afterGet = func(id string) {
|
||||||
|
if id == "leaf" {
|
||||||
|
cancelDuringTraversal()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := resolver.GetProfile(duringTraversal, "leaf"); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("during traversal error = %v", err)
|
||||||
|
}
|
||||||
|
if got := repo.callCount("base"); got != 2 {
|
||||||
|
t.Fatalf("base calls after cancellation = %d, want 2", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
terminalLookup, cancelTerminalLookup := context.WithCancel(context.Background())
|
||||||
|
repo.afterGet = func(id string) {
|
||||||
|
if id == "base" {
|
||||||
|
cancelTerminalLookup()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := resolver.GetProfile(terminalLookup, "leaf"); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("terminal lookup cancellation error = %v", err)
|
||||||
|
}
|
||||||
|
if got := repo.callCount("base"); got != 3 {
|
||||||
|
t.Fatalf("base calls after terminal cancellation = %d, want 3", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
repo.afterGet = nil
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
errors := make(chan error, 8)
|
||||||
|
for index := 0; index < cap(errors); index++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
resolved, err := resolver.GetProfile(context.Background(), "leaf")
|
||||||
|
if err != nil {
|
||||||
|
errors <- err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
resolved.ExtraParams["nested"].(map[string]any)["value"] = "mutated"
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
close(errors)
|
||||||
|
for err := range errors {
|
||||||
|
t.Errorf("concurrent resolution: %v", err)
|
||||||
|
}
|
||||||
|
latest, err := resolver.GetProfile(context.Background(), "leaf")
|
||||||
|
if err != nil || latest.ExtraParams["nested"].(map[string]any)["value"] != "second" {
|
||||||
|
t.Fatalf("latest result=(%+v, %v)", latest, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type resolvingTestRepository struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
profiles map[string]*domain.ExecutionProfile
|
||||||
|
errors map[string]error
|
||||||
|
calls map[string]int
|
||||||
|
afterGet func(string)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *resolvingTestRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
r.mu.Lock()
|
||||||
|
if r.calls == nil {
|
||||||
|
r.calls = make(map[string]int)
|
||||||
|
}
|
||||||
|
r.calls[id]++
|
||||||
|
err := r.errors[id]
|
||||||
|
profile := r.profiles[id]
|
||||||
|
afterGet := r.afterGet
|
||||||
|
r.mu.Unlock()
|
||||||
|
if afterGet != nil {
|
||||||
|
afterGet(id)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if profile == nil {
|
||||||
|
if _, exists := r.profiles[id]; exists {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, ErrProfileNotFound
|
||||||
|
}
|
||||||
|
copy := *profile
|
||||||
|
return ©, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *resolvingTestRepository) set(id string, profile *domain.ExecutionProfile) {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
r.profiles[id] = profile
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *resolvingTestRepository) callCount(id string) int {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
return r.calls[id]
|
||||||
|
}
|
||||||
|
|
||||||
|
func profileTestFS(files map[string]string) fs.FS {
|
||||||
|
fsys := make(fstest.MapFS, len(files))
|
||||||
|
for name, content := range files {
|
||||||
|
fsys[name] = profileMapFile(content)
|
||||||
|
}
|
||||||
|
return fsys
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user