Compare commits
89 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a4264bf4b4 | |||
| 81f41564a2 | |||
| 6b5a2497cc | |||
| 422cc6c978 | |||
| f48e042565 | |||
| d9442850ef | |||
| 2e828157b6 | |||
| 4189146536 | |||
| 59be507d2f | |||
| b45cea4084 | |||
| efe885c6b2 | |||
| 404a4c331d | |||
| 174516cb39 | |||
| b0999112cc | |||
| ed0f7527d5 | |||
| 5064cf833d | |||
| 8745d256bd | |||
| 2e76003fd5 | |||
| 36ce5a5099 | |||
| 465dc1389d | |||
| ae6f1a9865 | |||
| ee99dc9478 | |||
| 00ee5893e9 | |||
| 64d1cffd89 | |||
| f9e8afa2c3 | |||
| e827631d8c | |||
| 115fe8ba58 | |||
| c53250f023 | |||
| 3d99483219 | |||
| 67f788b1e2 | |||
| 764103a2e2 | |||
| a08dd83d1f | |||
| e8922d8ec5 | |||
| 2d44305a8a | |||
| a11c80291e | |||
| 3239567297 | |||
| c239304c2a | |||
| 159b02116f | |||
| e5b7adfb49 | |||
| fa2384e696 | |||
| af0bd3f31a | |||
| 5fff8cd623 | |||
| b2a6c47778 | |||
| a1805fe550 | |||
| 93af155254 | |||
| d783b687a5 | |||
| 4ca3be2c14 | |||
| 227fb35f99 | |||
| e291b8bfe9 | |||
| 2b6a7f83c4 | |||
| 3a43550f70 | |||
| c281f721bc | |||
| 350b0e76d9 | |||
| e43350fd0d | |||
| 20d3e3b5ee | |||
| a93b799236 | |||
| e83a3ce179 | |||
| a04a3bbc5f | |||
| 731b66cff5 | |||
| 70e0ea0cf0 | |||
| d45c474c1e | |||
| 25f1ba0b30 | |||
| a718762da1 | |||
| 58ac3ce298 | |||
| 57f2ce1ce4 | |||
| c8b6d5c490 | |||
| abeb50b525 | |||
| 1cb07c7d91 | |||
| 8cfc71c351 | |||
| 5ccfa4a345 | |||
| 14e03f19d0 | |||
| 7c562a9374 | |||
| ef97d85ac9 | |||
| 805e48c873 | |||
| e9e126dcba | |||
| 32e7a3557c | |||
| 1d1b04e2e0 | |||
| 9748897751 | |||
| 9d020039d5 | |||
| 5247ce0b73 | |||
| c434aa1dae | |||
| ac9b3f3d80 | |||
| 4f12a89a1b | |||
| df31e7f58e | |||
| 0678d242b9 | |||
| 3b4ea21208 | |||
| 1430e85147 | |||
| 34d7a19da5 | |||
| ebf1602635 |
14
README.md
14
README.md
@@ -33,7 +33,19 @@ boundary and constraints that framework work must preserve.
|
||||
|
||||
## Release Guidance
|
||||
|
||||
Consumers upgrading from `v0.4.0` to `v0.5.0` should read the
|
||||
Consumers upgrading from `v0.8.0` to `v0.9.0` should read the
|
||||
[v0.9.0 changelog and migration guide](docs/releases/v0.9.0.md).
|
||||
|
||||
Consumers upgrading from `v0.7.0` to `v0.8.0` can consult the
|
||||
[v0.8.0 changelog and migration guide](docs/releases/v0.8.0.md).
|
||||
|
||||
Earlier adopters can consult the
|
||||
[v0.7.0 changelog and migration guide](docs/releases/v0.7.0.md).
|
||||
|
||||
Consumers upgrading from `v0.5.0` to `v0.6.0` can consult the
|
||||
[v0.6.0 changelog and migration guide](docs/releases/v0.6.0.md).
|
||||
|
||||
Consumers upgrading from `v0.4.0` to `v0.5.0` can consult the
|
||||
[v0.5.0 changelog and migration guide](docs/releases/v0.5.0.md).
|
||||
|
||||
Consumers upgrading from `v0.3.0` to `v0.4.0` should read the
|
||||
|
||||
145
appended_messages_contract_test.go
Normal file
145
appended_messages_contract_test.go
Normal file
@@ -0,0 +1,145 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestAppendedMessageRoleConstantsAreStrings(t *testing.T) {
|
||||
var (
|
||||
developer string = promptkit.RoleDeveloper
|
||||
system string = promptkit.RoleSystem
|
||||
user string = promptkit.RoleUser
|
||||
assistant string = promptkit.RoleAssistant
|
||||
)
|
||||
if developer != "developer" || system != "system" || user != "user" || assistant != "assistant" {
|
||||
t.Fatalf("unexpected role constants: %q %q %q %q", developer, system, user, assistant)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendedMessagesComposeFrozenEffectivePrompt(t *testing.T) {
|
||||
cacheControl := &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}
|
||||
appended := []promptkit.RenderedMessage{
|
||||
{Role: " \tAsSiStAnT\n", Content: "previous response"},
|
||||
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: cacheControl},
|
||||
}
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "output"}}
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "configured prompt"), "."),
|
||||
promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
request := promptkit.RunRequest{PromptID: "prompt", AppendedMessages: appended}
|
||||
|
||||
plain, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare plain request: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare appended request: %v", err)
|
||||
}
|
||||
expected := []promptkit.RenderedMessage{
|
||||
{Role: promptkit.RoleUser, Content: "configured prompt"},
|
||||
{Role: promptkit.RoleAssistant, Content: "previous response"},
|
||||
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}},
|
||||
}
|
||||
if !reflect.DeepEqual(prepared.Messages, expected) || prepared.PromptHash != plain.PromptHash || prepared.RenderedPromptHash == plain.RenderedPromptHash {
|
||||
t.Fatalf("prepared effective prompt = %#v, plain = %#v", prepared, plain)
|
||||
}
|
||||
|
||||
equivalent, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{
|
||||
{Role: promptkit.RoleAssistant, Content: "previous response"},
|
||||
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}},
|
||||
}})
|
||||
if err != nil || equivalent.RenderedPromptHash != prepared.RenderedPromptHash {
|
||||
t.Fatalf("normalized-equivalent request = (%#v, %v), want matching rendered hash %q", equivalent, err, prepared.RenderedPromptHash)
|
||||
}
|
||||
|
||||
for _, variant := range []promptkit.RunRequest{
|
||||
{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleAssistant, Content: "changed"}, expected[2]}},
|
||||
{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{expected[2], expected[1]}},
|
||||
{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleAssistant, Content: "previous response"}, {Role: promptkit.RoleUser, Content: "corrective request"}}},
|
||||
} {
|
||||
variantPrepared, prepareErr := engine.Prepare(context.Background(), variant)
|
||||
if prepareErr != nil || variantPrepared.RenderedPromptHash == prepared.RenderedPromptHash || variantPrepared.PromptHash != prepared.PromptHash {
|
||||
t.Fatalf("variant preparation = (%#v, %v)", variantPrepared, prepareErr)
|
||||
}
|
||||
}
|
||||
|
||||
result, err := engine.Run(context.Background(), request)
|
||||
if err != nil || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(client.requests[0].Prompt.Messages, expected) {
|
||||
t.Fatalf("run result = (%#v, %v), request = %#v", result, err, client.requests)
|
||||
}
|
||||
|
||||
execution, err := engine.PrepareExecution(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
appended[0].Content = "changed caller content"
|
||||
cacheControl.TTL = ""
|
||||
firstDetails := execution.Details()
|
||||
firstDetails.Messages[1].Content = "changed details content"
|
||||
firstDetails.Messages[2].CacheControl.Type = "changed details cache"
|
||||
secondDetails := execution.Details()
|
||||
if !reflect.DeepEqual(secondDetails.Messages, expected) || secondDetails.RenderedPromptHash != prepared.RenderedPromptHash {
|
||||
t.Fatalf("prepared execution details = %#v, want frozen %#v", secondDetails, expected)
|
||||
}
|
||||
|
||||
result, err = engine.RunPrepared(context.Background(), execution)
|
||||
if err != nil || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(client.requests[1].Prompt.Messages, expected) {
|
||||
t.Fatalf("prepared result = (%#v, %v), requests = %#v", result, err, client.requests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidAppendedMessagesFailBeforeSourceOrModelWork(t *testing.T) {
|
||||
promptSource := &inspectionCountingFS{}
|
||||
var modelCalls atomic.Int64
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFS(promptSource, "."),
|
||||
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
request := promptkit.RunRequest{
|
||||
PromptID: "unreached",
|
||||
AppendedMessages: []promptkit.RenderedMessage{{
|
||||
Role: "unsupported-role",
|
||||
Content: "sensitive appended content",
|
||||
}},
|
||||
}
|
||||
|
||||
operations := []struct {
|
||||
name string
|
||||
run func() error
|
||||
}{
|
||||
{name: "Prepare", run: func() error { _, err := engine.Prepare(context.Background(), request); return err }},
|
||||
{name: "PrepareExecution", run: func() error { _, err := engine.PrepareExecution(context.Background(), request); return err }},
|
||||
{name: "Run", run: func() error { _, err := engine.Run(context.Background(), request); return err }},
|
||||
}
|
||||
for _, operation := range operations {
|
||||
t.Run(operation.name, func(t *testing.T) {
|
||||
err := operation.run()
|
||||
if !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("error = %v, want ErrInvalidRequest", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
if promptSource.opens.Load() != 0 {
|
||||
t.Fatalf("invalid appended message opened prompt sources %d times", promptSource.opens.Load())
|
||||
}
|
||||
if modelCalls.Load() != 0 {
|
||||
t.Fatalf("invalid appended message invoked the model %d times", modelCalls.Load())
|
||||
}
|
||||
}
|
||||
28
backends.go
28
backends.go
@@ -9,6 +9,10 @@ import (
|
||||
// backend.
|
||||
const BackendOpenRouter = backend.OpenRouterID
|
||||
|
||||
// BackendRakestrawHome is the reserved ID of Promptkit's built-in
|
||||
// Rakestrawhome backend.
|
||||
const BackendRakestrawHome = backend.RakestrawHomeID
|
||||
|
||||
// BackendLocal is the case-sensitive conventional ID used by [LocalBackend].
|
||||
// It is not a built-in or reserved backend and must be registered with
|
||||
// [WithBackend].
|
||||
@@ -20,21 +24,25 @@ const BackendLocal = "local"
|
||||
// to this configuration value do not break source compatibility.
|
||||
type Backend struct {
|
||||
// ID is the stable, case-sensitive registry key. NewEngine trims it and
|
||||
// requires a non-blank value. BackendOpenRouter is reserved.
|
||||
// requires a non-blank value. Built-in backend IDs are reserved.
|
||||
ID string
|
||||
// Endpoint is the OpenAI-compatible base endpoint. NewEngine trims it and
|
||||
// requires an absolute HTTP or HTTPS URL with a host and without user
|
||||
// information, a query string, or a fragment. Paths are allowed.
|
||||
Endpoint string
|
||||
// APIKeyEnv optionally names the environment variable containing the API
|
||||
// key. NewEngine trims it and requires the portable form
|
||||
// [A-Za-z_][A-Za-z0-9_]*. Store only the name, never a credential value.
|
||||
// APIKeyEnv optionally names an environment lookup source for an API key.
|
||||
// NewEngine trims it and requires the portable form [A-Za-z_][A-Za-z0-9_]*.
|
||||
// A direct RunRequest.APIKey takes precedence. When no usable credential is
|
||||
// available, the built-in client omits Authorization; injected clients own
|
||||
// their own credential-resolution behavior. Store only the name, never a
|
||||
// credential value.
|
||||
APIKeyEnv string
|
||||
// ExtraParams contains backend-wide request defaults. Values must be
|
||||
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
|
||||
// must not be model, session_id, messages, temperature, max_tokens, top_p,
|
||||
// service_tier, reasoning_effort, or response_format. An empty map supplies
|
||||
// no defaults. NewEngine deeply copies the map.
|
||||
// no defaults. NewEngine deeply copies the map and rejects excessively deep
|
||||
// or large values for safety.
|
||||
ExtraParams map[string]any
|
||||
// ConcurrencyLimit is the maximum number of simultaneous model-generation
|
||||
// calls allowed for this backend within one Engine. Zero leaves the backend
|
||||
@@ -73,11 +81,11 @@ func LocalBackend(endpoint string, concurrencyLimit int) Backend {
|
||||
//
|
||||
// Registrations accumulate in option order. Every normalized ID must be unique
|
||||
// across consumer registrations and built-ins; a duplicate or invalid
|
||||
// definition makes NewEngine fail with ErrInvalidConfig. In particular,
|
||||
// BackendOpenRouter cannot be replaced. The immutable registration is scoped
|
||||
// to the resulting Engine and cannot be enumerated, replaced, removed, or
|
||||
// mutated after construction. WithBackend does not install package-global
|
||||
// state.
|
||||
// definition makes NewEngine fail with ErrInvalidConfig. Built-in IDs,
|
||||
// including [BackendOpenRouter] and [BackendRakestrawHome], cannot be
|
||||
// replaced. The immutable registration is scoped to the resulting Engine and
|
||||
// cannot be enumerated, replaced, removed, or mutated after construction.
|
||||
// WithBackend does not install package-global state.
|
||||
func WithBackend(backend Backend) Option {
|
||||
queueCapacity := 0
|
||||
queueCapacitySet := backend.QueueCapacity != nil
|
||||
|
||||
63
convert.go
63
convert.go
@@ -1,7 +1,9 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
@@ -12,19 +14,52 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
||||
if err != nil {
|
||||
return domain.RunRequest{}, err
|
||||
}
|
||||
appendedMessages, err := toDomainAppendedMessages(req.AppendedMessages)
|
||||
if err != nil {
|
||||
return domain.RunRequest{}, err
|
||||
}
|
||||
return domain.RunRequest{
|
||||
PromptID: req.PromptID,
|
||||
PromptVersion: req.PromptVersion,
|
||||
ProfileID: req.ProfileID,
|
||||
SessionID: req.SessionID,
|
||||
APIKey: req.APIKey,
|
||||
Inputs: toDomainArtifactRefMap(req.Inputs),
|
||||
Vars: copyStringMap(req.Vars),
|
||||
Execution: execution,
|
||||
Validation: toDomainOutputContractPtr(req.Validation),
|
||||
PromptID: req.PromptID,
|
||||
PromptVersion: req.PromptVersion,
|
||||
ProfileID: req.ProfileID,
|
||||
SessionID: req.SessionID,
|
||||
APIKey: req.APIKey,
|
||||
Inputs: toDomainArtifactRefMap(req.Inputs),
|
||||
Vars: copyStringMap(req.Vars),
|
||||
Execution: execution,
|
||||
Validation: toDomainOutputContractPtr(req.Validation),
|
||||
AppendedMessages: appendedMessages,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func toDomainAppendedMessages(messages []RenderedMessage) ([]domain.RenderedMessage, error) {
|
||||
if messages == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
converted := make([]domain.RenderedMessage, len(messages))
|
||||
for index, message := range messages {
|
||||
if !utf8.ValidString(message.Content) {
|
||||
return nil, fmt.Errorf("appended message %d content must be valid UTF-8", index)
|
||||
}
|
||||
role, err := domain.NormalizeMessageRole(message.Role)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("appended message %d role: %w", index, err)
|
||||
}
|
||||
|
||||
cacheControl, err := domain.NormalizeCacheControl(toDomainCacheControl(message.CacheControl))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("appended message %d cache_control: %w", index, err)
|
||||
}
|
||||
converted[index] = domain.RenderedMessage{
|
||||
Role: role,
|
||||
Content: message.Content,
|
||||
CacheControl: cacheControl,
|
||||
}
|
||||
}
|
||||
return converted, nil
|
||||
}
|
||||
|
||||
func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
||||
if prepared == nil {
|
||||
return nil
|
||||
@@ -285,6 +320,16 @@ func fromDomainRenderedMessages(messages []domain.RenderedMessage) []RenderedMes
|
||||
return out
|
||||
}
|
||||
|
||||
func toDomainCacheControl(cacheControl *CacheControl) *domain.CacheControl {
|
||||
if cacheControl == nil {
|
||||
return nil
|
||||
}
|
||||
return &domain.CacheControl{
|
||||
Type: domain.CacheControlType(cacheControl.Type),
|
||||
TTL: cacheControl.TTL,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl {
|
||||
if cacheControl == nil {
|
||||
return nil
|
||||
|
||||
108
convert_appended_messages_test.go
Normal file
108
convert_appended_messages_test.go
Normal file
@@ -0,0 +1,108 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestToDomainAppendedMessages(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
messages []RenderedMessage
|
||||
want []domain.RenderedMessage
|
||||
wantErr string
|
||||
privateRole string
|
||||
privateText string
|
||||
}{
|
||||
{
|
||||
name: "normalizes supported roles",
|
||||
messages: []RenderedMessage{
|
||||
{Role: " DeVeLoPeR ", Content: "developer"},
|
||||
{Role: "SyStEm", Content: "system"},
|
||||
{Role: "\tUsEr\n", Content: "user"},
|
||||
{Role: "assistant", Content: "assistant"},
|
||||
},
|
||||
want: []domain.RenderedMessage{
|
||||
{Role: domain.RoleDeveloper, Content: "developer"},
|
||||
{Role: domain.RoleSystem, Content: "system"},
|
||||
{Role: domain.RoleUser, Content: "user"},
|
||||
{Role: domain.RoleAssistant, Content: "assistant"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid role UTF-8",
|
||||
messages: []RenderedMessage{{Role: string([]byte{0xff}), Content: "private-content"}},
|
||||
wantErr: "appended message 0 role",
|
||||
},
|
||||
{
|
||||
name: "invalid content UTF-8",
|
||||
messages: []RenderedMessage{{Role: "private-role", Content: string([]byte{0xff})}},
|
||||
wantErr: "appended message 0 content",
|
||||
},
|
||||
{
|
||||
name: "unsupported role",
|
||||
messages: []RenderedMessage{{Role: "private-role", Content: "private-content"}},
|
||||
wantErr: "appended message 0 role",
|
||||
privateRole: "private-role",
|
||||
privateText: "private-content",
|
||||
},
|
||||
{
|
||||
name: "invalid cache control",
|
||||
messages: []RenderedMessage{{Role: RoleUser, Content: "private-content", CacheControl: &CacheControl{Type: "private-cache"}}},
|
||||
wantErr: "appended message 0 cache_control",
|
||||
privateText: "private-content",
|
||||
},
|
||||
{
|
||||
name: "empty and whitespace content",
|
||||
messages: []RenderedMessage{
|
||||
{Role: RoleUser, Content: ""},
|
||||
{Role: RoleAssistant, Content: " \t\n "},
|
||||
},
|
||||
want: []domain.RenderedMessage{
|
||||
{Role: domain.RoleUser, Content: ""},
|
||||
{Role: domain.RoleAssistant, Content: " \t\n "},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, err := toDomainAppendedMessages(test.messages)
|
||||
if test.wantErr != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), test.wantErr) {
|
||||
t.Fatalf("error = %v, want %q", err, test.wantErr)
|
||||
}
|
||||
for _, privateValue := range []string{test.privateRole, test.privateText} {
|
||||
if privateValue != "" && strings.Contains(err.Error(), privateValue) {
|
||||
t.Fatalf("error exposed appended message data %q: %v", privateValue, err)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("toDomainAppendedMessages() error = %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, test.want) {
|
||||
t.Fatalf("toDomainAppendedMessages() = %#v, want %#v", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestToDomainAppendedMessagesCopiesCacheControl(t *testing.T) {
|
||||
cacheControl := &CacheControl{Type: CacheControlEphemeral, TTL: "1h"}
|
||||
messages := []RenderedMessage{{Role: RoleUser, Content: "content", CacheControl: cacheControl}}
|
||||
request, err := toDomainRunRequest(RunRequest{AppendedMessages: messages})
|
||||
if err != nil {
|
||||
t.Fatalf("toDomainRunRequest() error = %v", err)
|
||||
}
|
||||
|
||||
messages[0].Content = "changed"
|
||||
cacheControl.TTL = ""
|
||||
if request.AppendedMessages[0].Content != "content" || request.AppendedMessages[0].CacheControl.TTL != "1h" {
|
||||
t.Fatalf("domain request did not retain an independent appended-message copy: %#v", request.AppendedMessages)
|
||||
}
|
||||
}
|
||||
16
doc.go
16
doc.go
@@ -23,8 +23,9 @@
|
||||
// InspectProfile return copied inspection values. Returned values and values
|
||||
// passed to extension interfaces are likewise isolated from engine state.
|
||||
// Callers own those copies and may mutate them after the call that supplied or
|
||||
// returned them. Returned structured errors are likewise caller-owned and may
|
||||
// be mutated without affecting engine state or another error.
|
||||
// returned them. [CapacityError] values are caller-owned and may be mutated
|
||||
// without affecting engine state or another error. Immutable [GenerationError]
|
||||
// values are also caller-owned and do not retain shared engine state.
|
||||
//
|
||||
// # Security and sensitive data
|
||||
//
|
||||
@@ -53,9 +54,14 @@
|
||||
// Construction, inspection, handle, and error values, including [Config],
|
||||
// [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile],
|
||||
// [OpenAICompatibleProfileConfig], [ProfileInspection],
|
||||
// [PromptInputDefinition], [PromptInspection], [PreparedExecution], and
|
||||
// [CapacityError], do not have stable JSON representations. Direct API keys
|
||||
// are nevertheless excluded from JSON for every public value.
|
||||
// [PromptInputDefinition], [PromptInspection], [PreparedExecution],
|
||||
// [CapacityError], and [GenerationError], do not have stable JSON
|
||||
// representations. Direct API keys are nevertheless excluded from JSON for
|
||||
// every public value.
|
||||
|
||||
// Provider-derived [GenerationError] accessor values are untrusted and can
|
||||
// contain sensitive request or schema fragments. Applications must apply their
|
||||
// own disclosure policy before logging, displaying, or returning them.
|
||||
//
|
||||
// JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
|
||||
// PreparedRun and RunResult durations are encoded as integer milliseconds in
|
||||
|
||||
@@ -173,6 +173,49 @@ semantics. The
|
||||
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
|
||||
owns the built-in client's outbound HTTP behavior.
|
||||
|
||||
### Repair A Structured Result
|
||||
|
||||
Set a small additional-call budget when a structurally invalid result can be
|
||||
corrected automatically:
|
||||
|
||||
```go
|
||||
request.Validation = &promptkit.OutputContract{
|
||||
Format: promptkit.FormatJSON,
|
||||
ValidationMode: promptkit.ValidationJSONSchema,
|
||||
SchemaPath: "events.schema.json",
|
||||
RepairAttempts: 1,
|
||||
}
|
||||
```
|
||||
|
||||
Each repair attempt is another model call, so it can increase latency and
|
||||
usage; `RunResult.Usage` is cumulative and `Validation.RepairAttempts` reports
|
||||
calls actually started. Exhaustion still returns the final failed validation
|
||||
result. `basic` validation can also repair an empty candidate, but structural
|
||||
validity is not evidence of factual or domain correctness. See the
|
||||
[output-contract format reference](../formats.md#output-contract) and
|
||||
[`OutputContract` GoDoc](../../types.go) for the exact budget and eligibility
|
||||
rules.
|
||||
|
||||
### Append Already-Rendered Messages
|
||||
|
||||
An application can include an earlier assistant response and its own corrective
|
||||
instruction in a fresh request without changing the configured prompt:
|
||||
|
||||
```go
|
||||
request.AppendedMessages = []promptkit.RenderedMessage{
|
||||
{Role: promptkit.RoleAssistant, Content: previousResponse},
|
||||
{Role: promptkit.RoleUser, Content: correction},
|
||||
}
|
||||
result, err := engine.Run(ctx, request)
|
||||
```
|
||||
|
||||
These messages are already rendered: Promptkit does not template or resolve
|
||||
files in them, and they can contain sensitive model output or application
|
||||
feedback. Promptkit remains stateless; every `Run` call re-resolves its current
|
||||
sources and the application owns any semantic retry budget. When a
|
||||
pre-execution equality check is required, use `PrepareExecution` and compare
|
||||
its opaque rendered-prompt hash before invoking `RunPrepared`.
|
||||
|
||||
## Inputs, Profiles, And Overrides
|
||||
|
||||
Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request
|
||||
@@ -189,6 +232,53 @@ For programmatic profiles,
|
||||
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
|
||||
OpenAI-compatible settings into a value accepted by `WithProfiles`.
|
||||
|
||||
### Alias A Built-In Profile
|
||||
|
||||
Give an application-owned profile ID a built-in base when prompts should select
|
||||
the application ID while inheriting the built-in target. The child can override
|
||||
only the setting it owns:
|
||||
|
||||
```go
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "weather-light",
|
||||
BaseProfileID: "deepseek-4-flash",
|
||||
ReasoningEffort: "high",
|
||||
})
|
||||
```
|
||||
|
||||
Select `weather-light` in a prompt or `RunRequest.ProfileID`; it remains the
|
||||
reported selected profile. See the [profile inheritance format
|
||||
reference](../formats.md#profile-inheritance) and the
|
||||
[`Profile` GoDoc](../../types.go) for exact lookup, merging, and validation
|
||||
behavior.
|
||||
|
||||
### Use The Rakestrawhome Built-In Profile
|
||||
|
||||
Set `RAKESTRAWHOME_INFERENCE_API_KEY` in the application environment, then
|
||||
select `rakestrawhome-gemma-4-31b` as an ordinary profile ID. For example, a
|
||||
prepared result identifies the selected built-in through
|
||||
`BackendRakestrawHome`:
|
||||
|
||||
```go
|
||||
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
|
||||
PromptID: "meeting.summary",
|
||||
ProfileID: "rakestrawhome-gemma-4-31b",
|
||||
Inputs: inputs,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if prepared.SelectedBackendID != promptkit.BackendRakestrawHome {
|
||||
return fmt.Errorf("unexpected backend %q", prepared.SelectedBackendID)
|
||||
}
|
||||
```
|
||||
|
||||
Do not register `rakestrawhome` manually. When adopting this built-in, remove
|
||||
an existing `WithBackend` registration with that exact ID; retaining it causes
|
||||
the intentional duplicate-ID configuration error. Direct request credentials
|
||||
and runtime endpoint overrides remain supported under their ordinary GoDoc and
|
||||
format contracts.
|
||||
|
||||
### Inspect A Profile Before Prompt Work
|
||||
|
||||
Use [`Engine.InspectProfile`](../../engine.go) to validate one configured
|
||||
@@ -204,7 +294,7 @@ if err != nil {
|
||||
|
||||
target := inspection.EffectiveModelParams
|
||||
if target.APIKeyEnv != "" {
|
||||
// Apply application policy for the named environment variable.
|
||||
// This is a configured optional environment lookup source.
|
||||
} else if inspection.APIKeyRequired {
|
||||
// Arrange a direct credential before later execution.
|
||||
}
|
||||
@@ -213,9 +303,11 @@ if target.APIKeyEnv != "" {
|
||||
Use this configuration-time boundary when only the profile and its target need
|
||||
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
||||
or rendering work; use prepared execution when that work must remain tied to a
|
||||
later execution. Inspection reports credential requirements but leaves the
|
||||
timing of credential enforcement to the application. The method's
|
||||
[GoDoc](../../engine.go) owns its exact result and error contract.
|
||||
later execution. A reported `APIKeyEnv` is a configured optional source, while
|
||||
`APIKeyRequired` is the explicit local requirement. The
|
||||
[credential format reference](../formats.md#credentials) and the method's
|
||||
[GoDoc](../../engine.go) own the exact precedence, timing, result, and error
|
||||
contracts.
|
||||
|
||||
### Set A Per-Run Session And Reasoning
|
||||
|
||||
@@ -423,6 +515,24 @@ status; those choices remain with the consuming application. The
|
||||
contract, while the [`Engine.Run` and error GoDoc](../../engine.go) owns broad
|
||||
error and cancellation identities.
|
||||
|
||||
For a non-2xx response from the built-in OpenAI-compatible client, inspect the
|
||||
status and deliberately selected provider diagnostic when useful:
|
||||
|
||||
```go
|
||||
var generationErr *promptkit.GenerationError
|
||||
if errors.As(err, &generationErr) {
|
||||
status := generationErr.StatusCode()
|
||||
message := generationErr.ProviderMessage()
|
||||
_, _ = status, message // Apply application retry and presentation policy.
|
||||
}
|
||||
```
|
||||
|
||||
All provider fields are untrusted and can contain sensitive request or schema
|
||||
fragments. Do not log, display, or return them without an application-specific
|
||||
disclosure policy. Promptkit does not assign retry or presentation behavior.
|
||||
The [`GenerationError` GoDoc](../../generation_error.go) owns the exact typed
|
||||
error contract.
|
||||
|
||||
## Application Boundary
|
||||
|
||||
Promptkit is an importable library. It does not own a command, inbound HTTP
|
||||
|
||||
@@ -34,6 +34,7 @@ Start with:
|
||||
| Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. |
|
||||
| Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). |
|
||||
| Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. |
|
||||
| Maintained external catalogs or built-ins | The [internal source document](internal/sources.md), [framework format reference](formats.md), [testing policy](policy/testing.md), both catalog module repositories, and each catalog's release procedure. |
|
||||
| Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. |
|
||||
| Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. |
|
||||
| Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. |
|
||||
@@ -44,3 +45,190 @@ Start with:
|
||||
For cross-cutting changes, follow every applicable row. Do not create
|
||||
placeholder documents for packages, APIs, or integrations that do not yet
|
||||
exist.
|
||||
|
||||
## Maintainer Validation
|
||||
|
||||
This section is the canonical local validation workflow for Promptkit. Run
|
||||
every command from the repository root before accepting a change. The test
|
||||
suite and maintained examples are deterministic, offline, and require no real
|
||||
provider credentials.
|
||||
|
||||
### Tests, Analysis, Build, And Examples
|
||||
|
||||
Run the ordinary and race-enabled suites, static analysis, the build, and both
|
||||
maintained consumer examples:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go test -race ./...
|
||||
go vet ./...
|
||||
go build ./...
|
||||
go run ./examples/go-library/prepare
|
||||
go run ./examples/go-library/run
|
||||
```
|
||||
|
||||
Both examples must exit successfully. Review their JSON output: preparation
|
||||
must report the selected offline prompt, profile, model, and message count;
|
||||
execution must report the deterministic generated output, successful
|
||||
validation, selected offline model, and usage. Neither command may contact a
|
||||
provider or require credentials.
|
||||
|
||||
### Go Formatting
|
||||
|
||||
Check every tracked Go file. The final command must succeed and the captured
|
||||
list must be empty:
|
||||
|
||||
```sh
|
||||
unformatted=$(
|
||||
git ls-files '*.go' |
|
||||
while IFS= read -r go_file
|
||||
do
|
||||
gofmt -l "$go_file"
|
||||
done
|
||||
)
|
||||
test -z "$unformatted"
|
||||
```
|
||||
|
||||
### Local Markdown Links
|
||||
|
||||
Use the Python standard library to verify every repository-relative Markdown
|
||||
target and local heading fragment. The check is offline and prints nothing on
|
||||
success:
|
||||
|
||||
```sh
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from urllib.parse import unquote
|
||||
|
||||
root = Path.cwd().resolve()
|
||||
markdown_files = [
|
||||
root / name
|
||||
for name in subprocess.check_output(
|
||||
["git", "ls-files", "*.md"], text=True
|
||||
).splitlines()
|
||||
]
|
||||
link_pattern = re.compile(r"!?\[[^]]*\]\(([^)]+)\)")
|
||||
heading_pattern = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$")
|
||||
scheme_pattern = re.compile(r"^[a-z][a-z0-9+.-]*:", re.IGNORECASE)
|
||||
|
||||
|
||||
def markdown_lines(path):
|
||||
in_fence = False
|
||||
fence = ""
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
stripped = line.lstrip()
|
||||
marker = stripped[:3]
|
||||
if marker in {"```", "~~~"}:
|
||||
if not in_fence:
|
||||
in_fence = True
|
||||
fence = marker
|
||||
elif marker == fence:
|
||||
in_fence = False
|
||||
fence = ""
|
||||
continue
|
||||
if not in_fence:
|
||||
yield line
|
||||
|
||||
|
||||
anchor_cache = {}
|
||||
|
||||
|
||||
def anchors(path):
|
||||
if path in anchor_cache:
|
||||
return anchor_cache[path]
|
||||
found = set()
|
||||
counts = {}
|
||||
for line in markdown_lines(path):
|
||||
match = heading_pattern.match(line)
|
||||
if not match:
|
||||
continue
|
||||
heading = re.sub(r"<[^>]+>", "", match.group(1)).replace("`", "")
|
||||
base = re.sub(r"[^\w\- ]", "", heading.lower()).replace(" ", "-")
|
||||
count = counts.get(base, 0)
|
||||
counts[base] = count + 1
|
||||
found.add(base if count == 0 else f"{base}-{count}")
|
||||
anchor_cache[path] = found
|
||||
return found
|
||||
|
||||
|
||||
failures = []
|
||||
for source in markdown_files:
|
||||
text = "\n".join(markdown_lines(source))
|
||||
for match in link_pattern.finditer(text):
|
||||
target = match.group(1).strip()
|
||||
if target.startswith("<") and target.endswith(">"):
|
||||
target = target[1:-1]
|
||||
if scheme_pattern.match(target) or target.startswith("//"):
|
||||
continue
|
||||
path_text, separator, fragment = target.partition("#")
|
||||
destination = source if not path_text else source.parent / unquote(path_text)
|
||||
try:
|
||||
destination = destination.resolve()
|
||||
destination.relative_to(root)
|
||||
except ValueError:
|
||||
failures.append(f"{source.relative_to(root)}: escapes repository: {target}")
|
||||
continue
|
||||
if not destination.exists():
|
||||
failures.append(f"{source.relative_to(root)}: missing target: {target}")
|
||||
continue
|
||||
if separator and destination.suffix.lower() == ".md":
|
||||
fragment = unquote(fragment).lower()
|
||||
if fragment not in anchors(destination):
|
||||
failures.append(f"{source.relative_to(root)}: missing anchor: {target}")
|
||||
|
||||
if failures:
|
||||
print("\n".join(failures), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
```
|
||||
|
||||
### Repository Hygiene And Review
|
||||
|
||||
Reject an active Go workspace, tracked workspace files, a vendor tree, or a
|
||||
module replacement:
|
||||
|
||||
```sh
|
||||
case "$(go env GOWORK)" in
|
||||
''|off) ;;
|
||||
*) printf '%s\n' 'an active Go workspace is not allowed' >&2; exit 1 ;;
|
||||
esac
|
||||
test -z "$(git ls-files go.work go.work.sum)"
|
||||
test ! -e vendor
|
||||
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||
then
|
||||
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
Check whitespace in both unstaged and staged changes. List ignored files and
|
||||
scan tracked content for common credential forms:
|
||||
|
||||
```sh
|
||||
git diff --check
|
||||
git diff --cached --check
|
||||
test -z "$(git ls-files --others --ignored --exclude-standard)"
|
||||
credential_pattern='-----BEGIN ([A-Z0-9]+ )?PRIV''ATE KEY-----|AKI''A[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{36,}|sk-[A-Za-z0-9]{32,}'
|
||||
if git grep -nEI -e "$credential_pattern" -- .
|
||||
then
|
||||
printf '%s\n' 'possible credential found' >&2
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
Inspect `git status --short --untracked-files=all` and the complete diff before
|
||||
accepting a change. The status may contain only the intended source changes
|
||||
during development. Reject credentials, private keys, environment files,
|
||||
generated binaries, test or coverage output, downloaded assets, template
|
||||
residue, and any other artifact that does not belong in source control. The
|
||||
credential scan catches common forms but does not replace inspection of the
|
||||
actual change.
|
||||
|
||||
After committing the accepted change, require a clean candidate:
|
||||
|
||||
```sh
|
||||
test -z "$(git status --porcelain)"
|
||||
```
|
||||
|
||||
187
docs/formats.md
187
docs/formats.md
@@ -9,9 +9,10 @@ explains how to select these sources and invoke the engine. The
|
||||
owns the resulting outbound wire behavior.
|
||||
|
||||
Prompt and profile sources recursively discover files ending in `.yaml` or
|
||||
`.yml`. YAML decoding is strict: unknown fields are errors for the selected
|
||||
definition. Definitions are selected by their YAML `id`, not their file name
|
||||
or directory.
|
||||
`.yml`. Each prompt-definition and profile file contains exactly one YAML
|
||||
document; comments and trailing whitespace are allowed. YAML decoding is
|
||||
strict: unknown fields are errors for the selected definition. Definitions are
|
||||
selected by their YAML `id`, not their file name or directory.
|
||||
|
||||
## Prompt Definitions
|
||||
|
||||
@@ -81,14 +82,27 @@ allowed.
|
||||
|
||||
### Messages And Templates
|
||||
|
||||
Each message has a non-empty `role` and exactly one of:
|
||||
Each message has a `role` that Promptkit trims and lowercases. It must then be
|
||||
exactly one of `developer`, `system`, `user`, or `assistant`; blank, custom,
|
||||
`tool`, and `function` roles are invalid. This intentionally tightens the
|
||||
previous nonblank-string rule. Consumers migrating to the next minor release
|
||||
must update any nonstandard prompt-definition roles before upgrading.
|
||||
|
||||
Each message also has exactly one of:
|
||||
|
||||
- `content`, containing an inline Go template; or
|
||||
- `content_file`, naming a file whose contents are the Go template.
|
||||
|
||||
For directory and `fs.FS` prompt sources, `content_file` resolves relative to
|
||||
the prompt file and remains within the source root. `WithPromptFile` also
|
||||
resolves it relative to that file.
|
||||
`content_file` must be a relative path. It resolves from the directory that
|
||||
contains the prompt file and must remain within the configured prompt source
|
||||
root; parent components are allowed only when the resolved target remains
|
||||
inside that root. Absolute paths and paths that escape the root are rejected.
|
||||
Operating-system directory and single-file sources also reject symlink targets
|
||||
outside the root, while injected `fs.FS` sources apply containment in that
|
||||
filesystem's relative path namespace. For `WithPromptFile`, the source root is
|
||||
the directory containing the selected prompt file. Promptkit uses the parsed
|
||||
path text exactly after checking separately that it is not blank, so leading
|
||||
and trailing whitespace can name real filesystem entries.
|
||||
|
||||
Request variables are the template data, so a variable named `audience` is
|
||||
referenced as `{{.audience}}`. The `{{input "note"}}` helper renders the body
|
||||
@@ -118,7 +132,7 @@ outbound integration determines its wire representation.
|
||||
| `format` | yes | `text`, `markdown`, or `json`. |
|
||||
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema`. |
|
||||
| `schema_path` | for `json_schema` | Path to a schema in the configured schema source. |
|
||||
| `repair_attempts` | no | Integer zero or greater; omitted means zero. |
|
||||
| `repair_attempts` | no | Integer from zero through three; omitted means zero. A positive value requires `basic`, `json`, or `json_schema` validation. |
|
||||
|
||||
The validation modes behave as follows:
|
||||
|
||||
@@ -128,14 +142,31 @@ The validation modes behave as follows:
|
||||
- `json_schema` requires valid JSON that satisfies the selected schema.
|
||||
|
||||
`format` controls output artifact metadata. JSON Schema mode also supplies the
|
||||
schema to compatible model clients as structured-output metadata. The public
|
||||
engine does not install an output repairer, so its validation is single-pass
|
||||
even when a positive `repair_attempts` value is present.
|
||||
schema to compatible model clients as structured-output metadata. Plain `json`
|
||||
validation accepts every valid JSON value and does not request a provider-native
|
||||
JSON-object constraint.
|
||||
|
||||
`repair_attempts` counts additional generation calls after a failed validation.
|
||||
Zero is single-pass. With a positive eligible budget, Promptkit stops at the
|
||||
first valid candidate. If the budget is exhausted, it returns the final
|
||||
candidate and its complete failed validation result; generation and operational
|
||||
validation failures remain errors. `none` never permits repair.
|
||||
|
||||
A request-level `OutputContract` replaces the complete prompt output contract.
|
||||
It does not merge individual fields. If its format is empty, Promptkit uses
|
||||
`text`.
|
||||
|
||||
## Built-In Backends
|
||||
|
||||
Every engine provides these reserved OpenAI-compatible backend IDs. Consumers
|
||||
must not register either ID with `WithBackend`; exact registration and
|
||||
reservation behavior belongs to the [`Backend` GoDoc](../backends.go).
|
||||
|
||||
| ID | Base endpoint | API-key environment variable | Active generation limit | Default queue capacity |
|
||||
| --- | --- | --- | ---: | ---: |
|
||||
| `openrouter` | `https://openrouter.ai/api/v1` | `OPENROUTER_API_KEY` | 16 | 1024 |
|
||||
| `rakestrawhome` | `https://inference.ai.rakestrawhome.com/v1` | `RAKESTRAWHOME_INFERENCE_API_KEY` | 4 | 1024 |
|
||||
|
||||
## Profile Definitions
|
||||
|
||||
A profile supplies model execution settings:
|
||||
@@ -154,11 +185,21 @@ extra_params:
|
||||
provider_option: enabled
|
||||
```
|
||||
|
||||
A derived profile can use a named base and override only the settings it owns:
|
||||
|
||||
```yaml
|
||||
id: local-summary-fast
|
||||
base_profile: local-summary
|
||||
timeout_seconds: 30
|
||||
reasoning_effort: low
|
||||
```
|
||||
|
||||
| Field | Required | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `id` | yes | Non-empty profile identifier. IDs must be unique within one source. |
|
||||
| `id` | yes | Profile identifier, trimmed before selection and publication. It must be non-empty after trimming and unique within one source after normalization. |
|
||||
| `base_profile` | no | One optional parent profile ID. A derived profile may inherit target fields from it. |
|
||||
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared or inspected. |
|
||||
| `endpoint` | unless `backend` is present | Non-empty OpenAI-compatible base URL, including an API version path when required. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
|
||||
| `endpoint` | unless `backend` is present | OpenAI-compatible base URL, including an API version path when required. A nonempty value is trimmed and must be absolute HTTP or HTTPS with a host and without user information, a query, or a fragment. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
|
||||
| `model` | yes | Non-empty provider model name. |
|
||||
| `temperature` | no | Number from 0 through 2. |
|
||||
| `max_tokens` | no | Integer zero or greater. |
|
||||
@@ -166,16 +207,21 @@ extra_params:
|
||||
| `timeout_seconds` | no | Per-generation deadline in whole seconds; integer zero or greater. |
|
||||
| `service_tier` | no | Provider-specific request tier. |
|
||||
| `reasoning_effort` | no | Provider-specific reasoning setting. |
|
||||
| `api_key_env` | no | Name of an environment variable containing the API key. |
|
||||
| `api_key_env` | no | Optional environment-variable lookup source for an API key. |
|
||||
| `extra_params` | no | JSON-compatible provider-specific outbound fields. |
|
||||
|
||||
Raw `api_key` is prohibited in profile YAML. Store only an environment
|
||||
variable name in `api_key_env`.
|
||||
|
||||
A standalone profile must provide a model and at least one of `backend` or
|
||||
`endpoint`. A derived profile may omit those target fields because its selected
|
||||
base chain can provide them. Local parsing still validates a derived profile's
|
||||
own ID, supplied endpoint, execution-setting bounds, and `extra_params`.
|
||||
|
||||
Promptkit does not infer a backend from a model or endpoint. Endpoint-only
|
||||
profiles remain supported and have no effective backend ID.
|
||||
The engine always provides the built-in `openrouter` ID. Consumers can add
|
||||
engine-scoped IDs with
|
||||
The engine always provides the built-in `openrouter` and `rakestrawhome` IDs.
|
||||
Consumers can add engine-scoped IDs with
|
||||
[`WithBackend`](../backends.go); exact registration validation belongs to its
|
||||
GoDoc.
|
||||
|
||||
@@ -183,6 +229,7 @@ GoDoc.
|
||||
objects with string keys. Keys must be non-empty. With the built-in client,
|
||||
they also cannot collide with the standard fields listed in the
|
||||
[outbound request contract](integrations/openai-compatible-chat.md#request-body).
|
||||
Excessively deep or large JSON-shaped values are rejected for safety.
|
||||
|
||||
### Defaults And Overrides
|
||||
|
||||
@@ -236,7 +283,7 @@ Profile sources resolve matching IDs in this order:
|
||||
2. the ordinary configured source selected by a profile file, `fs.FS`, or
|
||||
configured profile directory;
|
||||
3. application fallback profiles supplied with `WithFallbackProfileFS`; and
|
||||
4. embedded built-in profiles.
|
||||
4. maintained external catalog profiles.
|
||||
|
||||
A profile source supplies a complete definition; definitions and their fields
|
||||
are not merged across sources. A higher-precedence source falls back only when
|
||||
@@ -246,41 +293,66 @@ profiles. They use `APIKeyRequired` for request-scoped credentials instead of
|
||||
`api_key_env`. Preparation and exact profile inspection use this same source
|
||||
precedence.
|
||||
|
||||
When a selected definition names `base_profile`, every profile ID in that
|
||||
chain is looked up through this same precedence order. A higher-precedence
|
||||
definition therefore shadows a lower-precedence definition of the same base
|
||||
ID, including a built-in. References are not source-qualified.
|
||||
|
||||
### Profile Inheritance
|
||||
|
||||
Promptkit resolves one linear base chain of at most 32 profiles, including the
|
||||
selected profile. It merges settings from the root base to the selected leaf.
|
||||
The leaf's `id` remains the selected profile identity. Nonblank string fields
|
||||
(`backend`, `endpoint`, `model`, `service_tier`, `reasoning_effort`, and
|
||||
`api_key_env`) and nonzero numeric fields replace inherited values. A nonempty
|
||||
`extra_params` map replaces the complete inherited map rather than merging
|
||||
keys, and `APIKeyRequired: true` remains true through the chain. Backend and
|
||||
endpoint are independent: replacing one does not clear the other.
|
||||
|
||||
There is no profile-level clearing syntax. Blank strings, zero numbers, false,
|
||||
and empty maps remain unspecified and inherit from a base. Use existing
|
||||
presence-aware request overrides where an execution needs an explicit zero or
|
||||
empty reasoning setting.
|
||||
|
||||
An absent directly selected profile reports the ordinary not-found error. Once
|
||||
the selected profile exists, a missing base, cycle, overlong chain, or
|
||||
incomplete resolved target is a profile-load failure. Ordinary operations
|
||||
resolve chains afresh; prepared execution retains the fully resolved target.
|
||||
|
||||
## Built-In Profile Catalog
|
||||
|
||||
Every built-in selects the `openrouter` backend. The engine's built-in backend
|
||||
registry supplies `https://openrouter.ai/api/v1` and the environment-variable
|
||||
name `OPENROUTER_API_KEY`, so individual profiles contain only model and
|
||||
generation settings. Built-in profile files do not repeat those connection
|
||||
values. A configured, application fallback, or in-memory profile with the same
|
||||
profile ID takes precedence.
|
||||
Every built-in profile selects one maintained built-in backend and inherits
|
||||
that backend's connection and credential metadata. Profile files do not repeat
|
||||
those values. A configured, application fallback, or in-memory profile with
|
||||
the same profile ID takes precedence.
|
||||
|
||||
| Provider | ID | Model |
|
||||
| --- | --- | --- |
|
||||
| aion-labs | `aion-2` | `aion-labs/aion-2.0` |
|
||||
| anthropic | `claude-fable-latest` | `~anthropic/claude-fable-latest` |
|
||||
| anthropic | `claude-haiku-latest` | `~anthropic/claude-haiku-latest` |
|
||||
| anthropic | `claude-opus-latest` | `~anthropic/claude-opus-latest` |
|
||||
| anthropic | `claude-sonnet-latest` | `~anthropic/claude-sonnet-latest` |
|
||||
| deepseek | `deepseek-3-2` | `deepseek/deepseek-v3.2` |
|
||||
| deepseek | `deepseek-4-flash` | `deepseek/deepseek-v4-flash` |
|
||||
| deepseek | `deepseek-4-pro` | `deepseek/deepseek-v4-pro` |
|
||||
| google | `gemini-2-flash` | `google/gemini-2.5-flash` |
|
||||
| google | `gemini-2-flash-lite` | `google/gemini-2.5-flash-lite` |
|
||||
| google | `gemini-2-pro` | `google/gemini-2.5-pro` |
|
||||
| google | `gemini-3-flash-lite` | `google/gemini-3.1-flash-lite` |
|
||||
| google | `gemini-flash-latest` | `~google/gemini-flash-latest` |
|
||||
| google | `gemini-pro-latest` | `~google/gemini-pro-latest` |
|
||||
| google | `gemma-4-31b` | `google/gemma-4-31b-it:exacto` |
|
||||
| minimax | `minimax-m2` | `minimax/minimax-m2.5` |
|
||||
| minimax | `minimax-m3` | `minimax/minimax-m3` |
|
||||
| mistral | `mistral-large-2512` | `mistralai/mistral-large-2512` |
|
||||
| mistral | `mistral-medium-3-5` | `mistralai/mistral-medium-3-5` |
|
||||
| mistral | `mistral-small-3` | `mistralai/mistral-small-3.2-24b-instruct` |
|
||||
| mistral | `mistral-small-4` | `mistralai/mistral-small-2603` |
|
||||
| nvidia | `nemotron-3-ultra` | `nvidia/nemotron-3-ultra-550b-a55b` |
|
||||
| openai | `gpt-5-mini` | `openai/gpt-5.4-mini` |
|
||||
| openai | `gpt-5-nano` | `openai/gpt-5.4-nano` |
|
||||
| Provider | ID | Backend | Model |
|
||||
| --- | --- | --- | --- |
|
||||
| aion-labs | `aion-2` | `openrouter` | `aion-labs/aion-2.0` |
|
||||
| anthropic | `claude-fable-latest` | `openrouter` | `~anthropic/claude-fable-latest` |
|
||||
| anthropic | `claude-haiku-latest` | `openrouter` | `~anthropic/claude-haiku-latest` |
|
||||
| anthropic | `claude-opus-latest` | `openrouter` | `~anthropic/claude-opus-latest` |
|
||||
| anthropic | `claude-sonnet-latest` | `openrouter` | `~anthropic/claude-sonnet-latest` |
|
||||
| deepseek | `deepseek-3-2` | `openrouter` | `deepseek/deepseek-v3.2` |
|
||||
| deepseek | `deepseek-4-flash` | `openrouter` | `deepseek/deepseek-v4-flash` |
|
||||
| deepseek | `deepseek-4-pro` | `openrouter` | `deepseek/deepseek-v4-pro` |
|
||||
| google | `gemini-2-flash` | `openrouter` | `google/gemini-2.5-flash` |
|
||||
| google | `gemini-2-flash-lite` | `openrouter` | `google/gemini-2.5-flash-lite` |
|
||||
| google | `gemini-2-pro` | `openrouter` | `google/gemini-2.5-pro` |
|
||||
| google | `gemini-3-flash-lite` | `openrouter` | `google/gemini-3.1-flash-lite` |
|
||||
| google | `gemini-flash-latest` | `openrouter` | `~google/gemini-flash-latest` |
|
||||
| google | `gemini-pro-latest` | `openrouter` | `~google/gemini-pro-latest` |
|
||||
| google | `gemma-4-31b` | `openrouter` | `google/gemma-4-31b-it:exacto` |
|
||||
| google | `rakestrawhome-gemma-4-31b` | `rakestrawhome` | `google/gemma-4-31b-it` |
|
||||
| minimax | `minimax-m2` | `openrouter` | `minimax/minimax-m2.5` |
|
||||
| minimax | `minimax-m3` | `openrouter` | `minimax/minimax-m3` |
|
||||
| mistral | `mistral-large-2512` | `openrouter` | `mistralai/mistral-large-2512` |
|
||||
| mistral | `mistral-medium-3-5` | `openrouter` | `mistralai/mistral-medium-3-5` |
|
||||
| mistral | `mistral-small-3` | `openrouter` | `mistralai/mistral-small-3.2-24b-instruct` |
|
||||
| mistral | `mistral-small-4` | `openrouter` | `mistralai/mistral-small-2603` |
|
||||
| nvidia | `nemotron-3-ultra` | `openrouter` | `nvidia/nemotron-3-ultra-550b-a55b` |
|
||||
| openai | `gpt-5-mini` | `openrouter` | `openai/gpt-5.4-mini` |
|
||||
| openai | `gpt-5-nano` | `openrouter` | `openai/gpt-5.4-nano` |
|
||||
|
||||
## Schemas
|
||||
|
||||
@@ -299,16 +371,25 @@ schema produces a failed validation result.
|
||||
Credential values belong at the request or environment boundary, never in
|
||||
prompt, profile, schema, or example files:
|
||||
|
||||
- a file profile names an environment variable with `api_key_env`;
|
||||
- an in-memory profile may set `APIKeyRequired`;
|
||||
- a request can provide a direct `APIKey` or override `APIKeyEnv`; and
|
||||
- a backend or file profile can name an optional environment lookup source
|
||||
with `APIKeyEnv` or `api_key_env`;
|
||||
- an in-memory profile may set `APIKeyRequired` as an explicit local
|
||||
requirement;
|
||||
- a request can provide a direct `APIKey` or override the optional `APIKeyEnv`
|
||||
source; and
|
||||
- a direct request key takes precedence over environment lookup.
|
||||
|
||||
After a direct request key, the credential-source precedence is request
|
||||
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
|
||||
profile with `APIKeyRequired` clears an inherited backend environment name and
|
||||
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
|
||||
Promptkit validates required credential availability during preparation.
|
||||
Named environment sources are optional: when the selected source is absent,
|
||||
empty, or whitespace-only, the built-in client omits the `Authorization`
|
||||
header and handles the provider response normally. `APIKeyRequired` is the
|
||||
only explicit local availability requirement. Promptkit validates required
|
||||
credential availability during preparation and rechecks it when a prepared
|
||||
execution runs. Injected clients receive resolved source metadata but define
|
||||
their own credential-resolution behavior.
|
||||
Direct keys are excluded from JSON results and redacted by public string
|
||||
formatters. Environment-variable names may appear in prepared metadata, but
|
||||
their values do not.
|
||||
|
||||
@@ -14,10 +14,16 @@ that produce these outbound settings.
|
||||
|
||||
Generation sends an HTTP `POST` with `Content-Type: application/json`.
|
||||
Before the client is called, the engine resolves framework, backend, profile,
|
||||
and request values into one execution target. A non-empty endpoint from that
|
||||
target overrides the client's configured base URL. After trailing slashes are
|
||||
removed, `/chat/completions` is appended. Generation fails before sending when
|
||||
neither source supplies an endpoint.
|
||||
and request values into one execution target. Endpoint configuration is trimmed
|
||||
and must be an absolute HTTP or HTTPS URL with a host and without user
|
||||
information, a query, or a fragment. A non-empty endpoint from the target
|
||||
overrides the client's configured base URL. The final selected endpoint is
|
||||
validated again before transport.
|
||||
|
||||
The completion URL is composed through parsed URL path operations. Nested base
|
||||
paths are retained, repeated trailing slashes are normalized, and the result
|
||||
has exactly one appended `/chat/completions` suffix. Generation fails before
|
||||
sending when neither source supplies a valid endpoint.
|
||||
|
||||
The target's backend ID is routing metadata for prepared values, results, and
|
||||
injected clients. The built-in client does not derive the URL from that ID and
|
||||
@@ -25,11 +31,13 @@ does not serialize it in the provider request.
|
||||
|
||||
## Authentication
|
||||
|
||||
A non-empty API key supplied directly on the execution target takes
|
||||
precedence. Otherwise, when an API-key environment-variable name is supplied,
|
||||
the client reads that variable and requires a non-empty value. The selected
|
||||
key is sent as `Authorization: Bearer <key>`. No authorization header is sent
|
||||
when neither mechanism is configured.
|
||||
A usable API key supplied directly on the execution target takes precedence.
|
||||
Otherwise, when an API-key environment-variable name is supplied, the client
|
||||
reads and trims that variable. A bearer header is sent only when the resolved
|
||||
direct or environment credential is non-empty. When neither source is usable,
|
||||
the client omits `Authorization` and handles the provider response normally.
|
||||
An explicitly required target with no usable source is rejected before
|
||||
transport.
|
||||
|
||||
The target contains the already resolved environment-variable name: an
|
||||
explicit request override takes precedence over profile metadata, which takes
|
||||
@@ -47,6 +55,12 @@ Each ordinary message contains its `role` and string `content`. A
|
||||
cache-controlled message instead uses a text content block containing `type`,
|
||||
`text`, and `cache_control`; an empty cache-control TTL is omitted.
|
||||
|
||||
Promptkit sends only `developer`, `system`, `user`, and `assistant` roles and
|
||||
does so without provider-specific translation. Tool and deprecated function
|
||||
payloads are outside this text-message contract. A backend or model that
|
||||
rejects an otherwise supported role or context returns its ordinary provider
|
||||
error, which follows the normal generation-error path.
|
||||
|
||||
The effective direct or prompt-rendered session ID is trimmed, limited to 256
|
||||
Unicode code points, and sent when nonempty as top-level `session_id`. It is
|
||||
never also sent as a session header.
|
||||
@@ -59,7 +73,8 @@ The client conditionally includes:
|
||||
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly
|
||||
disabled reasoning setting is empty and therefore omitted; and
|
||||
- `response_format` for JSON Schema structured output, including its name,
|
||||
strict flag, and schema document.
|
||||
strict flag, and schema document. Plain JSON validation does not add an
|
||||
object-only response constraint.
|
||||
|
||||
The engine resolves backend, profile, and request extra-parameter maps by
|
||||
whole-map replacement rather than key merging. The resulting effective map is
|
||||
@@ -82,13 +97,47 @@ request fields.
|
||||
|
||||
## Response Handling
|
||||
|
||||
Any 2xx response is decoded as an OpenAI-compatible chat response. The client
|
||||
returns the first choice's non-empty message content and maps prompt,
|
||||
completion, total, cached, and cache-write token counts.
|
||||
Any 2xx response body is limited to 16 MiB (16,777,216 bytes). A larger
|
||||
declared `Content-Length` is rejected before the body is read, and streamed,
|
||||
chunked, or underreported bodies are read through the same bound with at most
|
||||
one additional byte used to detect overflow. A body exactly at the limit is
|
||||
allowed. The body is closed on every outcome and an oversized stream is not
|
||||
drained.
|
||||
|
||||
Invalid JSON, absent choices, and empty first-choice content are malformed
|
||||
responses. For a non-2xx status, the error includes the status code but never
|
||||
the provider response body.
|
||||
The bounded body must contain exactly one OpenAI-compatible JSON response
|
||||
object followed only by JSON whitespace and EOF. The client returns the first
|
||||
choice's explicitly present string message content, including an empty or
|
||||
whitespace-only string, and maps prompt, completion, total, cached, and
|
||||
cache-write token counts. Invalid or truncated JSON, trailing non-whitespace
|
||||
data, a second JSON value, absent choices, missing content, `null` content,
|
||||
non-string content, and size overflow are malformed responses and return no
|
||||
partial result.
|
||||
|
||||
For a non-2xx status, Promptkit recognizes one JSON document with a top-level
|
||||
object-valued `error` member. Its optional `message` and `type` fields must be
|
||||
strings, and `code` may be a string or JSON number. Valid supported fields are
|
||||
handled independently, numeric codes retain their JSON number text, and
|
||||
unknown fields are ignored. Missing, invalid, malformed, or multiply framed
|
||||
envelopes contribute no provider detail.
|
||||
|
||||
Non-success bodies have a 65,536-byte limit. A larger declared
|
||||
`Content-Length` is not read; otherwise the client reads at most one additional
|
||||
byte to detect streamed or underreported overflow. Empty, unreadable,
|
||||
oversized, malformed, and unrecognized bodies retain only the received status.
|
||||
The body is always closed and no oversized stream is drained beyond that probe.
|
||||
|
||||
Extracted strings are made valid UTF-8, trimmed, and converted to one line by
|
||||
collapsing Unicode whitespace, control, and format-character runs. Blank
|
||||
values are omitted. Codes and types longer than 256 Unicode code points are
|
||||
omitted; messages longer than 4,096 code points are truncated at a code-point
|
||||
boundary with an ellipsis inside the limit. Promptkit never exposes raw bodies,
|
||||
headers, endpoints, credentials, request data, schemas, generated content, or
|
||||
unsupported provider metadata through this handling.
|
||||
|
||||
An outbound `http.Client.Do` failure retains both Promptkit's request-failure
|
||||
identity and the exact transport error for `errors.Is` and `errors.As` checks.
|
||||
The rendered error does not include the selected endpoint, request headers,
|
||||
request content, credentials, or provider body.
|
||||
|
||||
## Timeout And Cancellation
|
||||
|
||||
@@ -103,5 +152,7 @@ Timeouts are layered:
|
||||
timeout when the supplied value is not positive.
|
||||
|
||||
The earliest applicable caller, generation, or transport deadline controls the
|
||||
request. Constructing the internal client does not mutate a supplied
|
||||
`http.Client`.
|
||||
request. Caller cancellation retains `context.Canceled`; caller, generation,
|
||||
and whole-request timeout failures retain `context.DeadlineExceeded`, together
|
||||
with the request-failure identity. Constructing the internal client does not
|
||||
mutate a supplied `http.Client`.
|
||||
|
||||
@@ -56,7 +56,10 @@ reserve another bounded slot.
|
||||
|
||||
`NewClient` wraps the engine's selected internal model client after public
|
||||
client adaptation or built-in client construction. Initial generation and the
|
||||
default repairer receive the same wrapper.
|
||||
default repairer receive the same wrapper. Their requests retain the same
|
||||
effective backend, credential, numeric-presence metadata, and structured-output
|
||||
settings, so scheduling does not change provider omission semantics between
|
||||
calls.
|
||||
|
||||
For each `Generate` call, the wrapper selects a pool from the request's
|
||||
effective backend ID. An unlimited call passes directly to the next client. A
|
||||
@@ -71,7 +74,9 @@ other backend IDs.
|
||||
|
||||
The wrapper passes generation requests, responses, and collaborator errors
|
||||
through unchanged. It owns scheduling only; the concrete model client remains
|
||||
responsible for provider transport behavior.
|
||||
responsible for provider transport behavior. The runner, rather than the
|
||||
capacity layer, sums all five usage fields from the initial response and every
|
||||
completed repair response into the successful run result.
|
||||
|
||||
## Cancellation And Release
|
||||
|
||||
|
||||
@@ -21,20 +21,26 @@ uses internal domain values for rendered prompts, execution targets,
|
||||
structured output, responses, and token usage.
|
||||
|
||||
The runner supplies a fully resolved target after applying backend, profile,
|
||||
and request precedence. The client uses its endpoint, credential metadata,
|
||||
generation fields, and extra parameters. `BackendID` remains routing metadata
|
||||
and request precedence, plus canonical provider-bound text messages. The
|
||||
client uses its endpoint, credential metadata, generation fields, and extra
|
||||
parameters. `BackendID` remains routing metadata
|
||||
for the generation boundary and is not mapped into the provider payload.
|
||||
|
||||
Construction validates the configured base URL and clones any supplied
|
||||
`http.Client` so Promptkit can apply its timeout default without mutating the
|
||||
caller's client. Generation then:
|
||||
Construction trims and validates a nonempty configured base URL and clones any
|
||||
supplied `http.Client` so Promptkit can apply its timeout default without
|
||||
mutating the caller's client. An empty configured base remains valid because a
|
||||
resolved request target may supply the endpoint. Generation then:
|
||||
|
||||
1. validates request-level timeout and endpoint requirements;
|
||||
1. validates shared execution-setting invariants and the final selected base
|
||||
endpoint;
|
||||
2. maps the internal request into the OpenAI-compatible chat payload;
|
||||
3. validates and merges extra parameters;
|
||||
4. resolves authentication;
|
||||
5. performs the outbound request under the applicable deadlines; and
|
||||
6. decodes the first response choice and token usage.
|
||||
4. composes `/chat/completions` through parsed URL path operations;
|
||||
5. resolves authentication;
|
||||
6. performs the outbound request under the applicable deadlines; and
|
||||
7. decodes one strictly framed, size-bounded successful response object and
|
||||
maps its first choice and token usage, or decodes bounded structured
|
||||
non-success detail.
|
||||
|
||||
`internal/llm` owns the set of reserved OpenAI-compatible request fields used
|
||||
when validating extra parameters. Backend registration consumes the same rule
|
||||
@@ -50,12 +56,14 @@ the target, rendered messages, and structured-output constraint retained by
|
||||
executable preparation. Execution does not reopen or rerender consumer
|
||||
sources.
|
||||
|
||||
Before backend admission, the runner rechecks that the frozen credential
|
||||
environment-variable name is available. The handle does not retain the
|
||||
environment value; the model client resolves the value visible when generation
|
||||
begins. A direct request key remains in private execution state only until the
|
||||
claimed execution finishes or an unclaimed handle is discarded. Exact public
|
||||
ownership and redaction semantics belong to the
|
||||
Before backend admission, the runner rechecks a frozen credential
|
||||
environment-variable name only when the target explicitly requires a
|
||||
credential. The handle does not retain the environment value; the model client
|
||||
resolves the value visible when generation begins. For optional sources with no
|
||||
usable value, the built-in client omits `Authorization` and continues to the
|
||||
provider. A direct request key remains in private execution state only until
|
||||
the claimed execution finishes or an unclaimed handle is discarded. Exact
|
||||
public ownership and redaction semantics belong to the
|
||||
[`PreparedExecution` GoDoc](../../prepared_execution.go).
|
||||
|
||||
## Failure Categories
|
||||
@@ -63,11 +71,47 @@ ownership and redaction semantics belong to the
|
||||
The package preserves distinct error identities for invalid client
|
||||
configuration, invalid generation requests, request execution failures,
|
||||
non-success provider statuses, and malformed successful responses. Provider
|
||||
response bodies are not included in non-success errors.
|
||||
response bodies are never exposed in raw form through non-success errors.
|
||||
|
||||
Caller cancellation and deadline failures during the outbound request are
|
||||
reported as request execution failures. The runner classifies these identities
|
||||
without depending on HTTP status mapping.
|
||||
Invalid nonempty configured endpoints are configuration failures. A missing or
|
||||
invalid final selected endpoint is an invalid generation request and is
|
||||
rejected before transport.
|
||||
|
||||
Authentication resolves a trimmed direct key before a trimmed configured
|
||||
environment value. Optional missing, empty, or whitespace-only sources do not
|
||||
block transport and produce no `Authorization` header. An explicitly required
|
||||
target with no usable source is rejected before transport with the existing
|
||||
invalid-request diagnostics.
|
||||
|
||||
Successful response bodies have a fixed 16 MiB limit enforced by declared
|
||||
length and by reading at most one byte beyond the boundary. The decoder accepts
|
||||
exactly one JSON object plus trailing whitespace and EOF. Size overflow,
|
||||
truncation, malformed JSON, trailing data, and a second value are malformed
|
||||
responses with no partial result or provider content in the error. Every body
|
||||
is closed, and an unbounded oversized stream is not drained.
|
||||
|
||||
After framing succeeds, the first choice must contain an explicitly present
|
||||
string `message.content`. The string is returned exactly, including empty or
|
||||
whitespace-only content. Missing choices, missing or `null` content, and
|
||||
non-string content are malformed responses. Output validation and correction
|
||||
eligibility remain outside this package.
|
||||
|
||||
For a non-success response, `ProviderHTTPError` retains the HTTP status and
|
||||
only normalized detail from the bounded recognized envelope. It retains
|
||||
`ErrUnexpectedStatus` through unwrapping. The client owns response closure;
|
||||
its bounded reader and parser never close or drain a body themselves. The root
|
||||
facade converts this concrete internal error into the public
|
||||
[`GenerationError`](../../generation_error.go), while arbitrary injected-client
|
||||
errors continue through the ordinary generation-error mapping unchanged.
|
||||
|
||||
An `http.Client.Do` failure is represented by a redacting multi-cause error:
|
||||
the package request-failure sentinel and the exact returned transport error are
|
||||
both available through `errors.Is` and `errors.As`, while the rendered text
|
||||
does not expose the endpoint, headers, request content, credential, transport
|
||||
detail, or provider body. Caller cancellation retains `context.Canceled`;
|
||||
caller deadlines, generation deadlines, and whole-request client timeouts
|
||||
retain `context.DeadlineExceeded`. The runner adds its generation category
|
||||
without discarding those identities or depending on HTTP status mapping.
|
||||
|
||||
## Test Ownership
|
||||
|
||||
@@ -75,7 +119,15 @@ The
|
||||
[OpenAI-compatible client tests](../../internal/llm/openai_compatible_client_test.go)
|
||||
own configuration, client cloning, deterministic deadline precedence,
|
||||
authentication, request and response mapping, malformed data, error identity,
|
||||
cancellation, and response-body suppression. The root transport contract test
|
||||
also verifies that resolved backend settings reach this client without
|
||||
serializing backend identity. All use local test servers or test transports;
|
||||
the default suite makes no live or paid provider requests.
|
||||
cancellation, endpoint selection and composition, pre-transport rejection, and
|
||||
bounded single-document successful-response framing, closure, and
|
||||
response-body suppression. The focused
|
||||
[provider HTTP error tests](../../internal/llm/provider_http_error_test.go)
|
||||
own envelope parsing, normalization, and bounded-reader cases; their
|
||||
[transport tests](../../internal/llm/provider_http_error_transport_test.go)
|
||||
own non-success response closure and integration. Root transport contract tests
|
||||
own public `GenerationError` conversion, while also verifying that resolved
|
||||
backend settings reach this client without serializing backend identity and
|
||||
that ordinary-run cancellation retains its public generation and context
|
||||
identities. All use local test servers or controlled test transports; the
|
||||
default suite makes no live or paid provider requests.
|
||||
|
||||
@@ -11,23 +11,23 @@ contributor workflow and validation.
|
||||
|
||||
| Component | Implemented responsibility | References |
|
||||
| --- | --- | --- |
|
||||
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, and profile-inspection values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, typed capacity errors, public error mapping, and engine-local profile-source assembly including application fallbacks. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
|
||||
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, and profile-inspection values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, typed capacity and generation error mapping, engine-local profile-source assembly including application fallbacks, and bounded output-repair assembly. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
|
||||
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||
| `internal/backend` | Constructs each engine's immutable registry from the built-in OpenRouter definition and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
|
||||
| `internal/backend` | Constructs each engine's immutable registry from maintained definitions and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
|
||||
| `internal/catalog` | Strictly validates imported immutable maintained backend and profile catalog assets before engine assembly uses them. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) |
|
||||
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation. | [Domain declarations](../../internal/domain/domain.go) |
|
||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||
| `internal/jsonvalue` | Validates and deeply copies JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
|
||||
| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
|
||||
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
||||
| `internal/profile` | Loads strictly decoded, locally validated execution profiles from filesystem and `fs.FS` sources, overlays raw sources with error-preserving fallback, and resolves inherited profiles. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go), [internal sources](sources.md#profiles-and-built-ins) |
|
||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates frozen validation plans for prepared execution. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, generation, validation, capacity, and optional repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||
|
||||
The root package assembles these internal components without exposing their
|
||||
representations. Consumers depend only on the root facade.
|
||||
|
||||
@@ -20,10 +20,13 @@ and override semantics consumed by the runner.
|
||||
profiles, backend resolution, artifacts, rendering, model generation, and
|
||||
validation. The root engine supplies one immutable registry containing the
|
||||
built-in backend and validated consumer additions, one engine-local run
|
||||
admitter, and a model client wrapped by the same capacity manager. Schema
|
||||
documents are loaded through the validator's optional schema-loader interface.
|
||||
An output repairer can be injected internally, but the ordinary runner
|
||||
constructor does not enable one.
|
||||
admitter, and a model client wrapped by the same capacity manager. Validation
|
||||
plans and provider-facing schema metadata come from the validator's preparation
|
||||
interface.
|
||||
The root engine supplies one default output repairer through the explicit
|
||||
runner constructor, using the same capacity-wrapped client as initial
|
||||
generation. The no-repair runner constructor remains available for focused
|
||||
internal callers and tests.
|
||||
|
||||
Each invocation carries its state in request, prepared-run, and result values.
|
||||
The runner has no durable run or session store.
|
||||
@@ -69,17 +72,20 @@ performs only the work needed to validate routing and admission:
|
||||
5. resolve application-neutral defaults, backend defaults, profile values,
|
||||
and explicit request overrides in that order;
|
||||
6. validate endpoint, model, numeric overrides, and credential requirements;
|
||||
7. resolve the effective output contract without loading its schema; and
|
||||
7. resolve and validate the effective output contract without loading its
|
||||
schema; and
|
||||
8. retain the definition, source identities, effective settings, output
|
||||
contract, and preparation start time in invocation-local state.
|
||||
|
||||
The completion phase consumes that state without reloading the prompt,
|
||||
profile, or backend:
|
||||
|
||||
1. load structured-output schema metadata when required;
|
||||
1. create one operation-local validation plan and derive structured-output
|
||||
schema metadata from it when required;
|
||||
2. load and hash input artifacts;
|
||||
3. render messages and the prompt-defined session;
|
||||
4. apply any direct session ID;
|
||||
4. apply any direct session ID, then append already-normalized request messages
|
||||
after the rendered definition messages;
|
||||
5. hash the effective rendered prompt; and
|
||||
6. construct the prepared value and preparation timing.
|
||||
|
||||
@@ -87,7 +93,10 @@ profile, or backend:
|
||||
`Run` performs backend admission between the phases. This structure preserves
|
||||
one execution-precedence and error-ordering implementation while allowing a
|
||||
full backend pool to reject work before expensive schema, artifact, and
|
||||
rendering operations.
|
||||
rendering operations. `Prepare` discards the plan after returning its public
|
||||
metadata. `Run` retains the plan through initial and repaired-output validation
|
||||
and discards it when the operation ends. Prepared execution stores the same
|
||||
kind of plan only in its private payload.
|
||||
|
||||
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
||||
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
||||
@@ -105,7 +114,10 @@ the prompt session template, and is applied after ordinary message rendering.
|
||||
A blank direct value retains prompt-template behavior. The runner clears the
|
||||
template only on a value copy of the definition, so the definition hash always
|
||||
describes the original source while the rendered-prompt hash includes the
|
||||
effective direct or rendered session.
|
||||
effective direct or rendered session and the complete effective message
|
||||
sequence. The rendered-prompt hash uses a versioned, length-framed SHA-256
|
||||
encoding of that session, every role and content value, and cache-control
|
||||
presence and values; its hexadecimal value is opaque.
|
||||
|
||||
The registry is read-only after engine construction. Concurrent `Prepare` and
|
||||
`Run` calls resolve independent defensive backend values and keep all
|
||||
@@ -118,8 +130,17 @@ its `RunAdmitter` to reserve capacity for the effective backend ID. A nil
|
||||
admitter is an internal unlimited fallback. After successful admission, `Run`
|
||||
immediately defers the returned release function, performs the completion
|
||||
phase, makes one initial generation call, builds the named output artifact,
|
||||
and validates that artifact. Invalid generated content remains a validation
|
||||
result; an inability to perform validation is an operational error.
|
||||
and validates that artifact with the plan compiled during completion. Invalid
|
||||
generated content remains a validation result; an inability to generate or
|
||||
perform validation is an operational error.
|
||||
|
||||
Validation preparation and execution honor cancellation at every
|
||||
Promptkit-controlled boundary and do not publish a partial plan or result.
|
||||
Schema reads are bounded and context-checked between chunks; JSON decoding,
|
||||
schema compilation, and schema execution are checked immediately before and
|
||||
after their synchronous calls. Promptkit does not move arbitrary filesystem or
|
||||
JSON Schema work to background goroutines, so an already-blocked dependency
|
||||
method must return before cancellation can take precedence over its outcome.
|
||||
|
||||
The admission lease covers completion-phase preparation, initial generation,
|
||||
validation, every repair, and every exit. It bounds accepted work without
|
||||
@@ -127,22 +148,38 @@ serializing preparation or validation behind the active-generation limit.
|
||||
The wrapped model client separately acquires a FIFO active permit only around
|
||||
each actual generation call.
|
||||
|
||||
When an internal repairer is present, a JSON or JSON Schema content failure can
|
||||
trigger bounded repair attempts. Repair receives the effective execution
|
||||
target and session ID, validation errors, prior output, and structured-output
|
||||
specification. The default repairer uses the same wrapped client as initial
|
||||
After a failed `basic`, JSON, or JSON Schema validation with a positive frozen
|
||||
budget, the installed repairer can make a bounded corrective call. Each request
|
||||
starts with a fresh copy of the complete effective message sequence (the
|
||||
configured prefix followed by the request suffix), includes only the latest
|
||||
nonempty candidate as an assistant message, and appends one corrective user
|
||||
message. Empty candidates omit that assistant message. The
|
||||
correction carries validation diagnostics as JSON data bounded to 64 KiB; the
|
||||
full diagnostics remain in the validation result.
|
||||
|
||||
Repair receives the effective execution target, explicit numeric-presence bits,
|
||||
credential, backend identity, session ID, and structured-output specification.
|
||||
The same request constructor supplies those common fields to initial and repair
|
||||
generation. The default repairer uses the same wrapped client as initial
|
||||
generation, so each repair reacquires the selected backend's active permit
|
||||
while remaining inside its original admission lease. Repair never performs a
|
||||
second bounded admission. This capability remains internal and is not a public
|
||||
option.
|
||||
second bounded admission, and repaired outputs use the operation's existing
|
||||
validation plan. The runner stops at the first valid candidate, sums completed
|
||||
generation usage, reports calls actually started, and returns the final failed
|
||||
validation result on exhaustion. A repair generation failure follows the
|
||||
ordinary generation-error category rather than becoming a validation error.
|
||||
|
||||
A successful result includes the output artifact and raw output, validation
|
||||
state, effective session ID, prompt and rendered-prompt hashes, selected
|
||||
profile and backend, effective settings, input hashes, token usage, a generated
|
||||
run identifier, and UTC timing. The same effective session reaches initial
|
||||
generation and any repair attempt through the rendered prompt. The same
|
||||
effective target, including backend identity, reaches generation and any
|
||||
repair attempt.
|
||||
effective target and presence metadata, including backend identity and direct
|
||||
credential during execution, reaches generation and every repair attempt.
|
||||
Result usage is the field-wise sum of all five usage values from the initial
|
||||
response and every completed repair response. Final raw output, artifact, and
|
||||
validation state still come from the last candidate. A repair error returns no
|
||||
partial run result or partial usage.
|
||||
|
||||
## Failure Categories
|
||||
|
||||
@@ -165,7 +202,8 @@ category. Deferred release restores the admission lease on preparation,
|
||||
generation, validation, repair, and cancellation failures.
|
||||
|
||||
Other context cancellation propagates through the invoked collaborator and is
|
||||
classified by the owning operation.
|
||||
classified by the owning operation. In particular, cancellation observed by
|
||||
validation retains the context identity through the validation error category.
|
||||
An overlong direct session is an invalid request before source loading, while
|
||||
an invalid or overlong prompt session template remains a prompt-render failure.
|
||||
An unknown selected backend, or a selected backend with no configured resolver,
|
||||
@@ -177,8 +215,9 @@ The [runner tests](../../internal/usecase/runner_test.go) own preparation order,
|
||||
selection and override precedence, the two-phase boundary, early admission,
|
||||
lease lifetime and release, direct-session resolution, schema-before-generation
|
||||
behavior, hashing, generation and validation outcomes, backend propagation,
|
||||
bounded repair, shared initial/repair capacity, credentials and redaction,
|
||||
error categories, artifact metadata, usage, and timing. The
|
||||
bounded repair progression, initial/repair request parity, cumulative usage,
|
||||
shared initial/repair capacity, credentials and redaction, error categories,
|
||||
artifact metadata, and timing. The
|
||||
[capacity subsystem document](capacity.md) identifies the focused pool,
|
||||
waiter, and wrapped-client tests.
|
||||
|
||||
|
||||
@@ -12,9 +12,27 @@ validation modes, built-in catalog, and source precedence.
|
||||
|
||||
## Prompt Definitions
|
||||
|
||||
`internal/promptdef` discovers YAML deterministically, decodes and validates
|
||||
definitions, selects an ID and optional version, and resolves file-backed
|
||||
message content within the selected operating-system or `fs.FS` source.
|
||||
`internal/promptdef` uses one source-neutral flow for prompt selection and
|
||||
normalization. That flow scans normalized YAML ID and version metadata,
|
||||
requires one strictly decoded document per file, classifies errors for the
|
||||
selected definition, detects duplicates, and normalizes the exact match.
|
||||
Small operating-system and `fs.FS` adapters own discovery, byte reads, display
|
||||
paths, content opening, and root containment. Each lookup remains a
|
||||
point-in-time scan: definitions and catalogs are not cached, and file-backed
|
||||
message content is opened only for the exact selected candidate.
|
||||
|
||||
Message roles are normalized through the shared domain owner by trimming
|
||||
Unicode whitespace and lowercasing. Only `developer`, `system`, `user`, and
|
||||
`assistant` are published; invalid roles remain selected prompt-definition
|
||||
failures rather than becoming request errors. Cache-control metadata uses the
|
||||
same shared domain normalization and defensive-copy rule.
|
||||
|
||||
Operating-system sources enforce containment against canonical roots and
|
||||
targets so symlinks cannot escape. Injected `fs.FS` sources enforce containment
|
||||
in their clean relative path namespace. A single-file source uses the selected
|
||||
prompt file's containing directory as its root. Every content path must be
|
||||
relative and is opened from its exact parsed text after a separate blank check;
|
||||
contained parent components and whitespace-bearing names remain valid.
|
||||
|
||||
Exact prompt inspection performs one point-in-time lookup through that same
|
||||
repository and validates referenced message content before returning declared
|
||||
@@ -27,39 +45,78 @@ duplicate detection, and source containment:
|
||||
|
||||
## Profiles And Built-Ins
|
||||
|
||||
`internal/profile` loads and validates execution profiles from an
|
||||
operating-system filesystem or an `fs.FS`. Its overlay repository consults the
|
||||
next repository only when the higher-precedence repository reports that a
|
||||
profile is absent. Strict YAML decoding recognizes the optional `backend`
|
||||
field, trims its value, and requires a model plus at least one non-blank
|
||||
backend or endpoint. Loading does not check registry membership because the
|
||||
available registry belongs to the assembled engine; the runner checks
|
||||
membership during preparation and exact profile inspection.
|
||||
`internal/profile` loads, locally validates, overlays, and resolves execution
|
||||
profiles from an operating-system filesystem or an `fs.FS`. A file contains
|
||||
exactly one YAML document and its trimmed YAML `id` is its only selection
|
||||
identity; filenames do not confer authority. Each point lookup reads discovered
|
||||
files once for their metadata and reuses the selected file's bytes for strict
|
||||
decoding; unrelated profiles are not fully decoded. Strict selected decoding
|
||||
recognizes `base_profile` and the optional `backend` field, trims their values,
|
||||
and permits inherited target fields only when a base is named. File-backed
|
||||
`extra_params` values are validated and defensively copied through the shared
|
||||
bounded JSON-value owner before a profile is published. OpenAI-compatible
|
||||
reserved-field policy remains with the model-client and backend-registry owners.
|
||||
|
||||
The root engine assembles profile repositories in precedence order: in-memory
|
||||
profiles, one ordinary configured source, an application fallback source, then
|
||||
the embedded built-in catalog. An explicit file or `fs.FS` profile source
|
||||
replaces `Config.ProfileDir` within the ordinary configured-source category.
|
||||
`LoadFSRepository` is the eager immutable loading boundary for internal
|
||||
catalog consumers. It discovers and strictly validates every raw profile once,
|
||||
preserves safe source metadata including explicitly present YAML fields, and
|
||||
publishes independently copied values from memory. It does not resolve profile
|
||||
inheritance. Configured consumer sources continue to use the lazy point lookup
|
||||
repositories described above.
|
||||
|
||||
Exact profile inspection performs one point-in-time lookup through those
|
||||
profile sources and checks the resolved target without reading prompt, input,
|
||||
`internal/catalog` validates the imported immutable OpenRouter and
|
||||
Rakestrawhome asset modules as one private adapter boundary. It enforces their
|
||||
manifest, layout, profile ownership, inheritance, and secret-safety rules
|
||||
before returning raw catalog sources. Root assembly uses those validated
|
||||
catalogs as the maintained lowest-precedence profile source.
|
||||
|
||||
The overlay repository consults the next repository only when the
|
||||
higher-precedence repository reports that a profile is absent. A reliably
|
||||
selected malformed profile stops fallback, while an unrelated malformed file
|
||||
does not become authoritative through its filename. Loading does not check
|
||||
backend registry membership because the available registry belongs to the
|
||||
assembled engine; the runner checks membership during preparation and exact
|
||||
profile inspection.
|
||||
|
||||
The root engine assembles one raw composite catalog in precedence order:
|
||||
in-memory profiles, one ordinary configured source, an application fallback
|
||||
source, then the maintained external catalog. An explicit file or `fs.FS` profile
|
||||
source replaces `Config.ProfileDir` within the ordinary configured-source
|
||||
category. One outer resolving repository wraps that complete raw catalog, so
|
||||
each base lookup observes the same precedence and shadowing rules.
|
||||
|
||||
The resolving repository traverses every selected chain afresh, retains no
|
||||
cache, detects cycles, limits a chain to 32 profiles, merges root-to-leaf into a
|
||||
new caller-owned value, and validates the final target before publishing it. It
|
||||
does not check backend registry membership. Exact `base_profile` syntax, merge
|
||||
rules, and consumer-visible failure behavior belong to the [framework format
|
||||
reference](../formats.md#profile-inheritance).
|
||||
|
||||
Exact profile inspection performs one point-in-time resolved lookup through
|
||||
those profile sources and checks the final target without reading prompt, input,
|
||||
or schema sources. It does not retain that lookup for a later execution.
|
||||
Prepared execution instead freezes the fully resolved target; a later ordinary
|
||||
operation performs a fresh traversal.
|
||||
|
||||
`internal/profile/builtin` embeds the maintained built-in profile catalog.
|
||||
Every embedded profile selects `openrouter` and inherits its endpoint and
|
||||
credential environment-variable name from the built-in backend registry rather
|
||||
than repeating those values. Profile loading and overlay behavior are owned by
|
||||
the [profile repository tests](../../internal/profile/repository_test.go),
|
||||
while catalog completeness, the backend-selection invariant, and duplicate IDs
|
||||
are owned by the
|
||||
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
|
||||
The maintained external catalog provides every built-in profile and its
|
||||
matching backend definition. Profile loading and overlay behavior are owned by the
|
||||
[profile repository tests](../../internal/profile/repository_test.go), while
|
||||
catalog completeness, backend selection, and duplicate IDs are owned by the
|
||||
[catalog adapter tests](../../internal/catalog/catalog_test.go).
|
||||
|
||||
## Ordinary Artifacts
|
||||
|
||||
`internal/artifact` resolves inline references and unrestricted,
|
||||
caller-selected file paths. It copies content into an artifact, records
|
||||
metadata and a content hash, applies a content-type fallback, and honors
|
||||
context cancellation.
|
||||
`internal/artifact` accepts explicitly typed inline references even when their
|
||||
body is empty. It also resolves unrestricted, caller-selected paths only when
|
||||
they identify regular operating-system files, checking that condition before
|
||||
and after opening the file. It copies content into an artifact, records
|
||||
metadata and an opaque content-equality value, and applies a content-type
|
||||
fallback.
|
||||
|
||||
Regular files are read synchronously in bounded chunks. Cancellation is
|
||||
checked before opening, before and after every read, and before publishing the
|
||||
artifact, so a canceled read never publishes partial content. The ordinary
|
||||
reader does not detach file reads into background goroutines.
|
||||
|
||||
This ordinary reader does not implement an inbound HTTP security boundary. In
|
||||
particular, it does not constrain files to an application root or impose an
|
||||
@@ -71,7 +128,17 @@ implemented reader behavior and failures.
|
||||
## Rendering
|
||||
|
||||
`internal/prompt` renders definition messages as Go templates using named
|
||||
artifacts and variables. It carries message roles, session IDs, and cache
|
||||
artifacts and variables. Within one render, each referenced artifact body is
|
||||
converted to text lazily and cached by input name for reuse across the session
|
||||
and every message; the cache is not shared across renders. Conversion uses
|
||||
bounded chunks and preserves the artifact bytes exactly.
|
||||
|
||||
Session and message parsing and execution remain synchronous. The renderer
|
||||
checks cancellation before and after each parse and execution boundary,
|
||||
between artifact conversion chunks, around each message, and before publishing
|
||||
the complete prompt. It cannot interrupt template work already in progress and
|
||||
never publishes a partial prompt after observing cancellation. It validates and
|
||||
canonicalizes message roles before carrying roles, session IDs, and cache
|
||||
control into the rendered prompt. The
|
||||
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
|
||||
|
||||
@@ -82,22 +149,35 @@ filesystem or an `fs.FS`. Invalid generated content is returned as a validation
|
||||
result; inability to load, register, or compile a schema is an operational
|
||||
error.
|
||||
|
||||
For executable preparation, the built-in validators create a frozen validation
|
||||
plan. None, basic, and JSON modes retain the effective output contract without
|
||||
source access. JSON Schema mode loads the root document, resolves and compiles
|
||||
every transitive reference during preparation, and retains the compiled
|
||||
validator. The provider-facing structured-output metadata uses that same
|
||||
captured root document.
|
||||
Every preparation operation creates one operation-local validation plan. None,
|
||||
basic, and JSON modes retain the effective output contract without source
|
||||
access. JSON Schema mode loads the root document once, resolves and compiles
|
||||
each transitive reference, and retains the compiled validator. Schema compiler
|
||||
resources use canonical escaped file or private-scheme URLs; loaders decode
|
||||
their paths once and enforce the configured source boundary. The
|
||||
provider-facing structured-output metadata uses the root document captured by
|
||||
the same plan.
|
||||
|
||||
`PrepareExecution` also completes prompt and profile selection, artifact
|
||||
loading and hashing, session and message rendering, and target resolution.
|
||||
`RunPrepared` uses the retained source-derived state and validation plan; it
|
||||
does not reopen prompt, profile, input, or schema sources and does not rerender
|
||||
the request. By contrast, ordinary `Prepare` produces a preparation value only:
|
||||
a later `Run` performs its own source resolution and preparation.
|
||||
Schema preparation and execution remain synchronous. Promptkit checks
|
||||
cancellation before and after source resolution, JSON decoding, compilation,
|
||||
and validation, and between bounded schema-read chunks. Once cancellation is
|
||||
observed it returns the context error without publishing a partial plan or
|
||||
validation result, even when a compiler or validator has just returned a
|
||||
different error or a successful result. An `fs.FS` method or JSON Schema
|
||||
dependency call already in progress cannot be preempted; Promptkit waits for
|
||||
that call to return and then gives cancellation precedence. Validation does
|
||||
not detach dependency work into background goroutines.
|
||||
|
||||
`Prepare` discards its validation plan after returning metadata. `Run` retains
|
||||
its plan for initial and repaired-output validation, then discards it with the
|
||||
operation. `PrepareExecution` retains the plan in its private frozen payload;
|
||||
`RunPrepared` uses that plan without reopening prompt, profile, input, or
|
||||
schema sources or rerendering the request. A later ordinary `Run` always
|
||||
performs fresh source resolution and preparation.
|
||||
|
||||
The [validator tests](../../internal/validate/standard_validator_test.go) own
|
||||
basic, JSON, JSON Schema, source resolution, schema loading, compilation,
|
||||
frozen-reference behavior, and content-failure behavior. Prepared execution
|
||||
frozen-reference behavior, content-failure behavior, and the synchronous
|
||||
cancellation boundary. Prepared execution
|
||||
orchestration is owned by the
|
||||
[use-case tests](../../internal/usecase/prepared_execution_test.go).
|
||||
|
||||
@@ -19,24 +19,24 @@ results, public values, extension interfaces, profiles, and error sentinels.
|
||||
|
||||
The implemented internal components consist of:
|
||||
|
||||
- `internal/domain`, which owns framework data values shared by later internal
|
||||
components;
|
||||
- `internal/domain`, which owns framework data values and source-neutral
|
||||
invariants shared by later internal components;
|
||||
- `internal/backend`, which owns validated immutable OpenAI-compatible backend
|
||||
definitions and the built-in OpenRouter definition;
|
||||
definitions;
|
||||
- `internal/capacity`, which owns engine-local bounded run admission and
|
||||
model-generation scheduling for limited backends;
|
||||
- `internal/defaults`, which owns application-neutral framework defaults and
|
||||
constructs the default execution target;
|
||||
- `internal/filecatalog`, which discovers YAML files and provides source-path
|
||||
helpers for filesystem and `fs.FS` consumers;
|
||||
- `internal/catalog`, which validates immutable external maintained-catalog
|
||||
assets before they are eligible for engine assembly;
|
||||
- `internal/jsonvalue`, which validates and defensively copies JSON-compatible
|
||||
extra-parameter trees;
|
||||
- `internal/promptdef`, which loads and validates prompt definitions from
|
||||
filesystem and `fs.FS` sources;
|
||||
- `internal/profile`, which loads, validates, and overlays execution profiles
|
||||
from filesystem and `fs.FS` sources;
|
||||
- `internal/profile/builtin`, which embeds the built-in execution profile
|
||||
catalog;
|
||||
- `internal/prompt`, which renders prompt messages from Go templates;
|
||||
- `internal/artifact`, which resolves ordinary inline and unrestricted
|
||||
caller-selected file references;
|
||||
@@ -54,13 +54,15 @@ packages or participate in internal assembly.
|
||||
The root facade assembles one immutable backend registry, one capacity manager,
|
||||
the internal repositories, renderer, validator, outbound client, and use-case
|
||||
runner while translating public values and errors at the library boundary. The
|
||||
registry contains built-ins plus validated engine-scoped consumer additions.
|
||||
registry contains validated maintained definitions plus engine-scoped consumer
|
||||
additions.
|
||||
The facade constructs the capacity manager from the registry's immutable
|
||||
policy snapshot, wraps the selected built-in or injected model client, and
|
||||
supplies bounded admission to the runner. The defaults and renderer depend on
|
||||
the domain model. Prompt-definition and profile repositories use the domain
|
||||
model, file catalog, and YAML decoder. The built-in profile repository supplies
|
||||
an embedded `fs.FS` to the profile package. Artifact reading uses the domain
|
||||
model, file catalog, and YAML decoder. The catalog adapter validates imported
|
||||
immutable maintained data before root assembly supplies it to the profile
|
||||
package. Artifact reading uses the domain
|
||||
model and application-neutral defaults. Validation uses the domain model, file
|
||||
catalog, and JSON Schema implementation. The model client uses the domain
|
||||
model, application-neutral defaults, and an injected or standard-library HTTP
|
||||
@@ -92,6 +94,13 @@ coordinates internal components and adapts the supported public extension
|
||||
interfaces to narrow internal abstractions. Internal components must not depend
|
||||
on consumers or on Scriptorium.
|
||||
|
||||
`internal/domain` owns source-neutral invariants for values shared across
|
||||
multiple input and execution boundaries, including execution-setting bounds,
|
||||
OpenAI-compatible base endpoints, session identifiers, and output-contract
|
||||
legality. Callers retain source parsing, required-field rules, other
|
||||
source-specific normalization, defaulting, error classification, and policy
|
||||
specific to their own boundary.
|
||||
|
||||
## Repository And Consumer Boundary
|
||||
|
||||
Scriptorium is a downstream application that consumes Promptkit through
|
||||
|
||||
@@ -50,19 +50,18 @@ Examples of appropriate seams include clocks, randomness, subprocesses, remote A
|
||||
## Test execution requirements
|
||||
|
||||
Promptkit currently uses maintainer-run validation rather than hosted CI.
|
||||
Maintainers run the repository-documented test, vet, build, formatting,
|
||||
documentation-link, and repository-hygiene checks before accepting changes.
|
||||
Maintainers run the complete local workflow in the
|
||||
[development guide](../development.md#maintainer-validation) before accepting
|
||||
changes. That guide is the canonical owner of exact commands, formatting,
|
||||
documentation-link validation, and repository-hygiene checks.
|
||||
Introducing hosted CI later would supplement, not silently redefine, this
|
||||
documented validation model.
|
||||
|
||||
The complete test sequence includes ordinary and race-enabled package tests.
|
||||
The maintained offline consumer workflow is also run from the repository root:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go test -race ./...
|
||||
go run ./examples/go-library/prepare
|
||||
```
|
||||
Maintainer validation must include ordinary and race-enabled package tests,
|
||||
static analysis, a complete build, and execution of both maintained offline
|
||||
consumer examples. The preparation example protects assembled preparation and
|
||||
inspection behavior. The execution example separately protects assembled
|
||||
`Run`, injected-client, validation, usage, and result behavior.
|
||||
|
||||
Tests in the default suite must be deterministic, offline, and independent of
|
||||
real credentials. They must not invoke paid APIs, use live network
|
||||
@@ -88,8 +87,9 @@ Use each test type where it protects a distinct risk:
|
||||
interaction, while replacing live or nondeterministic external boundaries.
|
||||
- External-package root tests exercise the public facade as a Go consumer,
|
||||
while internal package tests own focused implementation behavior.
|
||||
- The maintained offline preparation example protects one representative
|
||||
assembled consumer workflow without contacting a model provider.
|
||||
- The maintained offline preparation and execution examples protect distinct
|
||||
representative assembled consumer workflows without contacting a model
|
||||
provider.
|
||||
- Fixtures should be minimal, synthetic, versioned with the behavior they
|
||||
exercise, and free of credentials or private data.
|
||||
- Golden files are appropriate only when the complete output is intentionally
|
||||
|
||||
@@ -106,48 +106,12 @@ gitea.maximumdirect.net/eric/promptkit 1.25.5
|
||||
promptkit gitea.maximumdirect.net/eric/promptkit
|
||||
```
|
||||
|
||||
Run the complete maintainer validation required by the
|
||||
[development guide](development.md):
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go test -race ./...
|
||||
go vet ./...
|
||||
go build ./...
|
||||
go run ./examples/go-library/prepare
|
||||
```
|
||||
|
||||
Check every tracked Go file. This command must produce no output:
|
||||
|
||||
```sh
|
||||
unformatted=$(
|
||||
git ls-files '*.go' |
|
||||
while IFS= read -r go_file
|
||||
do
|
||||
gofmt -l "$go_file"
|
||||
done
|
||||
)
|
||||
test -z "$unformatted"
|
||||
```
|
||||
|
||||
Follow every maintained Markdown link and confirm that its local or published
|
||||
target exists. Review the repository for generated binaries, test or coverage
|
||||
output, credentials, template residue, downloaded assets, and other files that
|
||||
do not belong in source control.
|
||||
|
||||
Recheck module and repository hygiene, whitespace, and the clean checkout:
|
||||
|
||||
```sh
|
||||
test -z "$(git ls-files go.work go.work.sum)"
|
||||
test ! -e vendor
|
||||
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||
then
|
||||
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||
exit 1
|
||||
fi
|
||||
git diff --check
|
||||
test -z "$(git status --porcelain)"
|
||||
```
|
||||
As a release prerequisite, run the complete
|
||||
[maintainer validation workflow](development.md#maintainer-validation) against
|
||||
the clean candidate. Do not substitute a partial command list: the development
|
||||
guide owns the tests, race checks, analysis, build, both offline examples,
|
||||
formatting, Markdown links, generated-output and credential review, and
|
||||
repository hygiene. Record the successful workflow result with the candidate.
|
||||
|
||||
## Write The Release Note
|
||||
|
||||
@@ -182,6 +146,13 @@ grep -F 'Consumer action:' "$RELEASE_NOTES_FILE"
|
||||
Inspect the complete message and confirm that it accurately records the
|
||||
compatibility impact, public API changes, and required consumer action.
|
||||
|
||||
When a release introduces or updates the maintained external catalogs, state
|
||||
that it selects two independently versioned data dependencies, preserves the
|
||||
public API and configuration, requires no consumer migration, and guarantees
|
||||
only the catalog versions selected and tested by that Promptkit release. Link
|
||||
to the current architecture and format documentation rather than duplicating
|
||||
catalog contracts in the release note.
|
||||
|
||||
## Create And Inspect The Tag
|
||||
|
||||
Run the candidate guard again immediately before tag creation. This ensures
|
||||
|
||||
131
docs/releases/v0.6.0.md
Normal file
131
docs/releases/v0.6.0.md
Normal file
@@ -0,0 +1,131 @@
|
||||
# Promptkit v0.6.0
|
||||
|
||||
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||
changes from `v0.5.0` to `v0.6.0`. The annotated `v0.6.0` tag is the
|
||||
authoritative release record. Exact current contracts belong to the linked
|
||||
GoDoc and durable documentation.
|
||||
|
||||
## Summary
|
||||
|
||||
`v0.6.0` is a broad correctness, safety, efficiency, and maintainability
|
||||
release. It does not add or remove public declarations. The release:
|
||||
|
||||
- centralizes shared execution-setting, output-contract, endpoint, and
|
||||
JSON-compatible-value rules;
|
||||
- unifies prompt repository behavior and avoids unnecessary prompt and profile
|
||||
decoding;
|
||||
- bounds consumer-controlled JSON trees and successful provider responses;
|
||||
- hardens prompt content paths, artifact files, provider URLs, JSON framing,
|
||||
and error propagation;
|
||||
- reuses compiled schema plans and rendered artifact text within an operation;
|
||||
and
|
||||
- improves cancellation behavior, prepared-value ownership, deterministic
|
||||
transport testing, and maintainer validation.
|
||||
|
||||
## Compatibility
|
||||
|
||||
No public declaration was added, removed, or changed. Ordinary valid `v0.5.0`
|
||||
configurations and requests should continue to compile and behave as before.
|
||||
|
||||
The release intentionally rejects or reports several inputs that were
|
||||
previously accepted, altered, or misclassified:
|
||||
|
||||
- execution settings must be finite, within their documented ranges, and safe
|
||||
to convert to Go durations;
|
||||
- output formats, validation modes, repair counts, and JSON Schema dependencies
|
||||
are validated consistently;
|
||||
- file-backed prompt and profile identity comes from normalized YAML metadata,
|
||||
not filenames;
|
||||
- prompt `content_file` values must be exact relative paths contained by their
|
||||
configured source root;
|
||||
- built-in file artifacts must resolve to regular files;
|
||||
- selected provider endpoints must be absolute HTTP or HTTPS URLs without user
|
||||
information, query strings, or fragments;
|
||||
- JSON documents and successful provider responses must contain exactly one
|
||||
value, and successful provider bodies are limited to 16 MiB; and
|
||||
- excessively deep or expansive JSON-compatible values fail with ordinary
|
||||
validation errors.
|
||||
|
||||
These are compatibility corrections and safety boundaries rather than new
|
||||
consumer configuration requirements. Consumers relying on an invalid or
|
||||
ambiguous input should correct that input before upgrading.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Update the module dependency with:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.6.0
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
Run the consuming project's ordinary and race-enabled tests after upgrading.
|
||||
Applications with custom prompt/profile sources, local provider endpoints,
|
||||
unusual artifact paths, or assertions over provider error identities should
|
||||
pay particular attention to the compatibility notes below.
|
||||
|
||||
## Source Loading And Identity
|
||||
|
||||
Prompt definitions now share one source-neutral selection and normalization
|
||||
flow across operating-system and `fs.FS` sources. YAML `id` and `version`
|
||||
metadata are authoritative; filenames do not create a second identity system.
|
||||
Only selected content bodies are loaded, malformed unrelated definitions do
|
||||
not shadow valid exact matches, and per-file read failures are reported as
|
||||
prompt-load failures rather than false absence.
|
||||
|
||||
File-backed profiles likewise use normalized YAML IDs, reuse their metadata
|
||||
read for selected strict decoding, and avoid fully decoding unrelated files.
|
||||
Selected malformed definitions remain authoritative and do not silently fall
|
||||
through to a lower-precedence source.
|
||||
|
||||
Prompt `content_file` paths are opened exactly as declared after a separate
|
||||
blank check. They must remain relative to and contained by the configured
|
||||
prompt source root, including across operating-system symlinks.
|
||||
|
||||
See the [framework source and identity reference](../formats.md) and
|
||||
[internal source overview](../internal/sources.md) for the current contracts.
|
||||
|
||||
## Validation, Cancellation, And Efficiency
|
||||
|
||||
JSON Schema documents preserve exact JSON-number representations. Schema
|
||||
resource URLs safely escape legal filesystem names, and each operation loads
|
||||
and compiles its schema graph once. `Run` and prepared execution reuse that
|
||||
operation-local plan; Promptkit does not introduce a cross-operation cache.
|
||||
|
||||
Artifact reading, rendering, schema loading, compilation, and validation now
|
||||
check cancellation at the synchronous boundaries Promptkit controls. Rendering
|
||||
memoizes each artifact's text within one render operation, while plain JSON
|
||||
validation avoids materializing an unnecessary generic tree.
|
||||
|
||||
The shared JSON-compatible-value owner now limits nesting and produced work so
|
||||
unsafe consumer-controlled structures return errors instead of risking
|
||||
unbounded recursion or allocation. See the
|
||||
[architecture policy](../policy/architecture.md) for invariant ownership and
|
||||
the [format reference](../formats.md) for validation behavior.
|
||||
|
||||
## Provider Transport Hardening
|
||||
|
||||
OpenAI-compatible endpoints are parsed and composed structurally, including
|
||||
nested base paths. Underlying transport cancellation and deadline errors remain
|
||||
discoverable with `errors.Is` through Promptkit's generation error category.
|
||||
|
||||
Successful provider bodies are read with a fixed 16 MiB bound and must contain
|
||||
exactly one JSON response object followed only by whitespace. Oversized,
|
||||
truncated, malformed, or multiply framed responses fail without returning a
|
||||
partial result. See the
|
||||
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
|
||||
for the canonical request, endpoint, error, and response behavior.
|
||||
|
||||
## Public API Changes
|
||||
|
||||
None.
|
||||
|
||||
## Consumer Action
|
||||
|
||||
- Correct any configuration or request that depends on the formerly permissive
|
||||
cases described under Compatibility.
|
||||
- Confirm custom local provider endpoints are absolute HTTP or HTTPS base URLs
|
||||
without credentials, queries, or fragments.
|
||||
- Confirm prompt content paths remain within their configured source root and
|
||||
file artifacts resolve to regular files.
|
||||
- Run ordinary and race-enabled consumer tests after updating the dependency.
|
||||
155
docs/releases/v0.7.0.md
Normal file
155
docs/releases/v0.7.0.md
Normal file
@@ -0,0 +1,155 @@
|
||||
# Promptkit v0.7.0
|
||||
|
||||
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||
changes from `v0.6.0` to `v0.7.0`. The annotated `v0.7.0` tag is the
|
||||
authoritative release record. Exact current contracts belong to the linked
|
||||
GoDoc and durable documentation.
|
||||
|
||||
## Summary
|
||||
|
||||
`v0.7.0` expands provider integration and profile composition while making
|
||||
credential and generation-failure handling more flexible:
|
||||
|
||||
- Promptkit now includes the `rakestrawhome` backend and its Gemma profile;
|
||||
- built-in generation failures expose bounded structured provider details;
|
||||
- an unavailable optional API-key environment source no longer prevents a
|
||||
request from reaching an upstream that permits unauthenticated access; and
|
||||
- profiles can inherit from and selectively refine another profile.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This release adds public declarations and fields but removes none. Existing
|
||||
keyed configuration literals and ordinary `errors.Is` handling continue to
|
||||
work.
|
||||
|
||||
Adding `BaseProfileID` to `Profile` and `OpenAICompatibleProfileConfig` changes
|
||||
their struct shape. Consumers using positional composite literals for either
|
||||
type must convert them to keyed literals. Existing keyed literals require no
|
||||
change.
|
||||
|
||||
The `rakestrawhome` backend ID is now built in and reserved. A consumer that
|
||||
previously registered that exact ID with `WithBackend` must remove its manual
|
||||
registration before upgrading. Other custom backend registrations are
|
||||
unchanged.
|
||||
|
||||
When an optional backend, profile, or request `APIKeyEnv` is unset, empty, or
|
||||
whitespace-only, the built-in client now omits `Authorization` and sends the
|
||||
request. Previously this condition could fail before transport. Set
|
||||
`Profile.APIKeyRequired` when missing credentials must remain a local
|
||||
preflight error.
|
||||
|
||||
Provider non-success responses continue to match `ErrLLMGenerate`. Their
|
||||
rendered wording is not a compatibility contract; consumers can now use
|
||||
`errors.As` with `*GenerationError` when structured status information is
|
||||
needed.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Update the module dependency with:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.7.0
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
Remove any manual `rakestrawhome` backend registration, convert positional
|
||||
profile literals to keyed literals, and run the consuming project's ordinary
|
||||
and race-enabled tests.
|
||||
|
||||
## Rakestrawhome Built-In Backend And Profile
|
||||
|
||||
Every engine now includes the reserved `rakestrawhome` backend, identified by
|
||||
`BackendRakestrawHome`. The built-in `rakestrawhome-gemma-4-31b` profile
|
||||
selects that backend. Consumers can use the maintained endpoint, credential,
|
||||
capacity, and model defaults without registering either definition themselves.
|
||||
|
||||
See the [built-in backend and profile catalogs](../formats.md#built-in-backends)
|
||||
and the [consumer adoption example](../consumers/pkg-promptkit.md#use-the-rakestrawhome-built-in-profile)
|
||||
for the current contracts.
|
||||
|
||||
## Structured Generation Errors
|
||||
|
||||
Non-2xx responses from the built-in OpenAI-compatible client now return an
|
||||
immutable `*GenerationError`. Consumers can inspect the HTTP status and any
|
||||
safely extracted provider code, type, or message while retaining the ordinary
|
||||
generation-error category:
|
||||
|
||||
```go
|
||||
var generationErr *promptkit.GenerationError
|
||||
if errors.As(err, &generationErr) {
|
||||
status := generationErr.StatusCode()
|
||||
_ = status
|
||||
}
|
||||
```
|
||||
|
||||
Provider fields are bounded and normalized but remain untrusted and may
|
||||
contain sensitive request or schema details. Default and Go-syntax formatting
|
||||
omit those fields. Applications must apply their own disclosure policy before
|
||||
logging or presenting accessor values.
|
||||
|
||||
See the [`GenerationError` GoDoc](../../generation_error.go), the
|
||||
[consumer error-handling guide](../consumers/pkg-promptkit.md#handle-errors),
|
||||
and the [OpenAI-compatible response contract](../integrations/openai-compatible-chat.md#response-handling).
|
||||
|
||||
## Optional Credential Sources
|
||||
|
||||
`APIKeyEnv` names an optional environment lookup source unless the selected
|
||||
profile explicitly sets `APIKeyRequired`. When neither a direct request key nor
|
||||
a usable environment value exists, the built-in client omits the bearer header
|
||||
and handles the upstream response normally. This supports local and other
|
||||
OpenAI-compatible providers that permit unauthenticated requests without
|
||||
hiding an authentication error returned by a provider that requires one.
|
||||
|
||||
The [credential format reference](../formats.md#credentials), the
|
||||
[`Backend` GoDoc](../../backends.go), the
|
||||
[`ExecutionTargetOverride` GoDoc](../../types.go), and the
|
||||
[authentication integration contract](../integrations/openai-compatible-chat.md#authentication)
|
||||
define the current precedence and availability rules.
|
||||
|
||||
## Profile Inheritance
|
||||
|
||||
YAML profiles can name one parent with `base_profile`; in-memory profiles use
|
||||
`Profile.BaseProfileID`, and `OpenAICompatibleProfileConfig` forwards the same
|
||||
field. A profile can act as an application-owned alias of a built-in or refine
|
||||
selected inherited settings:
|
||||
|
||||
```go
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "weather-light",
|
||||
BaseProfileID: "deepseek-4-flash",
|
||||
ReasoningEffort: "high",
|
||||
})
|
||||
```
|
||||
|
||||
Base lookup observes the existing source precedence. Chains are linear,
|
||||
cycle-safe, and resolved afresh for ordinary operations. Prepared execution
|
||||
freezes the fully resolved target. The selected leaf ID remains public while
|
||||
effective execution settings reflect the resolved chain.
|
||||
|
||||
See the [profile inheritance format reference](../formats.md#profile-inheritance),
|
||||
the [consumer alias example](../consumers/pkg-promptkit.md#alias-a-built-in-profile),
|
||||
and the [`Profile` GoDoc](../../types.go) for exact merge and validation
|
||||
behavior.
|
||||
|
||||
## Public API Changes
|
||||
|
||||
The release adds:
|
||||
|
||||
- `BackendRakestrawHome`;
|
||||
- `GenerationError`, including `StatusCode`, `ProviderCode`, `ProviderType`,
|
||||
`ProviderMessage`, `Error`, `GoString`, and `Unwrap`;
|
||||
- `Profile.BaseProfileID`; and
|
||||
- `OpenAICompatibleProfileConfig.BaseProfileID`.
|
||||
|
||||
No public declaration was removed.
|
||||
|
||||
## Consumer Action
|
||||
|
||||
- Remove a manual backend registration whose ID is exactly `rakestrawhome`.
|
||||
- Convert positional `Profile` or `OpenAICompatibleProfileConfig` literals to
|
||||
keyed literals.
|
||||
- Set `Profile.APIKeyRequired` where a missing credential must fail locally
|
||||
instead of reaching the provider unauthenticated.
|
||||
- Treat `GenerationError` provider fields as untrusted and potentially
|
||||
sensitive when adopting the new accessors.
|
||||
- Run consumer ordinary and race-enabled tests after updating the module.
|
||||
109
docs/releases/v0.8.0.md
Normal file
109
docs/releases/v0.8.0.md
Normal file
@@ -0,0 +1,109 @@
|
||||
# Promptkit v0.8.0
|
||||
|
||||
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||
changes from `v0.7.0` to `v0.8.0`. The annotated `v0.8.0` tag is the
|
||||
authoritative release record. Exact current contracts belong to the linked
|
||||
GoDoc and durable documentation.
|
||||
|
||||
## Summary
|
||||
|
||||
`v0.8.0` activates Promptkit's bounded output-repair workflow:
|
||||
|
||||
- failed nonempty-text, JSON, and JSON Schema validation can make a limited
|
||||
number of corrective model calls;
|
||||
- corrective calls preserve the original rendered conversation, effective
|
||||
target, session, structured-output contract, and backend capacity policy;
|
||||
- results report cumulative usage and the number of corrective calls actually
|
||||
made; and
|
||||
- explicitly empty OpenAI-compatible response content now reaches output
|
||||
validation instead of being classified as a malformed provider envelope.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This release adds no public declarations or fields and removes none. Existing
|
||||
source code remains source-compatible.
|
||||
|
||||
The behavior of the existing `OutputContract.RepairAttempts` field and prompt
|
||||
YAML `repair_attempts` field has changed. A positive value now authorizes real
|
||||
additional model calls after eligible validation failures; earlier releases
|
||||
accepted the field but the public engine remained single-pass. Consumers that
|
||||
set a positive value should expect additional latency, token usage, and
|
||||
provider cost when repair is needed.
|
||||
|
||||
Repair budgets must now be between zero and three. A positive budget requires
|
||||
`basic`, `json`, or `json_schema` validation. Values above three and a positive
|
||||
budget paired with `none` are invalid contracts rather than ignored settings.
|
||||
|
||||
An explicitly present empty or whitespace-only string returned by the built-in
|
||||
OpenAI-compatible client is now a completed generation candidate. `none`
|
||||
validation permits it, while `basic`, `json`, and `json_schema` classify it
|
||||
under their ordinary validation rules and may repair it when configured.
|
||||
Missing, `null`, or non-string content remains a malformed provider response.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Update the module dependency with:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.8.0
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
Review every prompt definition and request override that sets a positive repair
|
||||
budget. Use zero or omit the field to retain single-pass execution. Ensure each
|
||||
positive budget is no greater than three and uses an eligible validation mode,
|
||||
then run the consuming project's ordinary and race-enabled tests.
|
||||
|
||||
## Bounded Output Repair
|
||||
|
||||
`repair_attempts` counts corrective calls in addition to the initial model
|
||||
call. Promptkit validates each completed candidate, stops at the first valid
|
||||
one, and never exceeds the configured bound. If every candidate remains
|
||||
invalid, the run completes successfully with the final candidate and its
|
||||
failed validation result rather than returning an operational error.
|
||||
|
||||
Each correction starts from the original rendered messages and includes only
|
||||
the latest invalid candidate and latest validation diagnostics. JSON Schema
|
||||
mode retains the provider-native structured-output request as its first line of
|
||||
defense. Promptkit performs only deterministic structural validation; a valid
|
||||
response is not necessarily factual or correct for an application's domain.
|
||||
|
||||
Usage in the final result is cumulative across the initial response and every
|
||||
completed corrective response. `ValidationResult.RepairAttempts` reports the
|
||||
number of corrective calls actually made. Corrective generation failures use
|
||||
the same public generation-error categories and structured provider details as
|
||||
an initial generation failure.
|
||||
|
||||
See the [output-contract format reference](../formats.md#output-contract), the
|
||||
[consumer repair example](../consumers/pkg-promptkit.md#repair-a-structured-result),
|
||||
and the [`OutputContract` and `ValidationResult` GoDoc](../../types.go) for the
|
||||
current contracts.
|
||||
|
||||
## Explicit Empty Content
|
||||
|
||||
The built-in OpenAI-compatible client now distinguishes an explicitly present
|
||||
empty string from a missing or malformed `content` field. This aligns built-in
|
||||
and injected clients by letting the selected output contract decide whether an
|
||||
empty candidate is acceptable, invalid, or eligible for repair.
|
||||
|
||||
See the
|
||||
[OpenAI-compatible response contract](../integrations/openai-compatible-chat.md#response-handling)
|
||||
for the exact envelope behavior.
|
||||
|
||||
## Public API Changes
|
||||
|
||||
None. This release activates and tightens the documented behavior of existing
|
||||
fields.
|
||||
|
||||
## Consumer Action
|
||||
|
||||
- Remove or set `repair_attempts` to zero where execution must remain
|
||||
single-pass.
|
||||
- Keep every positive repair budget at three or fewer and pair it with
|
||||
`basic`, `json`, or `json_schema` validation.
|
||||
- Account for additional latency, usage, and provider cost when enabling
|
||||
repair.
|
||||
- Continue checking the returned validation status because bounded repair can
|
||||
exhaust without producing a valid candidate.
|
||||
- Review workflows that previously treated explicit empty provider content as
|
||||
a generation error.
|
||||
125
docs/releases/v0.9.0.md
Normal file
125
docs/releases/v0.9.0.md
Normal file
@@ -0,0 +1,125 @@
|
||||
# Promptkit v0.9.0
|
||||
|
||||
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||
changes from `v0.8.0` to `v0.9.0`. The annotated `v0.9.0` tag is the
|
||||
authoritative release record. Exact current contracts belong to the linked
|
||||
GoDoc and durable documentation.
|
||||
|
||||
## Summary
|
||||
|
||||
`v0.9.0` adds stateless request-message composition and separates maintained
|
||||
provider data from Promptkit's core implementation:
|
||||
|
||||
- callers can append already-rendered messages to a configured prompt for
|
||||
application-owned conversations and semantic correction workflows;
|
||||
- the public package now publishes constants for the four supported text-chat
|
||||
roles, and prompt definitions use the same normalized role vocabulary; and
|
||||
- the OpenRouter and Rakestrawhome backend/profile catalogs now come from two
|
||||
independently versioned Go module dependencies.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This release adds one field and four constants to the public API and removes no
|
||||
public declarations. Existing keyed `RunRequest` literals that omit
|
||||
`AppendedMessages` retain their behavior. Adding the field changes the struct
|
||||
shape, so consumers using positional `RunRequest` literals must convert them
|
||||
to keyed literals.
|
||||
|
||||
Message roles in prompt definitions are now trimmed, lowercased, and required
|
||||
to be `developer`, `system`, `user`, or `assistant`. Definitions using another
|
||||
role that earlier releases accepted as an arbitrary nonblank string now fail
|
||||
prompt loading. In particular, the text-only message contract does not support
|
||||
`tool` or the deprecated `function` role. Otherwise valid roles with different
|
||||
case or surrounding whitespace are normalized rather than rejected.
|
||||
|
||||
The catalog extraction preserves Promptkit's public API, built-in backend and
|
||||
profile IDs, configuration, precedence, credential handling, and capacity
|
||||
behavior. Consumers do not import or register either catalog themselves, and
|
||||
no configuration migration is required. Promptkit now selects two
|
||||
independently versioned data dependencies and guarantees only the catalog
|
||||
versions selected and tested by this Promptkit release.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Update the module dependency with:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.9.0
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
Convert any positional `RunRequest` literals to keyed literals. Review prompt
|
||||
definitions for unsupported roles, then run the consuming project's ordinary
|
||||
and race-enabled tests.
|
||||
|
||||
## Appended Request Messages
|
||||
|
||||
`RunRequest.AppendedMessages` accepts caller-owned `RenderedMessage` values
|
||||
that Promptkit validates, copies, and appends after every rendered
|
||||
prompt-definition message in caller order. Promptkit does not template this
|
||||
content, retain conversation state between calls, impose a retry policy, or
|
||||
apply a message-count, byte-size, token, or context-window limit. Upstream
|
||||
rejections continue through the ordinary generation-error boundary.
|
||||
|
||||
This primitive supports application-owned conversation continuations and
|
||||
domain-aware correction loops while preserving Promptkit's existing
|
||||
preparation, hashing, prepared-execution, structural repair, backend-capacity,
|
||||
credential, and cancellation behavior. Appended content can include sensitive
|
||||
model output or application feedback; prepared values expose the complete
|
||||
effective messages by design, while default request formatting reports only
|
||||
the appended-message count.
|
||||
|
||||
See the
|
||||
[consumer appended-message example](../consumers/pkg-promptkit.md#append-already-rendered-messages),
|
||||
the [`RunRequest`, `RenderedMessage`, and `CacheControl` GoDoc](../../types.go),
|
||||
and the [OpenAI-compatible request contract](../integrations/openai-compatible-chat.md#request-body)
|
||||
for current behavior.
|
||||
|
||||
## Supported Message Roles
|
||||
|
||||
The new `RoleDeveloper`, `RoleSystem`, `RoleUser`, and `RoleAssistant`
|
||||
constants identify the complete role vocabulary accepted by Promptkit's
|
||||
text-chat message model. The same validation and normalization now apply to
|
||||
prompt-definition messages and request-supplied appended messages. Promptkit
|
||||
does not translate between roles; provider- or model-specific rejection of an
|
||||
otherwise supported role remains an upstream generation error.
|
||||
|
||||
See the [message format reference](../formats.md#messages-and-templates) for
|
||||
the canonical prompt-definition contract.
|
||||
|
||||
## Independently Versioned Backend Catalogs
|
||||
|
||||
Promptkit imports immutable catalog data for its maintained OpenRouter and
|
||||
Rakestrawhome backends and profiles. Engine construction validates and
|
||||
assembles both catalogs behind the existing built-in registry and profile
|
||||
precedence rules. Promptkit no longer keeps duplicate embedded profile assets
|
||||
or hard-coded definitions for those maintained backends.
|
||||
|
||||
The module versions in Promptkit's `go.mod` identify the catalog releases
|
||||
tested with this release. The [built-in backend and profile format
|
||||
reference](../formats.md#built-in-backends) remains the canonical consumer
|
||||
contract, while the [internal source documentation](../internal/sources.md#profiles-and-built-ins)
|
||||
describes the dependency boundary.
|
||||
|
||||
## Public API Changes
|
||||
|
||||
The release adds:
|
||||
|
||||
- `RunRequest.AppendedMessages`;
|
||||
- `RoleDeveloper`;
|
||||
- `RoleSystem`;
|
||||
- `RoleUser`; and
|
||||
- `RoleAssistant`.
|
||||
|
||||
No public declaration was removed.
|
||||
|
||||
## Consumer Action
|
||||
|
||||
- Convert positional `RunRequest` literals to keyed literals.
|
||||
- Replace unsupported prompt-definition roles with an appropriate supported
|
||||
role, or keep richer tool-call protocols in an application-owned client.
|
||||
- Treat appended messages and prepared effective messages according to the
|
||||
application's sensitive-data policy.
|
||||
- Do not add direct catalog imports or registration calls; existing Promptkit
|
||||
construction and configuration remain correct.
|
||||
- Run consumer ordinary and race-enabled tests after updating the module.
|
||||
@@ -59,24 +59,3 @@ if supported at all, should be an explicit option and must not expose secrets.
|
||||
|
||||
Reconsider this idea after downstream use of `InspectPrompt`,
|
||||
`InspectProfile`, and fixture-based preparation demonstrates a concrete gap.
|
||||
|
||||
### Structured Generation Errors
|
||||
|
||||
**Reason for deferral:** Existing `ErrLLMGenerate` classification, preserved
|
||||
injected-client errors, and prepared execution details currently provide the
|
||||
necessary failure boundary. A typed error should wait for stronger downstream
|
||||
demand and a transport-neutral field design.
|
||||
|
||||
Promptkit could expose safe structured generation context through
|
||||
`errors.As` while preserving `errors.Is(err, ErrLLMGenerate)`. Potential
|
||||
fields include the selected backend ID and model plus an optional HTTP status
|
||||
when the built-in OpenAI-compatible transport supplies one.
|
||||
|
||||
The design must not expose provider response bodies, endpoints, credential
|
||||
environment names or values, request content, or generated content. It should
|
||||
not duplicate prompt and profile provenance already available from a prepared
|
||||
execution, and it must preserve the identity of errors returned by injected
|
||||
clients. Retry and backoff policy remains a consumer responsibility.
|
||||
|
||||
Reconsider this idea when consumers need structured generation diagnostics
|
||||
beyond the existing sentinel, wrapped client error, and preparation record.
|
||||
|
||||
@@ -38,7 +38,8 @@ consumers.
|
||||
|
||||
## Ideas
|
||||
|
||||
No ideas currently await selection.
|
||||
No ideas are currently awaiting selection. Active feature work belongs in its
|
||||
focused roadmap rather than this catalog.
|
||||
|
||||
## Entry Format
|
||||
|
||||
|
||||
108
engine.go
108
engine.go
@@ -11,14 +11,16 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter"
|
||||
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome"
|
||||
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/catalog"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile/builtin"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||
@@ -53,9 +55,9 @@ var (
|
||||
// an execution profile or resolve its backend, except for the profile
|
||||
// not-found case represented by ErrProfileNotFound.
|
||||
ErrProfileLoad = errors.New("failed to load execution profile")
|
||||
// ErrAPIKeyEnvMissing identifies an APIKeyEnv whose environment variable is
|
||||
// unset or empty when no direct RunRequest.APIKey takes precedence. Such an
|
||||
// error also matches ErrInvalidRequest.
|
||||
// ErrAPIKeyEnvMissing identifies an explicitly required APIKeyEnv whose
|
||||
// environment variable is unset or empty after direct RunRequest.APIKey
|
||||
// precedence is applied. Such an error also matches ErrInvalidRequest.
|
||||
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
||||
// ErrArtifactLoad identifies a failure to resolve an input artifact. Errors
|
||||
// returned by an injected ArtifactReader remain available through errors.Is.
|
||||
@@ -69,8 +71,9 @@ var (
|
||||
// request, an LLM or provider rate-limit response, or ErrLLMGenerate.
|
||||
ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
||||
// ErrLLMGenerate identifies a model-client failure or a nil successful
|
||||
// response. Errors returned by an injected LLMClient remain available
|
||||
// through errors.Is.
|
||||
// response. A built-in OpenAI-compatible non-2xx response is available as a
|
||||
// [GenerationError]. Errors returned by an injected LLMClient remain
|
||||
// available through errors.Is.
|
||||
ErrLLMGenerate = errors.New("failed to generate output")
|
||||
// ErrValidation identifies an operational failure to load or compile a
|
||||
// schema or validate output. A completed validation whose Status is
|
||||
@@ -99,7 +102,7 @@ type Config struct {
|
||||
// prompt source.
|
||||
PromptDir string
|
||||
// ProfileDir is an optional ordinary configured source whose profiles take
|
||||
// precedence over application fallback and embedded built-in profiles. An
|
||||
// precedence over application fallback and maintained catalog profiles. An
|
||||
// empty value selects the lower-precedence sources unless a profile-source
|
||||
// option supplies the ordinary source.
|
||||
ProfileDir string
|
||||
@@ -219,7 +222,7 @@ func WithPromptFile(path string) Option {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
options.promptDefs = promptdef.NewFSRepository(fsys, root)
|
||||
options.promptDefs = promptdef.NewFileRepository(fsys, root, filepath.Dir(path))
|
||||
options.promptSource = true
|
||||
return nil
|
||||
})
|
||||
@@ -272,7 +275,7 @@ func WithProfileFile(path string) Option {
|
||||
//
|
||||
// Profile lookup checks, in order, profiles supplied by WithProfiles; the
|
||||
// ordinary configured source selected by WithProfileFile, WithProfileFS, or
|
||||
// Config.ProfileDir; this fallback source; and Promptkit's embedded built-in
|
||||
// Config.ProfileDir; this fallback source; and Promptkit's maintained catalog
|
||||
// profiles. Each source supplies a complete profile definition; profile fields
|
||||
// are not merged between sources. Only an absent profile ID proceeds to the
|
||||
// next source. A matching read, parse, duplicate, validation, or credential
|
||||
@@ -303,10 +306,12 @@ func WithFallbackProfileFS(fsys fs.FS, root string) Option {
|
||||
// WithProfiles configures in-memory profiles that take precedence over
|
||||
// ordinary configured, application fallback, and built-in profiles.
|
||||
//
|
||||
// NewEngine validates and copies every profile. IDs must be unique within one
|
||||
// call. An invalid profile, duplicate ID, or unsupported ExtraParams value
|
||||
// makes construction fail with ErrInvalidConfig. Repeating WithProfiles
|
||||
// replaces the complete earlier in-memory set rather than merging it.
|
||||
// NewEngine locally validates and copies every profile. IDs must be unique
|
||||
// within one call. An invalid local definition, duplicate ID, or unsupported
|
||||
// ExtraParams value makes construction fail with ErrInvalidConfig. A derived
|
||||
// profile's base reference and resolved target completeness are checked when it
|
||||
// is selected or inspected. Repeating WithProfiles replaces the complete
|
||||
// earlier in-memory set rather than merging it.
|
||||
func WithProfiles(profiles ...Profile) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
repo, err := newMemoryProfileRepository(profiles)
|
||||
@@ -387,9 +392,17 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
|
||||
}
|
||||
|
||||
profiles := newProfileRepository(cfg.ProfileDir, options)
|
||||
maintainedCatalogs, err := catalog.Load(
|
||||
catalog.Source{Name: "OpenRouter", ExpectedBackendID: backend.OpenRouterID, FS: openrouter.FS(), Root: openrouter.Root},
|
||||
catalog.Source{Name: "Rakestrawhome", ExpectedBackendID: backend.RakestrawHomeID, FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to load maintained catalogs: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
|
||||
backendRegistry, err := backend.NewRegistry(options.backends)
|
||||
profiles := newProfileRepository(cfg.ProfileDir, options, maintainedCatalogs.Profiles)
|
||||
|
||||
backendRegistry, err := backend.NewRegistry(maintainedCatalogs.Backends, options.backends)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
@@ -427,7 +440,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
}
|
||||
|
||||
return &Engine{
|
||||
runner: usecase.NewRunner(
|
||||
runner: usecase.NewRunnerWithRepairer(
|
||||
promptDefs,
|
||||
profiles,
|
||||
backendRegistry,
|
||||
@@ -435,13 +448,14 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
prompt.NewGoRenderer(),
|
||||
llmClient,
|
||||
validator,
|
||||
usecase.NewDefaultOutputRepairer(llmClient),
|
||||
capacityManager,
|
||||
),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newProfileRepository(profileDir string, options engineOptions) profile.Repository {
|
||||
repository := builtin.NewRepository()
|
||||
func newProfileRepository(profileDir string, options engineOptions, maintained profile.Repository) profile.Repository {
|
||||
repository := maintained
|
||||
|
||||
if options.fallbackProfileSource {
|
||||
repository = profile.NewOverlayRepository(options.fallbackProfiles, repository)
|
||||
@@ -457,25 +471,24 @@ func newProfileRepository(profileDir string, options engineOptions) profile.Repo
|
||||
repository = profile.NewOverlayRepository(options.memoryProfiles, repository)
|
||||
}
|
||||
|
||||
return repository
|
||||
return profile.NewResolvingRepository(repository)
|
||||
}
|
||||
|
||||
func fileSource(name string) (fs.FS, string, error) {
|
||||
cleanName := strings.TrimSpace(name)
|
||||
if cleanName == "" {
|
||||
if strings.TrimSpace(name) == "" {
|
||||
return nil, "", ErrInvalidConfig
|
||||
}
|
||||
dir := filepath.Dir(cleanName)
|
||||
base := filepath.Base(cleanName)
|
||||
if base == "." || base == string(filepath.Separator) || strings.TrimSpace(base) == "" {
|
||||
dir := filepath.Dir(name)
|
||||
base := filepath.Base(name)
|
||||
if base == "." || base == string(filepath.Separator) {
|
||||
return nil, "", ErrInvalidConfig
|
||||
}
|
||||
info, err := os.Stat(cleanName)
|
||||
info, err := os.Stat(name)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, cleanName, err)
|
||||
return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, name, err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, cleanName)
|
||||
return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, name)
|
||||
}
|
||||
return os.DirFS(dir), filepath.ToSlash(base), nil
|
||||
}
|
||||
@@ -568,6 +581,8 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
|
||||
}
|
||||
|
||||
// Prepare resolves and renders a prompt request without calling an LLM.
|
||||
// It appends any validated RunRequest.AppendedMessages after rendered
|
||||
// definition messages in the returned caller-owned snapshot.
|
||||
//
|
||||
// Prepare selects the prompt and profile, resolves any selected backend and
|
||||
// effective execution settings, resolves the output contract, loads and hashes
|
||||
@@ -602,7 +617,8 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
||||
}
|
||||
|
||||
// PrepareExecution completely prepares a prompt request without calling the
|
||||
// configured LLMClient or reserving backend admission capacity.
|
||||
// configured LLMClient or reserving backend admission capacity. Validated
|
||||
// RunRequest.AppendedMessages are included in the frozen effective messages.
|
||||
//
|
||||
// The returned opaque handle is bound to this Engine and permits one
|
||||
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
|
||||
@@ -634,24 +650,29 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
|
||||
}
|
||||
|
||||
// Run prepares a request, invokes the configured LLMClient, and validates the
|
||||
// generated output.
|
||||
// generated output. Each call resolves current sources and composes a fresh,
|
||||
// stateless effective prompt with any validated RunRequest.AppendedMessages.
|
||||
//
|
||||
// A content-validation failure is a successful run whose
|
||||
// RunResult.Validation has Status ValidationFailed. An inability to perform
|
||||
// validation returns an error matching ErrValidation and no partial result.
|
||||
// The public Engine does not perform output repair, so validation is
|
||||
// single-pass even when OutputContract.RepairAttempts is positive.
|
||||
// RunResult.Validation has Status ValidationFailed. When its output contract
|
||||
// has a positive repair budget, a failed eligible validation can make bounded
|
||||
// additional model calls and stops at the first valid candidate. Exhaustion
|
||||
// returns the final failed validation result with cumulative usage and actual
|
||||
// repair attempts. An inability to generate or validate returns an error and
|
||||
// no partial result.
|
||||
//
|
||||
// Run can return every error category documented by [Engine.Prepare], plus
|
||||
// ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is
|
||||
// discoverable as [CapacityError] and still matches ErrCapacityExceeded. It
|
||||
// occurs before artifacts, schemas, rendering, or model generation because the
|
||||
// selected backend's admission capacity is full; it does not match
|
||||
// ErrInvalidRequest or ErrLLMGenerate. Errors from injected clients remain
|
||||
// available through errors.Is. Cancellation while waiting for model-generation
|
||||
// capacity matches both ErrLLMGenerate and the context error. Cancellation
|
||||
// otherwise follows the active collaborator's documented behavior. A nil
|
||||
// Engine returns ErrInvalidConfig. Run returns no partial result on error.
|
||||
// ErrInvalidRequest or ErrLLMGenerate. A built-in OpenAI-compatible non-2xx
|
||||
// response is discoverable as [GenerationError]. Errors from injected clients
|
||||
// remain available through errors.Is. Cancellation while waiting for
|
||||
// model-generation capacity matches both ErrLLMGenerate and the context error.
|
||||
// Cancellation otherwise follows the active collaborator's documented
|
||||
// behavior. A nil Engine returns ErrInvalidConfig. Run returns no partial
|
||||
// result on error.
|
||||
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
@@ -682,16 +703,17 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
//
|
||||
// The supplied context governs this execution attempt independently of the
|
||||
// preparation context. It covers credential revalidation, admission,
|
||||
// generation, validation, and any internal repair. Result timing begins after
|
||||
// the claim and excludes preparation and consumer-held delay.
|
||||
// generation, validation, and any bounded output repair. Result timing begins
|
||||
// after the claim and excludes preparation and consumer-held delay.
|
||||
//
|
||||
// RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing,
|
||||
// ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while
|
||||
// preserving documented collaborator and context identities. An engine
|
||||
// admission rejection is discoverable as [CapacityError] and still matches
|
||||
// ErrCapacityExceeded. A completed content-validation rejection is returned
|
||||
// in RunResult, not as an operational error. An operational error returns no
|
||||
// partial RunResult.
|
||||
// ErrCapacityExceeded. A built-in OpenAI-compatible non-2xx response is
|
||||
// discoverable as [GenerationError]. A completed content-validation rejection,
|
||||
// including repair exhaustion, is returned in RunResult, not as an operational
|
||||
// error. An operational error returns no partial RunResult.
|
||||
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
|
||||
1289
engine_test.go
1289
engine_test.go
File diff suppressed because it is too large
Load Diff
14
errors.go
14
errors.go
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||
@@ -21,6 +22,19 @@ func mapPublicError(err error) error {
|
||||
return &CapacityError{BackendID: internalCapacityError.BackendID}
|
||||
}
|
||||
publicErr := publicErrorFor(err)
|
||||
var providerHTTPError *llm.ProviderHTTPError
|
||||
if errors.As(err, &providerHTTPError) && providerHTTPError != nil {
|
||||
generationErr := newGenerationError(
|
||||
providerHTTPError.StatusCode(),
|
||||
providerHTTPError.ProviderCode(),
|
||||
providerHTTPError.ProviderType(),
|
||||
providerHTTPError.ProviderMessage(),
|
||||
)
|
||||
if publicErr != nil && !errors.Is(publicErr, ErrLLMGenerate) {
|
||||
return fmt.Errorf("%w: %w", publicErr, generationErr)
|
||||
}
|
||||
return generationErr
|
||||
}
|
||||
if publicErr == nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||
)
|
||||
|
||||
@@ -48,3 +49,27 @@ func TestMapPublicErrorTranslatesCapacityError(t *testing.T) {
|
||||
t.Fatalf("mapped backend ID changed with source error: %q", publicErr.BackendID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapPublicErrorPreservesValidationAroundGenerationError(t *testing.T) {
|
||||
internalErr := fmt.Errorf(
|
||||
"%w: %w",
|
||||
usecase.ErrValidation,
|
||||
&llm.ProviderHTTPError{},
|
||||
)
|
||||
|
||||
err := mapPublicError(internalErr)
|
||||
if !errors.Is(err, ErrValidation) {
|
||||
t.Fatalf("mapped error=%v, want ErrValidation", err)
|
||||
}
|
||||
if !errors.Is(err, ErrLLMGenerate) {
|
||||
t.Fatalf("mapped error=%v, want ErrLLMGenerate", err)
|
||||
}
|
||||
var generationErr *GenerationError
|
||||
if !errors.As(err, &generationErr) || generationErr == nil {
|
||||
t.Fatalf("mapped error=%v, want GenerationError", err)
|
||||
}
|
||||
var leakedInternalErr *llm.ProviderHTTPError
|
||||
if errors.As(err, &leakedInternalErr) {
|
||||
t.Fatalf("mapped error exposes internal ProviderHTTPError: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ func (r RunRequest) GoString() string {
|
||||
|
||||
func (r RunRequest) redactedString() string {
|
||||
return fmt.Sprintf(
|
||||
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t}",
|
||||
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t AppendedMessages:%d}",
|
||||
r.PromptID,
|
||||
r.PromptVersion,
|
||||
r.ProfileID,
|
||||
@@ -27,6 +27,7 @@ func (r RunRequest) redactedString() string {
|
||||
len(r.Vars),
|
||||
r.Execution != nil,
|
||||
r.Validation != nil,
|
||||
len(r.AppendedMessages),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
87
generation_error.go
Normal file
87
generation_error.go
Normal file
@@ -0,0 +1,87 @@
|
||||
package promptkit
|
||||
|
||||
import "fmt"
|
||||
|
||||
// GenerationError reports a non-2xx response from Promptkit's built-in
|
||||
// OpenAI-compatible client during [Engine.Run] or [Engine.RunPrepared].
|
||||
//
|
||||
// Engine-produced values are immutable, caller-owned values. Use errors.Is to
|
||||
// match [ErrLLMGenerate] and errors.As with a *GenerationError target to obtain
|
||||
// this type. The four provider accessors expose untrusted provider-controlled
|
||||
// values that can contain sensitive request or schema fragments. Applications
|
||||
// must apply their own disclosure policy before logging, displaying, or
|
||||
// returning them to another caller.
|
||||
//
|
||||
// Accessors, Error, GoString, and Unwrap are safe on a nil receiver and a zero
|
||||
// value. Default and Go-syntax formatting deliberately redact provider details.
|
||||
// GenerationError has no stable JSON representation.
|
||||
type GenerationError struct {
|
||||
statusCode int
|
||||
providerCode string
|
||||
providerType string
|
||||
providerMessage string
|
||||
}
|
||||
|
||||
func newGenerationError(statusCode int, providerCode, providerType, providerMessage string) *GenerationError {
|
||||
return &GenerationError{
|
||||
statusCode: statusCode,
|
||||
providerCode: providerCode,
|
||||
providerType: providerType,
|
||||
providerMessage: providerMessage,
|
||||
}
|
||||
}
|
||||
|
||||
// StatusCode returns the received provider HTTP status code, or zero for a nil
|
||||
// receiver or zero value.
|
||||
func (e *GenerationError) StatusCode() int {
|
||||
if e == nil {
|
||||
return 0
|
||||
}
|
||||
return e.statusCode
|
||||
}
|
||||
|
||||
// ProviderCode returns the normalized provider error code, if present. Its
|
||||
// value is untrusted and may contain sensitive data.
|
||||
func (e *GenerationError) ProviderCode() string {
|
||||
if e == nil {
|
||||
return ""
|
||||
}
|
||||
return e.providerCode
|
||||
}
|
||||
|
||||
// ProviderType returns the normalized provider error type, if present. Its
|
||||
// value is untrusted and may contain sensitive data.
|
||||
func (e *GenerationError) ProviderType() string {
|
||||
if e == nil {
|
||||
return ""
|
||||
}
|
||||
return e.providerType
|
||||
}
|
||||
|
||||
// ProviderMessage returns the bounded normalized provider diagnostic, if
|
||||
// present. Its value is untrusted and may contain sensitive data.
|
||||
func (e *GenerationError) ProviderMessage() string {
|
||||
if e == nil {
|
||||
return ""
|
||||
}
|
||||
return e.providerMessage
|
||||
}
|
||||
|
||||
// Error returns a redacted diagnostic that is not a parsing contract.
|
||||
func (e *GenerationError) Error() string {
|
||||
if e == nil || e.statusCode == 0 {
|
||||
return ErrLLMGenerate.Error()
|
||||
}
|
||||
return fmt.Sprintf("%s: provider returned HTTP status %d", ErrLLMGenerate, e.statusCode)
|
||||
}
|
||||
|
||||
// GoString returns the same redacted diagnostic as Error.
|
||||
func (e *GenerationError) GoString() string {
|
||||
return e.Error()
|
||||
}
|
||||
|
||||
// Unwrap returns ErrLLMGenerate. It is safe to call on a nil receiver or zero
|
||||
// value.
|
||||
func (e *GenerationError) Unwrap() error {
|
||||
return ErrLLMGenerate
|
||||
}
|
||||
150
generation_error_contract_test.go
Normal file
150
generation_error_contract_test.go
Normal file
@@ -0,0 +1,150 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestBuiltInGenerationError(t *testing.T) {
|
||||
const (
|
||||
codeMarker = "provider-code-marker"
|
||||
typeMarker = "provider-type-marker"
|
||||
messageMarker = "provider-message-marker"
|
||||
)
|
||||
engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity,
|
||||
`{"error":{"code":"`+codeMarker+`","type":"`+typeMarker+`","message":"`+messageMarker+`"}}`)
|
||||
|
||||
result, err := engine.Run(context.Background(), generationErrorRunRequest())
|
||||
if result != nil {
|
||||
t.Fatalf("Run result = %#v, want nil", result)
|
||||
}
|
||||
assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker)
|
||||
|
||||
preparedEngine := newBuiltInGenerationErrorEngine(t, http.StatusServiceUnavailable, `{"error":{}}`)
|
||||
prepared, err := preparedEngine.PrepareExecution(context.Background(), generationErrorRunRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("PrepareExecution: %v", err)
|
||||
}
|
||||
result, err = preparedEngine.RunPrepared(context.Background(), prepared)
|
||||
if result != nil {
|
||||
t.Fatalf("RunPrepared result = %#v, want nil", result)
|
||||
}
|
||||
assertGenerationError(t, err, http.StatusServiceUnavailable, "", "", "")
|
||||
}
|
||||
|
||||
func TestBuiltInGenerationErrorWithAppendedMessages(t *testing.T) {
|
||||
const messageMarker = "combined-request-provider-marker"
|
||||
engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity,
|
||||
`{"error":{"message":"`+messageMarker+`"}}`)
|
||||
request := generationErrorRunRequest()
|
||||
request.AppendedMessages = []promptkit.RenderedMessage{
|
||||
{Role: promptkit.RoleAssistant, Content: "previous response"},
|
||||
{Role: promptkit.RoleUser, Content: "consumer correction"},
|
||||
}
|
||||
|
||||
result, err := engine.Run(context.Background(), request)
|
||||
if result != nil {
|
||||
t.Fatalf("Run result = %#v, want nil", result)
|
||||
}
|
||||
assertGenerationError(t, err, http.StatusUnprocessableEntity, "", "", messageMarker)
|
||||
}
|
||||
|
||||
func TestBuiltInRepairGenerationError(t *testing.T) {
|
||||
const (
|
||||
codeMarker = "repair-code-marker"
|
||||
typeMarker = "repair-type-marker"
|
||||
messageMarker = "repair-message-marker"
|
||||
)
|
||||
calls := 0
|
||||
config := contractConfig(frameworkSchemaDir)
|
||||
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
body := `{"choices":[{"message":{"content":"not-json"}}]}`
|
||||
return &http.Response{StatusCode: http.StatusOK, ContentLength: int64(len(body)), Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||
}
|
||||
body := `{"error":{"code":"` + codeMarker + `","type":"` + typeMarker + `","message":"` + messageMarker + `"}}`
|
||||
return &http.Response{StatusCode: http.StatusUnprocessableEntity, ContentLength: int64(len(body)), Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(config)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine: %v", err)
|
||||
}
|
||||
req := generationErrorRunRequest()
|
||||
req.Validation = &promptkit.OutputContract{
|
||||
Format: promptkit.FormatJSON,
|
||||
ValidationMode: promptkit.ValidationJSON,
|
||||
RepairAttempts: 1,
|
||||
}
|
||||
|
||||
result, err := engine.Run(context.Background(), req)
|
||||
if result != nil {
|
||||
t.Fatalf("Run result = %#v, want nil", result)
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("provider calls = %d, want 2", calls)
|
||||
}
|
||||
assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker)
|
||||
}
|
||||
|
||||
func assertGenerationError(t *testing.T, err error, statusCode int, code, providerType, message string) {
|
||||
t.Helper()
|
||||
|
||||
if !errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||
t.Fatalf("errors.Is(%v, ErrLLMGenerate) = false", err)
|
||||
}
|
||||
var generationErr *promptkit.GenerationError
|
||||
if !errors.As(err, &generationErr) || generationErr == nil {
|
||||
t.Fatalf("error = %T, want *GenerationError", err)
|
||||
}
|
||||
if generationErr.StatusCode() != statusCode || generationErr.ProviderCode() != code || generationErr.ProviderType() != providerType || generationErr.ProviderMessage() != message {
|
||||
t.Fatalf("GenerationError = %#v", generationErr)
|
||||
}
|
||||
|
||||
wantFormatted := fmt.Sprintf("failed to generate output: provider returned HTTP status %d", statusCode)
|
||||
for _, rendered := range []string{fmt.Sprintf("%v", generationErr), fmt.Sprintf("%+v", generationErr), fmt.Sprintf("%#v", generationErr)} {
|
||||
if rendered != wantFormatted {
|
||||
t.Fatalf("formatted error = %q, want %q", rendered, wantFormatted)
|
||||
}
|
||||
for _, marker := range []string{code, providerType, message} {
|
||||
if marker != "" && strings.Contains(rendered, marker) {
|
||||
t.Fatalf("formatted error exposed provider marker %q: %q", marker, rendered)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newBuiltInGenerationErrorEngine(t *testing.T, statusCode int, body string) *promptkit.Engine {
|
||||
t.Helper()
|
||||
|
||||
config := contractConfig(frameworkSchemaDir)
|
||||
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
StatusCode: statusCode,
|
||||
ContentLength: int64(len(body)),
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
}, nil
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(config)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func generationErrorRunRequest() promptkit.RunRequest {
|
||||
return promptkit.RunRequest{
|
||||
PromptID: frameworkMarkdownSummaryPromptID,
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||
},
|
||||
}
|
||||
}
|
||||
32
generation_error_internal_test.go
Normal file
32
generation_error_internal_test.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGenerationErrorNilAndZeroValue(t *testing.T) {
|
||||
var nilError *GenerationError
|
||||
zeroError := &GenerationError{}
|
||||
|
||||
for name, err := range map[string]*GenerationError{
|
||||
"nil": nilError,
|
||||
"zero": zeroError,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if err.StatusCode() != 0 || err.ProviderCode() != "" || err.ProviderType() != "" || err.ProviderMessage() != "" {
|
||||
t.Fatalf("accessors returned provider details: %#v", err)
|
||||
}
|
||||
if err.Error() != "failed to generate output" || err.GoString() != "failed to generate output" {
|
||||
t.Fatalf("redacted formatting = (%q, %q)", err.Error(), err.GoString())
|
||||
}
|
||||
if fmt.Sprintf("%v", err) != "failed to generate output" || fmt.Sprintf("%#v", err) != "failed to generate output" {
|
||||
t.Fatalf("formatted error = (%q, %q)", fmt.Sprintf("%v", err), fmt.Sprintf("%#v", err))
|
||||
}
|
||||
if !errors.Is(err, ErrLLMGenerate) {
|
||||
t.Fatalf("errors.Is(%v, ErrLLMGenerate) = false", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
2
go.mod
2
go.mod
@@ -3,6 +3,8 @@ module gitea.maximumdirect.net/eric/promptkit
|
||||
go 1.25.5
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
4
go.sum
4
go.sum
@@ -1,3 +1,7 @@
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 h1:lc062euk2qseO//D762i3JaFyulDNML3eQQX7DkYTho=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0/go.mod h1:AIa7kAu2mfrRQgcspe4L+DW51WqgnALQT60lqkEywJI=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 h1:j9YY7wsTVjzke2kHH4YAzpU0oUpM+x+nXwl1IeS+2eg=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0/go.mod h1:4RNS+LILDg4JbS4Ts9Lwy1C92wauXJIbeQaalps4Koo=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
|
||||
@@ -16,10 +16,12 @@ import (
|
||||
|
||||
var (
|
||||
ErrUnsupportedRefType = errors.New("unsupported artifact reference type")
|
||||
ErrMissingInlineBody = errors.New("missing body for inline artifact")
|
||||
ErrMissingFilePath = errors.New("missing file path for file artifact")
|
||||
ErrUnsupportedFile = errors.New("file artifact path is not a regular file")
|
||||
)
|
||||
|
||||
const fileReadChunkSize = 64 * 1024
|
||||
|
||||
// Reader resolves artifact references into actual artifacts.
|
||||
type Reader interface {
|
||||
Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error)
|
||||
@@ -34,7 +36,7 @@ type CompositeReader struct {
|
||||
func NewCompositeReader() Reader {
|
||||
return &CompositeReader{
|
||||
inlineReader: &inlineReader{},
|
||||
fileReader: &fileReader{},
|
||||
fileReader: &fileReader{open: openArtifactFile},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,10 +66,6 @@ func (r *inlineReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domai
|
||||
default:
|
||||
}
|
||||
|
||||
if ref.Body == "" {
|
||||
return nil, ErrMissingInlineBody
|
||||
}
|
||||
|
||||
body := []byte(ref.Body)
|
||||
return &domain.Artifact{
|
||||
ContentType: defaults.ContentTypeTextPlain,
|
||||
@@ -78,7 +76,15 @@ func (r *inlineReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domai
|
||||
}, nil
|
||||
}
|
||||
|
||||
type fileReader struct{}
|
||||
type artifactFile interface {
|
||||
Read([]byte) (int, error)
|
||||
Stat() (os.FileInfo, error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
type fileReader struct {
|
||||
open func(string) (artifactFile, error)
|
||||
}
|
||||
|
||||
func (r *fileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
||||
select {
|
||||
@@ -91,25 +97,71 @@ func (r *fileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.
|
||||
return nil, ErrMissingFilePath
|
||||
}
|
||||
|
||||
return readFileArtifact(ref.URI)
|
||||
return readFileArtifact(ctx, ref.URI, r.open)
|
||||
}
|
||||
|
||||
func readFileArtifact(path string) (*domain.Artifact, error) {
|
||||
file, err := os.Open(path)
|
||||
func openArtifactFile(path string) (artifactFile, error) {
|
||||
return os.Open(path)
|
||||
}
|
||||
|
||||
func readFileArtifact(ctx context.Context, path string, open func(string) (artifactFile, error)) (*domain.Artifact, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
file, err := open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
data, err := io.ReadAll(file)
|
||||
openedInfo, err := file.Stat()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
||||
return nil, fmt.Errorf("failed to inspect opened file %s: %w", path, err)
|
||||
}
|
||||
if !openedInfo.Mode().IsRegular() {
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path)
|
||||
}
|
||||
|
||||
data := make([]byte, 0)
|
||||
chunk := make([]byte, fileReadChunkSize)
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n, readErr := file.Read(chunk)
|
||||
if n > 0 {
|
||||
data = append(data, chunk[:n]...)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if errors.Is(readErr, io.EOF) {
|
||||
break
|
||||
}
|
||||
if readErr != nil {
|
||||
return nil, fmt.Errorf("failed to read file %s: %w", path, readErr)
|
||||
}
|
||||
}
|
||||
|
||||
contentType := mime.TypeByExtension(filepath.Ext(path))
|
||||
if contentType == "" {
|
||||
contentType = defaults.ContentTypeTextPlain
|
||||
}
|
||||
hash := fmt.Sprintf("%x", sha256.Sum256(data))
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &domain.Artifact{
|
||||
Name: filepath.Base(path),
|
||||
@@ -117,6 +169,6 @@ func readFileArtifact(path string) (*domain.Artifact, error) {
|
||||
Body: data,
|
||||
URI: path,
|
||||
Size: int64(len(data)),
|
||||
Hash: fmt.Sprintf("%x", sha256.Sum256(data)),
|
||||
Hash: hash,
|
||||
}, nil
|
||||
}
|
||||
|
||||
43
internal/artifact/reader_fifo_linux_test.go
Normal file
43
internal/artifact/reader_fifo_linux_test.go
Normal file
@@ -0,0 +1,43 @@
|
||||
//go:build linux
|
||||
|
||||
package artifact
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestFileReaderRejectsFIFOBeforeOpen(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "artifact.fifo")
|
||||
if err := syscall.Mkfifo(path, 0o600); err != nil {
|
||||
t.Fatalf("create fifo: %v", err)
|
||||
}
|
||||
|
||||
type result struct {
|
||||
artifact *domain.Artifact
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
artifact, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefFile,
|
||||
URI: path,
|
||||
})
|
||||
done <- result{artifact: artifact, err: err}
|
||||
}()
|
||||
|
||||
select {
|
||||
case got := <-done:
|
||||
if got.artifact != nil || !errors.Is(got.err, ErrUnsupportedFile) {
|
||||
t.Fatalf("artifact=%#v err=%v, want nil/ErrUnsupportedFile", got.artifact, got.err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("FIFO read blocked instead of rejecting the non-regular file")
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package artifact
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
@@ -11,54 +12,97 @@ import (
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestCompositeReader_Read(t *testing.T) {
|
||||
func TestCompositeReaderRejectsUnsupportedReferences(t *testing.T) {
|
||||
_, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefType("unsupported"),
|
||||
URI: "unsupported://bucket/key",
|
||||
})
|
||||
if !errors.Is(err, ErrUnsupportedRefType) {
|
||||
t.Fatalf("expected ErrUnsupportedRefType, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompositeReaderSourceParityAndOpaqueHashes(t *testing.T) {
|
||||
reader := NewCompositeReader()
|
||||
ctx := context.Background()
|
||||
hashes := make(map[string]string)
|
||||
tests := []struct {
|
||||
name string
|
||||
content string
|
||||
}{
|
||||
{name: "empty", content: ""},
|
||||
{name: "ordinary", content: "same content"},
|
||||
{name: "changed", content: "changed content"},
|
||||
}
|
||||
|
||||
t.Run("inline artifact", func(t *testing.T) {
|
||||
ref := domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefInline,
|
||||
Body: "hello world",
|
||||
}
|
||||
art, err := reader.Read(ctx, ref)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if string(art.Body) != "hello world" {
|
||||
t.Errorf("expected 'hello world', got %s", string(art.Body))
|
||||
}
|
||||
if art.ContentType != "text/plain" {
|
||||
t.Errorf("expected text/plain content type, got %q", art.ContentType)
|
||||
}
|
||||
if art.Hash != "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" {
|
||||
t.Errorf("unexpected hash: %s", art.Hash)
|
||||
}
|
||||
if art.Size != int64(len(ref.Body)) {
|
||||
t.Errorf("expected size %d, got %d", len(ref.Body), art.Size)
|
||||
}
|
||||
})
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
filePath := filepath.Join(t.TempDir(), "artifact.txt")
|
||||
if err := os.WriteFile(filePath, []byte(tc.content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Run("inline artifact missing body", func(t *testing.T) {
|
||||
ref := domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefInline,
|
||||
Body: "",
|
||||
}
|
||||
_, err := reader.Read(ctx, ref)
|
||||
if !errors.Is(err, ErrMissingInlineBody) {
|
||||
t.Errorf("expected ErrMissingInlineBody, got %v", err)
|
||||
}
|
||||
})
|
||||
sources := []struct {
|
||||
name string
|
||||
ref domain.ArtifactRef
|
||||
wantURI string
|
||||
}{
|
||||
{
|
||||
name: "inline",
|
||||
ref: domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: tc.content},
|
||||
},
|
||||
{
|
||||
name: "inline with uri",
|
||||
ref: domain.ArtifactRef{Type: domain.ArtifactRefInline, URI: "memory://input", Body: tc.content},
|
||||
wantURI: "memory://input",
|
||||
},
|
||||
{
|
||||
name: "file",
|
||||
ref: domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: filePath},
|
||||
wantURI: filePath,
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("unsupported ref type", func(t *testing.T) {
|
||||
ref := domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefType("unsupported"),
|
||||
URI: "unsupported://bucket/key",
|
||||
}
|
||||
_, err := reader.Read(ctx, ref)
|
||||
if !errors.Is(err, ErrUnsupportedRefType) {
|
||||
t.Error("expected error for unsupported type")
|
||||
}
|
||||
})
|
||||
var sourceHash string
|
||||
for _, source := range sources {
|
||||
t.Run(source.name, func(t *testing.T) {
|
||||
first, err := reader.Read(context.Background(), source.ref)
|
||||
if err != nil {
|
||||
t.Fatalf("first read: %v", err)
|
||||
}
|
||||
second, err := reader.Read(context.Background(), source.ref)
|
||||
if err != nil {
|
||||
t.Fatalf("second read: %v", err)
|
||||
}
|
||||
if string(first.Body) != tc.content || first.Size != int64(len(tc.content)) {
|
||||
t.Fatalf("body=%q size=%d, want %q/%d", first.Body, first.Size, tc.content, len(tc.content))
|
||||
}
|
||||
if first.URI != source.wantURI {
|
||||
t.Fatalf("URI = %q, want %q", first.URI, source.wantURI)
|
||||
}
|
||||
if first.Hash == "" || first.Hash != second.Hash {
|
||||
t.Fatalf("hashes are not non-empty and stable: %q/%q", first.Hash, second.Hash)
|
||||
}
|
||||
if sourceHash == "" {
|
||||
sourceHash = first.Hash
|
||||
} else if first.Hash != sourceHash {
|
||||
t.Fatalf("equal content hashes differ: %q/%q", sourceHash, first.Hash)
|
||||
}
|
||||
if source.ref.Type == domain.ArtifactRefFile {
|
||||
if first.Name != filepath.Base(filePath) || !strings.HasPrefix(first.ContentType, "text/plain") {
|
||||
t.Fatalf("unexpected file metadata: %+v", first)
|
||||
}
|
||||
} else if first.ContentType != "text/plain" {
|
||||
t.Fatalf("inline content type = %q", first.ContentType)
|
||||
}
|
||||
})
|
||||
}
|
||||
hashes[tc.name] = sourceHash
|
||||
})
|
||||
}
|
||||
|
||||
if hashes["empty"] == hashes["ordinary"] || hashes["ordinary"] == hashes["changed"] {
|
||||
t.Fatalf("changed content did not change opaque hash: %#v", hashes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompositeReaderCopiesInlineData(t *testing.T) {
|
||||
@@ -87,94 +131,154 @@ func TestCompositeReaderCopiesInlineData(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompositeReaderHonorsCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
func TestCompositeReaderHonorsPreCancellation(t *testing.T) {
|
||||
filePath := filepath.Join(t.TempDir(), "artifact.txt")
|
||||
if err := os.WriteFile(filePath, []byte("ignored"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
ref domain.ArtifactRef
|
||||
}{
|
||||
{name: "inline", ref: domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: "ignored"}},
|
||||
{name: "file", ref: domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: filePath}},
|
||||
}
|
||||
|
||||
_, err := NewCompositeReader().Read(ctx, domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefInline,
|
||||
Body: "ignored",
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected context cancellation, got %v", err)
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
artifact, err := NewCompositeReader().Read(ctx, tc.ref)
|
||||
if artifact != nil || !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("artifact=%#v err=%v, want nil/context.Canceled", artifact, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileReader_Read(t *testing.T) {
|
||||
content := []byte("test file content")
|
||||
filePath := filepath.Join(t.TempDir(), "artifact.txt")
|
||||
if err := os.WriteFile(filePath, content, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
func TestFileReaderFailuresAndMetadata(t *testing.T) {
|
||||
reader := NewCompositeReader()
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("file artifact loading", func(t *testing.T) {
|
||||
ref := domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefFile,
|
||||
URI: filePath,
|
||||
}
|
||||
art, err := reader.Read(ctx, ref)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if string(art.Body) != string(content) {
|
||||
t.Errorf("expected %s, got %s", string(content), string(art.Body))
|
||||
}
|
||||
if art.Name != filepath.Base(filePath) {
|
||||
t.Errorf("expected name %q, got %q", filepath.Base(filePath), art.Name)
|
||||
}
|
||||
if !strings.HasPrefix(art.ContentType, "text/plain") {
|
||||
t.Errorf("expected text content type, got %q", art.ContentType)
|
||||
}
|
||||
if art.URI != filePath {
|
||||
t.Errorf("expected URI %q, got %q", filePath, art.URI)
|
||||
}
|
||||
if art.Size != int64(len(content)) {
|
||||
t.Errorf("expected size %d, got %d", len(content), art.Size)
|
||||
}
|
||||
if art.Hash != "60f5237ed4049f0382661ef009d2bc42e48c3ceb3edb6600f7024e7ab3b838f3" {
|
||||
t.Errorf("unexpected hash: %s", art.Hash)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing file path", func(t *testing.T) {
|
||||
ref := domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefFile,
|
||||
URI: "",
|
||||
}
|
||||
_, err := reader.Read(ctx, ref)
|
||||
_, err := reader.Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefFile})
|
||||
if !errors.Is(err, ErrMissingFilePath) {
|
||||
t.Errorf("expected ErrMissingFilePath, got %v", err)
|
||||
t.Fatalf("expected ErrMissingFilePath, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing file", func(t *testing.T) {
|
||||
ref := domain.ArtifactRef{
|
||||
_, err := reader.Read(context.Background(), domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefFile,
|
||||
URI: filepath.Join(t.TempDir(), "missing.txt"),
|
||||
}
|
||||
if _, err := reader.Read(ctx, ref); err == nil {
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected missing file error")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("directory rejected before open", func(t *testing.T) {
|
||||
artifact, err := reader.Read(context.Background(), domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefFile,
|
||||
URI: t.TempDir(),
|
||||
})
|
||||
if artifact != nil || !errors.Is(err, ErrUnsupportedFile) {
|
||||
t.Fatalf("artifact=%#v err=%v, want nil/ErrUnsupportedFile", artifact, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("non-regular opened target rejected", func(t *testing.T) {
|
||||
filePath := filepath.Join(t.TempDir(), "artifact.txt")
|
||||
if err := os.WriteFile(filePath, []byte("content"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
directoryInfo, err := os.Stat(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fileReader := &fileReader{open: func(path string) (artifactFile, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &reportedInfoFile{artifactFile: file, info: directoryInfo}, nil
|
||||
}}
|
||||
|
||||
artifact, err := fileReader.Read(context.Background(), domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefFile,
|
||||
URI: filePath,
|
||||
})
|
||||
if artifact != nil || !errors.Is(err, ErrUnsupportedFile) {
|
||||
t.Fatalf("artifact=%#v err=%v, want nil/ErrUnsupportedFile", artifact, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unknown extension uses text fallback", func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "artifact.unknownextension")
|
||||
if err := os.WriteFile(path, content, 0o600); err != nil {
|
||||
filePath := filepath.Join(t.TempDir(), "artifact.unknownextension")
|
||||
if err := os.WriteFile(filePath, []byte("content"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
art, err := reader.Read(ctx, domain.ArtifactRef{
|
||||
artifact, err := reader.Read(context.Background(), domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefFile,
|
||||
URI: path,
|
||||
URI: filePath,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
t.Fatalf("read artifact: %v", err)
|
||||
}
|
||||
if art.ContentType != "text/plain" {
|
||||
t.Errorf("expected text/plain fallback, got %q", art.ContentType)
|
||||
if artifact.ContentType != "text/plain" {
|
||||
t.Fatalf("content type = %q", artifact.ContentType)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFileReaderCancelsAfterReadProgress(t *testing.T) {
|
||||
filePath := filepath.Join(t.TempDir(), "artifact.bin")
|
||||
content := bytes.Repeat([]byte("x"), fileReadChunkSize*2)
|
||||
if err := os.WriteFile(filePath, content, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
var opened *cancelAfterProgressFile
|
||||
reader := &fileReader{open: func(path string) (artifactFile, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opened = &cancelAfterProgressFile{artifactFile: file, cancel: cancel}
|
||||
return opened, nil
|
||||
}}
|
||||
|
||||
artifact, err := reader.Read(ctx, domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: filePath})
|
||||
if artifact != nil || !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("artifact=%#v err=%v, want nil/context.Canceled", artifact, err)
|
||||
}
|
||||
if opened == nil || opened.reads != 1 {
|
||||
t.Fatalf("read count = %v, want one progressing read", opened)
|
||||
}
|
||||
}
|
||||
|
||||
type reportedInfoFile struct {
|
||||
artifactFile
|
||||
info os.FileInfo
|
||||
}
|
||||
|
||||
func (f *reportedInfoFile) Stat() (os.FileInfo, error) {
|
||||
return f.info, nil
|
||||
}
|
||||
|
||||
type cancelAfterProgressFile struct {
|
||||
artifactFile
|
||||
cancel context.CancelFunc
|
||||
reads int
|
||||
}
|
||||
|
||||
func (f *cancelAfterProgressFile) Read(buffer []byte) (int, error) {
|
||||
n, err := f.artifactFile.Read(buffer)
|
||||
if n > 0 {
|
||||
f.reads++
|
||||
f.cancel()
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ package backend
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -19,12 +18,11 @@ const (
|
||||
// OpenRouterID is the reserved ID of Promptkit's built-in OpenRouter
|
||||
// backend.
|
||||
OpenRouterID = "openrouter"
|
||||
// RakestrawHomeID is the reserved ID of Promptkit's built-in Rakestrawhome
|
||||
// backend.
|
||||
RakestrawHomeID = "rakestrawhome"
|
||||
|
||||
openRouterEndpoint = "https://openrouter.ai/api/v1"
|
||||
openRouterAPIKeyEnv = "OPENROUTER_API_KEY"
|
||||
|
||||
openRouterConcurrencyLimit = 16
|
||||
defaultQueueCapacity = 1024
|
||||
defaultQueueCapacity = 1024
|
||||
)
|
||||
|
||||
// ErrBackendNotFound identifies a registry lookup for an unknown backend ID.
|
||||
@@ -37,20 +35,15 @@ type Registry struct {
|
||||
backends map[string]domain.Backend
|
||||
}
|
||||
|
||||
// NewRegistry constructs a registry containing the built-in OpenRouter
|
||||
// definition followed by the supplied additions. Every ID must be unique.
|
||||
func NewRegistry(additions []domain.Backend) (*Registry, error) {
|
||||
// NewRegistry constructs a registry containing maintained definitions followed
|
||||
// by consumer additions. Every ID must be unique across both groups.
|
||||
func NewRegistry(maintained, additions []domain.Backend) (*Registry, error) {
|
||||
registry := &Registry{
|
||||
backends: make(map[string]domain.Backend, len(additions)+1),
|
||||
backends: make(map[string]domain.Backend, len(maintained)+len(additions)),
|
||||
}
|
||||
|
||||
definitions := make([]domain.Backend, 0, len(additions)+1)
|
||||
definitions = append(definitions, domain.Backend{
|
||||
ID: OpenRouterID,
|
||||
Endpoint: openRouterEndpoint,
|
||||
APIKeyEnv: openRouterAPIKeyEnv,
|
||||
ConcurrencyLimit: openRouterConcurrencyLimit,
|
||||
})
|
||||
definitions := make([]domain.Backend, 0, len(maintained)+len(additions))
|
||||
definitions = append(definitions, maintained...)
|
||||
definitions = append(definitions, additions...)
|
||||
|
||||
for _, definition := range definitions {
|
||||
@@ -62,7 +55,7 @@ func NewRegistry(additions []domain.Backend) (*Registry, error) {
|
||||
return nil, fmt.Errorf("backend ID %q is already registered", definition.ID)
|
||||
}
|
||||
|
||||
normalized, err := normalizeBackend(definition)
|
||||
normalized, err := NormalizeDefinition(definition)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -108,11 +101,13 @@ func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy {
|
||||
return policies
|
||||
}
|
||||
|
||||
func normalizeBackend(definition domain.Backend) (domain.Backend, error) {
|
||||
definition.Endpoint = strings.TrimSpace(definition.Endpoint)
|
||||
if err := validateEndpoint(definition.Endpoint); err != nil {
|
||||
// NormalizeDefinition validates and defensively copies one backend definition.
|
||||
func NormalizeDefinition(definition domain.Backend) (domain.Backend, error) {
|
||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(definition.Endpoint)
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err)
|
||||
}
|
||||
definition.Endpoint = endpoint
|
||||
|
||||
definition.APIKeyEnv = strings.TrimSpace(definition.APIKeyEnv)
|
||||
if definition.APIKeyEnv != "" && !environmentVariableName.MatchString(definition.APIKeyEnv) {
|
||||
@@ -182,31 +177,3 @@ func normalizeBackend(definition domain.Backend) (domain.Backend, error) {
|
||||
definition.ExtraParams = extraParams
|
||||
return definition, nil
|
||||
}
|
||||
|
||||
func validateEndpoint(endpoint string) error {
|
||||
if endpoint == "" {
|
||||
return errors.New("must not be blank")
|
||||
}
|
||||
if strings.Contains(endpoint, "#") {
|
||||
return errors.New("must not contain a fragment")
|
||||
}
|
||||
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return fmt.Errorf("must be a valid URL: %w", err)
|
||||
}
|
||||
scheme := strings.ToLower(parsed.Scheme)
|
||||
if scheme != "http" && scheme != "https" {
|
||||
return errors.New("must use http or https")
|
||||
}
|
||||
if !parsed.IsAbs() || parsed.Hostname() == "" {
|
||||
return errors.New("must be absolute and include a host")
|
||||
}
|
||||
if parsed.User != nil {
|
||||
return errors.New("must not contain user information")
|
||||
}
|
||||
if parsed.RawQuery != "" || parsed.ForceQuery {
|
||||
return errors.New("must not contain a query string")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package backend_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -11,32 +12,39 @@ import (
|
||||
|
||||
const validEndpoint = "https://backend.example/v1"
|
||||
|
||||
func TestRegistryIncludesExactOpenRouterDefinition(t *testing.T) {
|
||||
registry, err := backend.NewRegistry(nil)
|
||||
func TestRegistryIncludesMaintainedDefinitions(t *testing.T) {
|
||||
maintained := []domain.Backend{
|
||||
{ID: backend.OpenRouterID, Endpoint: validEndpoint, ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true},
|
||||
{ID: backend.RakestrawHomeID, Endpoint: "https://second.example/v1", ConcurrencyLimit: 4, QueueCapacity: 5, QueueCapacitySet: true},
|
||||
}
|
||||
registry, err := backend.NewRegistry(maintained, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
}
|
||||
|
||||
definition, err := registry.GetBackend(backend.OpenRouterID)
|
||||
if err != nil {
|
||||
t.Fatalf("look up OpenRouter: %v", err)
|
||||
}
|
||||
if definition.ID != "openrouter" ||
|
||||
definition.Endpoint != "https://openrouter.ai/api/v1" ||
|
||||
definition.APIKeyEnv != "OPENROUTER_API_KEY" ||
|
||||
definition.ConcurrencyLimit != 16 ||
|
||||
definition.QueueCapacity != 1024 ||
|
||||
!definition.QueueCapacitySet ||
|
||||
definition.ExtraParams != nil {
|
||||
t.Fatalf("unexpected OpenRouter definition: %#v", definition)
|
||||
for _, expected := range maintained {
|
||||
t.Run(expected.ID, func(t *testing.T) {
|
||||
definition, err := registry.GetBackend(expected.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("look up maintained definition: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(definition, expected) {
|
||||
t.Fatalf("unexpected maintained definition: %#v", definition)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
policies := registry.CapacityPolicies()
|
||||
if len(policies) != 1 ||
|
||||
policies["openrouter"] != (domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 16,
|
||||
QueueCapacity: 1024,
|
||||
if len(policies) != 2 ||
|
||||
policies[backend.OpenRouterID] != (domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 2,
|
||||
QueueCapacity: 3,
|
||||
}) ||
|
||||
policies[backend.RakestrawHomeID] != (domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 4,
|
||||
QueueCapacity: 5,
|
||||
}) {
|
||||
t.Fatalf("unexpected OpenRouter capacity policies: %#v", policies)
|
||||
t.Fatalf("unexpected built-in capacity policies: %#v", policies)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,7 +54,7 @@ func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
|
||||
"count": int64(7),
|
||||
"nested": nested,
|
||||
}
|
||||
registry, err := backend.NewRegistry([]domain.Backend{
|
||||
registry, err := backend.NewRegistry(nil, []domain.Backend{
|
||||
{
|
||||
ID: " custom ",
|
||||
Endpoint: " https://custom.example/openai/v1 ",
|
||||
@@ -109,11 +117,11 @@ func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
|
||||
}
|
||||
|
||||
policies := registry.CapacityPolicies()
|
||||
if len(policies) != 2 {
|
||||
if len(policies) != 1 {
|
||||
t.Fatalf("unexpected capacity policy count: %#v", policies)
|
||||
}
|
||||
policies["custom"] = domain.BackendCapacityPolicy{}
|
||||
delete(policies, backend.OpenRouterID)
|
||||
delete(policies, "custom")
|
||||
againPolicies := registry.CapacityPolicies()
|
||||
if againPolicies["custom"] != (domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 3,
|
||||
@@ -121,9 +129,6 @@ func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
|
||||
}) {
|
||||
t.Fatalf("capacity policy map mutated registry state: %#v", againPolicies)
|
||||
}
|
||||
if _, ok := againPolicies[backend.OpenRouterID]; !ok {
|
||||
t.Fatalf("capacity policy deletion mutated registry state: %#v", againPolicies)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) {
|
||||
@@ -199,7 +204,7 @@ func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
tc.definition.ID = "custom"
|
||||
tc.definition.Endpoint = validEndpoint
|
||||
registry, err := backend.NewRegistry([]domain.Backend{tc.definition})
|
||||
registry, err := backend.NewRegistry(nil, []domain.Backend{tc.definition})
|
||||
if tc.wantError {
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid capacity policy error")
|
||||
@@ -242,11 +247,14 @@ func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
|
||||
wantID string
|
||||
}{
|
||||
{
|
||||
name: "built-in collision after normalization",
|
||||
additions: []domain.Backend{{
|
||||
ID: " openrouter ",
|
||||
}},
|
||||
wantID: "openrouter",
|
||||
name: "OpenRouter collision after normalization",
|
||||
additions: []domain.Backend{{ID: " openrouter "}},
|
||||
wantID: backend.OpenRouterID,
|
||||
},
|
||||
{
|
||||
name: "Rakestrawhome collision after normalization",
|
||||
additions: []domain.Backend{{ID: " rakestrawhome "}},
|
||||
wantID: backend.RakestrawHomeID,
|
||||
},
|
||||
{
|
||||
name: "consumer collision after normalization",
|
||||
@@ -260,7 +268,7 @@ func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry(tc.additions)
|
||||
_, err := backend.NewRegistry(nil, tc.additions)
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate ID error")
|
||||
}
|
||||
@@ -274,7 +282,7 @@ func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
|
||||
func TestNewRegistryValidatesIDs(t *testing.T) {
|
||||
for _, id := range []string{"", " \t\n "} {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
_, err := backend.NewRegistry(nil, []domain.Backend{{
|
||||
ID: id,
|
||||
Endpoint: validEndpoint,
|
||||
}})
|
||||
@@ -303,7 +311,7 @@ func TestNewRegistryValidatesEndpoints(t *testing.T) {
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
_, err := backend.NewRegistry(nil, []domain.Backend{{
|
||||
ID: "custom",
|
||||
Endpoint: tc.endpoint,
|
||||
}})
|
||||
@@ -317,7 +325,7 @@ func TestNewRegistryValidatesEndpoints(t *testing.T) {
|
||||
func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) {
|
||||
for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
_, err := backend.NewRegistry(nil, []domain.Backend{{
|
||||
ID: "custom",
|
||||
Endpoint: validEndpoint,
|
||||
APIKeyEnv: name,
|
||||
@@ -340,7 +348,7 @@ func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
_, err := backend.NewRegistry(nil, []domain.Backend{{
|
||||
ID: "custom",
|
||||
Endpoint: validEndpoint,
|
||||
ExtraParams: tc.extraParams,
|
||||
@@ -353,7 +361,7 @@ func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRegistryLookupReportsNotFound(t *testing.T) {
|
||||
registry, err := backend.NewRegistry(nil)
|
||||
registry, err := backend.NewRegistry(nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
}
|
||||
|
||||
246
internal/catalog/catalog.go
Normal file
246
internal/catalog/catalog.go
Normal file
@@ -0,0 +1,246 @@
|
||||
// Package catalog validates immutable maintained backend catalog assets.
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
|
||||
// Source identifies one immutable backend catalog asset tree.
|
||||
type Source struct {
|
||||
Name string
|
||||
ExpectedBackendID string
|
||||
FS fs.FS
|
||||
Root string
|
||||
}
|
||||
|
||||
// Set is the validated maintained backend and raw profile catalog.
|
||||
type Set struct {
|
||||
Backends []domain.Backend
|
||||
Profiles profile.Repository
|
||||
profileIDs []string
|
||||
}
|
||||
|
||||
// Load validates and combines immutable catalog sources in source order.
|
||||
func Load(sources ...Source) (Set, error) {
|
||||
if len(sources) == 0 {
|
||||
return Set{}, errors.New("at least one catalog source is required")
|
||||
}
|
||||
loaded := Set{Backends: make([]domain.Backend, 0, len(sources))}
|
||||
names := map[string]bool{}
|
||||
backendIDs := map[string]bool{}
|
||||
profileIDs := map[string]bool{}
|
||||
for _, source := range sources {
|
||||
if err := validateSource(source, names); err != nil {
|
||||
return Set{}, err
|
||||
}
|
||||
names[source.Name] = true
|
||||
definition, err := loadBackend(source)
|
||||
if err != nil {
|
||||
return Set{}, err
|
||||
}
|
||||
if backendIDs[definition.ID] {
|
||||
return Set{}, fmt.Errorf("catalog %s: backend ID duplicates an earlier catalog", source.Name)
|
||||
}
|
||||
backendIDs[definition.ID] = true
|
||||
repository, metadata, err := profile.LoadFSRepository(context.Background(), source.FS, path.Join(source.Root, "profiles"))
|
||||
if err != nil {
|
||||
return Set{}, fmt.Errorf("catalog %s: %w", source.Name, err)
|
||||
}
|
||||
if len(metadata) == 0 {
|
||||
return Set{}, fmt.Errorf("catalog %s: profiles must not be empty", source.Name)
|
||||
}
|
||||
resolvingRepository := profile.NewResolvingRepository(repository)
|
||||
for _, entry := range metadata {
|
||||
if profileIDs[entry.ID] {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s duplicates an earlier profile ID", source.Name, entry.Path)
|
||||
}
|
||||
if containsField(entry.ExplicitFields, "endpoint") || containsField(entry.ExplicitFields, "api_key_env") {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s contains connection metadata", source.Name, entry.Path)
|
||||
}
|
||||
value, err := repository.GetProfile(context.Background(), entry.ID)
|
||||
if err != nil {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s: %w", source.Name, entry.Path, err)
|
||||
}
|
||||
if err := rejectSecretKeys(value.ExtraParams); err != nil {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s: prohibited extra parameter key", source.Name, entry.Path)
|
||||
}
|
||||
resolved, err := resolvingRepository.GetProfile(context.Background(), entry.ID)
|
||||
if err != nil {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s has invalid profile inheritance", source.Name, entry.Path)
|
||||
}
|
||||
if resolved.BackendID != definition.ID {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s selects a different backend", source.Name, entry.Path)
|
||||
}
|
||||
profileIDs[entry.ID] = true
|
||||
loaded.profileIDs = append(loaded.profileIDs, entry.ID)
|
||||
}
|
||||
if loaded.Profiles == nil {
|
||||
loaded.Profiles = repository
|
||||
} else {
|
||||
loaded.Profiles = profile.NewOverlayRepository(loaded.Profiles, repository)
|
||||
}
|
||||
loaded.Backends = append(loaded.Backends, definition)
|
||||
}
|
||||
sort.Strings(loaded.profileIDs)
|
||||
return loaded, nil
|
||||
}
|
||||
|
||||
func validateSource(source Source, names map[string]bool) error {
|
||||
if strings.TrimSpace(source.Name) == "" || source.Name != strings.TrimSpace(source.Name) {
|
||||
return errors.New("catalog source name must not be blank")
|
||||
}
|
||||
if names[source.Name] {
|
||||
return fmt.Errorf("duplicate catalog source name %q", source.Name)
|
||||
}
|
||||
if source.FS == nil {
|
||||
return fmt.Errorf("catalog %s: filesystem is nil", source.Name)
|
||||
}
|
||||
if source.Root == "." || source.Root == "" || !fs.ValidPath(source.Root) {
|
||||
return fmt.Errorf("catalog %s: asset root is invalid", source.Name)
|
||||
}
|
||||
if strings.TrimSpace(source.ExpectedBackendID) == "" {
|
||||
return fmt.Errorf("catalog %s: expected backend ID is blank", source.Name)
|
||||
}
|
||||
return validateLayout(source)
|
||||
}
|
||||
|
||||
func validateLayout(source Source) error {
|
||||
manifestPath := path.Join(source.Root, "backend.json")
|
||||
profilesRoot := path.Join(source.Root, "profiles")
|
||||
return fs.WalkDir(source.FS, source.Root, func(assetPath string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if assetPath == source.Root {
|
||||
if !entry.IsDir() {
|
||||
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if entry.IsDir() {
|
||||
if assetPath == profilesRoot || strings.HasPrefix(assetPath, profilesRoot+"/") {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
|
||||
}
|
||||
if assetPath == manifestPath && entry.Type().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
if strings.HasPrefix(assetPath, profilesRoot+"/") && entry.Type().IsRegular() && strings.HasSuffix(assetPath, ".yml") {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
|
||||
})
|
||||
}
|
||||
|
||||
func loadBackend(source Source) (domain.Backend, error) {
|
||||
data, err := fs.ReadFile(source.FS, path.Join(source.Root, "backend.json"))
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: %w", source.Name, err)
|
||||
}
|
||||
type manifest struct {
|
||||
SchemaVersion *int `json:"schema_version"`
|
||||
ID *string `json:"id"`
|
||||
Endpoint *string `json:"endpoint"`
|
||||
APIKeyEnv *string `json:"api_key_env"`
|
||||
ConcurrencyLimit *int `json:"concurrency_limit"`
|
||||
QueueCapacity *int `json:"queue_capacity"`
|
||||
ExtraParams json.RawMessage `json:"extra_params"`
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.DisallowUnknownFields()
|
||||
var value manifest
|
||||
if err := decoder.Decode(&value); err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
|
||||
}
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
|
||||
}
|
||||
if value.SchemaVersion == nil || *value.SchemaVersion != 1 || value.ID == nil || value.Endpoint == nil || value.APIKeyEnv == nil || value.ConcurrencyLimit == nil || value.QueueCapacity == nil || value.ExtraParams == nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: required field is missing or unsupported", source.Name)
|
||||
}
|
||||
if *value.ID != source.ExpectedBackendID {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend ID does not match expected ID", source.Name)
|
||||
}
|
||||
if strings.TrimSpace(*value.APIKeyEnv) == "" {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: api key environment variable must not be blank", source.Name)
|
||||
}
|
||||
extraParams, err := decodeExtraParams(value.ExtraParams)
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid extra parameters", source.Name)
|
||||
}
|
||||
if err := rejectSecretKeys(extraParams); err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: prohibited extra parameter key", source.Name)
|
||||
}
|
||||
normalized, err := backend.NormalizeDefinition(domain.Backend{ID: *value.ID, Endpoint: *value.Endpoint, APIKeyEnv: *value.APIKeyEnv, ExtraParams: extraParams, ConcurrencyLimit: *value.ConcurrencyLimit, QueueCapacity: *value.QueueCapacity, QueueCapacitySet: true})
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid backend definition", source.Name)
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
func decodeExtraParams(data []byte) (map[string]any, error) {
|
||||
if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
|
||||
return nil, nil
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.UseNumber()
|
||||
var value map[string]any
|
||||
if err := decoder.Decode(&value); err != nil || value == nil {
|
||||
return nil, errors.New("extra parameters must be an object or null")
|
||||
}
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return nil, errors.New("extra parameters must contain exactly one JSON value")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func containsField(fields []string, target string) bool {
|
||||
for _, field := range fields {
|
||||
if field == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func rejectSecretKeys(value map[string]any) error {
|
||||
return rejectSecretValue(value)
|
||||
}
|
||||
|
||||
func rejectSecretValue(value any) error {
|
||||
switch value := value.(type) {
|
||||
case map[string]any:
|
||||
for key, child := range value {
|
||||
switch strings.ToLower(key) {
|
||||
case "api_key", "apikey", "authorization", "credential", "credentials", "password", "secret", "token", "access_token":
|
||||
return errors.New("prohibited key")
|
||||
}
|
||||
if err := rejectSecretValue(child); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, child := range value {
|
||||
if err := rejectSecretValue(child); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
364
internal/catalog/catalog_test.go
Normal file
364
internal/catalog/catalog_test.go
Normal file
@@ -0,0 +1,364 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter"
|
||||
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome"
|
||||
)
|
||||
|
||||
func TestLoadPublishedCatalogsMatchCompatibilityFixture(t *testing.T) {
|
||||
loaded, err := Load(
|
||||
Source{Name: "OpenRouter", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
|
||||
Source{Name: "Rakestrawhome", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("load published catalogs: %v", err)
|
||||
}
|
||||
expected := loadCompatibilityFixture(t)
|
||||
expectedIDs := fixtureProfileIDs(t, expected)
|
||||
if !reflect.DeepEqual(loaded.profileIDs, expectedIDs) {
|
||||
t.Fatalf("published profile IDs differ from compatibility fixture: got %q, want %q", loaded.profileIDs, expectedIDs)
|
||||
}
|
||||
actual := catalogValue(t, loaded)
|
||||
actualJSON, err := json.Marshal(actual)
|
||||
if err != nil {
|
||||
t.Fatalf("encode loaded catalogs: %v", err)
|
||||
}
|
||||
expectedJSON, err := json.Marshal(expected)
|
||||
if err != nil {
|
||||
t.Fatalf("encode compatibility fixture: %v", err)
|
||||
}
|
||||
if !bytes.Equal(actualJSON, expectedJSON) {
|
||||
t.Fatalf("published catalogs differ from compatibility fixture: got %#v, want %#v", actual, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidSources(t *testing.T) {
|
||||
for name, sources := range map[string][]Source{
|
||||
"none": nil,
|
||||
"blank name": {{Name: " ", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root}},
|
||||
"duplicate name": {
|
||||
{Name: "same", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
|
||||
{Name: "same", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
|
||||
},
|
||||
"nil filesystem": {{Name: "missing", ExpectedBackendID: "openrouter", Root: openrouter.Root}},
|
||||
"invalid root": {{Name: "invalid-root", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: "."}},
|
||||
"blank expected backend": {{Name: "blank-backend", ExpectedBackendID: " ", FS: openrouter.FS(), Root: openrouter.Root}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := Load(sources...); err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidLayouts(t *testing.T) {
|
||||
tests := map[string]func(fstest.MapFS){
|
||||
"unexpected file": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/notes.txt"] = &fstest.MapFile{Data: []byte("unexpected")}
|
||||
},
|
||||
"unexpected directory": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/unexpected"] = &fstest.MapFile{Mode: fs.ModeDir}
|
||||
},
|
||||
"nonregular manifest": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/backend.json"].Mode = fs.ModeSymlink
|
||||
},
|
||||
"wrong profile extension": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/extra.yaml"] = &fstest.MapFile{Data: []byte(validProfile("extra", "one"))}
|
||||
},
|
||||
"nonregular profile": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Mode = fs.ModeSymlink
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fsys := validCatalogFS("one")
|
||||
mutate(fsys)
|
||||
if _, err := Load(testSource("one", fsys)); err == nil {
|
||||
t.Fatal("expected invalid layout error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidManifests(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"malformed": `{`,
|
||||
"trailing value": validManifest("one", "TEST_API_KEY", "null") + `{}`,
|
||||
"missing fields": `{"schema_version":1,"id":"one"}`,
|
||||
"unsupported version": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"schema_version":1`, `"schema_version":2`, 1),
|
||||
"unknown field": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"extra_params":null`, `"extra_params":null,"unknown":true`, 1),
|
||||
"blank API key env": validManifest("one", " ", "null"),
|
||||
"invalid API key env": validManifest("one", "LEAK-MARKER", "null"),
|
||||
"invalid endpoint": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `https://one.example/v1`, `ftp://leak-marker.invalid/v1`, 1),
|
||||
"zero concurrency": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"concurrency_limit":2`, `"concurrency_limit":0`, 1),
|
||||
"non-object parameters": validManifest("one", "TEST_API_KEY", `[]`),
|
||||
"secret parameter": validManifest("one", "TEST_API_KEY", `{"nested":{"token":"leak-marker"}}`),
|
||||
}
|
||||
for name, manifest := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fsys := validCatalogFS("one")
|
||||
fsys["catalog/backend.json"].Data = []byte(manifest)
|
||||
_, err := Load(testSource("one", fsys))
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid manifest error")
|
||||
}
|
||||
if strings.Contains(strings.ToLower(err.Error()), "leak-marker") {
|
||||
t.Fatalf("catalog error exposed manifest content: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPreservesManifestJSONNumbers(t *testing.T) {
|
||||
fsys := validCatalogFS("one")
|
||||
fsys["catalog/backend.json"].Data = []byte(validManifest(
|
||||
"one",
|
||||
"TEST_API_KEY",
|
||||
`{"large":9007199254740993,"nested":[1.25]}`,
|
||||
))
|
||||
loaded, err := Load(testSource("one", fsys))
|
||||
if err != nil {
|
||||
t.Fatalf("load catalog: %v", err)
|
||||
}
|
||||
if got := loaded.Backends[0].ExtraParams["large"]; got != json.Number("9007199254740993") {
|
||||
t.Fatalf("large JSON integer = %#v, want preserved json.Number", got)
|
||||
}
|
||||
nested := loaded.Backends[0].ExtraParams["nested"].([]any)
|
||||
if nested[0] != json.Number("1.25") {
|
||||
t.Fatalf("nested JSON number = %#v, want preserved json.Number", nested[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidCatalogProfiles(t *testing.T) {
|
||||
tests := map[string]func(fstest.MapFS){
|
||||
"empty": func(fsys fstest.MapFS) {
|
||||
delete(fsys, "catalog/profiles/one-profile.yml")
|
||||
fsys["catalog/profiles"] = &fstest.MapFile{Mode: fs.ModeDir}
|
||||
},
|
||||
"malformed": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: [")
|
||||
},
|
||||
"raw API key": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key: leak-marker\n")
|
||||
},
|
||||
"endpoint field": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "endpoint: ''\n")
|
||||
},
|
||||
"API key environment field": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key_env: ''\n")
|
||||
},
|
||||
"owner mismatch": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "other"))
|
||||
},
|
||||
"missing base": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: leak-marker\n")
|
||||
},
|
||||
"cyclic base": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: second\n")
|
||||
fsys["catalog/profiles/second.yml"] = &fstest.MapFile{Data: []byte("id: second\nbase_profile: one-profile\n")}
|
||||
},
|
||||
"secret profile parameter": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "extra_params:\n nested:\n password: leak-marker\n")
|
||||
},
|
||||
"unknown field is redacted": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "leak_marker: leak-marker\n")
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fsys := validCatalogFS("one")
|
||||
mutate(fsys)
|
||||
_, err := Load(testSource("one", fsys))
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid profile error")
|
||||
}
|
||||
if strings.Contains(strings.ToLower(err.Error()), "leak-marker") {
|
||||
t.Fatalf("catalog error exposed profile content: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsCrossCatalogConflicts(t *testing.T) {
|
||||
t.Run("duplicate backend", func(t *testing.T) {
|
||||
second := catalogFS("one", map[string]string{
|
||||
"catalog/profiles/second.yml": validProfile("second", "one"),
|
||||
}, "null")
|
||||
_, err := Load(
|
||||
Source{Name: "first", ExpectedBackendID: "one", FS: validCatalogFS("one"), Root: "catalog"},
|
||||
Source{Name: "second", ExpectedBackendID: "one", FS: second, Root: "catalog"},
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate backend error")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("duplicate profile", func(t *testing.T) {
|
||||
first := catalogFS("one", map[string]string{
|
||||
"catalog/profiles/shared.yml": validProfile("shared", "one"),
|
||||
}, "null")
|
||||
second := catalogFS("two", map[string]string{
|
||||
"catalog/profiles/shared.yml": validProfile("shared", "two"),
|
||||
}, "null")
|
||||
_, err := Load(testSource("one", first), testSource("two", second))
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate profile error")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("cross-catalog base", func(t *testing.T) {
|
||||
first := catalogFS("one", map[string]string{
|
||||
"catalog/profiles/base.yml": validProfile("base", "one"),
|
||||
}, "null")
|
||||
second := catalogFS("two", map[string]string{
|
||||
"catalog/profiles/child.yml": "id: child\nbase_profile: base\n",
|
||||
}, "null")
|
||||
_, err := Load(testSource("one", first), testSource("two", second))
|
||||
if err == nil {
|
||||
t.Fatal("expected cross-catalog base error")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestLoadReturnsDefensiveCatalogValues(t *testing.T) {
|
||||
fsys := catalogFS("one", map[string]string{
|
||||
"catalog/profiles/one-profile.yml": validProfile("one-profile", "one") + "extra_params:\n nested:\n value: profile\n",
|
||||
}, `{"nested":{"value":"backend"}}`)
|
||||
loaded, err := Load(testSource("one", fsys))
|
||||
if err != nil {
|
||||
t.Fatalf("load catalog: %v", err)
|
||||
}
|
||||
loaded.Backends[0].ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||
profileValue, err := loaded.Profiles.GetProfile(context.Background(), "one-profile")
|
||||
if err != nil {
|
||||
t.Fatalf("load profile: %v", err)
|
||||
}
|
||||
profileValue.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||
|
||||
again, err := Load(testSource("one", fsys))
|
||||
if err != nil {
|
||||
t.Fatalf("reload catalog: %v", err)
|
||||
}
|
||||
if got := again.Backends[0].ExtraParams["nested"].(map[string]any)["value"]; got != "backend" {
|
||||
t.Fatalf("backend mutation escaped returned set: %#v", got)
|
||||
}
|
||||
againProfile, err := loaded.Profiles.GetProfile(context.Background(), "one-profile")
|
||||
if err != nil {
|
||||
t.Fatalf("reload profile: %v", err)
|
||||
}
|
||||
if got := againProfile.ExtraParams["nested"].(map[string]any)["value"]; got != "profile" {
|
||||
t.Fatalf("profile mutation escaped returned value: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func loadCompatibilityFixture(t *testing.T) map[string]any {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join("..", "..", "testdata", "builtin-catalog-v1.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read fixture: %v", err)
|
||||
}
|
||||
var value map[string]any
|
||||
if err := json.Unmarshal(data, &value); err != nil {
|
||||
t.Fatalf("decode fixture: %v", err)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func fixtureProfileIDs(t *testing.T, fixture map[string]any) []string {
|
||||
t.Helper()
|
||||
profiles, ok := fixture["profiles"].([]any)
|
||||
if !ok {
|
||||
t.Fatal("compatibility fixture profiles are malformed")
|
||||
}
|
||||
ids := make([]string, 0, len(profiles))
|
||||
for _, entry := range profiles {
|
||||
profileValue, ok := entry.(map[string]any)
|
||||
if !ok {
|
||||
t.Fatal("compatibility fixture profile is malformed")
|
||||
}
|
||||
id, ok := profileValue["id"].(string)
|
||||
if !ok {
|
||||
t.Fatal("compatibility fixture profile ID is malformed")
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
}
|
||||
|
||||
func catalogValue(t *testing.T, loaded Set) map[string]any {
|
||||
t.Helper()
|
||||
backends := make([]any, 0, len(loaded.Backends))
|
||||
for _, backend := range loaded.Backends {
|
||||
backends = append(backends, map[string]any{"id": backend.ID, "endpoint": backend.Endpoint, "api_key_env": backend.APIKeyEnv, "extra_params": interfaceValue(backend.ExtraParams), "concurrency_limit": backend.ConcurrencyLimit, "queue_capacity": backend.QueueCapacity, "queue_capacity_set": backend.QueueCapacitySet})
|
||||
}
|
||||
sort.Slice(backends, func(left, right int) bool {
|
||||
return backends[left].(map[string]any)["id"].(string) < backends[right].(map[string]any)["id"].(string)
|
||||
})
|
||||
actualProfiles := make([]any, 0, len(loaded.profileIDs))
|
||||
for _, id := range loaded.profileIDs {
|
||||
profile, err := loaded.Profiles.GetProfile(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("load profile %q: %v", id, err)
|
||||
}
|
||||
actualProfiles = append(actualProfiles, map[string]any{"id": profile.ID, "base_profile": profile.BaseProfileID, "backend": profile.BackendID, "endpoint": profile.Endpoint, "model": profile.Model, "temperature": profile.Temperature, "max_tokens": profile.MaxTokens, "top_p": profile.TopP, "timeout_seconds": profile.TimeoutSeconds, "service_tier": profile.ServiceTier, "reasoning_effort": profile.ReasoningEffort, "api_key_env": profile.APIKeyEnv, "api_key_required": profile.APIKeyRequired, "extra_params": interfaceValue(profile.ExtraParams)})
|
||||
}
|
||||
return map[string]any{"backends": backends, "profiles": actualProfiles}
|
||||
}
|
||||
|
||||
func interfaceValue(value map[string]any) any {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func testSource(id string, fsys fs.FS) Source {
|
||||
return Source{Name: id, ExpectedBackendID: id, FS: fsys, Root: "catalog"}
|
||||
}
|
||||
|
||||
func validCatalogFS(id string) fstest.MapFS {
|
||||
return catalogFS(id, map[string]string{
|
||||
"catalog/profiles/" + id + "-profile.yml": validProfile(id+"-profile", id),
|
||||
}, "null")
|
||||
}
|
||||
|
||||
func catalogFS(id string, profiles map[string]string, extraParams string) fstest.MapFS {
|
||||
fsys := fstest.MapFS{
|
||||
"catalog/backend.json": &fstest.MapFile{Data: []byte(validManifest(id, "TEST_API_KEY", extraParams))},
|
||||
}
|
||||
for name, content := range profiles {
|
||||
fsys[name] = &fstest.MapFile{Data: []byte(content)}
|
||||
}
|
||||
return fsys
|
||||
}
|
||||
|
||||
func validManifest(id, apiKeyEnv, extraParams string) string {
|
||||
return fmt.Sprintf(
|
||||
`{"schema_version":1,"id":%q,"endpoint":%q,"api_key_env":%q,"concurrency_limit":2,"queue_capacity":3,"extra_params":%s}`,
|
||||
id,
|
||||
"https://"+id+".example/v1",
|
||||
apiKeyEnv,
|
||||
extraParams,
|
||||
)
|
||||
}
|
||||
|
||||
func validProfile(id, backendID string) string {
|
||||
return fmt.Sprintf("id: %s\nbackend: %s\nmodel: test-model\n", id, backendID)
|
||||
}
|
||||
|
||||
var _ fs.FS = openrouter.FS()
|
||||
@@ -15,10 +15,7 @@ const (
|
||||
OpenAIChatCompletionsPath = "/chat/completions"
|
||||
|
||||
ExecutionDefaultTimeoutSeconds = 600
|
||||
)
|
||||
|
||||
var (
|
||||
LLMRequestTimeoutDefault = 10 * time.Minute
|
||||
LLMRequestTimeoutDefault = 10 * time.Minute
|
||||
)
|
||||
|
||||
func ExecutionTargetDefault() domain.ExecutionTarget {
|
||||
|
||||
@@ -60,15 +60,16 @@ type CacheControl struct {
|
||||
|
||||
// RunRequest represents a request to generate a single artifact.
|
||||
type RunRequest struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
SessionID string
|
||||
APIKey string `json:"-" yaml:"-"`
|
||||
Inputs map[string]ArtifactRef
|
||||
Vars map[string]string
|
||||
Execution *ExecutionTargetOverride
|
||||
Validation *OutputContract
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
SessionID string
|
||||
APIKey string `json:"-" yaml:"-"`
|
||||
Inputs map[string]ArtifactRef
|
||||
Vars map[string]string
|
||||
Execution *ExecutionTargetOverride
|
||||
Validation *OutputContract
|
||||
AppendedMessages []RenderedMessage
|
||||
}
|
||||
|
||||
// RunResult represents the complete result of a prompt execution run.
|
||||
@@ -97,22 +98,22 @@ type RunResult struct {
|
||||
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
|
||||
// It must never include resolved API key values, model output, or validation data.
|
||||
type PreparedRun struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
TargetPresence ExecutionTargetPresence `json:"-"`
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
EndTime time.Time `json:"end_time,omitempty"`
|
||||
DurationMS int64 `json:"duration_ms,omitempty"`
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
PromptHash string
|
||||
SelectedProfileID string
|
||||
SelectedBackendID string
|
||||
EffectiveModelParams ExecutionTarget
|
||||
TargetPresence ExecutionTargetPresence
|
||||
OutputContract OutputContract
|
||||
StructuredOutput *StructuredOutputSpec
|
||||
InputHashes map[string]string
|
||||
SessionID string
|
||||
RenderedPromptHash string
|
||||
Messages []RenderedMessage
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
DurationMS int64
|
||||
}
|
||||
|
||||
// ArtifactRef represents a reference to an input artifact.
|
||||
@@ -192,6 +193,7 @@ type BackendCapacityPolicy struct {
|
||||
// ExecutionProfile describes how and where to execute a model.
|
||||
type ExecutionProfile struct {
|
||||
ID string `yaml:"id"`
|
||||
BaseProfileID string `yaml:"base_profile"`
|
||||
BackendID string `yaml:"backend"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Model string `yaml:"model"`
|
||||
|
||||
39
internal/domain/endpoint.go
Normal file
39
internal/domain/endpoint.go
Normal file
@@ -0,0 +1,39 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// NormalizeOpenAICompatibleBaseEndpoint trims and validates a source-neutral
|
||||
// OpenAI-compatible provider base endpoint.
|
||||
func NormalizeOpenAICompatibleBaseEndpoint(endpoint string) (string, error) {
|
||||
endpoint = strings.TrimSpace(endpoint)
|
||||
if endpoint == "" {
|
||||
return "", errors.New("endpoint must not be blank")
|
||||
}
|
||||
if strings.Contains(endpoint, "#") {
|
||||
return "", errors.New("endpoint must not contain a fragment")
|
||||
}
|
||||
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return "", errors.New("endpoint must be a valid URL")
|
||||
}
|
||||
parsed.Scheme = strings.ToLower(parsed.Scheme)
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return "", errors.New("endpoint must use http or https")
|
||||
}
|
||||
if !parsed.IsAbs() || parsed.Hostname() == "" {
|
||||
return "", errors.New("endpoint must be absolute and include a host")
|
||||
}
|
||||
if parsed.User != nil {
|
||||
return "", errors.New("endpoint must not contain user information")
|
||||
}
|
||||
if parsed.RawQuery != "" || parsed.ForceQuery {
|
||||
return "", errors.New("endpoint must not contain a query string")
|
||||
}
|
||||
|
||||
return parsed.String(), nil
|
||||
}
|
||||
47
internal/domain/endpoint_test.go
Normal file
47
internal/domain/endpoint_test.go
Normal file
@@ -0,0 +1,47 @@
|
||||
package domain
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNormalizeOpenAICompatibleBaseEndpoint(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "http host", endpoint: "http://provider.example", want: "http://provider.example"},
|
||||
{name: "https nested path and whitespace", endpoint: " HTTPS://provider.example/api/openai/v1 ", want: "https://provider.example/api/openai/v1"},
|
||||
{name: "IPv4 host and port", endpoint: "http://127.0.0.1:8080/v1", want: "http://127.0.0.1:8080/v1"},
|
||||
{name: "IPv6 host and port", endpoint: "https://[::1]:8443/v1", want: "https://[::1]:8443/v1"},
|
||||
{name: "repeated trailing slashes", endpoint: "https://provider.example/v1///", want: "https://provider.example/v1///"},
|
||||
{name: "blank", endpoint: " \t\n ", wantErr: true},
|
||||
{name: "relative path", endpoint: "/api/v1", wantErr: true},
|
||||
{name: "scheme relative", endpoint: "//provider.example/v1", wantErr: true},
|
||||
{name: "missing host", endpoint: "https:///v1", wantErr: true},
|
||||
{name: "unsupported scheme", endpoint: "ftp://provider.example/v1", wantErr: true},
|
||||
{name: "user information", endpoint: "https://user:secret@provider.example/v1", wantErr: true},
|
||||
{name: "query", endpoint: "https://provider.example/v1?mode=chat", wantErr: true},
|
||||
{name: "empty query", endpoint: "https://provider.example/v1?", wantErr: true},
|
||||
{name: "fragment", endpoint: "https://provider.example/v1#chat", wantErr: true},
|
||||
{name: "empty fragment", endpoint: "https://provider.example/v1#", wantErr: true},
|
||||
{name: "malformed URL", endpoint: "https://provider.example/%zz", wantErr: true},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := NormalizeOpenAICompatibleBaseEndpoint(tc.endpoint)
|
||||
if tc.wantErr {
|
||||
if err == nil {
|
||||
t.Fatalf("expected endpoint error, got %q", got)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("normalize endpoint: %v", err)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Fatalf("normalized endpoint = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
34
internal/domain/execution_settings.go
Normal file
34
internal/domain/execution_settings.go
Normal file
@@ -0,0 +1,34 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math"
|
||||
"time"
|
||||
)
|
||||
|
||||
const maxExecutionTimeoutSeconds int64 = math.MaxInt64 / int64(time.Second)
|
||||
|
||||
// ValidateExecutionTargetSettings validates source-neutral execution-setting
|
||||
// invariants on a resolved target.
|
||||
func ValidateExecutionTargetSettings(target ExecutionTarget) error {
|
||||
if !isFinite(target.Temperature) || target.Temperature < 0 || target.Temperature > 2 {
|
||||
return errors.New("temperature must be finite and between 0 and 2")
|
||||
}
|
||||
if target.MaxTokens < 0 {
|
||||
return errors.New("max_tokens must be greater than or equal to 0")
|
||||
}
|
||||
if !isFinite(target.TopP) || target.TopP < 0 || target.TopP > 1 {
|
||||
return errors.New("top_p must be finite and between 0 and 1")
|
||||
}
|
||||
if target.TimeoutSeconds < 0 {
|
||||
return errors.New("timeout_seconds must be greater than or equal to 0")
|
||||
}
|
||||
if int64(target.TimeoutSeconds) > maxExecutionTimeoutSeconds {
|
||||
return errors.New("timeout_seconds exceeds the maximum supported duration")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isFinite(value float64) bool {
|
||||
return !math.IsNaN(value) && !math.IsInf(value, 0)
|
||||
}
|
||||
73
internal/domain/execution_settings_test.go
Normal file
73
internal/domain/execution_settings_test.go
Normal file
@@ -0,0 +1,73 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidateExecutionTargetSettings(t *testing.T) {
|
||||
valid := ExecutionTarget{
|
||||
Temperature: 1,
|
||||
MaxTokens: 1,
|
||||
TopP: 0.5,
|
||||
TimeoutSeconds: 1,
|
||||
}
|
||||
|
||||
type testCase struct {
|
||||
name string
|
||||
change func(*ExecutionTarget)
|
||||
wantErr string
|
||||
}
|
||||
tests := []testCase{
|
||||
{name: "temperature lower boundary", change: func(v *ExecutionTarget) { v.Temperature = 0 }},
|
||||
{name: "temperature finite lower neighbor", change: func(v *ExecutionTarget) { v.Temperature = math.Nextafter(0, 1) }},
|
||||
{name: "temperature finite upper neighbor", change: func(v *ExecutionTarget) { v.Temperature = math.Nextafter(2, 0) }},
|
||||
{name: "temperature upper boundary", change: func(v *ExecutionTarget) { v.Temperature = 2 }},
|
||||
{name: "temperature below lower boundary", change: func(v *ExecutionTarget) { v.Temperature = math.Nextafter(0, math.Inf(-1)) }, wantErr: "temperature"},
|
||||
{name: "temperature above upper boundary", change: func(v *ExecutionTarget) { v.Temperature = math.Nextafter(2, math.Inf(1)) }, wantErr: "temperature"},
|
||||
{name: "temperature NaN", change: func(v *ExecutionTarget) { v.Temperature = math.NaN() }, wantErr: "temperature"},
|
||||
{name: "temperature positive infinity", change: func(v *ExecutionTarget) { v.Temperature = math.Inf(1) }, wantErr: "temperature"},
|
||||
{name: "temperature negative infinity", change: func(v *ExecutionTarget) { v.Temperature = math.Inf(-1) }, wantErr: "temperature"},
|
||||
{name: "max tokens lower boundary", change: func(v *ExecutionTarget) { v.MaxTokens = 0 }},
|
||||
{name: "max tokens finite neighbor", change: func(v *ExecutionTarget) { v.MaxTokens = 1 }},
|
||||
{name: "max tokens below lower boundary", change: func(v *ExecutionTarget) { v.MaxTokens = -1 }, wantErr: "max_tokens"},
|
||||
{name: "top p lower boundary", change: func(v *ExecutionTarget) { v.TopP = 0 }},
|
||||
{name: "top p finite lower neighbor", change: func(v *ExecutionTarget) { v.TopP = math.Nextafter(0, 1) }},
|
||||
{name: "top p finite upper neighbor", change: func(v *ExecutionTarget) { v.TopP = math.Nextafter(1, 0) }},
|
||||
{name: "top p upper boundary", change: func(v *ExecutionTarget) { v.TopP = 1 }},
|
||||
{name: "top p below lower boundary", change: func(v *ExecutionTarget) { v.TopP = math.Nextafter(0, math.Inf(-1)) }, wantErr: "top_p"},
|
||||
{name: "top p above upper boundary", change: func(v *ExecutionTarget) { v.TopP = math.Nextafter(1, math.Inf(1)) }, wantErr: "top_p"},
|
||||
{name: "top p NaN", change: func(v *ExecutionTarget) { v.TopP = math.NaN() }, wantErr: "top_p"},
|
||||
{name: "top p positive infinity", change: func(v *ExecutionTarget) { v.TopP = math.Inf(1) }, wantErr: "top_p"},
|
||||
{name: "top p negative infinity", change: func(v *ExecutionTarget) { v.TopP = math.Inf(-1) }, wantErr: "top_p"},
|
||||
{name: "timeout lower boundary", change: func(v *ExecutionTarget) { v.TimeoutSeconds = 0 }},
|
||||
{name: "timeout finite neighbor", change: func(v *ExecutionTarget) { v.TimeoutSeconds = 1 }},
|
||||
{name: "timeout below lower boundary", change: func(v *ExecutionTarget) { v.TimeoutSeconds = -1 }, wantErr: "timeout_seconds"},
|
||||
}
|
||||
if strconv.IntSize == 64 {
|
||||
durationLimit := maxExecutionTimeoutSeconds
|
||||
tests = append(tests,
|
||||
testCase{name: "timeout duration boundary", change: func(v *ExecutionTarget) { v.TimeoutSeconds = int(durationLimit) }},
|
||||
testCase{name: "timeout above duration boundary", change: func(v *ExecutionTarget) { v.TimeoutSeconds = int(durationLimit) + 1 }, wantErr: "timeout_seconds"},
|
||||
)
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
target := valid
|
||||
tt.change(&target)
|
||||
err := ValidateExecutionTargetSettings(target)
|
||||
if tt.wantErr == "" {
|
||||
if err != nil {
|
||||
t.Fatalf("validate execution settings: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("error = %v, want diagnostic containing %q", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
82
internal/domain/message.go
Normal file
82
internal/domain/message.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
RoleDeveloper = "developer"
|
||||
RoleSystem = "system"
|
||||
RoleUser = "user"
|
||||
RoleAssistant = "assistant"
|
||||
)
|
||||
|
||||
// NormalizeMessageRole validates and canonicalizes a provider-bound chat role.
|
||||
func NormalizeMessageRole(role string) (string, error) {
|
||||
if !utf8.ValidString(role) {
|
||||
return "", errors.New("message role must be valid UTF-8")
|
||||
}
|
||||
|
||||
normalized := strings.ToLower(strings.TrimSpace(role))
|
||||
switch normalized {
|
||||
case RoleDeveloper, RoleSystem, RoleUser, RoleAssistant:
|
||||
return normalized, nil
|
||||
default:
|
||||
return "", errors.New("message role must be developer, system, user, or assistant")
|
||||
}
|
||||
}
|
||||
|
||||
// NormalizeCacheControl validates, canonicalizes, and copies cache metadata.
|
||||
func NormalizeCacheControl(control *CacheControl) (*CacheControl, error) {
|
||||
if control == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if !utf8.ValidString(string(control.Type)) {
|
||||
return nil, errors.New("cache control type must be valid UTF-8")
|
||||
}
|
||||
if !utf8.ValidString(control.TTL) {
|
||||
return nil, errors.New("cache control ttl must be valid UTF-8")
|
||||
}
|
||||
|
||||
cacheType := strings.TrimSpace(string(control.Type))
|
||||
if cacheType == "" {
|
||||
return nil, errors.New("cache control type is required")
|
||||
}
|
||||
if CacheControlType(cacheType) != CacheControlEphemeral {
|
||||
return nil, errors.New("unsupported type")
|
||||
}
|
||||
|
||||
ttl := strings.TrimSpace(control.TTL)
|
||||
if ttl != "" && ttl != "1h" {
|
||||
return nil, errors.New("unsupported ttl")
|
||||
}
|
||||
|
||||
return &CacheControl{Type: CacheControlType(cacheType), TTL: ttl}, nil
|
||||
}
|
||||
|
||||
// CloneRenderedMessages returns a deep copy of rendered messages.
|
||||
func CloneRenderedMessages(messages []RenderedMessage) []RenderedMessage {
|
||||
cloned := make([]RenderedMessage, len(messages))
|
||||
copyRenderedMessages(cloned, messages)
|
||||
return cloned
|
||||
}
|
||||
|
||||
// ConcatRenderedMessages returns an independently owned concatenation of messages.
|
||||
func ConcatRenderedMessages(prefix, suffix []RenderedMessage) []RenderedMessage {
|
||||
messages := make([]RenderedMessage, len(prefix)+len(suffix))
|
||||
copyRenderedMessages(messages, prefix)
|
||||
copyRenderedMessages(messages[len(prefix):], suffix)
|
||||
return messages
|
||||
}
|
||||
|
||||
func copyRenderedMessages(destination, source []RenderedMessage) {
|
||||
for index, message := range source {
|
||||
destination[index] = message
|
||||
if message.CacheControl != nil {
|
||||
cacheControl := *message.CacheControl
|
||||
destination[index].CacheControl = &cacheControl
|
||||
}
|
||||
}
|
||||
}
|
||||
121
internal/domain/message_test.go
Normal file
121
internal/domain/message_test.go
Normal file
@@ -0,0 +1,121 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNormalizeMessageRole(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "developer", input: RoleDeveloper, want: RoleDeveloper},
|
||||
{name: "system", input: RoleSystem, want: RoleSystem},
|
||||
{name: "user", input: RoleUser, want: RoleUser},
|
||||
{name: "assistant", input: RoleAssistant, want: RoleAssistant},
|
||||
{name: "surrounding whitespace and mixed case", input: " \u2003UsEr\u2003 ", want: RoleUser},
|
||||
{name: "blank", input: " \t\n ", wantErr: true},
|
||||
{name: "tool", input: "tool", wantErr: true},
|
||||
{name: "function", input: "function", wantErr: true},
|
||||
{name: "custom", input: "custom-role", wantErr: true},
|
||||
{name: "invalid UTF-8", input: string([]byte{0xff}), wantErr: true},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, err := NormalizeMessageRole(test.input)
|
||||
if test.wantErr {
|
||||
if err == nil {
|
||||
t.Fatal("expected an error")
|
||||
}
|
||||
if strings.Contains(err.Error(), test.input) {
|
||||
t.Fatalf("error exposed the input: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("NormalizeMessageRole() error = %v", err)
|
||||
}
|
||||
if got != test.want {
|
||||
t.Fatalf("NormalizeMessageRole() = %q, want %q", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeCacheControl(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input *CacheControl
|
||||
want *CacheControl
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "nil", input: nil, want: nil},
|
||||
{
|
||||
name: "canonical values",
|
||||
input: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"},
|
||||
want: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"},
|
||||
},
|
||||
{
|
||||
name: "trims values",
|
||||
input: &CacheControl{Type: " ephemeral ", TTL: " 1h\t"},
|
||||
want: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"},
|
||||
},
|
||||
{name: "empty type", input: &CacheControl{}, wantErr: true},
|
||||
{name: "unsupported type", input: &CacheControl{Type: "persistent"}, wantErr: true},
|
||||
{name: "unsupported ttl", input: &CacheControl{Type: CacheControlEphemeral, TTL: "5m"}, wantErr: true},
|
||||
{name: "invalid type UTF-8", input: &CacheControl{Type: CacheControlType(string([]byte{0xff}))}, wantErr: true},
|
||||
{name: "invalid ttl UTF-8", input: &CacheControl{Type: CacheControlEphemeral, TTL: string([]byte{0xff})}, wantErr: true},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, err := NormalizeCacheControl(test.input)
|
||||
if test.wantErr {
|
||||
if err == nil {
|
||||
t.Fatal("expected an error")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("NormalizeCacheControl() error = %v", err)
|
||||
}
|
||||
if got == nil || test.want == nil {
|
||||
if got != test.want {
|
||||
t.Fatalf("NormalizeCacheControl() = %#v, want %#v", got, test.want)
|
||||
}
|
||||
return
|
||||
}
|
||||
if *got != *test.want {
|
||||
t.Fatalf("NormalizeCacheControl() = %#v, want %#v", got, test.want)
|
||||
}
|
||||
if got == test.input {
|
||||
t.Fatal("normalized cache control aliases its input")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderedMessageCloning(t *testing.T) {
|
||||
prefix := []RenderedMessage{{Role: RoleSystem, Content: "prefix", CacheControl: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"}}}
|
||||
suffix := []RenderedMessage{{Role: RoleUser, Content: " suffix "}, {Role: RoleAssistant, Content: "", CacheControl: &CacheControl{Type: CacheControlEphemeral}}}
|
||||
|
||||
cloned := CloneRenderedMessages(prefix)
|
||||
combined := ConcatRenderedMessages(prefix, suffix)
|
||||
if len(combined) != 3 || combined[0].Content != "prefix" || combined[1].Content != " suffix " || combined[2].Content != "" {
|
||||
t.Fatalf("unexpected combined messages: %#v", combined)
|
||||
}
|
||||
if cloned[0].CacheControl == prefix[0].CacheControl || combined[0].CacheControl == prefix[0].CacheControl || combined[2].CacheControl == suffix[1].CacheControl {
|
||||
t.Fatal("cloned cache controls alias their inputs")
|
||||
}
|
||||
|
||||
prefix[0].Content = "changed"
|
||||
prefix[0].CacheControl.TTL = ""
|
||||
suffix[1].CacheControl.Type = "changed"
|
||||
if cloned[0].Content != "prefix" || cloned[0].CacheControl.TTL != "1h" || combined[0].Content != "prefix" || combined[0].CacheControl.TTL != "1h" || combined[2].CacheControl.Type != CacheControlEphemeral {
|
||||
t.Fatalf("cloned messages changed with their inputs: cloned=%#v combined=%#v", cloned, combined)
|
||||
}
|
||||
}
|
||||
38
internal/domain/output_contract.go
Normal file
38
internal/domain/output_contract.go
Normal file
@@ -0,0 +1,38 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const maxOutputRepairAttempts = 3
|
||||
|
||||
// ValidateOutputContract validates source-neutral output-contract invariants.
|
||||
func ValidateOutputContract(contract OutputContract) error {
|
||||
switch contract.Format {
|
||||
case FormatText, FormatMarkdown, FormatJSON:
|
||||
default:
|
||||
return fmt.Errorf("invalid output format: %q", contract.Format)
|
||||
}
|
||||
|
||||
switch contract.ValidationMode {
|
||||
case ValidationNone, ValidationBasic, ValidationJSON, ValidationJSONSchema:
|
||||
default:
|
||||
return fmt.Errorf("invalid validation mode: %q", contract.ValidationMode)
|
||||
}
|
||||
|
||||
if contract.ValidationMode == ValidationJSONSchema && strings.TrimSpace(contract.SchemaPath) == "" {
|
||||
return errors.New("schema_path is required when validation_mode is json_schema")
|
||||
}
|
||||
if contract.RepairAttempts < 0 {
|
||||
return errors.New("repair_attempts must be greater than or equal to 0")
|
||||
}
|
||||
if contract.RepairAttempts > maxOutputRepairAttempts {
|
||||
return fmt.Errorf("repair_attempts must be less than or equal to %d", maxOutputRepairAttempts)
|
||||
}
|
||||
if contract.ValidationMode == ValidationNone && contract.RepairAttempts > 0 {
|
||||
return errors.New("repair_attempts requires basic, json, or json_schema validation")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
87
internal/domain/output_contract_test.go
Normal file
87
internal/domain/output_contract_test.go
Normal file
@@ -0,0 +1,87 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidateOutputContract(t *testing.T) {
|
||||
valid := OutputContract{
|
||||
Format: FormatText,
|
||||
ValidationMode: ValidationNone,
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
change func(*OutputContract)
|
||||
wantErr string
|
||||
}{
|
||||
{name: "text format", change: func(c *OutputContract) { c.Format = FormatText }},
|
||||
{name: "markdown format", change: func(c *OutputContract) { c.Format = FormatMarkdown }},
|
||||
{name: "json format", change: func(c *OutputContract) { c.Format = FormatJSON }},
|
||||
{name: "empty format", change: func(c *OutputContract) { c.Format = "" }, wantErr: "format"},
|
||||
{name: "unsupported format", change: func(c *OutputContract) { c.Format = OutputFormat("binary") }, wantErr: "format"},
|
||||
{name: "none validation", change: func(c *OutputContract) { c.ValidationMode = ValidationNone }},
|
||||
{name: "basic validation", change: func(c *OutputContract) { c.ValidationMode = ValidationBasic }},
|
||||
{name: "json validation", change: func(c *OutputContract) { c.ValidationMode = ValidationJSON }},
|
||||
{name: "json schema validation", change: func(c *OutputContract) {
|
||||
c.ValidationMode = ValidationJSONSchema
|
||||
c.SchemaPath = "schema.json"
|
||||
}},
|
||||
{name: "empty validation mode", change: func(c *OutputContract) { c.ValidationMode = "" }, wantErr: "validation mode"},
|
||||
{name: "unsupported validation mode", change: func(c *OutputContract) { c.ValidationMode = ValidationMode("unknown") }, wantErr: "validation mode"},
|
||||
{name: "negative repair attempts", change: func(c *OutputContract) {
|
||||
c.ValidationMode = ValidationBasic
|
||||
c.RepairAttempts = -1
|
||||
}, wantErr: "repair_attempts"},
|
||||
{name: "zero repair attempts", change: func(c *OutputContract) { c.RepairAttempts = 0 }},
|
||||
{name: "one repair attempt", change: func(c *OutputContract) {
|
||||
c.ValidationMode = ValidationBasic
|
||||
c.RepairAttempts = 1
|
||||
}},
|
||||
{name: "maximum repair attempts", change: func(c *OutputContract) {
|
||||
c.ValidationMode = ValidationJSON
|
||||
c.RepairAttempts = 3
|
||||
}},
|
||||
{name: "too many repair attempts", change: func(c *OutputContract) {
|
||||
c.ValidationMode = ValidationJSONSchema
|
||||
c.SchemaPath = "schema.json"
|
||||
c.RepairAttempts = 4
|
||||
}, wantErr: "repair_attempts"},
|
||||
{name: "none validation with repair attempts", change: func(c *OutputContract) {
|
||||
c.ValidationMode = ValidationNone
|
||||
c.RepairAttempts = 1
|
||||
}, wantErr: "repair_attempts"},
|
||||
{name: "json schema empty path", change: func(c *OutputContract) {
|
||||
c.ValidationMode = ValidationJSONSchema
|
||||
c.SchemaPath = ""
|
||||
}, wantErr: "schema_path"},
|
||||
{name: "json schema whitespace path", change: func(c *OutputContract) {
|
||||
c.ValidationMode = ValidationJSONSchema
|
||||
c.SchemaPath = " \t "
|
||||
}, wantErr: "schema_path"},
|
||||
{name: "json schema nonblank path", change: func(c *OutputContract) {
|
||||
c.ValidationMode = ValidationJSONSchema
|
||||
c.SchemaPath = " schema.json "
|
||||
}},
|
||||
{name: "non-schema empty path", change: func(c *OutputContract) { c.SchemaPath = "" }},
|
||||
{name: "non-schema populated path", change: func(c *OutputContract) { c.SchemaPath = "ignored.json" }},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
contract := valid
|
||||
tt.change(&contract)
|
||||
err := ValidateOutputContract(contract)
|
||||
if tt.wantErr == "" {
|
||||
if err != nil {
|
||||
t.Fatalf("validate output contract: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("error = %v, want diagnostic containing %q", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,141 +0,0 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
|
||||
const envName = "PROMPTKIT_TEST_API_KEY"
|
||||
const secret = "super-secret-value"
|
||||
t.Setenv(envName, secret)
|
||||
|
||||
prepared := PreparedRun{
|
||||
PromptID: "prompt.id",
|
||||
PromptVersion: "v1",
|
||||
PromptHash: "prompt-hash",
|
||||
SelectedProfileID: "local-fast",
|
||||
EffectiveModelParams: ExecutionTarget{
|
||||
Endpoint: "http://llm/v1",
|
||||
Model: "gpt-test",
|
||||
APIKeyEnv: envName,
|
||||
APIKey: secret,
|
||||
},
|
||||
InputHashes: map[string]string{"transcript": "hash-1"},
|
||||
RenderedPromptHash: "rendered-hash",
|
||||
Messages: []RenderedMessage{
|
||||
{Role: "system", Content: "You are helpful."},
|
||||
{Role: "user", Content: "Summarize this."},
|
||||
},
|
||||
}
|
||||
|
||||
b, err := json.Marshal(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal failed: %v", err)
|
||||
}
|
||||
|
||||
out := string(b)
|
||||
if strings.Contains(out, secret) {
|
||||
t.Fatalf("prepared run JSON unexpectedly contains secret value: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, `"api_key_env":"`+envName+`"`) {
|
||||
t.Fatalf("prepared run JSON should include api_key_env name: %s", out)
|
||||
}
|
||||
|
||||
var top map[string]any
|
||||
if err := json.Unmarshal(b, &top); err != nil {
|
||||
t.Fatalf("unmarshal failed: %v", err)
|
||||
}
|
||||
|
||||
for _, forbidden := range []string{"raw_output", "validation", "artifact"} {
|
||||
if _, ok := top[forbidden]; ok {
|
||||
t.Fatalf("prepared run JSON should not include %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedRunJSONIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
||||
prepared := PreparedRun{
|
||||
PromptID: "prompt.id",
|
||||
SelectedProfileID: "local-fast",
|
||||
EffectiveModelParams: ExecutionTarget{
|
||||
Endpoint: "http://llm/v1",
|
||||
Model: "gpt-test",
|
||||
},
|
||||
RenderedPromptHash: "rendered-hash",
|
||||
Messages: []RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "You are helpful.",
|
||||
CacheControl: &CacheControl{
|
||||
Type: CacheControlEphemeral,
|
||||
TTL: "1h",
|
||||
},
|
||||
},
|
||||
{Role: "user", Content: "Summarize this."},
|
||||
},
|
||||
}
|
||||
|
||||
b, err := json.Marshal(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal failed: %v", err)
|
||||
}
|
||||
|
||||
var decoded struct {
|
||||
Messages []map[string]any `json:"messages"`
|
||||
}
|
||||
if err := json.Unmarshal(b, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal failed: %v", err)
|
||||
}
|
||||
if len(decoded.Messages) != 2 {
|
||||
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
|
||||
}
|
||||
|
||||
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected cache_control on first message, got %#v", decoded.Messages[0])
|
||||
}
|
||||
if cacheControl["type"] != string(CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
||||
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
||||
}
|
||||
if _, ok := decoded.Messages[1]["cache_control"]; ok {
|
||||
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedRunJSONIncludesSessionIDOnlyWhenPresent(t *testing.T) {
|
||||
prepared := PreparedRun{
|
||||
PromptID: "prompt.id",
|
||||
SelectedProfileID: "local-fast",
|
||||
EffectiveModelParams: ExecutionTarget{
|
||||
Endpoint: "http://llm/v1",
|
||||
Model: "gpt-test",
|
||||
},
|
||||
SessionID: "session-123",
|
||||
RenderedPromptHash: "rendered-hash",
|
||||
Messages: []RenderedMessage{{Role: "user", Content: "Summarize this."}},
|
||||
}
|
||||
|
||||
b, err := json.Marshal(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal failed: %v", err)
|
||||
}
|
||||
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(b, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal failed: %v", err)
|
||||
}
|
||||
if decoded["session_id"] != "session-123" {
|
||||
t.Fatalf("expected session_id in prepared run JSON, got %#v", decoded["session_id"])
|
||||
}
|
||||
|
||||
prepared.SessionID = ""
|
||||
b, err = json.Marshal(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal failed: %v", err)
|
||||
}
|
||||
if strings.Contains(string(b), "session_id") {
|
||||
t.Fatalf("expected empty session_id to be omitted, got %s", b)
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,9 @@ import (
|
||||
|
||||
// NormalizeSessionID applies the shared session identifier rule.
|
||||
func NormalizeSessionID(raw string) (string, error) {
|
||||
if !utf8.ValidString(raw) {
|
||||
return "", fmt.Errorf("session_id must contain valid UTF-8")
|
||||
}
|
||||
normalized := strings.TrimSpace(raw)
|
||||
if normalized == "" {
|
||||
return "", nil
|
||||
|
||||
@@ -7,10 +7,10 @@ import (
|
||||
|
||||
func TestNormalizeSessionID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
want string
|
||||
wantErr bool
|
||||
name string
|
||||
raw string
|
||||
want string
|
||||
wantErrContains string
|
||||
}{
|
||||
{
|
||||
name: "trims surrounding Unicode whitespace",
|
||||
@@ -28,21 +28,24 @@ func TestNormalizeSessionID(t *testing.T) {
|
||||
want: strings.Repeat("界", SessionIDMaxLength),
|
||||
},
|
||||
{
|
||||
name: "one Unicode code point over maximum is rejected",
|
||||
raw: strings.Repeat("界", SessionIDMaxLength+1),
|
||||
wantErr: true,
|
||||
name: "one Unicode code point over maximum is rejected",
|
||||
raw: strings.Repeat("界", SessionIDMaxLength+1),
|
||||
wantErrContains: "exceeds maximum",
|
||||
},
|
||||
{name: "invalid UTF-8 before valid content", raw: string([]byte{0xff}) + "session", wantErrContains: "valid UTF-8"},
|
||||
{name: "invalid UTF-8 within valid content", raw: "ses" + string([]byte{0xff}) + "sion", wantErrContains: "valid UTF-8"},
|
||||
{name: "invalid UTF-8 after valid content", raw: "session" + string([]byte{0xff}), wantErrContains: "valid UTF-8"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := NormalizeSessionID(tt.raw)
|
||||
if tt.wantErr {
|
||||
if tt.wantErrContains != "" {
|
||||
if err == nil {
|
||||
t.Fatal("expected normalization error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "exceeds maximum") {
|
||||
t.Fatalf("expected useful length diagnostic, got %v", err)
|
||||
if !strings.Contains(err.Error(), tt.wantErrContains) {
|
||||
t.Fatalf("expected diagnostic containing %q, got %v", tt.wantErrContains, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -36,7 +36,8 @@ func FindYAMLFiles(ctx context.Context, root string) ([]string, error) {
|
||||
return files, err
|
||||
}
|
||||
|
||||
// FindFSYAMLFiles returns sorted paths for .yaml and .yml files under root in fsys.
|
||||
// FindFSYAMLFiles returns root itself when it names a file. For a directory
|
||||
// root, it returns sorted paths for .yaml and .yml files beneath that root.
|
||||
func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, error) {
|
||||
cleanRoot := CleanFSRoot(root)
|
||||
var files []string
|
||||
@@ -52,6 +53,10 @@ func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, er
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if name == cleanRoot {
|
||||
files = append(files, name)
|
||||
return nil
|
||||
}
|
||||
if !IsYAMLFile(d.Name()) {
|
||||
return nil
|
||||
}
|
||||
@@ -71,10 +76,10 @@ func RelativePath(root string, filePath string) string {
|
||||
return filepath.Clean(rel)
|
||||
}
|
||||
|
||||
// CleanFSRoot normalizes a root path for use with fs.FS.
|
||||
// CleanFSRoot normalizes a root path for use with fs.FS while preserving
|
||||
// nonblank leading and trailing whitespace.
|
||||
func CleanFSRoot(root string) string {
|
||||
root = strings.TrimSpace(root)
|
||||
if root == "" || root == "." {
|
||||
if strings.TrimSpace(root) == "" || root == "." {
|
||||
return "."
|
||||
}
|
||||
return path.Clean(root)
|
||||
@@ -97,22 +102,21 @@ func DisplayPath(root string, name string) string {
|
||||
// ResolveFSPath resolves userPath from baseDir and keeps it inside root.
|
||||
func ResolveFSPath(root string, baseDir string, userPath string) (string, string, error) {
|
||||
cleanRoot := CleanFSRoot(root)
|
||||
cleanBase := path.Clean(strings.TrimSpace(baseDir))
|
||||
if cleanBase == "" {
|
||||
cleanBase := path.Clean(baseDir)
|
||||
if strings.TrimSpace(baseDir) == "" {
|
||||
cleanBase = cleanRoot
|
||||
}
|
||||
if !containsFSPath(cleanRoot, cleanBase) {
|
||||
return "", "", fmt.Errorf("base path %q is outside source root %q", cleanBase, cleanRoot)
|
||||
}
|
||||
|
||||
cleanUserPath := strings.TrimSpace(userPath)
|
||||
if cleanUserPath == "" {
|
||||
if strings.TrimSpace(userPath) == "" {
|
||||
return "", "", fmt.Errorf("path is required")
|
||||
}
|
||||
cleanUserPath = path.Clean(cleanUserPath)
|
||||
if path.IsAbs(cleanUserPath) {
|
||||
if path.IsAbs(userPath) {
|
||||
return "", "", fmt.Errorf("path %q must be relative", userPath)
|
||||
}
|
||||
cleanUserPath := path.Clean(userPath)
|
||||
|
||||
resolved := path.Clean(path.Join(cleanBase, cleanUserPath))
|
||||
if !containsFSPath(cleanRoot, resolved) {
|
||||
@@ -130,13 +134,6 @@ func containsFSPath(root string, name string) bool {
|
||||
return name == root || strings.HasPrefix(name, strings.TrimSuffix(root, "/")+"/")
|
||||
}
|
||||
|
||||
// Stem strips .yaml or .yml from a file name.
|
||||
func Stem(name string) string {
|
||||
name = strings.TrimSuffix(name, ".yaml")
|
||||
name = strings.TrimSuffix(name, ".yml")
|
||||
return name
|
||||
}
|
||||
|
||||
func IsYAMLFile(name string) bool {
|
||||
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
|
||||
}
|
||||
|
||||
@@ -54,7 +54,7 @@ func TestFindFSYAMLFilesNestedSortedAndFiltered(t *testing.T) {
|
||||
"other/ignored.yaml": &fstest.MapFile{Data: []byte("id: ignored")},
|
||||
}
|
||||
|
||||
got, err := FindFSYAMLFiles(context.Background(), fsys, " prompts ")
|
||||
got, err := FindFSYAMLFiles(context.Background(), fsys, "prompts")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
@@ -98,8 +98,11 @@ func TestCleanFSRoot(t *testing.T) {
|
||||
want string
|
||||
}{
|
||||
{name: "empty", root: "", want: "."},
|
||||
{name: "whitespace only", root: " \t ", want: "."},
|
||||
{name: "dot", root: ".", want: "."},
|
||||
{name: "trimmed", root: " prompts/../profiles ", want: "profiles"},
|
||||
{name: "cleaned", root: "prompts/../profiles", want: "profiles"},
|
||||
{name: "leading whitespace preserved", root: " profiles", want: " profiles"},
|
||||
{name: "trailing whitespace preserved", root: "profiles ", want: "profiles "},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
@@ -158,6 +161,22 @@ func TestResolveFSPath(t *testing.T) {
|
||||
wantPath: "prompts/shared/user.tmpl",
|
||||
wantDisplay: "shared/user.tmpl",
|
||||
},
|
||||
{
|
||||
name: "leading whitespace preserved",
|
||||
root: "prompts",
|
||||
baseDir: "prompts/nested",
|
||||
userPath: " user.tmpl",
|
||||
wantPath: "prompts/nested/ user.tmpl",
|
||||
wantDisplay: "nested/ user.tmpl",
|
||||
},
|
||||
{
|
||||
name: "trailing whitespace preserved",
|
||||
root: "prompts",
|
||||
baseDir: "prompts/nested",
|
||||
userPath: "user.tmpl ",
|
||||
wantPath: "prompts/nested/user.tmpl ",
|
||||
wantDisplay: "nested/user.tmpl ",
|
||||
},
|
||||
{
|
||||
name: "escape rejected",
|
||||
root: "prompts",
|
||||
@@ -218,26 +237,6 @@ func TestResolveFSPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestStemStripsYAMLExtensions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{name: "yaml", in: "prompt.yaml", want: "prompt"},
|
||||
{name: "yml", in: "profile.yml", want: "profile"},
|
||||
{name: "other", in: "file.txt", want: "file.txt"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := Stem(tc.in); got != tc.want {
|
||||
t.Fatalf("expected %q, got %q", tc.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsYAMLFile(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Package jsonvalue validates and defensively copies JSON-compatible value
|
||||
// trees used by configuration, request, and prepared-state boundaries.
|
||||
// Package jsonvalue validates and defensively copies bounded JSON-compatible
|
||||
// value trees used by configuration, request, and prepared-state boundaries.
|
||||
package jsonvalue
|
||||
|
||||
import (
|
||||
@@ -8,29 +8,39 @@ import (
|
||||
"math"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
const maxSafeJSONInteger = 1<<53 - 1
|
||||
const (
|
||||
maxContainerDepth = 100
|
||||
maxProducedNodes = 100_000
|
||||
)
|
||||
|
||||
type visit struct {
|
||||
typ reflect.Type
|
||||
ptr uintptr
|
||||
}
|
||||
|
||||
type traversalState struct {
|
||||
active map[visit]struct{}
|
||||
producedNodes int
|
||||
}
|
||||
|
||||
// Copy validates and deeply copies a JSON-compatible value while preserving
|
||||
// compatible concrete map, slice, array, scalar, and number types.
|
||||
// compatible concrete map, slice, array, scalar, and number types. It rejects
|
||||
// cycles and values that exceed the package's traversal limits.
|
||||
func Copy(src any) (any, error) {
|
||||
return copyValue(reflect.ValueOf(src), "value", make(map[visit]struct{}), true)
|
||||
return copyValue(reflect.ValueOf(src), "value", newTraversalState(), true, 0)
|
||||
}
|
||||
|
||||
// CopyMap validates and deeply copies an extra-parameter map while preserving
|
||||
// compatible concrete map, slice, array, scalar, and number types.
|
||||
// compatible concrete map, slice, array, scalar, and number types. It rejects
|
||||
// empty object keys, cycles, and values that exceed the package's traversal
|
||||
// limits.
|
||||
func CopyMap(src map[string]any) (map[string]any, error) {
|
||||
if src == nil {
|
||||
return nil, nil
|
||||
}
|
||||
copied, err := copyValue(reflect.ValueOf(src), "extra_params", make(map[visit]struct{}), false)
|
||||
copied, err := copyValue(reflect.ValueOf(src), "extra_params", newTraversalState(), false, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -44,71 +54,90 @@ func CopyMap(src map[string]any) (map[string]any, error) {
|
||||
func copyValue(
|
||||
value reflect.Value,
|
||||
path string,
|
||||
seen map[visit]struct{},
|
||||
state *traversalState,
|
||||
allowEmptyMapKeys bool,
|
||||
containerDepth int,
|
||||
) (any, error) {
|
||||
if !value.IsValid() {
|
||||
resolved, cleanup, isNull, err := state.resolveIndirection(value, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cleanup()
|
||||
if isNull {
|
||||
if err := state.produceNode(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
if value.Kind() == reflect.Interface {
|
||||
if value.IsNil() {
|
||||
return nil, nil
|
||||
}
|
||||
return copyValue(value.Elem(), path, seen, allowEmptyMapKeys)
|
||||
}
|
||||
value = resolved
|
||||
if !value.CanInterface() {
|
||||
return nil, fmt.Errorf("%s: value cannot be copied", path)
|
||||
}
|
||||
if number, ok := value.Interface().(json.Number); ok {
|
||||
if _, err := json.Marshal(number); err != nil {
|
||||
if !validJSONNumber(number) {
|
||||
return nil, fmt.Errorf("%s: invalid JSON number", path)
|
||||
}
|
||||
f, err := strconv.ParseFloat(number.String(), 64)
|
||||
if err != nil || math.IsNaN(f) || math.IsInf(f, 0) {
|
||||
return nil, fmt.Errorf("%s: invalid JSON number", path)
|
||||
if err := state.produceNode(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return number, nil
|
||||
}
|
||||
|
||||
switch value.Kind() {
|
||||
case reflect.Bool, reflect.String:
|
||||
if err := state.produceNode(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return value.Interface(), nil
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
if value.Int() < -maxSafeJSONInteger || value.Int() > maxSafeJSONInteger {
|
||||
return nil, fmt.Errorf("%s: integer is outside the JSON-safe range", path)
|
||||
if err := state.produceNode(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return value.Interface(), nil
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
|
||||
if value.Uint() > maxSafeJSONInteger {
|
||||
return nil, fmt.Errorf("%s: integer is outside the JSON-safe range", path)
|
||||
if err := state.produceNode(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return value.Interface(), nil
|
||||
case reflect.Float32, reflect.Float64:
|
||||
number := value.Convert(reflect.TypeOf(float64(0))).Float()
|
||||
number := value.Float()
|
||||
if math.IsNaN(number) || math.IsInf(number, 0) {
|
||||
return nil, fmt.Errorf("%s: floating-point value must be finite", path)
|
||||
}
|
||||
if err := state.produceNode(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return value.Interface(), nil
|
||||
case reflect.Pointer:
|
||||
case reflect.Map:
|
||||
if value.IsNil() {
|
||||
if err := state.produceNode(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
current := visit{typ: value.Type(), ptr: value.Pointer()}
|
||||
if _, ok := seen[current]; ok {
|
||||
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||
nextDepth, err := state.enterContainer(path, containerDepth)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seen[current] = struct{}{}
|
||||
defer delete(seen, current)
|
||||
return copyValue(value.Elem(), path, seen, allowEmptyMapKeys)
|
||||
case reflect.Map:
|
||||
return copyMapValue(value, path, seen, allowEmptyMapKeys)
|
||||
return copyMapValue(value, path, state, allowEmptyMapKeys, nextDepth)
|
||||
case reflect.Slice:
|
||||
if value.IsNil() {
|
||||
if err := state.produceNode(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
return copySequenceValue(value, path, seen, allowEmptyMapKeys)
|
||||
nextDepth, err := state.enterContainer(path, containerDepth)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return copySequenceValue(value, path, state, allowEmptyMapKeys, nextDepth)
|
||||
case reflect.Array:
|
||||
return copySequenceValue(value, path, seen, allowEmptyMapKeys)
|
||||
nextDepth, err := state.enterContainer(path, containerDepth)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return copySequenceValue(value, path, state, allowEmptyMapKeys, nextDepth)
|
||||
default:
|
||||
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
|
||||
}
|
||||
@@ -117,22 +146,26 @@ func copyValue(
|
||||
func copyMapValue(
|
||||
value reflect.Value,
|
||||
path string,
|
||||
seen map[visit]struct{},
|
||||
state *traversalState,
|
||||
allowEmptyMapKeys bool,
|
||||
containerDepth int,
|
||||
) (any, error) {
|
||||
if value.IsNil() {
|
||||
return nil, nil
|
||||
}
|
||||
if value.Type().Key().Kind() != reflect.String {
|
||||
return nil, fmt.Errorf("%s: map key type %s is not supported", path, value.Type().Key())
|
||||
}
|
||||
if err := state.produceNode(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := state.ensureChildCapacity(path, value.Len()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
current := visit{typ: value.Type(), ptr: value.Pointer()}
|
||||
if _, ok := seen[current]; ok {
|
||||
if _, ok := state.active[current]; ok {
|
||||
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||
}
|
||||
seen[current] = struct{}{}
|
||||
defer delete(seen, current)
|
||||
state.active[current] = struct{}{}
|
||||
defer delete(state.active, current)
|
||||
|
||||
keys := value.MapKeys()
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
@@ -152,7 +185,13 @@ func copyMapValue(
|
||||
if name == "" && !allowEmptyMapKeys {
|
||||
return nil, fmt.Errorf("%s: map key must not be empty", path)
|
||||
}
|
||||
copied, err := copyValue(value.MapIndex(key), path+"."+name, seen, allowEmptyMapKeys)
|
||||
copied, err := copyValue(
|
||||
value.MapIndex(key),
|
||||
path+"."+name,
|
||||
state,
|
||||
allowEmptyMapKeys,
|
||||
containerDepth,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -190,17 +229,25 @@ func copyMapValue(
|
||||
func copySequenceValue(
|
||||
value reflect.Value,
|
||||
path string,
|
||||
seen map[visit]struct{},
|
||||
state *traversalState,
|
||||
allowEmptyMapKeys bool,
|
||||
containerDepth int,
|
||||
) (any, error) {
|
||||
if err := state.produceNode(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := state.ensureChildCapacity(path, value.Len()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var current visit
|
||||
if value.Kind() == reflect.Slice {
|
||||
current = visit{typ: value.Type(), ptr: value.Pointer()}
|
||||
if _, ok := seen[current]; ok {
|
||||
if _, ok := state.active[current]; ok {
|
||||
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||
}
|
||||
seen[current] = struct{}{}
|
||||
defer delete(seen, current)
|
||||
state.active[current] = struct{}{}
|
||||
defer delete(state.active, current)
|
||||
}
|
||||
|
||||
values := make([]any, value.Len())
|
||||
@@ -210,8 +257,9 @@ func copySequenceValue(
|
||||
copied, err := copyValue(
|
||||
value.Index(i),
|
||||
fmt.Sprintf("%s[%d]", path, i),
|
||||
seen,
|
||||
state,
|
||||
allowEmptyMapKeys,
|
||||
containerDepth,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -248,6 +296,73 @@ func copySequenceValue(
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func validJSONNumber(number json.Number) bool {
|
||||
var parsed json.Number
|
||||
if err := json.Unmarshal([]byte(number.String()), &parsed); err != nil {
|
||||
return false
|
||||
}
|
||||
return parsed.String() == number.String()
|
||||
}
|
||||
|
||||
func newTraversalState() *traversalState {
|
||||
return &traversalState{active: make(map[visit]struct{})}
|
||||
}
|
||||
|
||||
func (state *traversalState) produceNode(path string) error {
|
||||
if state.producedNodes >= maxProducedNodes {
|
||||
return fmt.Errorf("%s: JSON value work limit exceeded", path)
|
||||
}
|
||||
state.producedNodes++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (state *traversalState) enterContainer(path string, depth int) (int, error) {
|
||||
depth++
|
||||
if depth > maxContainerDepth {
|
||||
return 0, fmt.Errorf("%s: JSON container depth limit exceeded", path)
|
||||
}
|
||||
return depth, nil
|
||||
}
|
||||
|
||||
func (state *traversalState) ensureChildCapacity(path string, count int) error {
|
||||
if count > maxProducedNodes-state.producedNodes {
|
||||
return fmt.Errorf("%s: JSON value work limit exceeded", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (state *traversalState) resolveIndirection(
|
||||
value reflect.Value,
|
||||
path string,
|
||||
) (reflect.Value, func(), bool, error) {
|
||||
var visits []visit
|
||||
cleanup := func() {
|
||||
for _, current := range visits {
|
||||
delete(state.active, current)
|
||||
}
|
||||
}
|
||||
|
||||
for value.IsValid() && (value.Kind() == reflect.Interface || value.Kind() == reflect.Pointer) {
|
||||
if value.IsNil() {
|
||||
return reflect.Value{}, cleanup, true, nil
|
||||
}
|
||||
if value.Kind() == reflect.Pointer {
|
||||
current := visit{typ: value.Type(), ptr: value.Pointer()}
|
||||
if _, ok := state.active[current]; ok {
|
||||
cleanup()
|
||||
return reflect.Value{}, nil, false, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||
}
|
||||
state.active[current] = struct{}{}
|
||||
visits = append(visits, current)
|
||||
}
|
||||
value = value.Elem()
|
||||
}
|
||||
if !value.IsValid() {
|
||||
return reflect.Value{}, cleanup, true, nil
|
||||
}
|
||||
return value, cleanup, false, nil
|
||||
}
|
||||
|
||||
func canAssignNil(typ reflect.Type) bool {
|
||||
switch typ.Kind() {
|
||||
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
||||
|
||||
@@ -1,112 +1,332 @@
|
||||
package jsonvalue_test
|
||||
package jsonvalue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"math"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
)
|
||||
|
||||
func TestCopyMapPreservesTypesAndIsolatesMutations(t *testing.T) {
|
||||
nested := map[string]int{"limit": 2}
|
||||
sequence := []string{"one", "two"}
|
||||
input := map[string]any{
|
||||
"count": int64(7),
|
||||
"number": json.Number("-1.25e+2"),
|
||||
"nested": nested,
|
||||
"sequence": sequence,
|
||||
}
|
||||
|
||||
copied, err := jsonvalue.CopyMap(input)
|
||||
if err != nil {
|
||||
t.Fatalf("copy map: %v", err)
|
||||
}
|
||||
nested["limit"] = 99
|
||||
sequence[0] = "changed"
|
||||
input["added"] = true
|
||||
|
||||
if got, ok := copied["count"].(int64); !ok || got != 7 {
|
||||
t.Fatalf("integer type or value changed: %#v", copied["count"])
|
||||
}
|
||||
if got, ok := copied["number"].(json.Number); !ok || got != "-1.25e+2" {
|
||||
t.Fatalf("JSON number type or value changed: %#v", copied["number"])
|
||||
}
|
||||
if got := copied["nested"].(map[string]int)["limit"]; got != 2 {
|
||||
t.Fatalf("nested map was not isolated: %d", got)
|
||||
}
|
||||
if got := copied["sequence"].([]string)[0]; got != "one" {
|
||||
t.Fatalf("sequence was not isolated: %q", got)
|
||||
}
|
||||
if _, ok := copied["added"]; ok {
|
||||
t.Fatalf("top-level map was not isolated: %#v", copied)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyAllowsEmptyObjectKeysAndIsolatesMutations(t *testing.T) {
|
||||
nested := map[string]any{"": []any{"original"}}
|
||||
|
||||
copiedValue, err := jsonvalue.Copy(nested)
|
||||
if err != nil {
|
||||
t.Fatalf("copy value: %v", err)
|
||||
}
|
||||
nested[""].([]any)[0] = "changed"
|
||||
|
||||
copied := copiedValue.(map[string]any)
|
||||
if got := copied[""].([]any)[0]; got != "original" {
|
||||
t.Fatalf("copied value was not isolated: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyMapRejectsInvalidValues(t *testing.T) {
|
||||
cyclicMap := map[string]any{}
|
||||
cyclicMap["self"] = cyclicMap
|
||||
cyclicSlice := []any{nil}
|
||||
cyclicSlice[0] = cyclicSlice
|
||||
type (
|
||||
namedBool bool
|
||||
namedString string
|
||||
namedInt64 int64
|
||||
namedUint64 uint64
|
||||
namedFloat32 float32
|
||||
namedFloat64 float64
|
||||
namedKey string
|
||||
namedMap map[namedKey]namedInt64
|
||||
namedSlice []namedString
|
||||
namedArray [1]map[string]int
|
||||
)
|
||||
|
||||
func TestCopyPreservesSupportedScalarAndNumberTypes(t *testing.T) {
|
||||
maxInt := int(^uint(0) >> 1)
|
||||
minInt := -maxInt - 1
|
||||
tests := []struct {
|
||||
name string
|
||||
value any
|
||||
}{
|
||||
{name: "empty nested key", value: map[string]int{"": 1}},
|
||||
{name: "non-string map key", value: map[int]string{1: "one"}},
|
||||
{name: "unsupported value", value: make(chan int)},
|
||||
{name: "cyclic map", value: cyclicMap},
|
||||
{name: "cyclic slice", value: cyclicSlice},
|
||||
{name: "NaN", value: math.NaN()},
|
||||
{name: "positive infinity", value: math.Inf(1)},
|
||||
{name: "unsafe signed integer", value: int64(1 << 53)},
|
||||
{name: "unsafe unsigned integer", value: uint64(1 << 53)},
|
||||
{name: "bool", value: true},
|
||||
{name: "named bool", value: namedBool(true)},
|
||||
{name: "string", value: "value"},
|
||||
{name: "named string", value: namedString("value")},
|
||||
{name: "int", value: minInt},
|
||||
{name: "int8", value: int8(-1 << 7)},
|
||||
{name: "int16", value: int16(-1 << 15)},
|
||||
{name: "int32", value: int32(-1 << 31)},
|
||||
{name: "int64", value: int64(-1 << 63)},
|
||||
{name: "named int64", value: namedInt64(1<<63 - 1)},
|
||||
{name: "uint", value: ^uint(0)},
|
||||
{name: "uint8", value: ^uint8(0)},
|
||||
{name: "uint16", value: ^uint16(0)},
|
||||
{name: "uint32", value: ^uint32(0)},
|
||||
{name: "uint64", value: ^uint64(0)},
|
||||
{name: "uintptr", value: ^uintptr(0)},
|
||||
{name: "named uint64", value: namedUint64(^uint64(0))},
|
||||
{name: "float32", value: float32(1.25)},
|
||||
{name: "float64", value: float64(-2.5e100)},
|
||||
{name: "named float32", value: namedFloat32(3.5)},
|
||||
{name: "named float64", value: namedFloat64(-4.5e200)},
|
||||
{name: "JSON number integer", value: json.Number("18446744073709551615")},
|
||||
{name: "JSON number fraction", value: json.Number("-1.25e+2")},
|
||||
{name: "JSON number beyond float64", value: json.Number("1e9999")},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if _, err := jsonvalue.CopyMap(map[string]any{"value": tc.value}); err == nil {
|
||||
got, err := Copy(tc.value)
|
||||
if err != nil {
|
||||
t.Fatalf("copy value: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, tc.value) {
|
||||
t.Fatalf("value or concrete type changed: got %#v (%T), want %#v (%T)", got, got, tc.value, tc.value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyRejectsInvalidNumbers(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
value any
|
||||
}{
|
||||
{name: "float32 NaN", value: float32(math.NaN())},
|
||||
{name: "float64 NaN", value: math.NaN()},
|
||||
{name: "named float NaN", value: namedFloat64(math.NaN())},
|
||||
{name: "positive infinity", value: math.Inf(1)},
|
||||
{name: "negative infinity", value: math.Inf(-1)},
|
||||
{name: "empty JSON number", value: json.Number("")},
|
||||
{name: "leading zero JSON number", value: json.Number("01")},
|
||||
{name: "leading plus JSON number", value: json.Number("+1")},
|
||||
{name: "trailing decimal JSON number", value: json.Number("1.")},
|
||||
{name: "leading decimal JSON number", value: json.Number(".1")},
|
||||
{name: "non-number JSON number", value: json.Number("NaN")},
|
||||
{name: "spaced JSON number", value: json.Number(" 1")},
|
||||
{name: "quoted JSON number", value: json.Number(`"1"`)},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if _, err := Copy(tc.value); err == nil {
|
||||
t.Fatal("expected validation error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyMapValidatesJSONNumberSyntaxAndRange(t *testing.T) {
|
||||
for _, number := range []json.Number{"0", "-1", "1.25", "-1.25e+2"} {
|
||||
t.Run("valid "+number.String(), func(t *testing.T) {
|
||||
got, err := jsonvalue.CopyMap(map[string]any{"value": number})
|
||||
func TestCopyPreservesCompatibleCollectionsAndNilEmptyDistinctions(t *testing.T) {
|
||||
collections := []struct {
|
||||
name string
|
||||
value any
|
||||
}{
|
||||
{name: "unnamed map", value: map[string]int{"limit": 2}},
|
||||
{name: "named map", value: namedMap{"limit": 2}},
|
||||
{name: "unnamed slice", value: []string{"one", "two"}},
|
||||
{name: "named slice", value: namedSlice{"one", "two"}},
|
||||
{name: "unnamed array", value: [2]int{1, 2}},
|
||||
{name: "named array", value: namedArray{{"limit": 2}}},
|
||||
{name: "empty map", value: map[string]int{}},
|
||||
{name: "empty named map", value: namedMap{}},
|
||||
{name: "empty slice", value: []string{}},
|
||||
{name: "empty named slice", value: namedSlice{}},
|
||||
{name: "empty array", value: [0]string{}},
|
||||
}
|
||||
|
||||
for _, tc := range collections {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := Copy(tc.value)
|
||||
if err != nil {
|
||||
t.Fatalf("copy valid JSON number: %v", err)
|
||||
t.Fatalf("copy collection: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(got["value"], number) {
|
||||
t.Fatalf("JSON number changed: got %#v want %#v", got["value"], number)
|
||||
if !reflect.DeepEqual(got, tc.value) || reflect.TypeOf(got) != reflect.TypeOf(tc.value) {
|
||||
t.Fatalf("collection changed: got %#v (%T), want %#v (%T)", got, got, tc.value, tc.value)
|
||||
}
|
||||
kind := reflect.ValueOf(got).Kind()
|
||||
if (kind == reflect.Map || kind == reflect.Slice) && reflect.ValueOf(got).IsNil() {
|
||||
t.Fatal("non-nil collection became nil")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
for _, number := range []json.Number{"", "01", "+1", "1.", ".1", "1e9999", "not-a-number"} {
|
||||
t.Run("invalid "+number.String(), func(t *testing.T) {
|
||||
if _, err := jsonvalue.CopyMap(map[string]any{"value": number}); err == nil {
|
||||
t.Fatal("expected invalid JSON number error")
|
||||
var nilMap map[string]int
|
||||
var nilSlice []string
|
||||
var nilPointer *namedInt64
|
||||
for _, value := range []any{nil, nilMap, nilSlice, nilPointer} {
|
||||
got, err := Copy(value)
|
||||
if err != nil {
|
||||
t.Fatalf("copy null value: %v", err)
|
||||
}
|
||||
if got != nil {
|
||||
t.Fatalf("null value became %#v (%T)", got, got)
|
||||
}
|
||||
}
|
||||
|
||||
gotNil, err := CopyMap(nil)
|
||||
if err != nil || gotNil != nil {
|
||||
t.Fatalf("nil CopyMap result = %#v, %v", gotNil, err)
|
||||
}
|
||||
gotEmpty, err := CopyMap(map[string]any{})
|
||||
if err != nil || gotEmpty == nil || len(gotEmpty) != 0 {
|
||||
t.Fatalf("empty CopyMap result = %#v, %v", gotEmpty, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyHandlesIndirectionAndIsolatesNestedMutations(t *testing.T) {
|
||||
integer := namedInt64(7)
|
||||
nestedMap := namedMap{"limit": 2}
|
||||
nestedSlice := namedSlice{"original"}
|
||||
nestedArray := namedArray{{"limit": 3}}
|
||||
shared := []any{map[string]int{"value": 4}}
|
||||
input := map[string]any{
|
||||
"integer": &integer,
|
||||
"map": nestedMap,
|
||||
"slice": nestedSlice,
|
||||
"array": nestedArray,
|
||||
"first": shared,
|
||||
"second": shared,
|
||||
}
|
||||
|
||||
copiedValue, err := Copy(input)
|
||||
if err != nil {
|
||||
t.Fatalf("copy mixed tree: %v", err)
|
||||
}
|
||||
copied := copiedValue.(map[string]any)
|
||||
nestedMap["limit"] = 20
|
||||
nestedSlice[0] = "changed"
|
||||
nestedArray[0]["limit"] = 30
|
||||
shared[0].(map[string]int)["value"] = 40
|
||||
|
||||
if got, ok := copied["integer"].(namedInt64); !ok || got != 7 {
|
||||
t.Fatalf("pointer target changed: %#v", copied["integer"])
|
||||
}
|
||||
if got := copied["map"].(namedMap)["limit"]; got != 2 {
|
||||
t.Fatalf("nested map aliased input: %d", got)
|
||||
}
|
||||
if got := copied["slice"].(namedSlice)[0]; got != "original" {
|
||||
t.Fatalf("nested slice aliased input: %q", got)
|
||||
}
|
||||
if got := copied["array"].(namedArray)[0]["limit"]; got != 3 {
|
||||
t.Fatalf("nested array aliased input: %d", got)
|
||||
}
|
||||
first := copied["first"].([]any)
|
||||
second := copied["second"].([]any)
|
||||
if got := first[0].(map[string]int)["value"]; got != 4 {
|
||||
t.Fatalf("shared child aliased input: %d", got)
|
||||
}
|
||||
first[0].(map[string]int)["value"] = 99
|
||||
if got := second[0].(map[string]int)["value"]; got != 4 {
|
||||
t.Fatalf("repeated acyclic value shared copied output: %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyAndCopyMapApplyDistinctEmptyKeyRules(t *testing.T) {
|
||||
nested := map[string]any{"": []any{"original"}}
|
||||
copiedValue, err := Copy(nested)
|
||||
if err != nil {
|
||||
t.Fatalf("Copy rejected empty schema key: %v", err)
|
||||
}
|
||||
nested[""].([]any)[0] = "changed"
|
||||
if got := copiedValue.(map[string]any)[""].([]any)[0]; got != "original" {
|
||||
t.Fatalf("copied schema value was not isolated: %v", got)
|
||||
}
|
||||
|
||||
_, err = CopyMap(map[string]any{"nested": map[string]any{"": true}})
|
||||
if err == nil || !strings.Contains(err.Error(), "extra_params.nested") {
|
||||
t.Fatalf("CopyMap empty-key error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyRejectsUnsupportedValuesAndActiveCycles(t *testing.T) {
|
||||
cyclicMap := map[string]any{}
|
||||
cyclicMap["self"] = cyclicMap
|
||||
cyclicSlice := []any{nil}
|
||||
cyclicSlice[0] = cyclicSlice
|
||||
var cyclicPointer any
|
||||
cyclicPointer = &cyclicPointer
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
value any
|
||||
wantPath string
|
||||
}{
|
||||
{name: "non-string map key", value: map[int]string{1: "one"}, wantPath: "value"},
|
||||
{name: "unsupported channel", value: make(chan int), wantPath: "value"},
|
||||
{name: "deterministic map path", value: map[string]any{"z": make(chan int), "a": make(chan int)}, wantPath: "value.a"},
|
||||
{name: "cyclic map", value: cyclicMap, wantPath: "value.self"},
|
||||
{name: "cyclic slice", value: cyclicSlice, wantPath: "value[0]"},
|
||||
{name: "cyclic pointer", value: cyclicPointer, wantPath: "value"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := Copy(tc.value)
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantPath) {
|
||||
t.Fatalf("error = %v, want structural path %q", err, tc.wantPath)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyEnforcesContainerDepth(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
depth int
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "just below", depth: maxContainerDepth - 1},
|
||||
{name: "at limit", depth: maxContainerDepth},
|
||||
{name: "over limit", depth: maxContainerDepth + 1, wantErr: true},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := Copy(alternatingContainers(tc.depth))
|
||||
if tc.wantErr {
|
||||
if err == nil || !strings.HasPrefix(err.Error(), "value") || !strings.Contains(err.Error(), "container depth limit") {
|
||||
t.Fatalf("depth error = %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("copy depth %d: %v", tc.depth, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyEnforcesProducedNodeBudgetForRepeatedAcyclicValues(t *testing.T) {
|
||||
shared := []any{true}
|
||||
sharedOccurrences := (maxProducedNodes - 2) / 2
|
||||
justBelow := repeatedValues(shared, sharedOccurrences, 0)
|
||||
atLimit := repeatedValues(shared, sharedOccurrences, 1)
|
||||
overLimit := repeatedValues(shared, sharedOccurrences, 2)
|
||||
|
||||
for name, value := range map[string]any{
|
||||
"just below": justBelow,
|
||||
"at limit": atLimit,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := Copy(value); err != nil {
|
||||
t.Fatalf("copy value within work budget: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
_, err := Copy(overLimit)
|
||||
if err == nil || !strings.HasPrefix(err.Error(), "value[") || !strings.Contains(err.Error(), "value work limit") {
|
||||
t.Fatalf("work-budget error = %v", err)
|
||||
}
|
||||
|
||||
_, err = Copy(make([]any, maxProducedNodes))
|
||||
if err == nil || !strings.HasPrefix(err.Error(), "value:") || !strings.Contains(err.Error(), "value work limit") {
|
||||
t.Fatalf("flat work-budget error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func alternatingContainers(depth int) any {
|
||||
var value any = true
|
||||
for level := 0; level < depth; level++ {
|
||||
switch level % 3 {
|
||||
case 0:
|
||||
value = map[string]any{"child": value}
|
||||
case 1:
|
||||
value = []any{value}
|
||||
default:
|
||||
value = [1]any{value}
|
||||
}
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func repeatedValues(shared []any, occurrences, leadingScalars int) []any {
|
||||
values := make([]any, 0, leadingScalars+occurrences)
|
||||
for i := 0; i < leadingScalars; i++ {
|
||||
values = append(values, false)
|
||||
}
|
||||
for i := 0; i < occurrences; i++ {
|
||||
values = append(values, shared)
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
@@ -25,6 +25,20 @@ var (
|
||||
ErrMalformedResponse = errors.New("malformed llm response")
|
||||
)
|
||||
|
||||
const maxOpenAIChatResponseBytes int64 = 16 << 20
|
||||
|
||||
type requestFailedError struct {
|
||||
cause error
|
||||
}
|
||||
|
||||
func (e *requestFailedError) Error() string {
|
||||
return ErrRequestFailed.Error()
|
||||
}
|
||||
|
||||
func (e *requestFailedError) Unwrap() []error {
|
||||
return []error{ErrRequestFailed, e.cause}
|
||||
}
|
||||
|
||||
type OpenAICompatibleConfig struct {
|
||||
BaseURL string
|
||||
Model string
|
||||
@@ -39,9 +53,11 @@ type OpenAICompatibleClient struct {
|
||||
}
|
||||
|
||||
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) {
|
||||
baseURL := strings.TrimSpace(cfg.BaseURL)
|
||||
if baseURL != "" {
|
||||
if _, err := url.ParseRequestURI(baseURL); err != nil {
|
||||
baseURL := ""
|
||||
if strings.TrimSpace(cfg.BaseURL) != "" {
|
||||
var err error
|
||||
baseURL, err = domain.NormalizeOpenAICompatibleBaseEndpoint(cfg.BaseURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
}
|
||||
@@ -63,25 +79,29 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
|
||||
}
|
||||
|
||||
return &OpenAICompatibleClient{
|
||||
baseURL: strings.TrimRight(baseURL, "/"),
|
||||
baseURL: baseURL,
|
||||
defaultModel: cfg.Model,
|
||||
httpClient: client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
if req.Target.TimeoutSeconds < 0 {
|
||||
return nil, fmt.Errorf("%w: timeout_seconds must be greater than or equal to 0", ErrInvalidRequest)
|
||||
if err := domain.ValidateExecutionTargetSettings(req.Target); err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
endpoint := strings.TrimSpace(req.Target.Endpoint)
|
||||
if endpoint == "" {
|
||||
endpoint = c.baseURL
|
||||
selectedEndpoint := req.Target.Endpoint
|
||||
if strings.TrimSpace(selectedEndpoint) == "" {
|
||||
selectedEndpoint = c.baseURL
|
||||
}
|
||||
if endpoint == "" {
|
||||
return nil, fmt.Errorf("%w: endpoint is required", ErrInvalidRequest)
|
||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(selectedEndpoint)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: invalid endpoint: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
endpoint, err = url.JoinPath(endpoint, defaults.OpenAIChatCompletionsPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: invalid endpoint path: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
endpoint = strings.TrimRight(endpoint, "/") + defaults.OpenAIChatCompletionsPath
|
||||
|
||||
wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel)
|
||||
if err != nil {
|
||||
@@ -113,13 +133,18 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
||||
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
if apiKey := strings.TrimSpace(req.Target.APIKey); apiKey != "" {
|
||||
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
} else if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" {
|
||||
apiKey := strings.TrimSpace(os.Getenv(envName))
|
||||
if apiKey == "" {
|
||||
apiKey := strings.TrimSpace(req.Target.APIKey)
|
||||
envName := strings.TrimSpace(req.Target.APIKeyEnv)
|
||||
if apiKey == "" && envName != "" {
|
||||
apiKey = strings.TrimSpace(os.Getenv(envName))
|
||||
}
|
||||
if apiKey == "" && req.Target.APIKeyRequired {
|
||||
if envName != "" {
|
||||
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
|
||||
}
|
||||
return nil, fmt.Errorf("%w: api key is required", ErrInvalidRequest)
|
||||
}
|
||||
if apiKey != "" {
|
||||
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
}
|
||||
|
||||
@@ -130,30 +155,36 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
||||
|
||||
httpResp, err := httpClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrRequestFailed, err)
|
||||
return nil, &requestFailedError{cause: err}
|
||||
}
|
||||
defer httpResp.Body.Close()
|
||||
|
||||
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(httpResp.Body, 4096))
|
||||
return nil, fmt.Errorf("%w: status=%d", ErrUnexpectedStatus, httpResp.StatusCode)
|
||||
return nil, providerHTTPErrorFromBody(
|
||||
httpResp.StatusCode,
|
||||
httpResp.ContentLength,
|
||||
httpResp.Body,
|
||||
)
|
||||
}
|
||||
if httpResp.ContentLength > maxOpenAIChatResponseBytes {
|
||||
return nil, openAIChatResponseTooLargeError()
|
||||
}
|
||||
|
||||
var wireResp openAIChatResponse
|
||||
if err := json.NewDecoder(httpResp.Body).Decode(&wireResp); err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to decode response: %v", ErrMalformedResponse, err)
|
||||
wireResp, err := decodeOpenAIChatResponse(httpResp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(wireResp.Choices) == 0 {
|
||||
return nil, fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
|
||||
}
|
||||
content := wireResp.Choices[0].Message.Content
|
||||
if content == "" {
|
||||
return nil, fmt.Errorf("%w: first choice has empty message content", ErrMalformedResponse)
|
||||
if content == nil {
|
||||
return nil, fmt.Errorf("%w: first choice has missing message content", ErrMalformedResponse)
|
||||
}
|
||||
|
||||
return &domain.GenerateResponse{
|
||||
Content: content,
|
||||
Content: *content,
|
||||
Usage: domain.TokenUsage{
|
||||
PromptTokens: wireResp.Usage.PromptTokens,
|
||||
CompletionTokens: wireResp.Usage.CompletionTokens,
|
||||
@@ -164,6 +195,46 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
||||
}, nil
|
||||
}
|
||||
|
||||
func decodeOpenAIChatResponse(body io.Reader) (openAIChatResponse, error) {
|
||||
limited := &io.LimitedReader{
|
||||
R: body,
|
||||
N: maxOpenAIChatResponseBytes + 1,
|
||||
}
|
||||
decoder := json.NewDecoder(limited)
|
||||
|
||||
var response openAIChatResponse
|
||||
if err := decoder.Decode(&response); err != nil {
|
||||
if limited.N == 0 {
|
||||
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||
}
|
||||
return openAIChatResponse{}, fmt.Errorf("%w: failed to decode response", ErrMalformedResponse)
|
||||
}
|
||||
if limited.N == 0 {
|
||||
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||
}
|
||||
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
if limited.N == 0 {
|
||||
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||
}
|
||||
return openAIChatResponse{}, fmt.Errorf("%w: response contains trailing data", ErrMalformedResponse)
|
||||
}
|
||||
if limited.N == 0 {
|
||||
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||
}
|
||||
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func openAIChatResponseTooLargeError() error {
|
||||
return fmt.Errorf(
|
||||
"%w: response exceeds %d-byte limit",
|
||||
ErrMalformedResponse,
|
||||
maxOpenAIChatResponseBytes,
|
||||
)
|
||||
}
|
||||
|
||||
func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) {
|
||||
model := strings.TrimSpace(req.Target.Model)
|
||||
if model == "" {
|
||||
@@ -308,8 +379,8 @@ type openAICacheControl struct {
|
||||
}
|
||||
|
||||
type openAIChatResponseMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
Role string `json:"role"`
|
||||
Content *string `json:"content"`
|
||||
}
|
||||
|
||||
type openAIChatResponse struct {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
189
internal/llm/provider_http_error.go
Normal file
189
internal/llm/provider_http_error.go
Normal file
@@ -0,0 +1,189 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
const (
|
||||
maxProviderErrorResponseBytes int64 = 64 << 10
|
||||
maxProviderErrorIdentifierRunes = 256
|
||||
maxProviderErrorMessageRunes = 4096
|
||||
)
|
||||
|
||||
// ProviderHTTPError describes a non-success response from an LLM provider.
|
||||
type ProviderHTTPError struct {
|
||||
statusCode int
|
||||
providerCode string
|
||||
providerType string
|
||||
providerMessage string
|
||||
}
|
||||
|
||||
func (e *ProviderHTTPError) StatusCode() int {
|
||||
if e == nil {
|
||||
return 0
|
||||
}
|
||||
return e.statusCode
|
||||
}
|
||||
|
||||
func (e *ProviderHTTPError) ProviderCode() string {
|
||||
if e == nil {
|
||||
return ""
|
||||
}
|
||||
return e.providerCode
|
||||
}
|
||||
|
||||
func (e *ProviderHTTPError) ProviderType() string {
|
||||
if e == nil {
|
||||
return ""
|
||||
}
|
||||
return e.providerType
|
||||
}
|
||||
|
||||
func (e *ProviderHTTPError) ProviderMessage() string {
|
||||
if e == nil {
|
||||
return ""
|
||||
}
|
||||
return e.providerMessage
|
||||
}
|
||||
|
||||
func (e *ProviderHTTPError) Error() string {
|
||||
if e == nil || e.statusCode == 0 {
|
||||
return ErrUnexpectedStatus.Error()
|
||||
}
|
||||
return fmt.Sprintf("%s: status=%d", ErrUnexpectedStatus, e.statusCode)
|
||||
}
|
||||
|
||||
func (e *ProviderHTTPError) GoString() string {
|
||||
return e.Error()
|
||||
}
|
||||
|
||||
func (e *ProviderHTTPError) Unwrap() error {
|
||||
return ErrUnexpectedStatus
|
||||
}
|
||||
|
||||
type providerErrorDetails struct {
|
||||
providerCode string
|
||||
providerType string
|
||||
providerMessage string
|
||||
}
|
||||
|
||||
func newProviderHTTPError(statusCode int, details providerErrorDetails) *ProviderHTTPError {
|
||||
return &ProviderHTTPError{
|
||||
statusCode: statusCode,
|
||||
providerCode: details.providerCode,
|
||||
providerType: details.providerType,
|
||||
providerMessage: details.providerMessage,
|
||||
}
|
||||
}
|
||||
|
||||
func providerHTTPErrorFromBody(statusCode int, contentLength int64, body io.Reader) *ProviderHTTPError {
|
||||
if contentLength > maxProviderErrorResponseBytes {
|
||||
return newProviderHTTPError(statusCode, providerErrorDetails{})
|
||||
}
|
||||
|
||||
limited := &io.LimitedReader{
|
||||
R: body,
|
||||
N: maxProviderErrorResponseBytes + 1,
|
||||
}
|
||||
contents, err := io.ReadAll(limited)
|
||||
if err != nil || limited.N == 0 {
|
||||
return newProviderHTTPError(statusCode, providerErrorDetails{})
|
||||
}
|
||||
return newProviderHTTPError(statusCode, parseProviderErrorEnvelope(contents))
|
||||
}
|
||||
|
||||
func parseProviderErrorEnvelope(body []byte) providerErrorDetails {
|
||||
decoder := json.NewDecoder(strings.NewReader(string(body)))
|
||||
decoder.UseNumber()
|
||||
|
||||
var envelope map[string]json.RawMessage
|
||||
if err := decoder.Decode(&envelope); err != nil {
|
||||
return providerErrorDetails{}
|
||||
}
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return providerErrorDetails{}
|
||||
}
|
||||
|
||||
rawError, ok := envelope["error"]
|
||||
if !ok {
|
||||
return providerErrorDetails{}
|
||||
}
|
||||
var providerError map[string]json.RawMessage
|
||||
if err := json.Unmarshal(rawError, &providerError); err != nil || providerError == nil {
|
||||
return providerErrorDetails{}
|
||||
}
|
||||
|
||||
var details providerErrorDetails
|
||||
if raw, ok := providerError["message"]; ok {
|
||||
var value string
|
||||
if json.Unmarshal(raw, &value) == nil {
|
||||
details.providerMessage = normalizeProviderErrorMessage(value)
|
||||
}
|
||||
}
|
||||
if raw, ok := providerError["type"]; ok {
|
||||
var value string
|
||||
if json.Unmarshal(raw, &value) == nil {
|
||||
details.providerType = normalizeProviderErrorIdentifier(value)
|
||||
}
|
||||
}
|
||||
if raw, ok := providerError["code"]; ok {
|
||||
var value any
|
||||
fieldDecoder := json.NewDecoder(strings.NewReader(string(raw)))
|
||||
fieldDecoder.UseNumber()
|
||||
if fieldDecoder.Decode(&value) == nil {
|
||||
switch value := value.(type) {
|
||||
case string:
|
||||
details.providerCode = normalizeProviderErrorIdentifier(value)
|
||||
case json.Number:
|
||||
details.providerCode = normalizeProviderErrorIdentifier(value.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return details
|
||||
}
|
||||
|
||||
func normalizeProviderErrorIdentifier(value string) string {
|
||||
normalized := normalizeProviderErrorText(value)
|
||||
if len([]rune(normalized)) > maxProviderErrorIdentifierRunes {
|
||||
return ""
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
func normalizeProviderErrorMessage(value string) string {
|
||||
normalized := normalizeProviderErrorText(value)
|
||||
runes := []rune(normalized)
|
||||
if len(runes) <= maxProviderErrorMessageRunes {
|
||||
return normalized
|
||||
}
|
||||
return string(runes[:maxProviderErrorMessageRunes-1]) + "…"
|
||||
}
|
||||
|
||||
func normalizeProviderErrorText(value string) string {
|
||||
value = strings.ToValidUTF8(value, "<22>")
|
||||
|
||||
var result strings.Builder
|
||||
result.Grow(len(value))
|
||||
separatorPending := false
|
||||
for _, r := range value {
|
||||
if unicode.IsSpace(r) || unicode.IsControl(r) || unicode.In(r, unicode.Cf) {
|
||||
if result.Len() > 0 {
|
||||
separatorPending = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if separatorPending {
|
||||
result.WriteByte(' ')
|
||||
separatorPending = false
|
||||
}
|
||||
result.WriteRune(r)
|
||||
}
|
||||
return result.String()
|
||||
}
|
||||
255
internal/llm/provider_http_error_test.go
Normal file
255
internal/llm/provider_http_error_test.go
Normal file
@@ -0,0 +1,255 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
type guardedReader struct {
|
||||
reader io.Reader
|
||||
remaining int64
|
||||
bytes int64
|
||||
violated bool
|
||||
}
|
||||
|
||||
func (r *guardedReader) Read(buffer []byte) (int, error) {
|
||||
if int64(len(buffer)) > r.remaining {
|
||||
r.violated = true
|
||||
return 0, errors.New("reader was read past its allowed boundary")
|
||||
}
|
||||
n, err := r.reader.Read(buffer)
|
||||
r.bytes += int64(n)
|
||||
r.remaining -= int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
type failingReader struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (r failingReader) Read([]byte) (int, error) {
|
||||
return 0, r.err
|
||||
}
|
||||
|
||||
func TestProviderHTTPErrorEnvelopeParsing(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
body string
|
||||
want providerErrorDetails
|
||||
}{
|
||||
{
|
||||
name: "all supported string fields",
|
||||
body: `{"error":{"message":"diagnostic","type":"invalid_request_error","code":"unsupported_parameter"}}`,
|
||||
want: providerErrorDetails{providerMessage: "diagnostic", providerType: "invalid_request_error", providerCode: "unsupported_parameter"},
|
||||
},
|
||||
{
|
||||
name: "integer code",
|
||||
body: `{"error":{"code":17}}`,
|
||||
want: providerErrorDetails{providerCode: "17"},
|
||||
},
|
||||
{
|
||||
name: "fractional code",
|
||||
body: `{"error":{"code":1.25}}`,
|
||||
want: providerErrorDetails{providerCode: "1.25"},
|
||||
},
|
||||
{
|
||||
name: "exponent code",
|
||||
body: `{"error":{"code":6.02e+23}}`,
|
||||
want: providerErrorDetails{providerCode: "6.02e+23"},
|
||||
},
|
||||
{
|
||||
name: "invalid fields do not discard valid fields",
|
||||
body: `{"error":{"message":null,"type":"invalid_request_error","code":false}}`,
|
||||
want: providerErrorDetails{providerType: "invalid_request_error"},
|
||||
},
|
||||
{
|
||||
name: "unknown fields are ignored",
|
||||
body: `{"trace":"do not retain","error":{"param":"temperature","metadata":{"secret":"x"}}}`,
|
||||
want: providerErrorDetails{},
|
||||
},
|
||||
{name: "missing error", body: `{}`, want: providerErrorDetails{}},
|
||||
{name: "null error", body: `{"error":null}`, want: providerErrorDetails{}},
|
||||
{name: "scalar error", body: `{"error":"nope"}`, want: providerErrorDetails{}},
|
||||
{name: "empty error", body: `{"error":{}}`, want: providerErrorDetails{}},
|
||||
{name: "malformed", body: `{"error":`, want: providerErrorDetails{}},
|
||||
{name: "truncated", body: `{"error":{"message":"x"`, want: providerErrorDetails{}},
|
||||
{name: "trailing garbage", body: `{"error":{"message":"x"}} garbage`, want: providerErrorDetails{}},
|
||||
{name: "second document", body: `{"error":{"message":"x"}} {}`, want: providerErrorDetails{}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := parseProviderErrorEnvelope([]byte(tc.body)); !reflect.DeepEqual(got, tc.want) {
|
||||
t.Fatalf("parseProviderErrorEnvelope() = %#v, want %#v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderErrorTextNormalizationAndLimits(t *testing.T) {
|
||||
validIdentifier := strings.Repeat("界", maxProviderErrorIdentifierRunes)
|
||||
validMessage := strings.Repeat("界", maxProviderErrorMessageRunes)
|
||||
tests := []struct {
|
||||
name string
|
||||
got string
|
||||
want string
|
||||
}{
|
||||
{name: "multibyte text", got: "Grüße 世界", want: "Grüße 世界"},
|
||||
{name: "invalid UTF-8", got: string([]byte{'a', 0xff, 'b'}), want: "a<>b"},
|
||||
{name: "whitespace control and format runs", got: " \n\talpha\x00\u200b\u200bbeta \r ", want: "alpha beta"},
|
||||
{name: "blank normalization", got: "\t\u200b\n", want: ""},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := normalizeProviderErrorText(tc.got); got != tc.want {
|
||||
t.Fatalf("normalizeProviderErrorText() = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if got := normalizeProviderErrorIdentifier(validIdentifier); got != validIdentifier {
|
||||
t.Fatalf("exact identifier boundary = %q, want retained value", got)
|
||||
}
|
||||
if got := normalizeProviderErrorIdentifier(validIdentifier + "界"); got != "" {
|
||||
t.Fatalf("overlong identifier = %q, want empty", got)
|
||||
}
|
||||
if got := normalizeProviderErrorMessage(validMessage); got != validMessage {
|
||||
t.Fatalf("exact message boundary = %q, want retained value", got)
|
||||
}
|
||||
wantTruncatedMessage := strings.Repeat("界", maxProviderErrorMessageRunes-1) + "…"
|
||||
if got := normalizeProviderErrorMessage(validMessage + "界"); got != wantTruncatedMessage {
|
||||
t.Fatalf("overlong message length = %d, want %d", utf8.RuneCountInString(got), maxProviderErrorMessageRunes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderHTTPErrorIdentityAndFormatting(t *testing.T) {
|
||||
const marker = "provider-secret-marker"
|
||||
err := newProviderHTTPError(429, providerErrorDetails{
|
||||
providerCode: marker + "-code",
|
||||
providerType: marker + "-type",
|
||||
providerMessage: marker + "-message",
|
||||
})
|
||||
|
||||
if err.StatusCode() != 429 || err.ProviderCode() != marker+"-code" || err.ProviderType() != marker+"-type" || err.ProviderMessage() != marker+"-message" {
|
||||
t.Fatalf("accessors returned unexpected values: %#v", err)
|
||||
}
|
||||
if !errors.Is(err, ErrUnexpectedStatus) {
|
||||
t.Fatalf("errors.Is(%v, ErrUnexpectedStatus) = false", err)
|
||||
}
|
||||
for _, rendered := range []string{fmt.Sprintf("%v", err), fmt.Sprintf("%+v", err), fmt.Sprintf("%#v", err)} {
|
||||
if rendered != "llm returned non-success status: status=429" {
|
||||
t.Fatalf("formatted error = %q", rendered)
|
||||
}
|
||||
if strings.Contains(rendered, marker) {
|
||||
t.Fatalf("formatted error exposed provider marker: %q", rendered)
|
||||
}
|
||||
}
|
||||
|
||||
var nilError *ProviderHTTPError
|
||||
if nilError.StatusCode() != 0 || nilError.ProviderCode() != "" || nilError.ProviderType() != "" || nilError.ProviderMessage() != "" {
|
||||
t.Fatal("nil accessors returned provider values")
|
||||
}
|
||||
if nilError.Error() != "llm returned non-success status" || nilError.GoString() != "llm returned non-success status" || !errors.Is(nilError, ErrUnexpectedStatus) {
|
||||
t.Fatalf("nil error behavior is not safe: %v", nilError)
|
||||
}
|
||||
|
||||
zero := &ProviderHTTPError{}
|
||||
if zero.Error() != "llm returned non-success status" || zero.GoString() != "llm returned non-success status" || !errors.Is(zero, ErrUnexpectedStatus) {
|
||||
t.Fatalf("zero error behavior is not safe: %v", zero)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderHTTPErrorBodyBounds(t *testing.T) {
|
||||
const (
|
||||
statusCode = 502
|
||||
marker = "provider-body-marker"
|
||||
)
|
||||
|
||||
ordinaryBody := `{"error":{"message":"` + marker + `"}}`
|
||||
exactLimitBody := ordinaryBody + strings.Repeat(" ", int(maxProviderErrorResponseBytes)-len(ordinaryBody))
|
||||
overLimitBody := ordinaryBody + strings.Repeat(" ", int(maxProviderErrorResponseBytes)+1-len(ordinaryBody))
|
||||
tests := []struct {
|
||||
name string
|
||||
contentLength int64
|
||||
reader io.Reader
|
||||
wantRead int64
|
||||
wantMessage string
|
||||
}{
|
||||
{
|
||||
name: "recognized envelope",
|
||||
contentLength: int64(len(ordinaryBody)),
|
||||
reader: strings.NewReader(ordinaryBody),
|
||||
wantRead: int64(len(ordinaryBody)),
|
||||
wantMessage: marker,
|
||||
},
|
||||
{
|
||||
name: "exact limit",
|
||||
contentLength: maxProviderErrorResponseBytes,
|
||||
reader: strings.NewReader(exactLimitBody),
|
||||
wantRead: maxProviderErrorResponseBytes,
|
||||
wantMessage: marker,
|
||||
},
|
||||
{
|
||||
name: "declared oversize does not read",
|
||||
contentLength: maxProviderErrorResponseBytes + 1,
|
||||
reader: strings.NewReader(ordinaryBody),
|
||||
wantRead: 0,
|
||||
},
|
||||
{
|
||||
name: "unknown length oversize",
|
||||
contentLength: -1,
|
||||
reader: strings.NewReader(overLimitBody),
|
||||
wantRead: maxProviderErrorResponseBytes + 1,
|
||||
},
|
||||
{
|
||||
name: "underreported oversize",
|
||||
contentLength: maxProviderErrorResponseBytes,
|
||||
reader: strings.NewReader(overLimitBody),
|
||||
wantRead: maxProviderErrorResponseBytes + 1,
|
||||
},
|
||||
{
|
||||
name: "read failure",
|
||||
contentLength: -1,
|
||||
reader: failingReader{err: errors.New("read failure")},
|
||||
wantRead: 0,
|
||||
},
|
||||
{
|
||||
name: "empty body",
|
||||
contentLength: 0,
|
||||
reader: strings.NewReader(""),
|
||||
wantRead: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
reader := &guardedReader{
|
||||
reader: tc.reader,
|
||||
remaining: maxProviderErrorResponseBytes + 1,
|
||||
}
|
||||
err := providerHTTPErrorFromBody(statusCode, tc.contentLength, reader)
|
||||
if err == nil || err.StatusCode() != statusCode {
|
||||
t.Fatalf("error status = %v, want %d", err, statusCode)
|
||||
}
|
||||
if reader.bytes != tc.wantRead {
|
||||
t.Fatalf("body bytes read = %d, want %d", reader.bytes, tc.wantRead)
|
||||
}
|
||||
if reader.violated {
|
||||
t.Fatal("body reader was asked to read beyond the overflow probe")
|
||||
}
|
||||
if got := err.ProviderMessage(); got != tc.wantMessage {
|
||||
t.Fatalf("provider message = %q, want %q", got, tc.wantMessage)
|
||||
}
|
||||
if tc.wantMessage == "" {
|
||||
if err.ProviderCode() != "" || err.ProviderType() != "" || strings.Contains(err.Error(), marker) {
|
||||
t.Fatalf("discarded details were retained: %#v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
145
internal/llm/provider_http_error_transport_test.go
Normal file
145
internal/llm/provider_http_error_transport_test.go
Normal file
@@ -0,0 +1,145 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOpenAICompatibleClientStructuredNonSuccessResponse(t *testing.T) {
|
||||
body := `{"error":{"message":" provider\nmessage\u200b","type":"invalid\ttype","code":1.5e+4}}`
|
||||
responseBody := &countingReadCloser{reader: strings.NewReader(body)}
|
||||
client := newNonSuccessResponseClient(t, http.StatusBadRequest, int64(len(body)), responseBody)
|
||||
|
||||
response, err := client.Generate(context.Background(), ordinaryGenerateRequest())
|
||||
if response != nil {
|
||||
t.Fatalf("response = %#v, want nil", response)
|
||||
}
|
||||
if !errors.Is(err, ErrUnexpectedStatus) {
|
||||
t.Fatalf("errors.Is(%v, ErrUnexpectedStatus) = false", err)
|
||||
}
|
||||
var providerHTTPError *ProviderHTTPError
|
||||
if !errors.As(err, &providerHTTPError) {
|
||||
t.Fatalf("error = %T, want *ProviderHTTPError", err)
|
||||
}
|
||||
if providerHTTPError.StatusCode() != http.StatusBadRequest || providerHTTPError.ProviderCode() != "1.5e+4" || providerHTTPError.ProviderType() != "invalid type" || providerHTTPError.ProviderMessage() != "provider message" {
|
||||
t.Fatalf("provider error = %#v", providerHTTPError)
|
||||
}
|
||||
if !responseBody.closed {
|
||||
t.Fatal("non-success response body was not closed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientNonSuccessBodyOwnership(t *testing.T) {
|
||||
const marker = "provider-body-marker"
|
||||
normalBody := `{"error":{"message":"` + marker + `"}}`
|
||||
overLimitBody := normalBody + strings.Repeat(" ", int(maxProviderErrorResponseBytes)+1-len(normalBody))
|
||||
tests := []struct {
|
||||
name string
|
||||
contentLength int64
|
||||
reader io.Reader
|
||||
wantRead int64
|
||||
wantMessage string
|
||||
}{
|
||||
{
|
||||
name: "normal",
|
||||
contentLength: int64(len(normalBody)),
|
||||
reader: strings.NewReader(normalBody),
|
||||
wantRead: int64(len(normalBody)),
|
||||
wantMessage: marker,
|
||||
},
|
||||
{
|
||||
name: "declared oversize",
|
||||
contentLength: maxProviderErrorResponseBytes + 1,
|
||||
reader: strings.NewReader(normalBody),
|
||||
wantRead: 0,
|
||||
},
|
||||
{
|
||||
name: "streamed oversize",
|
||||
contentLength: -1,
|
||||
reader: &guardedReader{
|
||||
reader: strings.NewReader(overLimitBody),
|
||||
remaining: maxProviderErrorResponseBytes + 1,
|
||||
},
|
||||
wantRead: maxProviderErrorResponseBytes + 1,
|
||||
},
|
||||
{
|
||||
name: "underreported oversize",
|
||||
contentLength: maxProviderErrorResponseBytes,
|
||||
reader: &guardedReader{
|
||||
reader: strings.NewReader(overLimitBody),
|
||||
remaining: maxProviderErrorResponseBytes + 1,
|
||||
},
|
||||
wantRead: maxProviderErrorResponseBytes + 1,
|
||||
},
|
||||
{
|
||||
name: "malformed",
|
||||
contentLength: 1,
|
||||
reader: strings.NewReader("{"),
|
||||
wantRead: 1,
|
||||
},
|
||||
{
|
||||
name: "read failure",
|
||||
contentLength: -1,
|
||||
reader: failingReader{err: errors.New("response read failed")},
|
||||
wantRead: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
body := &countingReadCloser{reader: tc.reader}
|
||||
client := newNonSuccessResponseClient(t, http.StatusBadGateway, tc.contentLength, body)
|
||||
|
||||
response, err := client.Generate(context.Background(), ordinaryGenerateRequest())
|
||||
if response != nil {
|
||||
t.Fatalf("response = %#v, want nil", response)
|
||||
}
|
||||
var providerHTTPError *ProviderHTTPError
|
||||
if !errors.As(err, &providerHTTPError) {
|
||||
t.Fatalf("error = %T, want *ProviderHTTPError", err)
|
||||
}
|
||||
if !body.closed {
|
||||
t.Fatal("response body was not closed")
|
||||
}
|
||||
if body.bytesRead != tc.wantRead {
|
||||
t.Fatalf("body bytes read = %d, want %d", body.bytesRead, tc.wantRead)
|
||||
}
|
||||
if body.bytesRead > maxProviderErrorResponseBytes+1 {
|
||||
t.Fatalf("body bytes read = %d, exceeds overflow probe", body.bytesRead)
|
||||
}
|
||||
if guarded, ok := tc.reader.(*guardedReader); ok && guarded.violated {
|
||||
t.Fatal("body reader was asked to read beyond the overflow probe")
|
||||
}
|
||||
if got := providerHTTPError.ProviderMessage(); got != tc.wantMessage {
|
||||
t.Fatalf("provider message = %q, want %q", got, tc.wantMessage)
|
||||
}
|
||||
if tc.wantMessage == "" && (providerHTTPError.ProviderCode() != "" || providerHTTPError.ProviderType() != "" || strings.Contains(providerHTTPError.Error(), marker)) {
|
||||
t.Fatalf("discarded details were retained: %#v", providerHTTPError)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func newNonSuccessResponseClient(t *testing.T, statusCode int, contentLength int64, body io.ReadCloser) *OpenAICompatibleClient {
|
||||
t.Helper()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||
BaseURL: "https://provider.example/v1",
|
||||
Model: "m",
|
||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
StatusCode: statusCode,
|
||||
ContentLength: contentLength,
|
||||
Body: body,
|
||||
}, nil
|
||||
})},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("construct client: %v", err)
|
||||
}
|
||||
return client
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
id: aion-2
|
||||
backend: openrouter
|
||||
model: aion-labs/aion-2.0
|
||||
temperature: 0.72
|
||||
reasoning_effort: high
|
||||
top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: claude-fable-latest
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-fable-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 600
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: claude-haiku-latest
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-haiku-latest"
|
||||
reasoning_effort: medium
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: claude-opus-latest
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-opus-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: claude-sonnet-latest
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-sonnet-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: deepseek-3-2
|
||||
backend: openrouter
|
||||
model: deepseek/deepseek-v3.2
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: deepseek-4-flash
|
||||
backend: openrouter
|
||||
model: deepseek/deepseek-v4-flash
|
||||
#reasoning_effort: medium
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: deepseek-4-pro
|
||||
backend: openrouter
|
||||
model: deepseek/deepseek-v4-pro
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemini-2-flash-lite
|
||||
backend: openrouter
|
||||
model: "google/gemini-2.5-flash-lite"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemini-2-flash
|
||||
backend: openrouter
|
||||
model: "google/gemini-2.5-flash"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemini-2-pro
|
||||
backend: openrouter
|
||||
model: "google/gemini-2.5-pro"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemini-3-flash-lite
|
||||
backend: openrouter
|
||||
model: "google/gemini-3.1-flash-lite"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemini-flash-latest
|
||||
backend: openrouter
|
||||
model: "~google/gemini-flash-latest"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemini-pro-latest
|
||||
backend: openrouter
|
||||
model: "~google/gemini-pro-latest"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemma-4-31b
|
||||
backend: openrouter
|
||||
model: google/gemma-4-31b-it:exacto
|
||||
temperature: 0.15
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: minimax-m2
|
||||
backend: openrouter
|
||||
model: minimax/minimax-m2.5
|
||||
temperature: 0.5
|
||||
reasoning_effort: high
|
||||
top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: minimax-m3
|
||||
backend: openrouter
|
||||
model: minimax/minimax-m3
|
||||
#temperature: 0.5
|
||||
reasoning_effort: high
|
||||
#top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: mistral-large-2512
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-large-2512
|
||||
temperature: 0.15
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
@@ -1,7 +0,0 @@
|
||||
id: mistral-medium-3-5
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-medium-3-5
|
||||
temperature: 0.15
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
@@ -1,6 +0,0 @@
|
||||
id: mistral-small-3
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-small-3.2-24b-instruct
|
||||
temperature: 0.05
|
||||
top_p: 1.0
|
||||
timeout_seconds: 180
|
||||
@@ -1,7 +0,0 @@
|
||||
id: mistral-small-4
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-small-2603
|
||||
temperature: 0.1
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
@@ -1,6 +0,0 @@
|
||||
id: nemotron-3-ultra
|
||||
backend: openrouter
|
||||
model: nvidia/nemotron-3-ultra-550b-a55b
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: gpt-5-mini
|
||||
backend: openrouter
|
||||
model: "openai/gpt-5.4-mini"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: gpt-5-nano
|
||||
backend: openrouter
|
||||
model: "openai/gpt-5.4-nano"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,16 +0,0 @@
|
||||
package builtin
|
||||
|
||||
import (
|
||||
"embed"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
|
||||
const assetRoot = "assets"
|
||||
|
||||
//go:embed assets/**/*.yml
|
||||
var assets embed.FS
|
||||
|
||||
func NewRepository() profile.Repository {
|
||||
return profile.NewFSRepository(assets, assetRoot)
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
package builtin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
|
||||
repo := NewRepository()
|
||||
ids := loadBuiltInProfileIDs(t)
|
||||
if len(ids) == 0 {
|
||||
t.Fatal("expected built-in profiles")
|
||||
}
|
||||
|
||||
for id := range ids {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
p, err := repo.GetProfile(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("expected built-in profile %q to load, got %v", id, err)
|
||||
}
|
||||
if p.ID != id {
|
||||
t.Fatalf("expected profile id %q, got %q", id, p.ID)
|
||||
}
|
||||
if p.BackendID != backend.OpenRouterID {
|
||||
t.Fatalf("expected profile %q to select %q, got %q", id, backend.OpenRouterID, p.BackendID)
|
||||
}
|
||||
if p.Endpoint != "" || p.APIKeyEnv != "" {
|
||||
t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltInProfilesDoNotContainDuplicateIDsOrRawAPIKeys(t *testing.T) {
|
||||
loadBuiltInProfileIDs(t)
|
||||
}
|
||||
|
||||
func loadBuiltInProfileIDs(t *testing.T) map[string]string {
|
||||
t.Helper()
|
||||
|
||||
ids := map[string]string{}
|
||||
err := fs.WalkDir(assets, assetRoot, func(name string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() || !strings.HasSuffix(name, ".yml") {
|
||||
return nil
|
||||
}
|
||||
|
||||
data, err := assets.ReadFile(name)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read built-in profile %s: %v", name, err)
|
||||
}
|
||||
|
||||
var raw map[string]any
|
||||
if err := yaml.Unmarshal(data, &raw); err != nil {
|
||||
t.Fatalf("failed to decode built-in profile %s: %v", name, err)
|
||||
}
|
||||
if _, ok := raw["api_key"]; ok {
|
||||
t.Fatalf("built-in profile %s contains raw api_key", name)
|
||||
}
|
||||
if raw["backend"] != backend.OpenRouterID {
|
||||
t.Fatalf("built-in profile %s does not select %q", name, backend.OpenRouterID)
|
||||
}
|
||||
if _, ok := raw["endpoint"]; ok {
|
||||
t.Fatalf("built-in profile %s repeats endpoint", name)
|
||||
}
|
||||
if _, ok := raw["api_key_env"]; ok {
|
||||
t.Fatalf("built-in profile %s repeats api_key_env", name)
|
||||
}
|
||||
id, ok := raw["id"].(string)
|
||||
if !ok || strings.TrimSpace(id) == "" {
|
||||
t.Fatalf("built-in profile %s has missing id", name)
|
||||
}
|
||||
if previous, ok := ids[id]; ok {
|
||||
t.Fatalf("duplicate built-in profile id %q in %s and %s", id, previous, name)
|
||||
}
|
||||
ids[id] = name
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to walk built-in profiles: %v", err)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
47
internal/profile/definition.go
Normal file
47
internal/profile/definition.go
Normal file
@@ -0,0 +1,47 @@
|
||||
package profile
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
// NormalizeAndValidateDefinition normalizes and validates one source-local
|
||||
// profile definition without resolving a base profile.
|
||||
func NormalizeAndValidateDefinition(profile *domain.ExecutionProfile) error {
|
||||
if profile == nil {
|
||||
return errors.New("profile is required")
|
||||
}
|
||||
|
||||
profile.ID = strings.TrimSpace(profile.ID)
|
||||
profile.BaseProfileID = strings.TrimSpace(profile.BaseProfileID)
|
||||
profile.BackendID = strings.TrimSpace(profile.BackendID)
|
||||
profile.Endpoint = strings.TrimSpace(profile.Endpoint)
|
||||
|
||||
if profile.ID == "" {
|
||||
return errors.New("id is required")
|
||||
}
|
||||
if profile.Endpoint != "" {
|
||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(profile.Endpoint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
profile.Endpoint = endpoint
|
||||
}
|
||||
if profile.BaseProfileID == "" {
|
||||
if profile.BackendID == "" && profile.Endpoint == "" {
|
||||
return errors.New("backend or endpoint is required")
|
||||
}
|
||||
if strings.TrimSpace(profile.Model) == "" {
|
||||
return errors.New("model is required")
|
||||
}
|
||||
}
|
||||
|
||||
return domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{
|
||||
Temperature: profile.Temperature,
|
||||
MaxTokens: profile.MaxTokens,
|
||||
TopP: profile.TopP,
|
||||
TimeoutSeconds: profile.TimeoutSeconds,
|
||||
})
|
||||
}
|
||||
98
internal/profile/eager_repository.go
Normal file
98
internal/profile/eager_repository.go
Normal file
@@ -0,0 +1,98 @@
|
||||
package profile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
)
|
||||
|
||||
// LoadedProfileMetadata identifies one profile accepted by LoadFSRepository.
|
||||
type LoadedProfileMetadata struct {
|
||||
// ID is the normalized profile ID.
|
||||
ID string
|
||||
// Path is the safe root-relative source path.
|
||||
Path string
|
||||
// ExplicitFields lists the sorted top-level YAML fields present in source.
|
||||
ExplicitFields []string
|
||||
}
|
||||
|
||||
// LoadFSRepository eagerly validates every profile under root and returns an
|
||||
// immutable raw repository and independently owned source metadata.
|
||||
func LoadFSRepository(ctx context.Context, fsys fs.FS, root string) (Repository, []LoadedProfileMetadata, error) {
|
||||
if fsys == nil {
|
||||
return nil, nil, fmt.Errorf("failed to read profile directory: filesystem is nil")
|
||||
}
|
||||
paths, err := filecatalog.FindFSYAMLFiles(ctx, fsys, root)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to read profile directory: %w", err)
|
||||
}
|
||||
|
||||
repository := &loadedRepository{profiles: make(map[string]domain.ExecutionProfile, len(paths))}
|
||||
metadata := make([]LoadedProfileMetadata, 0, len(paths))
|
||||
for _, path := range paths {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
data, err := fs.ReadFile(fsys, path)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to read profile file %s: %w", filecatalog.DisplayPath(root, path), err)
|
||||
}
|
||||
fileMetadata, err := readProfileFileMetadata(data)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidYAML, filecatalog.DisplayPath(root, path))
|
||||
}
|
||||
if fileMetadata.hasRawAPIKey {
|
||||
return nil, nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, filecatalog.DisplayPath(root, path))
|
||||
}
|
||||
definition, err := decodeProfile(data)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidYAML, filecatalog.DisplayPath(root, path))
|
||||
}
|
||||
definition.ExtraParams, err = jsonvalue.CopyMap(definition.ExtraParams)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidProfile, filecatalog.DisplayPath(root, path))
|
||||
}
|
||||
if err := NormalizeAndValidateDefinition(definition); err != nil {
|
||||
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidProfile, filecatalog.DisplayPath(root, path))
|
||||
}
|
||||
if _, exists := repository.profiles[definition.ID]; exists {
|
||||
return nil, nil, fmt.Errorf("%w: %s: duplicate profile ID", ErrInvalidProfile, filecatalog.DisplayPath(root, path))
|
||||
}
|
||||
repository.profiles[definition.ID] = *definition
|
||||
fields := append([]string(nil), fileMetadata.explicitFields...)
|
||||
sort.Strings(fields)
|
||||
metadata = append(metadata, LoadedProfileMetadata{
|
||||
ID: definition.ID,
|
||||
Path: filecatalog.DisplayPath(root, path),
|
||||
ExplicitFields: fields,
|
||||
})
|
||||
}
|
||||
sort.Slice(metadata, func(left, right int) bool { return metadata[left].ID < metadata[right].ID })
|
||||
return repository, metadata, nil
|
||||
}
|
||||
|
||||
type loadedRepository struct {
|
||||
profiles map[string]domain.ExecutionProfile
|
||||
}
|
||||
|
||||
func (r *loadedRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
definition, found := r.profiles[strings.TrimSpace(id)]
|
||||
if !found {
|
||||
return nil, ErrProfileNotFound
|
||||
}
|
||||
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("copy loaded profile %q: %w", definition.ID, err)
|
||||
}
|
||||
definition.ExtraParams = extraParams
|
||||
return &definition, nil
|
||||
}
|
||||
110
internal/profile/eager_repository_test.go
Normal file
110
internal/profile/eager_repository_test.go
Normal file
@@ -0,0 +1,110 @@
|
||||
package profile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"reflect"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
)
|
||||
|
||||
func TestLoadFSRepositoryLoadsSortedIndependentProfiles(t *testing.T) {
|
||||
fsys := fstest.MapFS{
|
||||
"profiles/z.yml": {Data: []byte("id: z\nbackend: local\nmodel: z-model\nextra_params:\n nested:\n value: one\n")},
|
||||
"profiles/a.yml": {Data: []byte("id: a\nbackend: local\nmodel: a-model\nendpoint: ''\n")},
|
||||
}
|
||||
|
||||
repository, metadata, err := LoadFSRepository(context.Background(), fsys, "profiles")
|
||||
if err != nil {
|
||||
t.Fatalf("load repository: %v", err)
|
||||
}
|
||||
if len(metadata) != 2 || metadata[0].ID != "a" || metadata[1].ID != "z" || metadata[0].Path != "a.yml" {
|
||||
t.Fatalf("unexpected metadata: %#v", metadata)
|
||||
}
|
||||
if len(metadata[0].ExplicitFields) != 4 || metadata[0].ExplicitFields[0] != "backend" || metadata[0].ExplicitFields[1] != "endpoint" {
|
||||
t.Fatalf("expected explicitly empty endpoint metadata, got %#v", metadata[0].ExplicitFields)
|
||||
}
|
||||
metadata[1].ExplicitFields[0] = "changed"
|
||||
first, err := repository.GetProfile(context.Background(), "z")
|
||||
if err != nil {
|
||||
t.Fatalf("load profile: %v", err)
|
||||
}
|
||||
first.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||
second, err := repository.GetProfile(context.Background(), "z")
|
||||
if err != nil {
|
||||
t.Fatalf("reload profile: %v", err)
|
||||
}
|
||||
if second.ExtraParams["nested"].(map[string]any)["value"] != "one" {
|
||||
t.Fatalf("profile value was not defensively copied: %#v", second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFSRepositoryRejectsInvalidProfiles(t *testing.T) {
|
||||
tests := map[string]fstest.MapFS{
|
||||
"duplicate IDs": {
|
||||
"profiles/one.yml": {Data: []byte("id: duplicate\nbackend: local\nmodel: one\n")},
|
||||
"profiles/two.yml": {Data: []byte("id: duplicate\nbackend: local\nmodel: two\n")},
|
||||
},
|
||||
"raw API key": {
|
||||
"profiles/one.yml": {Data: []byte("id: one\nbackend: local\nmodel: one\napi_key: forbidden\n")},
|
||||
},
|
||||
"multiple documents": {
|
||||
"profiles/one.yml": {Data: []byte("id: one\nbackend: local\nmodel: one\n---\nid: two\n")},
|
||||
},
|
||||
}
|
||||
for name, fsys := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, _, err := LoadFSRepository(context.Background(), fsys, "profiles")
|
||||
if err == nil {
|
||||
t.Fatal("expected load failure")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadedRepositoryHonorsCancellationAndMissingProfiles(t *testing.T) {
|
||||
repository, _, err := LoadFSRepository(context.Background(), fstest.MapFS{
|
||||
"profiles/one.yml": {Data: []byte("id: one\nbackend: local\nmodel: one\n")},
|
||||
}, "profiles")
|
||||
if err != nil {
|
||||
t.Fatalf("load repository: %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := repository.GetProfile(ctx, "one"); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected cancellation, got %v", err)
|
||||
}
|
||||
if _, err := repository.GetProfile(context.Background(), "missing"); !errors.Is(err, ErrProfileNotFound) {
|
||||
t.Fatalf("expected missing profile, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFSRepositoryMatchesPointLookupForRawProfiles(t *testing.T) {
|
||||
fsys := fstest.MapFS{
|
||||
"profiles/base.yml": {Data: []byte("id: base\nbackend: local\nmodel: base-model\n")},
|
||||
"profiles/derived.yml": {Data: []byte("id: derived\nbase_profile: base\nreasoning_effort: high\n")},
|
||||
}
|
||||
eager, _, err := LoadFSRepository(context.Background(), fsys, "profiles")
|
||||
if err != nil {
|
||||
t.Fatalf("load eager repository: %v", err)
|
||||
}
|
||||
pointLookup := NewFSRepository(fsys, "profiles")
|
||||
for _, id := range []string{"base", "derived"} {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
got, err := eager.GetProfile(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("load eager profile: %v", err)
|
||||
}
|
||||
want, err := pointLookup.GetProfile(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("load point-in-time profile: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("raw profiles differ: got %#v, want %#v", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
var _ fs.FS = fstest.MapFS{}
|
||||
@@ -5,13 +5,14 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
@@ -73,7 +74,8 @@ func (r *overlayRepository) GetProfile(ctx context.Context, id string) (*domain.
|
||||
}
|
||||
|
||||
func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*domain.ExecutionProfile, error) {
|
||||
if strings.TrimSpace(id) == "" {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
|
||||
}
|
||||
if fsys == nil {
|
||||
@@ -94,42 +96,49 @@ func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*doma
|
||||
}
|
||||
|
||||
relPath := filecatalog.DisplayPath(root, fullPath)
|
||||
fileMatch := filecatalog.Stem(path.Base(fullPath)) == id
|
||||
data, err := fs.ReadFile(fsys, fullPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
|
||||
}
|
||||
metadata := readProfileFileMetadata(data)
|
||||
idMatch := fileMatch || metadata.id == id
|
||||
metadata, metadataErr := readProfileFileMetadata(data)
|
||||
idMatch := metadata.matchesID(id)
|
||||
if metadataErr != nil {
|
||||
if idMatch {
|
||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, metadataErr)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if metadata.hasRawAPIKey {
|
||||
if idMatch {
|
||||
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
var prof domain.ExecutionProfile
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||
decoder.KnownFields(true)
|
||||
if err := decoder.Decode(&prof); err != nil {
|
||||
if idMatch {
|
||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
|
||||
}
|
||||
if !idMatch {
|
||||
continue
|
||||
}
|
||||
|
||||
prof, err := decodeProfile(data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
|
||||
}
|
||||
|
||||
prof.ID = strings.TrimSpace(prof.ID)
|
||||
if prof.ID != id {
|
||||
continue
|
||||
}
|
||||
prof.BackendID = strings.TrimSpace(prof.BackendID)
|
||||
if err := validateProfile(&prof); err != nil {
|
||||
prof.ExtraParams, err = jsonvalue.CopyMap(prof.ExtraParams)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
|
||||
}
|
||||
if err := NormalizeAndValidateDefinition(prof); err != nil {
|
||||
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||
return nil, fmt.Errorf("%w: %s", err, relPath)
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
|
||||
}
|
||||
matches = append(matches, profileMatch{
|
||||
profile: &prof,
|
||||
profile: prof,
|
||||
path: relPath,
|
||||
})
|
||||
}
|
||||
@@ -155,15 +164,37 @@ type profileMatch struct {
|
||||
}
|
||||
|
||||
type profileFileMetadata struct {
|
||||
id string
|
||||
hasRawAPIKey bool
|
||||
ids []string
|
||||
hasRawAPIKey bool
|
||||
explicitFields []string
|
||||
}
|
||||
|
||||
func readProfileFileMetadata(data []byte) profileFileMetadata {
|
||||
func readProfileFileMetadata(data []byte) (profileFileMetadata, error) {
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||
var node yaml.Node
|
||||
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&node); err != nil {
|
||||
return profileFileMetadata{}
|
||||
if err := decoder.Decode(&node); err != nil {
|
||||
return profileFileMetadata{}, err
|
||||
}
|
||||
metadata := profileMetadataFromNode(&node)
|
||||
documentCount := 1
|
||||
for {
|
||||
var trailing yaml.Node
|
||||
err := decoder.Decode(&trailing)
|
||||
if errors.Is(err, io.EOF) {
|
||||
if documentCount == 1 {
|
||||
return metadata, nil
|
||||
}
|
||||
return metadata, errors.New("profile file must contain exactly one YAML document")
|
||||
}
|
||||
if err != nil {
|
||||
return metadata, err
|
||||
}
|
||||
documentCount++
|
||||
metadata.merge(profileMetadataFromNode(&trailing))
|
||||
}
|
||||
}
|
||||
|
||||
func profileMetadataFromNode(node *yaml.Node) profileFileMetadata {
|
||||
if node.Kind != yaml.DocumentNode || len(node.Content) == 0 {
|
||||
return profileFileMetadata{}
|
||||
}
|
||||
@@ -176,9 +207,10 @@ func readProfileFileMetadata(data []byte) profileFileMetadata {
|
||||
for i := 0; i+1 < len(mapping.Content); i += 2 {
|
||||
key := mapping.Content[i]
|
||||
value := mapping.Content[i+1]
|
||||
metadata.explicitFields = append(metadata.explicitFields, key.Value)
|
||||
switch key.Value {
|
||||
case "id":
|
||||
metadata.id = strings.TrimSpace(value.Value)
|
||||
metadata.ids = append(metadata.ids, strings.TrimSpace(value.Value))
|
||||
case "api_key":
|
||||
metadata.hasRawAPIKey = true
|
||||
}
|
||||
@@ -186,29 +218,42 @@ func readProfileFileMetadata(data []byte) profileFileMetadata {
|
||||
return metadata
|
||||
}
|
||||
|
||||
func validateProfile(p *domain.ExecutionProfile) error {
|
||||
if strings.TrimSpace(p.ID) == "" {
|
||||
return errors.New("id is required")
|
||||
func (m profileFileMetadata) matchesID(id string) bool {
|
||||
for _, candidate := range m.ids {
|
||||
if candidate == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(p.BackendID) == "" && strings.TrimSpace(p.Endpoint) == "" {
|
||||
return errors.New("backend or endpoint is required")
|
||||
}
|
||||
if strings.TrimSpace(p.Model) == "" {
|
||||
return errors.New("model is required")
|
||||
}
|
||||
|
||||
if p.Temperature < 0 || p.Temperature > 2 {
|
||||
return errors.New("temperature must be between 0 and 2")
|
||||
}
|
||||
if p.MaxTokens < 0 {
|
||||
return errors.New("max_tokens must be greater than or equal to 0")
|
||||
}
|
||||
if p.TopP < 0 || p.TopP > 1 {
|
||||
return errors.New("top_p must be between 0 and 1")
|
||||
}
|
||||
if p.TimeoutSeconds < 0 {
|
||||
return errors.New("timeout_seconds must be greater than or equal to 0")
|
||||
}
|
||||
|
||||
return nil
|
||||
return false
|
||||
}
|
||||
|
||||
func (m *profileFileMetadata) merge(other profileFileMetadata) {
|
||||
m.ids = append(m.ids, other.ids...)
|
||||
m.hasRawAPIKey = m.hasRawAPIKey || other.hasRawAPIKey
|
||||
m.explicitFields = append(m.explicitFields, other.explicitFields...)
|
||||
}
|
||||
|
||||
func decodeProfile(data []byte) (*domain.ExecutionProfile, error) {
|
||||
var prof domain.ExecutionProfile
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||
decoder.KnownFields(true)
|
||||
if err := decoder.Decode(&prof); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := requireYAMLStreamEnd(decoder); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &prof, nil
|
||||
}
|
||||
|
||||
func requireYAMLStreamEnd(decoder *yaml.Decoder) error {
|
||||
var trailing yaml.Node
|
||||
err := decoder.Decode(&trailing)
|
||||
if errors.Is(err, io.EOF) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return errors.New("profile file must contain exactly one YAML document")
|
||||
}
|
||||
|
||||
58
internal/profile/repository_benchmark_test.go
Normal file
58
internal/profile/repository_benchmark_test.go
Normal file
@@ -0,0 +1,58 @@
|
||||
package profile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
)
|
||||
|
||||
func BenchmarkProfileRepositoryLookup(b *testing.B) {
|
||||
for _, size := range []int{10, 1000} {
|
||||
b.Run(fmt.Sprintf("catalog-%d", size), func(b *testing.B) {
|
||||
files := fstest.MapFS{
|
||||
"target.yaml": profileMapFile(`
|
||||
id: target
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: target-model
|
||||
extra_params:
|
||||
selected: true
|
||||
`),
|
||||
}
|
||||
metadataNames := []string{"target.yaml"}
|
||||
for i := 1; i < size; i++ {
|
||||
name := fmt.Sprintf("profile-%04d.yaml", i)
|
||||
files[name] = profileMapFile(fmt.Sprintf(`
|
||||
id: profile-%04d
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: unrelated-model
|
||||
temperature: 0.5
|
||||
max_tokens: 500
|
||||
extra_params:
|
||||
provider:
|
||||
order:
|
||||
- first
|
||||
- second
|
||||
`, i))
|
||||
metadataNames = append(metadataNames, name)
|
||||
}
|
||||
|
||||
fsys := &recordingProfileFS{FS: files}
|
||||
repo := NewFSRepository(fsys, ".")
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
if _, err := repo.GetProfile(context.Background(), "target"); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
b.StopTimer()
|
||||
|
||||
for _, name := range metadataNames {
|
||||
if got := fsys.openCount(name); got != b.N {
|
||||
b.Fatalf("metadata %q opens = %d, want %d", name, got, b.N)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2,11 +2,13 @@ package profile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
@@ -61,7 +63,7 @@ func TestFilesystemRepository_GetProfile(t *testing.T) {
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"},
|
||||
{name: "endpoint only", connection: "endpoint: http://localhost:8000/v1", wantEndpoint: "http://localhost:8000/v1"},
|
||||
{name: "endpoint only", connection: "endpoint: ' https://localhost:8000/nested/v1 '", wantEndpoint: "https://localhost:8000/nested/v1"},
|
||||
{name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"},
|
||||
{name: "neither", wantErr: true},
|
||||
{name: "blank backend", connection: "backend: ' '", wantErr: true},
|
||||
@@ -126,63 +128,6 @@ temperature: 0.1
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid profile with JSON-compatible extra params", func(t *testing.T) {
|
||||
writeProfileTestFile(t, filepath.Join(tmpDir, "json-extra-params.yaml"), `
|
||||
id: json-extra-params
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: nested-model
|
||||
extra_params:
|
||||
string_value: enabled
|
||||
number_value: 42
|
||||
boolean_value: true
|
||||
object_value:
|
||||
nested: value
|
||||
count: 2
|
||||
array_value:
|
||||
- first
|
||||
- 3
|
||||
- false
|
||||
`)
|
||||
|
||||
p, err := repo.GetProfile(ctx, "json-extra-params")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
var got map[string]any
|
||||
encoded, err := json.Marshal(p.ExtraParams)
|
||||
if err != nil {
|
||||
t.Fatalf("expected extra_params to marshal as JSON, got %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(encoded, &got); err != nil {
|
||||
t.Fatalf("expected extra_params JSON to decode, got %v", err)
|
||||
}
|
||||
|
||||
if got["string_value"] != "enabled" {
|
||||
t.Fatalf("unexpected string extra param: %#v", got["string_value"])
|
||||
}
|
||||
if got["number_value"] != float64(42) {
|
||||
t.Fatalf("unexpected number extra param: %#v", got["number_value"])
|
||||
}
|
||||
if got["boolean_value"] != true {
|
||||
t.Fatalf("unexpected boolean extra param: %#v", got["boolean_value"])
|
||||
}
|
||||
objectValue, ok := got["object_value"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected object extra param, got %#v", got["object_value"])
|
||||
}
|
||||
if objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
|
||||
t.Fatalf("unexpected object extra param: %#v", objectValue)
|
||||
}
|
||||
arrayValue, ok := got["array_value"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected array extra param, got %#v", got["array_value"])
|
||||
}
|
||||
if len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
|
||||
t.Fatalf("unexpected array extra param: %#v", arrayValue)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
|
||||
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
|
||||
id: duplicate-profile
|
||||
@@ -245,10 +190,10 @@ api_key: secret
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid yaml", func(t *testing.T) {
|
||||
t.Run("unidentifiable invalid yaml is unrelated", func(t *testing.T) {
|
||||
_, err := repo.GetProfile(ctx, "invalid_yaml")
|
||||
if !errors.Is(err, ErrInvalidYAML) {
|
||||
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
||||
if !errors.Is(err, ErrProfileNotFound) {
|
||||
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -274,14 +219,14 @@ api_key: secret
|
||||
})
|
||||
|
||||
t.Run("unknown field", func(t *testing.T) {
|
||||
_, err := repo.GetProfile(ctx, "unknown_field")
|
||||
_, err := repo.GetProfile(ctx, "unknown-field")
|
||||
if !errors.Is(err, ErrInvalidYAML) {
|
||||
t.Fatalf("expected ErrInvalidYAML for strict decode unknown field, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("raw api_key rejected", func(t *testing.T) {
|
||||
_, err := repo.GetProfile(ctx, "raw_api_key")
|
||||
_, err := repo.GetProfile(ctx, "raw-api-key")
|
||||
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
||||
}
|
||||
@@ -406,6 +351,614 @@ model: second
|
||||
})
|
||||
}
|
||||
|
||||
func TestProfileRepositoriesRejectInvalidEndpoints(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
withBackend bool
|
||||
}{
|
||||
{name: "relative", endpoint: "/v1"},
|
||||
{name: "missing host", endpoint: "https:///v1"},
|
||||
{name: "unsupported scheme", endpoint: "ftp://provider.example/v1"},
|
||||
{name: "user information", endpoint: "https://user@provider.example/v1"},
|
||||
{name: "query", endpoint: "https://provider.example/v1?mode=chat"},
|
||||
{name: "fragment", endpoint: "https://provider.example/v1#chat"},
|
||||
{name: "backend with invalid override", endpoint: "/v1", withBackend: true},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
backend := ""
|
||||
if tc.withBackend {
|
||||
backend = "backend: openrouter\n"
|
||||
}
|
||||
repo := NewFSRepository(fstest.MapFS{
|
||||
"profiles/invalid.yaml": profileMapFile(fmt.Sprintf(
|
||||
"id: invalid-endpoint\nmodel: model\n%sendpoint: %q\n",
|
||||
backend,
|
||||
tc.endpoint,
|
||||
)),
|
||||
}, "profiles")
|
||||
|
||||
_, err := repo.GetProfile(context.Background(), "invalid-endpoint")
|
||||
if !errors.Is(err, ErrInvalidProfile) {
|
||||
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileRepositoriesValidateExtraParams(t *testing.T) {
|
||||
const validProfile = `
|
||||
id: selected-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: model
|
||||
extra_params:
|
||||
string_value: enabled
|
||||
object_value:
|
||||
nested: true
|
||||
array_value:
|
||||
- first
|
||||
- 3
|
||||
`
|
||||
tests := []struct {
|
||||
name string
|
||||
definition string
|
||||
wantErr bool
|
||||
diagnostics []string
|
||||
}{
|
||||
{name: "valid nested values", definition: validProfile},
|
||||
{
|
||||
name: "empty key",
|
||||
definition: `
|
||||
id: selected-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: model
|
||||
extra_params:
|
||||
"": value
|
||||
`,
|
||||
wantErr: true,
|
||||
diagnostics: []string{"extra_params", "key must not be empty"},
|
||||
},
|
||||
{
|
||||
name: "non-finite value",
|
||||
definition: `
|
||||
id: selected-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: model
|
||||
extra_params:
|
||||
invalid: .nan
|
||||
`,
|
||||
wantErr: true,
|
||||
diagnostics: []string{"extra_params.invalid", "must be finite"},
|
||||
},
|
||||
{
|
||||
name: "nested non-finite value",
|
||||
definition: `
|
||||
id: selected-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: model
|
||||
extra_params:
|
||||
outer:
|
||||
invalid: .inf
|
||||
`,
|
||||
wantErr: true,
|
||||
diagnostics: []string{"extra_params.outer.invalid", "must be finite"},
|
||||
},
|
||||
{
|
||||
name: "unsupported decoded value",
|
||||
definition: `
|
||||
id: selected-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: model
|
||||
extra_params:
|
||||
timestamp: 2026-08-11T12:34:56Z
|
||||
`,
|
||||
wantErr: true,
|
||||
diagnostics: []string{"extra_params.timestamp", "unsupported JSON value type"},
|
||||
},
|
||||
{
|
||||
name: "excessive nesting",
|
||||
definition: deeplyNestedExtraParamsProfile(101),
|
||||
wantErr: true,
|
||||
diagnostics: []string{"extra_params", "JSON container depth limit exceeded"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, source := range profileRepositorySources() {
|
||||
for _, tc := range tests {
|
||||
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
||||
repo := source.newRepository(t, map[string]string{"selected.yaml": tc.definition})
|
||||
got, err := repo.GetProfile(context.Background(), "selected-profile")
|
||||
if tc.wantErr {
|
||||
if !errors.Is(err, ErrInvalidProfile) {
|
||||
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "selected.yaml") {
|
||||
t.Fatalf("expected source path in error, got %v", err)
|
||||
}
|
||||
for _, diagnostic := range tc.diagnostics {
|
||||
if !strings.Contains(err.Error(), diagnostic) {
|
||||
t.Fatalf("expected error to contain %q, got %v", diagnostic, err)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("load valid profile: %v", err)
|
||||
}
|
||||
if got.ExtraParams["string_value"] != "enabled" {
|
||||
t.Fatalf("unexpected copied extra params: %#v", got.ExtraParams)
|
||||
}
|
||||
objectValue, objectOK := got.ExtraParams["object_value"].(map[string]any)
|
||||
arrayValue, arrayOK := got.ExtraParams["array_value"].([]any)
|
||||
if !objectOK || objectValue["nested"] != true ||
|
||||
!arrayOK || len(arrayValue) != 2 || arrayValue[0] != "first" || arrayValue[1] != 3 {
|
||||
t.Fatalf("unexpected copied nested extra params: %#v", got.ExtraParams)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileRepositoriesSelectCanonicalYAMLID(t *testing.T) {
|
||||
const validProfile = `
|
||||
id: selected-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: selected-model
|
||||
`
|
||||
tests := []struct {
|
||||
name string
|
||||
files map[string]string
|
||||
wantErr error
|
||||
diagnostics []string
|
||||
}{
|
||||
{
|
||||
name: "same stem unknown field with different id is unrelated",
|
||||
files: map[string]string{
|
||||
"selected-profile.yaml": `
|
||||
id: unrelated-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: unrelated
|
||||
unknown: true
|
||||
`,
|
||||
"valid.yaml": validProfile,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "same stem unidentifiable yaml is unrelated",
|
||||
files: map[string]string{
|
||||
"selected-profile.yaml": "id: [",
|
||||
"valid.yaml": validProfile,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "same stem raw key with different id is unrelated",
|
||||
files: map[string]string{
|
||||
"selected-profile.yaml": `
|
||||
id: unrelated-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: unrelated
|
||||
api_key: secret
|
||||
`,
|
||||
"valid.yaml": validProfile,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "leading and trailing whitespace is normalized",
|
||||
files: map[string]string{
|
||||
"padded.yaml": `
|
||||
id: " selected-profile "
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: selected-model
|
||||
`,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "blank id is unrelated",
|
||||
files: map[string]string{
|
||||
"selected-profile.yaml": `
|
||||
id: " "
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: unrelated
|
||||
`,
|
||||
},
|
||||
wantErr: ErrProfileNotFound,
|
||||
},
|
||||
{
|
||||
name: "normalized duplicates are ambiguous",
|
||||
files: map[string]string{
|
||||
"first.yaml": validProfile,
|
||||
"nested/second.yaml": `
|
||||
id: " selected-profile "
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: duplicate
|
||||
`,
|
||||
},
|
||||
wantErr: ErrInvalidProfile,
|
||||
diagnostics: []string{"duplicate execution profile id", "first.yaml", "nested/second.yaml"},
|
||||
},
|
||||
{
|
||||
name: "selected unknown field is authoritative",
|
||||
files: map[string]string{
|
||||
"malformed.yaml": `
|
||||
id: selected-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: selected-model
|
||||
unknown: true
|
||||
`,
|
||||
},
|
||||
wantErr: ErrInvalidYAML,
|
||||
diagnostics: []string{"malformed.yaml"},
|
||||
},
|
||||
{
|
||||
name: "selected raw key is authoritative",
|
||||
files: map[string]string{
|
||||
"insecure.yaml": `
|
||||
id: selected-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: selected-model
|
||||
api_key: secret
|
||||
`,
|
||||
},
|
||||
wantErr: ErrRawAPIKeyNotAllowed,
|
||||
diagnostics: []string{"insecure.yaml"},
|
||||
},
|
||||
{
|
||||
name: "selected identity in an additional document is authoritative",
|
||||
files: map[string]string{
|
||||
"additional-document.yaml": `
|
||||
---
|
||||
---
|
||||
id: selected-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: selected-model
|
||||
`,
|
||||
},
|
||||
wantErr: ErrInvalidYAML,
|
||||
diagnostics: []string{"additional-document.yaml", "exactly one YAML document"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, source := range profileRepositorySources() {
|
||||
for _, tc := range tests {
|
||||
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
||||
repo := source.newRepository(t, tc.files)
|
||||
got, err := repo.GetProfile(context.Background(), " selected-profile ")
|
||||
if tc.wantErr != nil {
|
||||
if !errors.Is(err, tc.wantErr) {
|
||||
t.Fatalf("expected %v, got %v", tc.wantErr, err)
|
||||
}
|
||||
for _, diagnostic := range tc.diagnostics {
|
||||
if !strings.Contains(err.Error(), diagnostic) {
|
||||
t.Fatalf("expected error to contain %q, got %v", diagnostic, err)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("load selected profile: %v", err)
|
||||
}
|
||||
if got.ID != "selected-profile" || got.Model != "selected-model" {
|
||||
t.Fatalf("unexpected selected profile: %+v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileRepositoriesRequireOneYAMLDocument(t *testing.T) {
|
||||
const profile = `
|
||||
id: selected-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: selected-model
|
||||
`
|
||||
tests := []struct {
|
||||
name string
|
||||
suffix string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "comments and trailing whitespace", suffix: "\n# trailing comment\n\n"},
|
||||
{name: "second populated document", suffix: "\n---\nid: another\n", wantErr: true},
|
||||
{name: "second empty document", suffix: "\n---\n", wantErr: true},
|
||||
{name: "malformed trailing yaml", suffix: "\n---\n[", wantErr: true},
|
||||
{name: "raw key in trailing document", suffix: "\n---\napi_key: secret\n", wantErr: true},
|
||||
}
|
||||
|
||||
for _, source := range profileRepositorySources() {
|
||||
for _, tc := range tests {
|
||||
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
||||
repo := source.newRepository(t, map[string]string{"definition.yaml": profile + tc.suffix})
|
||||
got, err := repo.GetProfile(context.Background(), "selected-profile")
|
||||
if tc.wantErr {
|
||||
if !errors.Is(err, ErrInvalidYAML) {
|
||||
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "definition.yaml") {
|
||||
t.Fatalf("expected source path in error, got %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("load one-document profile: %v", err)
|
||||
}
|
||||
if got.ID != "selected-profile" {
|
||||
t.Fatalf("unexpected profile: %+v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileRepositoriesPreserveOverlayFallbackRules(t *testing.T) {
|
||||
fallback := staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"selected-profile": {ID: "selected-profile", Endpoint: "http://fallback", Model: "fallback-model"},
|
||||
}}
|
||||
tests := []struct {
|
||||
name string
|
||||
files map[string]string
|
||||
wantModel string
|
||||
wantErr error
|
||||
}{
|
||||
{
|
||||
name: "same stem malformed different id falls back",
|
||||
files: map[string]string{
|
||||
"selected-profile.yaml": `
|
||||
id: unrelated-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: unrelated
|
||||
unknown: true
|
||||
`,
|
||||
},
|
||||
wantModel: "fallback-model",
|
||||
},
|
||||
{
|
||||
name: "blank id falls back",
|
||||
files: map[string]string{
|
||||
"selected-profile.yaml": `
|
||||
id: " "
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: unrelated
|
||||
`,
|
||||
},
|
||||
wantModel: "fallback-model",
|
||||
},
|
||||
{
|
||||
name: "selected malformed profile stops fallback",
|
||||
files: map[string]string{
|
||||
"other-name.yaml": `
|
||||
id: selected-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: selected
|
||||
unknown: true
|
||||
`,
|
||||
},
|
||||
wantErr: ErrInvalidYAML,
|
||||
},
|
||||
}
|
||||
|
||||
for _, source := range profileRepositorySources() {
|
||||
for _, tc := range tests {
|
||||
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
||||
primary := source.newRepository(t, tc.files)
|
||||
got, err := NewOverlayRepository(primary, fallback).GetProfile(context.Background(), "selected-profile")
|
||||
if tc.wantErr != nil {
|
||||
if !errors.Is(err, tc.wantErr) {
|
||||
t.Fatalf("expected %v, got %v", tc.wantErr, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("load fallback profile: %v", err)
|
||||
}
|
||||
if got.Model != tc.wantModel {
|
||||
t.Fatalf("model = %q, want %q", got.Model, tc.wantModel)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileRepositoryReadsSourcesFreshOnEveryLookup(t *testing.T) {
|
||||
newSource := func() (*recordingProfileFS, Repository) {
|
||||
fsys := &recordingProfileFS{FS: fstest.MapFS{
|
||||
"target.yaml": profileMapFile(`
|
||||
id: target
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: target-model
|
||||
`),
|
||||
"unrelated.yaml": profileMapFile(`
|
||||
id: unrelated
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: unrelated-model
|
||||
`),
|
||||
}}
|
||||
return fsys, NewFSRepository(fsys, ".")
|
||||
}
|
||||
|
||||
t.Run("selected source", func(t *testing.T) {
|
||||
fsys, repo := newSource()
|
||||
for lookup := 1; lookup <= 2; lookup++ {
|
||||
got, err := repo.GetProfile(context.Background(), "target")
|
||||
if err != nil {
|
||||
t.Fatalf("lookup %d: %v", lookup, err)
|
||||
}
|
||||
if got.Model != "target-model" {
|
||||
t.Fatalf("lookup %d model = %q", lookup, got.Model)
|
||||
}
|
||||
for _, name := range []string{"target.yaml", "unrelated.yaml"} {
|
||||
if count := fsys.openCount(name); count != lookup {
|
||||
t.Fatalf("%s opens after lookup %d = %d, want %d", name, lookup, count, lookup)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("overlay fallthrough", func(t *testing.T) {
|
||||
primaryFS := &recordingProfileFS{FS: fstest.MapFS{
|
||||
"unrelated.yaml": profileMapFile(`
|
||||
id: unrelated
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: unrelated-model
|
||||
`),
|
||||
}}
|
||||
fallbackFS, fallback := newSource()
|
||||
repo := NewOverlayRepository(NewFSRepository(primaryFS, "."), fallback)
|
||||
|
||||
for lookup := 1; lookup <= 2; lookup++ {
|
||||
got, err := repo.GetProfile(context.Background(), "target")
|
||||
if err != nil {
|
||||
t.Fatalf("lookup %d: %v", lookup, err)
|
||||
}
|
||||
if got.Model != "target-model" {
|
||||
t.Fatalf("lookup %d model = %q", lookup, got.Model)
|
||||
}
|
||||
if count := primaryFS.openCount("unrelated.yaml"); count != lookup {
|
||||
t.Fatalf("primary opens after lookup %d = %d, want %d", lookup, count, lookup)
|
||||
}
|
||||
if count := fallbackFS.openCount("target.yaml"); count != lookup {
|
||||
t.Fatalf("fallback opens after lookup %d = %d, want %d", lookup, count, lookup)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestProfileRepositoriesRejectInvalidExecutionSettings(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("operating-system filesystem", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeProfileTestFile(t, filepath.Join(dir, "invalid.yaml"), `
|
||||
id: invalid
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: model
|
||||
temperature: .nan
|
||||
`)
|
||||
|
||||
_, err := NewFilesystemRepository(dir).GetProfile(ctx, "invalid")
|
||||
if !errors.Is(err, ErrInvalidProfile) {
|
||||
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("fs.FS", func(t *testing.T) {
|
||||
repo := NewFSRepository(fstest.MapFS{
|
||||
"profiles/invalid.yaml": profileMapFile(`
|
||||
id: invalid
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: model
|
||||
top_p: .inf
|
||||
`),
|
||||
}, "profiles")
|
||||
|
||||
_, err := repo.GetProfile(ctx, "invalid")
|
||||
if !errors.Is(err, ErrInvalidProfile) {
|
||||
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestProfileRepositoriesValidateDerivedDefinitions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
files map[string]string
|
||||
wantError error
|
||||
wantBaseID string
|
||||
wantProfile bool
|
||||
}{
|
||||
{
|
||||
name: "alias is locally valid and normalizes base id",
|
||||
files: map[string]string{"alias.yaml": `
|
||||
id: selected-profile
|
||||
base_profile: " base-profile "
|
||||
`},
|
||||
wantBaseID: "base-profile",
|
||||
wantProfile: true,
|
||||
},
|
||||
{
|
||||
name: "derived endpoint remains valid",
|
||||
files: map[string]string{"invalid.yaml": `
|
||||
id: selected-profile
|
||||
base_profile: base-profile
|
||||
endpoint: /v1
|
||||
`},
|
||||
wantError: ErrInvalidProfile,
|
||||
},
|
||||
{
|
||||
name: "derived settings remain valid",
|
||||
files: map[string]string{"invalid.yaml": `
|
||||
id: selected-profile
|
||||
base_profile: base-profile
|
||||
top_p: 1.1
|
||||
`},
|
||||
wantError: ErrInvalidProfile,
|
||||
},
|
||||
{
|
||||
name: "derived extra params remain valid",
|
||||
files: map[string]string{"invalid.yaml": `
|
||||
id: selected-profile
|
||||
base_profile: base-profile
|
||||
extra_params:
|
||||
timestamp: 2026-08-11T12:34:56Z
|
||||
`},
|
||||
wantError: ErrInvalidProfile,
|
||||
},
|
||||
{
|
||||
name: "derived raw key remains prohibited",
|
||||
files: map[string]string{"invalid.yaml": `
|
||||
id: selected-profile
|
||||
base_profile: base-profile
|
||||
api_key: secret
|
||||
`},
|
||||
wantError: ErrRawAPIKeyNotAllowed,
|
||||
},
|
||||
{
|
||||
name: "derived duplicate id remains invalid",
|
||||
files: map[string]string{
|
||||
"first.yaml": "id: selected-profile\nbase_profile: first-base\n",
|
||||
"second.yaml": "id: selected-profile\nbase_profile: second-base\n",
|
||||
},
|
||||
wantError: ErrInvalidProfile,
|
||||
},
|
||||
{
|
||||
name: "derived extra document remains invalid",
|
||||
files: map[string]string{"invalid.yaml": `
|
||||
id: selected-profile
|
||||
base_profile: base-profile
|
||||
---
|
||||
id: other
|
||||
`},
|
||||
wantError: ErrInvalidYAML,
|
||||
},
|
||||
{
|
||||
name: "standalone profile remains complete",
|
||||
files: map[string]string{"invalid.yaml": "id: selected-profile\n"},
|
||||
wantError: ErrInvalidProfile,
|
||||
},
|
||||
}
|
||||
|
||||
for _, source := range profileRepositorySources() {
|
||||
for _, tc := range tests {
|
||||
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
||||
repo := source.newRepository(t, tc.files)
|
||||
got, err := repo.GetProfile(context.Background(), "selected-profile")
|
||||
if tc.wantError != nil {
|
||||
if !errors.Is(err, tc.wantError) {
|
||||
t.Fatalf("error = %v, want %v", err, tc.wantError)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil || !tc.wantProfile {
|
||||
t.Fatalf("profile = %+v, error = %v, want valid derived definition", got, err)
|
||||
}
|
||||
if got.BaseProfileID != tc.wantBaseID {
|
||||
t.Fatalf("BaseProfileID = %q, want %q", got.BaseProfileID, tc.wantBaseID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverlayRepository(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"}
|
||||
@@ -495,6 +1048,77 @@ func TestOverlayRepository(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
type profileRepositorySource struct {
|
||||
name string
|
||||
newRepository func(t *testing.T, files map[string]string) Repository
|
||||
}
|
||||
|
||||
type recordingProfileFS struct {
|
||||
fs.FS
|
||||
mu sync.Mutex
|
||||
opened []string
|
||||
}
|
||||
|
||||
func (f *recordingProfileFS) Open(name string) (fs.File, error) {
|
||||
f.mu.Lock()
|
||||
f.opened = append(f.opened, name)
|
||||
f.mu.Unlock()
|
||||
return f.FS.Open(name)
|
||||
}
|
||||
|
||||
func (f *recordingProfileFS) openCount(name string) int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
count := 0
|
||||
for _, opened := range f.opened {
|
||||
if opened == name {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func profileRepositorySources() []profileRepositorySource {
|
||||
return []profileRepositorySource{
|
||||
{
|
||||
name: "operating system",
|
||||
newRepository: func(t *testing.T, files map[string]string) Repository {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
for name, content := range files {
|
||||
filePath := filepath.Join(root, filepath.FromSlash(name))
|
||||
if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil {
|
||||
t.Fatalf("create profile directory: %v", err)
|
||||
}
|
||||
writeProfileTestFile(t, filePath, content)
|
||||
}
|
||||
return NewFilesystemRepository(root)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "filesystem",
|
||||
newRepository: func(t *testing.T, files map[string]string) Repository {
|
||||
t.Helper()
|
||||
fsys := make(fstest.MapFS, len(files))
|
||||
for name, content := range files {
|
||||
fsys[name] = profileMapFile(content)
|
||||
}
|
||||
return NewFSRepository(fsys, ".")
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func deeplyNestedExtraParamsProfile(depth int) string {
|
||||
var definition strings.Builder
|
||||
definition.WriteString("id: selected-profile\nendpoint: http://localhost:8000/v1\nmodel: model\nextra_params:\n")
|
||||
for level := 0; level < depth; level++ {
|
||||
fmt.Fprintf(&definition, "%slevel_%d:\n", strings.Repeat(" ", level+1), level)
|
||||
}
|
||||
fmt.Fprintf(&definition, "%svalue: true\n", strings.Repeat(" ", depth+1))
|
||||
return definition.String()
|
||||
}
|
||||
|
||||
func profileMapFile(content string) *fstest.MapFile {
|
||||
return &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))}
|
||||
}
|
||||
|
||||
160
internal/profile/resolving_repository.go
Normal file
160
internal/profile/resolving_repository.go
Normal file
@@ -0,0 +1,160 @@
|
||||
package profile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
)
|
||||
|
||||
const maximumProfileChainLength = 32
|
||||
|
||||
type resolvingRepository struct {
|
||||
source Repository
|
||||
}
|
||||
|
||||
// NewResolvingRepository resolves inherited profile definitions from source.
|
||||
func NewResolvingRepository(source Repository) Repository {
|
||||
return &resolvingRepository{source: source}
|
||||
}
|
||||
|
||||
func (r *resolvingRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
if r == nil || r.source == nil {
|
||||
return nil, fmt.Errorf("%w: profile repository is required", ErrInvalidProfile)
|
||||
}
|
||||
requestedID := strings.TrimSpace(id)
|
||||
if requestedID == "" {
|
||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
profile, err := r.getRawProfile(ctx, requestedID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if profile == nil {
|
||||
return nil, fmt.Errorf("%w: selected profile %q is nil", ErrInvalidProfile, requestedID)
|
||||
}
|
||||
|
||||
chain := []*domain.ExecutionProfile{profile}
|
||||
chainIDs := []string{requestedID}
|
||||
visited := map[string]struct{}{requestedID: {}}
|
||||
current := profile
|
||||
|
||||
for {
|
||||
baseID := strings.TrimSpace(current.BaseProfileID)
|
||||
if baseID == "" {
|
||||
break
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, seen := visited[baseID]; seen {
|
||||
return nil, fmt.Errorf("%w: profile inheritance cycle %s", ErrInvalidProfile, joinProfileChain(chainIDs, baseID))
|
||||
}
|
||||
if len(chain) >= maximumProfileChainLength {
|
||||
return nil, fmt.Errorf("%w: profile inheritance chain exceeds %d profiles: %s", ErrInvalidProfile, maximumProfileChainLength, joinProfileChain(chainIDs, baseID))
|
||||
}
|
||||
|
||||
base, err := r.getRawProfile(ctx, baseID)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrProfileNotFound) {
|
||||
return nil, fmt.Errorf("%w: base profile %q is missing in chain %s", ErrInvalidProfile, baseID, joinProfileChain(chainIDs, baseID))
|
||||
}
|
||||
return nil, fmt.Errorf("%w: failed to load base profile %q in chain %s: %w", ErrInvalidProfile, baseID, joinProfileChain(chainIDs, baseID), err)
|
||||
}
|
||||
if base == nil {
|
||||
return nil, fmt.Errorf("%w: base profile %q is nil in chain %s", ErrInvalidProfile, baseID, joinProfileChain(chainIDs, baseID))
|
||||
}
|
||||
|
||||
chain = append(chain, base)
|
||||
chainIDs = append(chainIDs, baseID)
|
||||
visited[baseID] = struct{}{}
|
||||
current = base
|
||||
}
|
||||
|
||||
resolved, err := mergeProfileChain(chain)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: resolved profile chain %s: %w", ErrInvalidProfile, strings.Join(chainIDs, " -> "), err)
|
||||
}
|
||||
if err := validateResolvedProfile(resolved); err != nil {
|
||||
return nil, fmt.Errorf("%w: resolved profile chain %s: %w", ErrInvalidProfile, strings.Join(chainIDs, " -> "), err)
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func (r *resolvingRepository) getRawProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
profile, err := r.source.GetProfile(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return profile, nil
|
||||
}
|
||||
|
||||
func joinProfileChain(chain []string, next string) string {
|
||||
return strings.Join(append(append([]string(nil), chain...), next), " -> ")
|
||||
}
|
||||
|
||||
func mergeProfileChain(chain []*domain.ExecutionProfile) (*domain.ExecutionProfile, error) {
|
||||
resolved := &domain.ExecutionProfile{ID: chain[0].ID}
|
||||
for index := len(chain) - 1; index >= 0; index-- {
|
||||
definition := chain[index]
|
||||
if strings.TrimSpace(definition.BackendID) != "" {
|
||||
resolved.BackendID = definition.BackendID
|
||||
}
|
||||
if strings.TrimSpace(definition.Endpoint) != "" {
|
||||
resolved.Endpoint = definition.Endpoint
|
||||
}
|
||||
if strings.TrimSpace(definition.Model) != "" {
|
||||
resolved.Model = definition.Model
|
||||
}
|
||||
if definition.Temperature != 0 {
|
||||
resolved.Temperature = definition.Temperature
|
||||
}
|
||||
if definition.MaxTokens != 0 {
|
||||
resolved.MaxTokens = definition.MaxTokens
|
||||
}
|
||||
if definition.TopP != 0 {
|
||||
resolved.TopP = definition.TopP
|
||||
}
|
||||
if definition.TimeoutSeconds != 0 {
|
||||
resolved.TimeoutSeconds = definition.TimeoutSeconds
|
||||
}
|
||||
if strings.TrimSpace(definition.ServiceTier) != "" {
|
||||
resolved.ServiceTier = definition.ServiceTier
|
||||
}
|
||||
if strings.TrimSpace(definition.ReasoningEffort) != "" {
|
||||
resolved.ReasoningEffort = definition.ReasoningEffort
|
||||
}
|
||||
if strings.TrimSpace(definition.APIKeyEnv) != "" {
|
||||
resolved.APIKeyEnv = definition.APIKeyEnv
|
||||
}
|
||||
resolved.APIKeyRequired = resolved.APIKeyRequired || definition.APIKeyRequired
|
||||
if len(definition.ExtraParams) != 0 {
|
||||
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resolved.ExtraParams = extraParams
|
||||
}
|
||||
}
|
||||
resolved.ID = chain[0].ID
|
||||
resolved.BaseProfileID = ""
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func validateResolvedProfile(profile *domain.ExecutionProfile) error {
|
||||
if profile == nil {
|
||||
return errors.New("resolved profile is required")
|
||||
}
|
||||
profile.BaseProfileID = ""
|
||||
return NormalizeAndValidateDefinition(profile)
|
||||
}
|
||||
418
internal/profile/resolving_repository_test.go
Normal file
418
internal/profile/resolving_repository_test.go
Normal file
@@ -0,0 +1,418 @@
|
||||
package profile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestResolvingRepositoryMergesProfileChain(t *testing.T) {
|
||||
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||
"leaf": {
|
||||
ID: "leaf",
|
||||
BaseProfileID: "middle",
|
||||
BackendID: "leaf-backend",
|
||||
TopP: 0.8,
|
||||
TimeoutSeconds: 45,
|
||||
ReasoningEffort: "high",
|
||||
},
|
||||
"middle": {
|
||||
ID: "middle",
|
||||
BaseProfileID: "root",
|
||||
Endpoint: "https://middle.example/v1",
|
||||
Model: "middle-model",
|
||||
MaxTokens: 256,
|
||||
APIKeyEnv: "MIDDLE_API_KEY",
|
||||
APIKeyRequired: true,
|
||||
ExtraParams: map[string]any{"middle": map[string]any{"value": "middle"}},
|
||||
},
|
||||
"root": {
|
||||
ID: "root",
|
||||
BackendID: "root-backend",
|
||||
Endpoint: "https://root.example/v1",
|
||||
Model: "root-model",
|
||||
Temperature: 0.3,
|
||||
ServiceTier: "priority",
|
||||
ExtraParams: map[string]any{"root": "value"},
|
||||
},
|
||||
}}
|
||||
|
||||
got, err := NewResolvingRepository(repo).GetProfile(context.Background(), "leaf")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve profile: %v", err)
|
||||
}
|
||||
want := &domain.ExecutionProfile{
|
||||
ID: "leaf",
|
||||
BackendID: "leaf-backend",
|
||||
Endpoint: "https://middle.example/v1",
|
||||
Model: "middle-model",
|
||||
Temperature: 0.3,
|
||||
MaxTokens: 256,
|
||||
TopP: 0.8,
|
||||
TimeoutSeconds: 45,
|
||||
ServiceTier: "priority",
|
||||
ReasoningEffort: "high",
|
||||
APIKeyEnv: "MIDDLE_API_KEY",
|
||||
APIKeyRequired: true,
|
||||
ExtraParams: map[string]any{"middle": map[string]any{"value": "middle"}},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("resolved profile:\n got %#v\nwant %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvingRepositoryRejectsMissingSourceAndProfileID(t *testing.T) {
|
||||
if _, err := NewResolvingRepository(nil).GetProfile(context.Background(), "profile"); !errors.Is(err, ErrInvalidProfile) {
|
||||
t.Fatalf("nil source error = %v, want ErrInvalidProfile", err)
|
||||
}
|
||||
|
||||
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{}}
|
||||
if _, err := NewResolvingRepository(repo).GetProfile(context.Background(), " \t "); !errors.Is(err, ErrInvalidProfile) {
|
||||
t.Fatalf("blank id error = %v, want ErrInvalidProfile", err)
|
||||
}
|
||||
if got := repo.callCount(" "); got != 0 {
|
||||
t.Fatalf("blank id looked up source %d times", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvingRepositoryCopiesExtraParams(t *testing.T) {
|
||||
baseParams := map[string]any{"nested": map[string]any{"value": "base"}}
|
||||
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||
"child": {ID: "child", BaseProfileID: "base"},
|
||||
"base": {
|
||||
ID: "base",
|
||||
Endpoint: "https://base.example/v1",
|
||||
Model: "model",
|
||||
ExtraParams: baseParams,
|
||||
},
|
||||
}}
|
||||
resolver := NewResolvingRepository(repo)
|
||||
|
||||
first, err := resolver.GetProfile(context.Background(), "child")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve inherited map: %v", err)
|
||||
}
|
||||
first.ExtraParams["nested"].(map[string]any)["value"] = "mutated"
|
||||
second, err := resolver.GetProfile(context.Background(), "child")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve inherited map again: %v", err)
|
||||
}
|
||||
if got := second.ExtraParams["nested"].(map[string]any)["value"]; got != "base" {
|
||||
t.Fatalf("later result retained mutation: %v", got)
|
||||
}
|
||||
if got := baseParams["nested"].(map[string]any)["value"]; got != "base" {
|
||||
t.Fatalf("source map retained mutation: %v", got)
|
||||
}
|
||||
|
||||
repo.set("child", &domain.ExecutionProfile{
|
||||
ID: "child",
|
||||
BaseProfileID: "base",
|
||||
ExtraParams: map[string]any{"child": "replacement"},
|
||||
})
|
||||
replaced, err := resolver.GetProfile(context.Background(), "child")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve replacement map: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(replaced.ExtraParams, map[string]any{"child": "replacement"}) {
|
||||
t.Fatalf("extra params = %#v, want complete child replacement", replaced.ExtraParams)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvingRepositoryUsesRawOverlayForEachLookup(t *testing.T) {
|
||||
leafSource := NewFSRepository(profileTestFS(map[string]string{
|
||||
"leaf.yaml": "id: leaf\nbase_profile: base\n",
|
||||
}), ".")
|
||||
fallback := NewFSRepository(profileTestFS(map[string]string{
|
||||
"base.yaml": "id: base\nendpoint: https://fallback.example/v1\nmodel: fallback-model\n",
|
||||
}), ".")
|
||||
overlay := NewOverlayRepository(leafSource, fallback)
|
||||
resolver := NewResolvingRepository(overlay)
|
||||
|
||||
got, err := resolver.GetProfile(context.Background(), "leaf")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve fallback base: %v", err)
|
||||
}
|
||||
if got.Model != "fallback-model" {
|
||||
t.Fatalf("fallback base model = %q", got.Model)
|
||||
}
|
||||
|
||||
shadowing := NewOverlayRepository(NewFSRepository(profileTestFS(map[string]string{
|
||||
"leaf.yaml": "id: leaf\nbase_profile: base\n",
|
||||
"base.yaml": "id: base\nendpoint: https://primary.example/v1\nmodel: primary-model\n",
|
||||
}), "."), fallback)
|
||||
got, err = NewResolvingRepository(shadowing).GetProfile(context.Background(), "leaf")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve shadowed base: %v", err)
|
||||
}
|
||||
if got.Model != "primary-model" || got.Endpoint != "https://primary.example/v1" {
|
||||
t.Fatalf("shadowed base = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvingRepositoryReportsSafetyAndSourceErrors(t *testing.T) {
|
||||
sourceErr := errors.New("source failure")
|
||||
tests := []struct {
|
||||
name string
|
||||
repo *resolvingTestRepository
|
||||
id string
|
||||
want []error
|
||||
wantNot error
|
||||
contains []string
|
||||
}{
|
||||
{
|
||||
name: "missing selected profile preserves not found",
|
||||
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{}},
|
||||
id: "missing",
|
||||
want: []error{ErrProfileNotFound},
|
||||
wantNot: ErrInvalidProfile,
|
||||
},
|
||||
{
|
||||
name: "missing base is invalid but not not found",
|
||||
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||
"leaf": {ID: "leaf", BaseProfileID: "missing"},
|
||||
}},
|
||||
id: "leaf",
|
||||
want: []error{ErrInvalidProfile},
|
||||
wantNot: ErrProfileNotFound,
|
||||
contains: []string{"missing", "leaf -> missing"},
|
||||
},
|
||||
{
|
||||
name: "direct cycle",
|
||||
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||
"a": {ID: "a", BaseProfileID: "a"},
|
||||
}},
|
||||
id: "a",
|
||||
want: []error{ErrInvalidProfile},
|
||||
contains: []string{"a -> a"},
|
||||
},
|
||||
{
|
||||
name: "indirect cycle",
|
||||
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||
"a": {ID: "a", BaseProfileID: "b"},
|
||||
"b": {ID: "b", BaseProfileID: "c"},
|
||||
"c": {ID: "c", BaseProfileID: "a"},
|
||||
}},
|
||||
id: "a",
|
||||
want: []error{ErrInvalidProfile},
|
||||
contains: []string{"a -> b -> c -> a"},
|
||||
},
|
||||
{
|
||||
name: "nil result",
|
||||
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||
"leaf": nil,
|
||||
}},
|
||||
id: "leaf",
|
||||
want: []error{ErrInvalidProfile},
|
||||
},
|
||||
{
|
||||
name: "incomplete resolved profile",
|
||||
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||
"leaf": {ID: "leaf", BaseProfileID: "base"},
|
||||
"base": {ID: "base", Model: "model"},
|
||||
}},
|
||||
id: "leaf",
|
||||
want: []error{ErrInvalidProfile},
|
||||
},
|
||||
{
|
||||
name: "base source error is retained",
|
||||
repo: &resolvingTestRepository{
|
||||
profiles: map[string]*domain.ExecutionProfile{"leaf": {ID: "leaf", BaseProfileID: "base"}},
|
||||
errors: map[string]error{"base": sourceErr},
|
||||
},
|
||||
id: "leaf",
|
||||
want: []error{ErrInvalidProfile, sourceErr},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := NewResolvingRepository(tc.repo).GetProfile(context.Background(), tc.id)
|
||||
for _, want := range tc.want {
|
||||
if !errors.Is(err, want) {
|
||||
t.Fatalf("error = %v, want %v", err, want)
|
||||
}
|
||||
}
|
||||
if tc.wantNot != nil && errors.Is(err, tc.wantNot) {
|
||||
t.Fatalf("error = %v, must not match %v", err, tc.wantNot)
|
||||
}
|
||||
for _, fragment := range tc.contains {
|
||||
if !strings.Contains(err.Error(), fragment) {
|
||||
t.Fatalf("error = %v, want %q", err, fragment)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvingRepositoryEnforcesChainLength(t *testing.T) {
|
||||
for _, count := range []int{maximumProfileChainLength, maximumProfileChainLength + 1} {
|
||||
t.Run(fmt.Sprintf("%d profiles", count), func(t *testing.T) {
|
||||
profiles := make(map[string]*domain.ExecutionProfile, count)
|
||||
for index := 1; index <= count; index++ {
|
||||
id := fmt.Sprintf("profile-%d", index)
|
||||
definition := &domain.ExecutionProfile{ID: id}
|
||||
if index == count {
|
||||
definition.Endpoint = "https://root.example/v1"
|
||||
definition.Model = "model"
|
||||
} else {
|
||||
definition.BaseProfileID = fmt.Sprintf("profile-%d", index+1)
|
||||
}
|
||||
profiles[id] = definition
|
||||
}
|
||||
|
||||
got, err := NewResolvingRepository(&resolvingTestRepository{profiles: profiles}).GetProfile(context.Background(), "profile-1")
|
||||
if count == maximumProfileChainLength {
|
||||
if err != nil || got == nil {
|
||||
t.Fatalf("profile = %+v, error = %v, want accepted chain", got, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !errors.Is(err, ErrInvalidProfile) {
|
||||
t.Fatalf("error = %v, want ErrInvalidProfile", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvingRepositoryIsFreshAndCancellationAware(t *testing.T) {
|
||||
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||
"leaf": {ID: "leaf", BaseProfileID: "base"},
|
||||
"base": {ID: "base", Endpoint: "https://base.example/v1", Model: "first", ExtraParams: map[string]any{"nested": map[string]any{"value": "first"}}},
|
||||
}}
|
||||
resolver := NewResolvingRepository(repo)
|
||||
|
||||
first, err := resolver.GetProfile(context.Background(), "leaf")
|
||||
if err != nil || first.Model != "first" {
|
||||
t.Fatalf("first result=(%+v, %v)", first, err)
|
||||
}
|
||||
repo.set("base", &domain.ExecutionProfile{ID: "base", Endpoint: "https://base.example/v1", Model: "second", ExtraParams: map[string]any{"nested": map[string]any{"value": "second"}}})
|
||||
second, err := resolver.GetProfile(context.Background(), "leaf")
|
||||
if err != nil || second.Model != "second" {
|
||||
t.Fatalf("second result=(%+v, %v)", second, err)
|
||||
}
|
||||
|
||||
canceled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := resolver.GetProfile(canceled, "leaf"); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("canceled lookup error = %v", err)
|
||||
}
|
||||
if got := repo.callCount("leaf"); got != 2 {
|
||||
t.Fatalf("calls after canceled lookup = %d, want 2", got)
|
||||
}
|
||||
|
||||
duringTraversal, cancelDuringTraversal := context.WithCancel(context.Background())
|
||||
repo.afterGet = func(id string) {
|
||||
if id == "leaf" {
|
||||
cancelDuringTraversal()
|
||||
}
|
||||
}
|
||||
if _, err := resolver.GetProfile(duringTraversal, "leaf"); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("during traversal error = %v", err)
|
||||
}
|
||||
if got := repo.callCount("base"); got != 2 {
|
||||
t.Fatalf("base calls after cancellation = %d, want 2", got)
|
||||
}
|
||||
|
||||
terminalLookup, cancelTerminalLookup := context.WithCancel(context.Background())
|
||||
repo.afterGet = func(id string) {
|
||||
if id == "base" {
|
||||
cancelTerminalLookup()
|
||||
}
|
||||
}
|
||||
if _, err := resolver.GetProfile(terminalLookup, "leaf"); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("terminal lookup cancellation error = %v", err)
|
||||
}
|
||||
if got := repo.callCount("base"); got != 3 {
|
||||
t.Fatalf("base calls after terminal cancellation = %d, want 3", got)
|
||||
}
|
||||
|
||||
repo.afterGet = nil
|
||||
var wg sync.WaitGroup
|
||||
errors := make(chan error, 8)
|
||||
for index := 0; index < cap(errors); index++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
resolved, err := resolver.GetProfile(context.Background(), "leaf")
|
||||
if err != nil {
|
||||
errors <- err
|
||||
return
|
||||
}
|
||||
resolved.ExtraParams["nested"].(map[string]any)["value"] = "mutated"
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errors)
|
||||
for err := range errors {
|
||||
t.Errorf("concurrent resolution: %v", err)
|
||||
}
|
||||
latest, err := resolver.GetProfile(context.Background(), "leaf")
|
||||
if err != nil || latest.ExtraParams["nested"].(map[string]any)["value"] != "second" {
|
||||
t.Fatalf("latest result=(%+v, %v)", latest, err)
|
||||
}
|
||||
}
|
||||
|
||||
type resolvingTestRepository struct {
|
||||
mu sync.Mutex
|
||||
profiles map[string]*domain.ExecutionProfile
|
||||
errors map[string]error
|
||||
calls map[string]int
|
||||
afterGet func(string)
|
||||
}
|
||||
|
||||
func (r *resolvingTestRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.mu.Lock()
|
||||
if r.calls == nil {
|
||||
r.calls = make(map[string]int)
|
||||
}
|
||||
r.calls[id]++
|
||||
err := r.errors[id]
|
||||
profile := r.profiles[id]
|
||||
afterGet := r.afterGet
|
||||
r.mu.Unlock()
|
||||
if afterGet != nil {
|
||||
afterGet(id)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if profile == nil {
|
||||
if _, exists := r.profiles[id]; exists {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, ErrProfileNotFound
|
||||
}
|
||||
copy := *profile
|
||||
return ©, nil
|
||||
}
|
||||
|
||||
func (r *resolvingTestRepository) set(id string, profile *domain.ExecutionProfile) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.profiles[id] = profile
|
||||
}
|
||||
|
||||
func (r *resolvingTestRepository) callCount(id string) int {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.calls[id]
|
||||
}
|
||||
|
||||
func profileTestFS(files map[string]string) fs.FS {
|
||||
fsys := make(fstest.MapFS, len(files))
|
||||
for name, content := range files {
|
||||
fsys[name] = profileMapFile(content)
|
||||
}
|
||||
return fsys
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"text/template"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
@@ -18,6 +19,8 @@ var (
|
||||
ErrInvalidMessageRole = errors.New("invalid or empty message role")
|
||||
)
|
||||
|
||||
const artifactTextChunkSize = 64 * 1024
|
||||
|
||||
type goRenderer struct{}
|
||||
|
||||
func NewGoRenderer() Renderer {
|
||||
@@ -25,11 +28,13 @@ func NewGoRenderer() Renderer {
|
||||
}
|
||||
|
||||
func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if definition == nil {
|
||||
return nil, fmt.Errorf("%w: nil prompt definition", ErrRenderFailure)
|
||||
}
|
||||
|
||||
// 1. Verify required inputs
|
||||
for _, in := range definition.Inputs {
|
||||
if !in.Required {
|
||||
continue
|
||||
@@ -40,51 +45,69 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Setup template functions
|
||||
resolver := newArtifactTextResolver(ctx, inputs)
|
||||
funcs := template.FuncMap{
|
||||
"input": func(name string) (string, error) {
|
||||
art, ok := inputs[name]
|
||||
if !ok || art == nil {
|
||||
return "", fmt.Errorf("%w: %s", ErrUnknownInput, name)
|
||||
}
|
||||
return string(art.Body), nil
|
||||
},
|
||||
"input": resolver.resolve,
|
||||
}
|
||||
|
||||
sessionID, err := renderSessionID(definition.SessionID, funcs, vars)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sessionID, err := renderSessionID(ctx, definition.SessionID, funcs, vars)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var renderedMessages []domain.RenderedMessage
|
||||
renderedMessages := make([]domain.RenderedMessage, 0, len(definition.Templates))
|
||||
|
||||
for i, tmplMsg := range definition.Templates {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if tmplMsg.Role == "" {
|
||||
return nil, fmt.Errorf("%w: message %d", ErrInvalidMessageRole, i)
|
||||
}
|
||||
|
||||
// Parse and execute template
|
||||
tmpl, err := template.New(fmt.Sprintf("msg_%d", i)).Funcs(funcs).Option("missingkey=error").Parse(tmplMsg.Content)
|
||||
role, err := domain.NormalizeMessageRole(tmplMsg.Role)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: message %d: %v", ErrInvalidTemplate, i, err)
|
||||
return nil, fmt.Errorf("%w: message %d: %v", ErrInvalidMessageRole, i, err)
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tmpl, parseErr := template.New(fmt.Sprintf("msg_%d", i)).Funcs(funcs).Option("missingkey=error").Parse(tmplMsg.Content)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if parseErr != nil {
|
||||
return nil, fmt.Errorf("%w: message %d: %v", ErrInvalidTemplate, i, parseErr)
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := tmpl.Execute(&buf, vars); err != nil {
|
||||
return nil, fmt.Errorf("%w: message %d: %w", ErrRenderFailure, i, err)
|
||||
executeErr := tmpl.Execute(&buf, vars)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if executeErr != nil {
|
||||
return nil, fmt.Errorf("%w: message %d: %w", ErrRenderFailure, i, executeErr)
|
||||
}
|
||||
|
||||
renderedMessages = append(renderedMessages, domain.RenderedMessage{
|
||||
Role: tmplMsg.Role,
|
||||
Role: role,
|
||||
Content: buf.String(),
|
||||
CacheControl: cloneCacheControl(tmplMsg.CacheControl),
|
||||
})
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &domain.RenderedPrompt{
|
||||
@@ -93,15 +116,75 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
||||
}, nil
|
||||
}
|
||||
|
||||
func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
|
||||
tmpl, err := template.New("session_id").Funcs(funcs).Option("missingkey=error").Parse(raw)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, err)
|
||||
type artifactTextResolver struct {
|
||||
ctx context.Context
|
||||
inputs map[string]*domain.Artifact
|
||||
textByName map[string]string
|
||||
}
|
||||
|
||||
func newArtifactTextResolver(ctx context.Context, inputs map[string]*domain.Artifact) *artifactTextResolver {
|
||||
return &artifactTextResolver{
|
||||
ctx: ctx,
|
||||
inputs: inputs,
|
||||
textByName: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *artifactTextResolver) resolve(name string) (string, error) {
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
artifact, ok := r.inputs[name]
|
||||
if !ok || artifact == nil {
|
||||
return "", fmt.Errorf("%w: %s", ErrUnknownInput, name)
|
||||
}
|
||||
if text, ok := r.textByName[name]; ok {
|
||||
return text, nil
|
||||
}
|
||||
|
||||
var builder strings.Builder
|
||||
builder.Grow(len(artifact.Body))
|
||||
for start := 0; start < len(artifact.Body); start += artifactTextChunkSize {
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
end := min(start+artifactTextChunkSize, len(artifact.Body))
|
||||
_, _ = builder.Write(artifact.Body[start:end])
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
text := builder.String()
|
||||
r.textByName[name] = text
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func renderSessionID(ctx context.Context, raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
tmpl, parseErr := template.New("session_id").Funcs(funcs).Option("missingkey=error").Parse(raw)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if parseErr != nil {
|
||||
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, parseErr)
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := tmpl.Execute(&buf, vars); err != nil {
|
||||
return "", fmt.Errorf("%w: session_id: %w", ErrRenderFailure, err)
|
||||
executeErr := tmpl.Execute(&buf, vars)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if executeErr != nil {
|
||||
return "", fmt.Errorf("%w: session_id: %w", ErrRenderFailure, executeErr)
|
||||
}
|
||||
|
||||
sessionID, err := domain.NormalizeSessionID(buf.String())
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package prompt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
@@ -228,6 +229,24 @@ func TestGoRenderer_Render(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("malformed rendered session id fails rendering", func(t *testing.T) {
|
||||
def := &domain.PromptDefinition{
|
||||
SessionID: "{{ .session_id }}",
|
||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||
Templates: []domain.PromptMessageTemplate{
|
||||
{Role: "system", Content: "Speak in a {{.tone}} tone."},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := renderer.Render(ctx, def, inputs, map[string]string{
|
||||
"tone": "concise",
|
||||
"session_id": "session" + string([]byte{0xff}),
|
||||
})
|
||||
if !errors.Is(err, ErrRenderFailure) {
|
||||
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("inserting required input artifact", func(t *testing.T) {
|
||||
def := &domain.PromptDefinition{
|
||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||
@@ -342,4 +361,216 @@ func TestGoRenderer_Render(t *testing.T) {
|
||||
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("canonicalizes directly supplied message roles", func(t *testing.T) {
|
||||
def := &domain.PromptDefinition{
|
||||
Templates: []domain.PromptMessageTemplate{
|
||||
{Role: " \u2003DeVeLoPeR\u2003 ", Content: "Hello"},
|
||||
},
|
||||
}
|
||||
res, err := renderer.Render(ctx, def, nil, vars)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got := res.Messages[0].Role; got != domain.RoleDeveloper {
|
||||
t.Fatalf("rendered role = %q, want %q", got, domain.RoleDeveloper)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects unsupported directly supplied message roles", func(t *testing.T) {
|
||||
const unsupportedRole = "consumer-private-role"
|
||||
def := &domain.PromptDefinition{
|
||||
Templates: []domain.PromptMessageTemplate{{Role: unsupportedRole, Content: "Hello"}},
|
||||
}
|
||||
_, err := renderer.Render(ctx, def, nil, vars)
|
||||
if !errors.Is(err, ErrInvalidMessageRole) {
|
||||
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), unsupportedRole) {
|
||||
t.Fatalf("renderer error exposed unsupported role: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestGoRendererCancellation(t *testing.T) {
|
||||
t.Run("before session parsing", func(t *testing.T) {
|
||||
definition := &domain.PromptDefinition{
|
||||
SessionID: "{{ malformed",
|
||||
Templates: []domain.PromptMessageTemplate{
|
||||
{Role: "user", Content: "not rendered"},
|
||||
},
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
result, err := NewGoRenderer().Render(ctx, definition, nil, nil)
|
||||
if result != nil || !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
|
||||
}
|
||||
if errors.Is(err, ErrInvalidTemplate) {
|
||||
t.Fatalf("pre-canceled render parsed the malformed session: %v", err)
|
||||
}
|
||||
|
||||
result, err = NewGoRenderer().Render(context.Background(), definition, nil, nil)
|
||||
if result != nil || !errors.Is(err, ErrInvalidTemplate) {
|
||||
t.Fatalf("active render result=%#v err=%v, want nil/ErrInvalidTemplate", result, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("during artifact text conversion", func(t *testing.T) {
|
||||
ctx := newCancelOnCheckContext(3)
|
||||
body := bytes.Repeat([]byte("x"), artifactTextChunkSize*2)
|
||||
original := append([]byte(nil), body...)
|
||||
resolver := newArtifactTextResolver(ctx, map[string]*domain.Artifact{
|
||||
"document": {Body: body},
|
||||
})
|
||||
|
||||
text, err := resolver.resolve("document")
|
||||
if text != "" || !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("text length=%d err=%v, want empty/context.Canceled", len(text), err)
|
||||
}
|
||||
if _, published := resolver.textByName["document"]; published {
|
||||
t.Fatal("canceled conversion published partial artifact text")
|
||||
}
|
||||
if !bytes.Equal(body, original) {
|
||||
t.Fatal("resolver mutated the artifact body")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("after final message execution", func(t *testing.T) {
|
||||
definition := &domain.PromptDefinition{
|
||||
Templates: []domain.PromptMessageTemplate{
|
||||
{Role: "user", Content: "fully rendered"},
|
||||
},
|
||||
}
|
||||
counter := &checkCountingContext{Context: context.Background()}
|
||||
if _, err := NewGoRenderer().Render(counter, definition, nil, nil); err != nil {
|
||||
t.Fatalf("count render checkpoints: %v", err)
|
||||
}
|
||||
|
||||
// The final three checks occur after template execution, after the
|
||||
// message is assembled, and immediately before publication.
|
||||
ctx := newCancelOnCheckContext(counter.checks - 2)
|
||||
result, err := NewGoRenderer().Render(ctx, definition, nil, nil)
|
||||
if result != nil || !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestGoRendererArtifactTextLifecycle(t *testing.T) {
|
||||
body := []byte{'a', 0xff, 'b', 0xfe}
|
||||
original := append([]byte(nil), body...)
|
||||
artifact := &domain.Artifact{Body: body}
|
||||
inputs := map[string]*domain.Artifact{"document": artifact}
|
||||
definition := &domain.PromptDefinition{
|
||||
Templates: []domain.PromptMessageTemplate{
|
||||
{Role: "user", Content: "{{input \"document\"}}|{{input \"document\"}}"},
|
||||
},
|
||||
}
|
||||
|
||||
first, err := NewGoRenderer().Render(context.Background(), definition, inputs, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("first render: %v", err)
|
||||
}
|
||||
wantFirst := append(append(append([]byte(nil), body...), '|'), body...)
|
||||
if !bytes.Equal([]byte(first.Messages[0].Content), wantFirst) {
|
||||
t.Fatalf("rendered bytes=%v, want %v", []byte(first.Messages[0].Content), wantFirst)
|
||||
}
|
||||
if !bytes.Equal(body, original) {
|
||||
t.Fatalf("renderer mutated artifact body: got %v want %v", body, original)
|
||||
}
|
||||
|
||||
body[0] = 'z'
|
||||
if bytes.Equal([]byte(first.Messages[0].Content), append(append(append([]byte(nil), body...), '|'), body...)) {
|
||||
t.Fatal("completed render aliases the artifact body")
|
||||
}
|
||||
second, err := NewGoRenderer().Render(context.Background(), definition, inputs, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("second render: %v", err)
|
||||
}
|
||||
wantSecond := append(append(append([]byte(nil), body...), '|'), body...)
|
||||
if !bytes.Equal([]byte(second.Messages[0].Content), wantSecond) {
|
||||
t.Fatalf("second render reused text from another call: got %v want %v", []byte(second.Messages[0].Content), wantSecond)
|
||||
}
|
||||
|
||||
nilInputs := map[string]*domain.Artifact{"document": nil}
|
||||
result, err := NewGoRenderer().Render(context.Background(), definition, nilInputs, nil)
|
||||
if result != nil || !errors.Is(err, ErrUnknownInput) || !errors.Is(err, ErrRenderFailure) {
|
||||
t.Fatalf("nil input result=%#v err=%v, want ErrUnknownInput and ErrRenderFailure", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkGoRendererArtifactReferences(b *testing.B) {
|
||||
body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content "))
|
||||
inputs := map[string]*domain.Artifact{
|
||||
"document": {Body: body},
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
definition *domain.PromptDefinition
|
||||
}{
|
||||
{
|
||||
name: "one reference",
|
||||
definition: &domain.PromptDefinition{
|
||||
Templates: []domain.PromptMessageTemplate{
|
||||
{Role: "user", Content: "{{input \"document\"}}"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "repeated across session and messages",
|
||||
definition: &domain.PromptDefinition{
|
||||
SessionID: "document-{{len (input \"document\")}}",
|
||||
Templates: []domain.PromptMessageTemplate{
|
||||
{Role: "system", Content: "{{input \"document\"}}"},
|
||||
{Role: "user", Content: "{{input \"document\"}} {{input \"document\"}}"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
b.Run(tc.name, func(b *testing.B) {
|
||||
renderer := NewGoRenderer()
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(body)))
|
||||
for range b.N {
|
||||
if _, err := renderer.Render(context.Background(), tc.definition, inputs, nil); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type checkCountingContext struct {
|
||||
context.Context
|
||||
checks int
|
||||
}
|
||||
|
||||
func (c *checkCountingContext) Err() error {
|
||||
c.checks++
|
||||
return c.Context.Err()
|
||||
}
|
||||
|
||||
type cancelOnCheckContext struct {
|
||||
context.Context
|
||||
cancel context.CancelFunc
|
||||
remaining int
|
||||
}
|
||||
|
||||
func newCancelOnCheckContext(checks int) *cancelOnCheckContext {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
return &cancelOnCheckContext{Context: ctx, cancel: cancel, remaining: checks}
|
||||
}
|
||||
|
||||
func (c *cancelOnCheckContext) Err() error {
|
||||
if c.Context.Err() == nil {
|
||||
c.remaining--
|
||||
if c.remaining == 0 {
|
||||
c.cancel()
|
||||
}
|
||||
}
|
||||
return c.Context.Err()
|
||||
}
|
||||
|
||||
98
internal/promptdef/content_source.go
Normal file
98
internal/promptdef/content_source.go
Normal file
@@ -0,0 +1,98 @@
|
||||
package promptdef
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
||||
)
|
||||
|
||||
type contentSourceRoot interface {
|
||||
readContentFile(sourcePath string, contentFile string) (string, string, error)
|
||||
}
|
||||
|
||||
type osContentSourceRoot struct {
|
||||
root string
|
||||
sourcePathsRelative bool
|
||||
}
|
||||
|
||||
func (r osContentSourceRoot) readContentFile(sourcePath string, contentFile string) (string, string, error) {
|
||||
if strings.TrimSpace(contentFile) == "" {
|
||||
return "", "", fmt.Errorf("path is required")
|
||||
}
|
||||
if filepath.IsAbs(contentFile) {
|
||||
return "", "", fmt.Errorf("path %q must be relative", contentFile)
|
||||
}
|
||||
|
||||
root, err := filepath.Abs(r.root)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("resolve source root %q: %w", r.root, err)
|
||||
}
|
||||
canonicalRoot, err := filepath.EvalSymlinks(root)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("resolve source root %q: %w", r.root, err)
|
||||
}
|
||||
|
||||
promptPath := sourcePath
|
||||
if r.sourcePathsRelative && !filepath.IsAbs(promptPath) {
|
||||
promptPath = filepath.Join(root, filepath.FromSlash(promptPath))
|
||||
} else {
|
||||
promptPath, err = filepath.Abs(promptPath)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("resolve prompt source %q: %w", sourcePath, err)
|
||||
}
|
||||
}
|
||||
resolvedPath := filepath.Clean(filepath.Join(filepath.Dir(promptPath), contentFile))
|
||||
if !containsOSPath(root, resolvedPath) {
|
||||
return "", "", fmt.Errorf("path %q escapes source root %q", contentFile, r.root)
|
||||
}
|
||||
|
||||
canonicalPath, err := filepath.EvalSymlinks(resolvedPath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if !containsOSPath(canonicalRoot, canonicalPath) {
|
||||
return "", "", fmt.Errorf("path %q escapes source root %q", contentFile, r.root)
|
||||
}
|
||||
|
||||
body, err := os.ReadFile(canonicalPath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return string(body), resolvedPath, nil
|
||||
}
|
||||
|
||||
type fsContentSourceRoot struct {
|
||||
fsys fs.FS
|
||||
root string
|
||||
}
|
||||
|
||||
func (r fsContentSourceRoot) readContentFile(sourcePath string, contentFile string) (string, string, error) {
|
||||
root := filecatalog.CleanFSRoot(r.root)
|
||||
cleanSourcePath := path.Clean(sourcePath)
|
||||
if cleanSourcePath == root {
|
||||
root = path.Dir(root)
|
||||
}
|
||||
|
||||
resolvedPath, _, err := filecatalog.ResolveFSPath(root, path.Dir(cleanSourcePath), contentFile)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
body, err := fs.ReadFile(r.fsys, resolvedPath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return string(body), resolvedPath, nil
|
||||
}
|
||||
|
||||
func containsOSPath(root string, name string) bool {
|
||||
relative, err := filepath.Rel(root, name)
|
||||
if err != nil || filepath.IsAbs(relative) {
|
||||
return false
|
||||
}
|
||||
return relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user