69 Commits

Author SHA1 Message Date
a4264bf4b4 Prepare the v0.9.0 release 2026-08-26 17:31:53 +00:00
81f41564a2 Harden external catalog validation 2026-08-26 16:51:31 +00:00
6b5a2497cc Document the external catalog boundary 2026-08-26 15:06:33 +00:00
422cc6c978 Use external catalogs for maintained backends 2026-08-26 15:04:50 +00:00
f48e042565 Validate the external backend catalogs 2026-08-26 15:00:50 +00:00
d9442850ef Add immutable profile catalog loading 2026-08-26 14:56:56 +00:00
2e828157b6 Mark the Rakestrawhome catalog release complete 2026-08-26 14:54:42 +00:00
4189146536 Mark the OpenRouter catalog release complete 2026-08-26 14:53:02 +00:00
59be507d2f Freeze the built-in catalog compatibility baseline 2026-08-26 14:49:25 +00:00
b45cea4084 Refine the external catalog implementation plan 2026-08-26 14:20:22 +00:00
efe885c6b2 Plan external catalog extraction 2026-08-26 13:58:22 +00:00
404a4c331d Clarify appended messages and streamline repair 2026-08-26 02:37:47 +00:00
174516cb39 Document appended request messages 2026-08-26 02:19:44 +00:00
b0999112cc Compose appended request messages 2026-08-26 02:16:16 +00:00
ed0f7527d5 Add appended message request validation 2026-08-26 02:10:01 +00:00
5064cf833d Centralize message role invariants 2026-08-26 02:07:15 +00:00
8745d256bd Plan appended request messages 2026-08-26 01:53:22 +00:00
2e76003fd5 Prepare documentation for the v0.8.0 release 2026-08-25 12:19:17 +00:00
36ce5a5099 Fix output repair diagnostics and concurrency tests 2026-08-25 11:54:50 +00:00
465dc1389d Document bounded output repair 2026-08-25 09:58:10 +00:00
ae6f1a9865 Enable bounded output repair in the engine 2026-08-25 09:55:19 +00:00
ee99dc9478 Run bounded output repair attempts 2026-08-25 09:51:28 +00:00
00ee5893e9 Build bounded output repair requests 2026-08-25 09:46:56 +00:00
64d1cffd89 Preserve explicit empty model responses 2026-08-25 09:42:28 +00:00
f9e8afa2c3 Enforce bounded output repair contracts 2026-08-25 09:41:15 +00:00
e827631d8c Plan the public output repair implementation 2026-08-25 09:38:06 +00:00
115fe8ba58 Plan bounded output repair 2026-08-25 03:33:33 +00:00
c53250f023 Prepare documentation for the v0.7.0 release 2026-08-25 02:13:43 +00:00
3d99483219 Preserve cancellation after profile lookups 2026-08-25 02:06:47 +00:00
67f788b1e2 Document profile inheritance behavior 2026-08-25 01:46:16 +00:00
764103a2e2 Resolve inherited profiles in engine workflows 2026-08-25 01:41:21 +00:00
a08dd83d1f Add profile inheritance resolver 2026-08-25 01:38:15 +00:00
e8922d8ec5 Add profile inheritance definition support 2026-08-25 01:34:40 +00:00
2d44305a8a Document optional API key environment behavior 2026-08-25 00:44:26 +00:00
a11c80291e Allow unauthenticated optional API key requests 2026-08-25 00:40:49 +00:00
3239567297 Make optional API key environments nonblocking 2026-08-25 00:39:26 +00:00
c239304c2a Document structured generation errors 2026-08-23 19:04:00 +00:00
159b02116f Expose structured generation errors to consumers 2026-08-23 19:01:38 +00:00
e5b7adfb49 Return structured errors for provider status failures 2026-08-23 18:58:15 +00:00
fa2384e696 Bound provider error response body reads 2026-08-23 18:56:15 +00:00
af0bd3f31a Add internal structured provider error parsing 2026-08-23 18:54:42 +00:00
5fff8cd623 Retire the completed Rakestrawhome backend roadmaps 2026-08-23 17:34:14 +00:00
b2a6c47778 Document the Rakestrawhome built-in backend 2026-08-23 17:19:12 +00:00
a1805fe550 Add the Rakestrawhome Gemma profile 2026-08-23 17:12:17 +00:00
93af155254 Add the Rakestrawhome built-in backend 2026-08-23 17:10:34 +00:00
d783b687a5 Plan the built-in Rakestrawhome backend 2026-08-23 17:00:01 +00:00
4ca3be2c14 Finish audit remediation and prepare v0.6.0 2026-08-12 13:01:05 +00:00
227fb35f99 Centralize the maintainer validation workflow 2026-08-12 00:04:48 +00:00
e291b8bfe9 Consolidate provider transport test scaffolding 2026-08-11 23:58:31 +00:00
2b6a7f83c4 Bound and strictly decode provider responses 2026-08-11 23:46:16 +00:00
3a43550f70 Validate and compose provider endpoints 2026-08-11 23:38:45 +00:00
c281f721bc Preserve transport error identities 2026-08-11 23:28:00 +00:00
350b0e76d9 Preserve repair settings and cumulative usage 2026-08-11 23:21:34 +00:00
e43350fd0d Make validation cancellation authoritative 2026-08-11 23:14:01 +00:00
20d3e3b5ee Escape schema resources and reuse compiled plans 2026-08-11 23:05:11 +00:00
a93b799236 Preserve exact JSON validation semantics 2026-08-11 22:55:17 +00:00
e83a3ce179 Honor cancellation while rendering prompts 2026-08-11 22:48:20 +00:00
a04a3bbc5f Correct artifact file and empty input handling 2026-08-11 22:41:37 +00:00
731b66cff5 Avoid decoding unrelated profiles 2026-08-11 22:32:33 +00:00
70e0ea0cf0 Correct profile source validation and identity 2026-08-11 22:28:48 +00:00
d45c474c1e Unify prompt repository source handling 2026-08-11 22:18:50 +00:00
25f1ba0b30 Correct prompt definition selection and decoding 2026-08-11 22:11:39 +00:00
a718762da1 Contain prompt content paths within source roots 2026-08-11 22:03:32 +00:00
58ac3ce298 Correct engine construction edge cases 2026-08-11 21:54:47 +00:00
57f2ce1ce4 Harden public ownership and diagnostic contracts 2026-08-11 21:45:54 +00:00
c8b6d5c490 Consolidate public JSON serialization 2026-08-11 21:40:12 +00:00
abeb50b525 Bound JSON-compatible value copying 2026-08-11 21:32:24 +00:00
1cb07c7d91 Centralize output contract validation 2026-08-11 21:21:33 +00:00
8cfc71c351 Centralize execution setting and session validation 2026-08-11 21:12:56 +00:00
132 changed files with 14373 additions and 9511 deletions

View File

@@ -33,7 +33,19 @@ boundary and constraints that framework work must preserve.
## Release Guidance ## Release Guidance
Consumers upgrading from `v0.4.0` to `v0.5.0` should read the Consumers upgrading from `v0.8.0` to `v0.9.0` should read the
[v0.9.0 changelog and migration guide](docs/releases/v0.9.0.md).
Consumers upgrading from `v0.7.0` to `v0.8.0` can consult the
[v0.8.0 changelog and migration guide](docs/releases/v0.8.0.md).
Earlier adopters can consult the
[v0.7.0 changelog and migration guide](docs/releases/v0.7.0.md).
Consumers upgrading from `v0.5.0` to `v0.6.0` can consult the
[v0.6.0 changelog and migration guide](docs/releases/v0.6.0.md).
Consumers upgrading from `v0.4.0` to `v0.5.0` can consult the
[v0.5.0 changelog and migration guide](docs/releases/v0.5.0.md). [v0.5.0 changelog and migration guide](docs/releases/v0.5.0.md).
Consumers upgrading from `v0.3.0` to `v0.4.0` should read the Consumers upgrading from `v0.3.0` to `v0.4.0` should read the

View File

@@ -0,0 +1,145 @@
package promptkit_test
import (
"context"
"errors"
"reflect"
"sync/atomic"
"testing"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestAppendedMessageRoleConstantsAreStrings(t *testing.T) {
var (
developer string = promptkit.RoleDeveloper
system string = promptkit.RoleSystem
user string = promptkit.RoleUser
assistant string = promptkit.RoleAssistant
)
if developer != "developer" || system != "system" || user != "user" || assistant != "assistant" {
t.Fatalf("unexpected role constants: %q %q %q %q", developer, system, user, assistant)
}
}
func TestAppendedMessagesComposeFrozenEffectivePrompt(t *testing.T) {
cacheControl := &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}
appended := []promptkit.RenderedMessage{
{Role: " \tAsSiStAnT\n", Content: "previous response"},
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: cacheControl},
}
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "output"}}
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "configured prompt"), "."),
promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
request := promptkit.RunRequest{PromptID: "prompt", AppendedMessages: appended}
plain, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("prepare plain request: %v", err)
}
prepared, err := engine.Prepare(context.Background(), request)
if err != nil {
t.Fatalf("prepare appended request: %v", err)
}
expected := []promptkit.RenderedMessage{
{Role: promptkit.RoleUser, Content: "configured prompt"},
{Role: promptkit.RoleAssistant, Content: "previous response"},
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}},
}
if !reflect.DeepEqual(prepared.Messages, expected) || prepared.PromptHash != plain.PromptHash || prepared.RenderedPromptHash == plain.RenderedPromptHash {
t.Fatalf("prepared effective prompt = %#v, plain = %#v", prepared, plain)
}
equivalent, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{
{Role: promptkit.RoleAssistant, Content: "previous response"},
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}},
}})
if err != nil || equivalent.RenderedPromptHash != prepared.RenderedPromptHash {
t.Fatalf("normalized-equivalent request = (%#v, %v), want matching rendered hash %q", equivalent, err, prepared.RenderedPromptHash)
}
for _, variant := range []promptkit.RunRequest{
{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleAssistant, Content: "changed"}, expected[2]}},
{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{expected[2], expected[1]}},
{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleAssistant, Content: "previous response"}, {Role: promptkit.RoleUser, Content: "corrective request"}}},
} {
variantPrepared, prepareErr := engine.Prepare(context.Background(), variant)
if prepareErr != nil || variantPrepared.RenderedPromptHash == prepared.RenderedPromptHash || variantPrepared.PromptHash != prepared.PromptHash {
t.Fatalf("variant preparation = (%#v, %v)", variantPrepared, prepareErr)
}
}
result, err := engine.Run(context.Background(), request)
if err != nil || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(client.requests[0].Prompt.Messages, expected) {
t.Fatalf("run result = (%#v, %v), request = %#v", result, err, client.requests)
}
execution, err := engine.PrepareExecution(context.Background(), request)
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
appended[0].Content = "changed caller content"
cacheControl.TTL = ""
firstDetails := execution.Details()
firstDetails.Messages[1].Content = "changed details content"
firstDetails.Messages[2].CacheControl.Type = "changed details cache"
secondDetails := execution.Details()
if !reflect.DeepEqual(secondDetails.Messages, expected) || secondDetails.RenderedPromptHash != prepared.RenderedPromptHash {
t.Fatalf("prepared execution details = %#v, want frozen %#v", secondDetails, expected)
}
result, err = engine.RunPrepared(context.Background(), execution)
if err != nil || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(client.requests[1].Prompt.Messages, expected) {
t.Fatalf("prepared result = (%#v, %v), requests = %#v", result, err, client.requests)
}
}
func TestInvalidAppendedMessagesFailBeforeSourceOrModelWork(t *testing.T) {
promptSource := &inspectionCountingFS{}
var modelCalls atomic.Int64
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(promptSource, "."),
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
request := promptkit.RunRequest{
PromptID: "unreached",
AppendedMessages: []promptkit.RenderedMessage{{
Role: "unsupported-role",
Content: "sensitive appended content",
}},
}
operations := []struct {
name string
run func() error
}{
{name: "Prepare", run: func() error { _, err := engine.Prepare(context.Background(), request); return err }},
{name: "PrepareExecution", run: func() error { _, err := engine.PrepareExecution(context.Background(), request); return err }},
{name: "Run", run: func() error { _, err := engine.Run(context.Background(), request); return err }},
}
for _, operation := range operations {
t.Run(operation.name, func(t *testing.T) {
err := operation.run()
if !errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("error = %v, want ErrInvalidRequest", err)
}
})
}
if promptSource.opens.Load() != 0 {
t.Fatalf("invalid appended message opened prompt sources %d times", promptSource.opens.Load())
}
if modelCalls.Load() != 0 {
t.Fatalf("invalid appended message invoked the model %d times", modelCalls.Load())
}
}

View File

@@ -9,6 +9,10 @@ import (
// backend. // backend.
const BackendOpenRouter = backend.OpenRouterID const BackendOpenRouter = backend.OpenRouterID
// BackendRakestrawHome is the reserved ID of Promptkit's built-in
// Rakestrawhome backend.
const BackendRakestrawHome = backend.RakestrawHomeID
// BackendLocal is the case-sensitive conventional ID used by [LocalBackend]. // BackendLocal is the case-sensitive conventional ID used by [LocalBackend].
// It is not a built-in or reserved backend and must be registered with // It is not a built-in or reserved backend and must be registered with
// [WithBackend]. // [WithBackend].
@@ -20,21 +24,25 @@ const BackendLocal = "local"
// to this configuration value do not break source compatibility. // to this configuration value do not break source compatibility.
type Backend struct { type Backend struct {
// ID is the stable, case-sensitive registry key. NewEngine trims it and // ID is the stable, case-sensitive registry key. NewEngine trims it and
// requires a non-blank value. BackendOpenRouter is reserved. // requires a non-blank value. Built-in backend IDs are reserved.
ID string ID string
// Endpoint is the OpenAI-compatible base endpoint. NewEngine trims it and // Endpoint is the OpenAI-compatible base endpoint. NewEngine trims it and
// requires an absolute HTTP or HTTPS URL with a host and without user // requires an absolute HTTP or HTTPS URL with a host and without user
// information, a query string, or a fragment. Paths are allowed. // information, a query string, or a fragment. Paths are allowed.
Endpoint string Endpoint string
// APIKeyEnv optionally names the environment variable containing the API // APIKeyEnv optionally names an environment lookup source for an API key.
// key. NewEngine trims it and requires the portable form // NewEngine trims it and requires the portable form [A-Za-z_][A-Za-z0-9_]*.
// [A-Za-z_][A-Za-z0-9_]*. Store only the name, never a credential value. // A direct RunRequest.APIKey takes precedence. When no usable credential is
// available, the built-in client omits Authorization; injected clients own
// their own credential-resolution behavior. Store only the name, never a
// credential value.
APIKeyEnv string APIKeyEnv string
// ExtraParams contains backend-wide request defaults. Values must be // ExtraParams contains backend-wide request defaults. Values must be
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys // JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
// must not be model, session_id, messages, temperature, max_tokens, top_p, // must not be model, session_id, messages, temperature, max_tokens, top_p,
// service_tier, reasoning_effort, or response_format. An empty map supplies // service_tier, reasoning_effort, or response_format. An empty map supplies
// no defaults. NewEngine deeply copies the map. // no defaults. NewEngine deeply copies the map and rejects excessively deep
// or large values for safety.
ExtraParams map[string]any ExtraParams map[string]any
// ConcurrencyLimit is the maximum number of simultaneous model-generation // ConcurrencyLimit is the maximum number of simultaneous model-generation
// calls allowed for this backend within one Engine. Zero leaves the backend // calls allowed for this backend within one Engine. Zero leaves the backend
@@ -73,11 +81,11 @@ func LocalBackend(endpoint string, concurrencyLimit int) Backend {
// //
// Registrations accumulate in option order. Every normalized ID must be unique // Registrations accumulate in option order. Every normalized ID must be unique
// across consumer registrations and built-ins; a duplicate or invalid // across consumer registrations and built-ins; a duplicate or invalid
// definition makes NewEngine fail with ErrInvalidConfig. In particular, // definition makes NewEngine fail with ErrInvalidConfig. Built-in IDs,
// BackendOpenRouter cannot be replaced. The immutable registration is scoped // including [BackendOpenRouter] and [BackendRakestrawHome], cannot be
// to the resulting Engine and cannot be enumerated, replaced, removed, or // replaced. The immutable registration is scoped to the resulting Engine and
// mutated after construction. WithBackend does not install package-global // cannot be enumerated, replaced, removed, or mutated after construction.
// state. // WithBackend does not install package-global state.
func WithBackend(backend Backend) Option { func WithBackend(backend Backend) Option {
queueCapacity := 0 queueCapacity := 0
queueCapacitySet := backend.QueueCapacity != nil queueCapacitySet := backend.QueueCapacity != nil

View File

@@ -1,7 +1,9 @@
package promptkit package promptkit
import ( import (
"fmt"
"reflect" "reflect"
"unicode/utf8"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue" "gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
@@ -12,19 +14,52 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
if err != nil { if err != nil {
return domain.RunRequest{}, err return domain.RunRequest{}, err
} }
appendedMessages, err := toDomainAppendedMessages(req.AppendedMessages)
if err != nil {
return domain.RunRequest{}, err
}
return domain.RunRequest{ return domain.RunRequest{
PromptID: req.PromptID, PromptID: req.PromptID,
PromptVersion: req.PromptVersion, PromptVersion: req.PromptVersion,
ProfileID: req.ProfileID, ProfileID: req.ProfileID,
SessionID: req.SessionID, SessionID: req.SessionID,
APIKey: req.APIKey, APIKey: req.APIKey,
Inputs: toDomainArtifactRefMap(req.Inputs), Inputs: toDomainArtifactRefMap(req.Inputs),
Vars: copyStringMap(req.Vars), Vars: copyStringMap(req.Vars),
Execution: execution, Execution: execution,
Validation: toDomainOutputContractPtr(req.Validation), Validation: toDomainOutputContractPtr(req.Validation),
AppendedMessages: appendedMessages,
}, nil }, nil
} }
func toDomainAppendedMessages(messages []RenderedMessage) ([]domain.RenderedMessage, error) {
if messages == nil {
return nil, nil
}
converted := make([]domain.RenderedMessage, len(messages))
for index, message := range messages {
if !utf8.ValidString(message.Content) {
return nil, fmt.Errorf("appended message %d content must be valid UTF-8", index)
}
role, err := domain.NormalizeMessageRole(message.Role)
if err != nil {
return nil, fmt.Errorf("appended message %d role: %w", index, err)
}
cacheControl, err := domain.NormalizeCacheControl(toDomainCacheControl(message.CacheControl))
if err != nil {
return nil, fmt.Errorf("appended message %d cache_control: %w", index, err)
}
converted[index] = domain.RenderedMessage{
Role: role,
Content: message.Content,
CacheControl: cacheControl,
}
}
return converted, nil
}
func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun { func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
if prepared == nil { if prepared == nil {
return nil return nil
@@ -285,6 +320,16 @@ func fromDomainRenderedMessages(messages []domain.RenderedMessage) []RenderedMes
return out return out
} }
func toDomainCacheControl(cacheControl *CacheControl) *domain.CacheControl {
if cacheControl == nil {
return nil
}
return &domain.CacheControl{
Type: domain.CacheControlType(cacheControl.Type),
TTL: cacheControl.TTL,
}
}
func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl { func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl {
if cacheControl == nil { if cacheControl == nil {
return nil return nil

View File

@@ -0,0 +1,108 @@
package promptkit
import (
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestToDomainAppendedMessages(t *testing.T) {
tests := []struct {
name string
messages []RenderedMessage
want []domain.RenderedMessage
wantErr string
privateRole string
privateText string
}{
{
name: "normalizes supported roles",
messages: []RenderedMessage{
{Role: " DeVeLoPeR ", Content: "developer"},
{Role: "SyStEm", Content: "system"},
{Role: "\tUsEr\n", Content: "user"},
{Role: "assistant", Content: "assistant"},
},
want: []domain.RenderedMessage{
{Role: domain.RoleDeveloper, Content: "developer"},
{Role: domain.RoleSystem, Content: "system"},
{Role: domain.RoleUser, Content: "user"},
{Role: domain.RoleAssistant, Content: "assistant"},
},
},
{
name: "invalid role UTF-8",
messages: []RenderedMessage{{Role: string([]byte{0xff}), Content: "private-content"}},
wantErr: "appended message 0 role",
},
{
name: "invalid content UTF-8",
messages: []RenderedMessage{{Role: "private-role", Content: string([]byte{0xff})}},
wantErr: "appended message 0 content",
},
{
name: "unsupported role",
messages: []RenderedMessage{{Role: "private-role", Content: "private-content"}},
wantErr: "appended message 0 role",
privateRole: "private-role",
privateText: "private-content",
},
{
name: "invalid cache control",
messages: []RenderedMessage{{Role: RoleUser, Content: "private-content", CacheControl: &CacheControl{Type: "private-cache"}}},
wantErr: "appended message 0 cache_control",
privateText: "private-content",
},
{
name: "empty and whitespace content",
messages: []RenderedMessage{
{Role: RoleUser, Content: ""},
{Role: RoleAssistant, Content: " \t\n "},
},
want: []domain.RenderedMessage{
{Role: domain.RoleUser, Content: ""},
{Role: domain.RoleAssistant, Content: " \t\n "},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got, err := toDomainAppendedMessages(test.messages)
if test.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), test.wantErr) {
t.Fatalf("error = %v, want %q", err, test.wantErr)
}
for _, privateValue := range []string{test.privateRole, test.privateText} {
if privateValue != "" && strings.Contains(err.Error(), privateValue) {
t.Fatalf("error exposed appended message data %q: %v", privateValue, err)
}
}
return
}
if err != nil {
t.Fatalf("toDomainAppendedMessages() error = %v", err)
}
if !reflect.DeepEqual(got, test.want) {
t.Fatalf("toDomainAppendedMessages() = %#v, want %#v", got, test.want)
}
})
}
}
func TestToDomainAppendedMessagesCopiesCacheControl(t *testing.T) {
cacheControl := &CacheControl{Type: CacheControlEphemeral, TTL: "1h"}
messages := []RenderedMessage{{Role: RoleUser, Content: "content", CacheControl: cacheControl}}
request, err := toDomainRunRequest(RunRequest{AppendedMessages: messages})
if err != nil {
t.Fatalf("toDomainRunRequest() error = %v", err)
}
messages[0].Content = "changed"
cacheControl.TTL = ""
if request.AppendedMessages[0].Content != "content" || request.AppendedMessages[0].CacheControl.TTL != "1h" {
t.Fatalf("domain request did not retain an independent appended-message copy: %#v", request.AppendedMessages)
}
}

16
doc.go
View File

@@ -23,8 +23,9 @@
// InspectProfile return copied inspection values. Returned values and values // InspectProfile return copied inspection values. Returned values and values
// passed to extension interfaces are likewise isolated from engine state. // passed to extension interfaces are likewise isolated from engine state.
// Callers own those copies and may mutate them after the call that supplied or // Callers own those copies and may mutate them after the call that supplied or
// returned them. Returned structured errors are likewise caller-owned and may // returned them. [CapacityError] values are caller-owned and may be mutated
// be mutated without affecting engine state or another error. // without affecting engine state or another error. Immutable [GenerationError]
// values are also caller-owned and do not retain shared engine state.
// //
// # Security and sensitive data // # Security and sensitive data
// //
@@ -53,9 +54,14 @@
// Construction, inspection, handle, and error values, including [Config], // Construction, inspection, handle, and error values, including [Config],
// [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile], // [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile],
// [OpenAICompatibleProfileConfig], [ProfileInspection], // [OpenAICompatibleProfileConfig], [ProfileInspection],
// [PromptInputDefinition], [PromptInspection], [PreparedExecution], and // [PromptInputDefinition], [PromptInspection], [PreparedExecution],
// [CapacityError], do not have stable JSON representations. Direct API keys // [CapacityError], and [GenerationError], do not have stable JSON
// are nevertheless excluded from JSON for every public value. // representations. Direct API keys are nevertheless excluded from JSON for
// every public value.
// Provider-derived [GenerationError] accessor values are untrusted and can
// contain sensitive request or schema fragments. Applications must apply their
// own disclosure policy before logging, displaying, or returning them.
// //
// JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero. // JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
// PreparedRun and RunResult durations are encoded as integer milliseconds in // PreparedRun and RunResult durations are encoded as integer milliseconds in

View File

@@ -173,6 +173,49 @@ semantics. The
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md) [OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
owns the built-in client's outbound HTTP behavior. owns the built-in client's outbound HTTP behavior.
### Repair A Structured Result
Set a small additional-call budget when a structurally invalid result can be
corrected automatically:
```go
request.Validation = &promptkit.OutputContract{
Format: promptkit.FormatJSON,
ValidationMode: promptkit.ValidationJSONSchema,
SchemaPath: "events.schema.json",
RepairAttempts: 1,
}
```
Each repair attempt is another model call, so it can increase latency and
usage; `RunResult.Usage` is cumulative and `Validation.RepairAttempts` reports
calls actually started. Exhaustion still returns the final failed validation
result. `basic` validation can also repair an empty candidate, but structural
validity is not evidence of factual or domain correctness. See the
[output-contract format reference](../formats.md#output-contract) and
[`OutputContract` GoDoc](../../types.go) for the exact budget and eligibility
rules.
### Append Already-Rendered Messages
An application can include an earlier assistant response and its own corrective
instruction in a fresh request without changing the configured prompt:
```go
request.AppendedMessages = []promptkit.RenderedMessage{
{Role: promptkit.RoleAssistant, Content: previousResponse},
{Role: promptkit.RoleUser, Content: correction},
}
result, err := engine.Run(ctx, request)
```
These messages are already rendered: Promptkit does not template or resolve
files in them, and they can contain sensitive model output or application
feedback. Promptkit remains stateless; every `Run` call re-resolves its current
sources and the application owns any semantic retry budget. When a
pre-execution equality check is required, use `PrepareExecution` and compare
its opaque rendered-prompt hash before invoking `RunPrepared`.
## Inputs, Profiles, And Overrides ## Inputs, Profiles, And Overrides
Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request
@@ -189,6 +232,53 @@ For programmatic profiles,
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary [`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
OpenAI-compatible settings into a value accepted by `WithProfiles`. OpenAI-compatible settings into a value accepted by `WithProfiles`.
### Alias A Built-In Profile
Give an application-owned profile ID a built-in base when prompts should select
the application ID while inheriting the built-in target. The child can override
only the setting it owns:
```go
promptkit.WithProfiles(promptkit.Profile{
ID: "weather-light",
BaseProfileID: "deepseek-4-flash",
ReasoningEffort: "high",
})
```
Select `weather-light` in a prompt or `RunRequest.ProfileID`; it remains the
reported selected profile. See the [profile inheritance format
reference](../formats.md#profile-inheritance) and the
[`Profile` GoDoc](../../types.go) for exact lookup, merging, and validation
behavior.
### Use The Rakestrawhome Built-In Profile
Set `RAKESTRAWHOME_INFERENCE_API_KEY` in the application environment, then
select `rakestrawhome-gemma-4-31b` as an ordinary profile ID. For example, a
prepared result identifies the selected built-in through
`BackendRakestrawHome`:
```go
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
PromptID: "meeting.summary",
ProfileID: "rakestrawhome-gemma-4-31b",
Inputs: inputs,
})
if err != nil {
return err
}
if prepared.SelectedBackendID != promptkit.BackendRakestrawHome {
return fmt.Errorf("unexpected backend %q", prepared.SelectedBackendID)
}
```
Do not register `rakestrawhome` manually. When adopting this built-in, remove
an existing `WithBackend` registration with that exact ID; retaining it causes
the intentional duplicate-ID configuration error. Direct request credentials
and runtime endpoint overrides remain supported under their ordinary GoDoc and
format contracts.
### Inspect A Profile Before Prompt Work ### Inspect A Profile Before Prompt Work
Use [`Engine.InspectProfile`](../../engine.go) to validate one configured Use [`Engine.InspectProfile`](../../engine.go) to validate one configured
@@ -204,7 +294,7 @@ if err != nil {
target := inspection.EffectiveModelParams target := inspection.EffectiveModelParams
if target.APIKeyEnv != "" { if target.APIKeyEnv != "" {
// Apply application policy for the named environment variable. // This is a configured optional environment lookup source.
} else if inspection.APIKeyRequired { } else if inspection.APIKeyRequired {
// Arrange a direct credential before later execution. // Arrange a direct credential before later execution.
} }
@@ -213,9 +303,11 @@ if target.APIKeyEnv != "" {
Use this configuration-time boundary when only the profile and its target need Use this configuration-time boundary when only the profile and its target need
checking. Use `Prepare` when the application also needs prompt, input, schema, checking. Use `Prepare` when the application also needs prompt, input, schema,
or rendering work; use prepared execution when that work must remain tied to a or rendering work; use prepared execution when that work must remain tied to a
later execution. Inspection reports credential requirements but leaves the later execution. A reported `APIKeyEnv` is a configured optional source, while
timing of credential enforcement to the application. The method's `APIKeyRequired` is the explicit local requirement. The
[GoDoc](../../engine.go) owns its exact result and error contract. [credential format reference](../formats.md#credentials) and the method's
[GoDoc](../../engine.go) own the exact precedence, timing, result, and error
contracts.
### Set A Per-Run Session And Reasoning ### Set A Per-Run Session And Reasoning
@@ -423,6 +515,24 @@ status; those choices remain with the consuming application. The
contract, while the [`Engine.Run` and error GoDoc](../../engine.go) owns broad contract, while the [`Engine.Run` and error GoDoc](../../engine.go) owns broad
error and cancellation identities. error and cancellation identities.
For a non-2xx response from the built-in OpenAI-compatible client, inspect the
status and deliberately selected provider diagnostic when useful:
```go
var generationErr *promptkit.GenerationError
if errors.As(err, &generationErr) {
status := generationErr.StatusCode()
message := generationErr.ProviderMessage()
_, _ = status, message // Apply application retry and presentation policy.
}
```
All provider fields are untrusted and can contain sensitive request or schema
fragments. Do not log, display, or return them without an application-specific
disclosure policy. Promptkit does not assign retry or presentation behavior.
The [`GenerationError` GoDoc](../../generation_error.go) owns the exact typed
error contract.
## Application Boundary ## Application Boundary
Promptkit is an importable library. It does not own a command, inbound HTTP Promptkit is an importable library. It does not own a command, inbound HTTP

View File

@@ -34,6 +34,7 @@ Start with:
| Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. | | Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. |
| Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). | | Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). |
| Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. | | Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. |
| Maintained external catalogs or built-ins | The [internal source document](internal/sources.md), [framework format reference](formats.md), [testing policy](policy/testing.md), both catalog module repositories, and each catalog's release procedure. |
| Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. | | Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. |
| Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. | | Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. |
| Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. | | Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. |
@@ -44,3 +45,190 @@ Start with:
For cross-cutting changes, follow every applicable row. Do not create For cross-cutting changes, follow every applicable row. Do not create
placeholder documents for packages, APIs, or integrations that do not yet placeholder documents for packages, APIs, or integrations that do not yet
exist. exist.
## Maintainer Validation
This section is the canonical local validation workflow for Promptkit. Run
every command from the repository root before accepting a change. The test
suite and maintained examples are deterministic, offline, and require no real
provider credentials.
### Tests, Analysis, Build, And Examples
Run the ordinary and race-enabled suites, static analysis, the build, and both
maintained consumer examples:
```sh
go test ./...
go test -race ./...
go vet ./...
go build ./...
go run ./examples/go-library/prepare
go run ./examples/go-library/run
```
Both examples must exit successfully. Review their JSON output: preparation
must report the selected offline prompt, profile, model, and message count;
execution must report the deterministic generated output, successful
validation, selected offline model, and usage. Neither command may contact a
provider or require credentials.
### Go Formatting
Check every tracked Go file. The final command must succeed and the captured
list must be empty:
```sh
unformatted=$(
git ls-files '*.go' |
while IFS= read -r go_file
do
gofmt -l "$go_file"
done
)
test -z "$unformatted"
```
### Local Markdown Links
Use the Python standard library to verify every repository-relative Markdown
target and local heading fragment. The check is offline and prints nothing on
success:
```sh
python3 - <<'PY'
from pathlib import Path
import re
import subprocess
import sys
from urllib.parse import unquote
root = Path.cwd().resolve()
markdown_files = [
root / name
for name in subprocess.check_output(
["git", "ls-files", "*.md"], text=True
).splitlines()
]
link_pattern = re.compile(r"!?\[[^]]*\]\(([^)]+)\)")
heading_pattern = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$")
scheme_pattern = re.compile(r"^[a-z][a-z0-9+.-]*:", re.IGNORECASE)
def markdown_lines(path):
in_fence = False
fence = ""
for line in path.read_text(encoding="utf-8").splitlines():
stripped = line.lstrip()
marker = stripped[:3]
if marker in {"```", "~~~"}:
if not in_fence:
in_fence = True
fence = marker
elif marker == fence:
in_fence = False
fence = ""
continue
if not in_fence:
yield line
anchor_cache = {}
def anchors(path):
if path in anchor_cache:
return anchor_cache[path]
found = set()
counts = {}
for line in markdown_lines(path):
match = heading_pattern.match(line)
if not match:
continue
heading = re.sub(r"<[^>]+>", "", match.group(1)).replace("`", "")
base = re.sub(r"[^\w\- ]", "", heading.lower()).replace(" ", "-")
count = counts.get(base, 0)
counts[base] = count + 1
found.add(base if count == 0 else f"{base}-{count}")
anchor_cache[path] = found
return found
failures = []
for source in markdown_files:
text = "\n".join(markdown_lines(source))
for match in link_pattern.finditer(text):
target = match.group(1).strip()
if target.startswith("<") and target.endswith(">"):
target = target[1:-1]
if scheme_pattern.match(target) or target.startswith("//"):
continue
path_text, separator, fragment = target.partition("#")
destination = source if not path_text else source.parent / unquote(path_text)
try:
destination = destination.resolve()
destination.relative_to(root)
except ValueError:
failures.append(f"{source.relative_to(root)}: escapes repository: {target}")
continue
if not destination.exists():
failures.append(f"{source.relative_to(root)}: missing target: {target}")
continue
if separator and destination.suffix.lower() == ".md":
fragment = unquote(fragment).lower()
if fragment not in anchors(destination):
failures.append(f"{source.relative_to(root)}: missing anchor: {target}")
if failures:
print("\n".join(failures), file=sys.stderr)
raise SystemExit(1)
PY
```
### Repository Hygiene And Review
Reject an active Go workspace, tracked workspace files, a vendor tree, or a
module replacement:
```sh
case "$(go env GOWORK)" in
''|off) ;;
*) printf '%s\n' 'an active Go workspace is not allowed' >&2; exit 1 ;;
esac
test -z "$(git ls-files go.work go.work.sum)"
test ! -e vendor
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
then
printf '%s\n' 'go.mod contains a replacement' >&2
exit 1
fi
```
Check whitespace in both unstaged and staged changes. List ignored files and
scan tracked content for common credential forms:
```sh
git diff --check
git diff --cached --check
test -z "$(git ls-files --others --ignored --exclude-standard)"
credential_pattern='-----BEGIN ([A-Z0-9]+ )?PRIV''ATE KEY-----|AKI''A[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{36,}|sk-[A-Za-z0-9]{32,}'
if git grep -nEI -e "$credential_pattern" -- .
then
printf '%s\n' 'possible credential found' >&2
exit 1
fi
```
Inspect `git status --short --untracked-files=all` and the complete diff before
accepting a change. The status may contain only the intended source changes
during development. Reject credentials, private keys, environment files,
generated binaries, test or coverage output, downloaded assets, template
residue, and any other artifact that does not belong in source control. The
credential scan catches common forms but does not replace inspection of the
actual change.
After committing the accepted change, require a clean candidate:
```sh
test -z "$(git status --porcelain)"
```

View File

@@ -9,9 +9,10 @@ explains how to select these sources and invoke the engine. The
owns the resulting outbound wire behavior. owns the resulting outbound wire behavior.
Prompt and profile sources recursively discover files ending in `.yaml` or Prompt and profile sources recursively discover files ending in `.yaml` or
`.yml`. YAML decoding is strict: unknown fields are errors for the selected `.yml`. Each prompt-definition and profile file contains exactly one YAML
definition. Definitions are selected by their YAML `id`, not their file name document; comments and trailing whitespace are allowed. YAML decoding is
or directory. strict: unknown fields are errors for the selected definition. Definitions are
selected by their YAML `id`, not their file name or directory.
## Prompt Definitions ## Prompt Definitions
@@ -81,14 +82,27 @@ allowed.
### Messages And Templates ### Messages And Templates
Each message has a non-empty `role` and exactly one of: Each message has a `role` that Promptkit trims and lowercases. It must then be
exactly one of `developer`, `system`, `user`, or `assistant`; blank, custom,
`tool`, and `function` roles are invalid. This intentionally tightens the
previous nonblank-string rule. Consumers migrating to the next minor release
must update any nonstandard prompt-definition roles before upgrading.
Each message also has exactly one of:
- `content`, containing an inline Go template; or - `content`, containing an inline Go template; or
- `content_file`, naming a file whose contents are the Go template. - `content_file`, naming a file whose contents are the Go template.
For directory and `fs.FS` prompt sources, `content_file` resolves relative to `content_file` must be a relative path. It resolves from the directory that
the prompt file and remains within the source root. `WithPromptFile` also contains the prompt file and must remain within the configured prompt source
resolves it relative to that file. root; parent components are allowed only when the resolved target remains
inside that root. Absolute paths and paths that escape the root are rejected.
Operating-system directory and single-file sources also reject symlink targets
outside the root, while injected `fs.FS` sources apply containment in that
filesystem's relative path namespace. For `WithPromptFile`, the source root is
the directory containing the selected prompt file. Promptkit uses the parsed
path text exactly after checking separately that it is not blank, so leading
and trailing whitespace can name real filesystem entries.
Request variables are the template data, so a variable named `audience` is Request variables are the template data, so a variable named `audience` is
referenced as `{{.audience}}`. The `{{input "note"}}` helper renders the body referenced as `{{.audience}}`. The `{{input "note"}}` helper renders the body
@@ -118,7 +132,7 @@ outbound integration determines its wire representation.
| `format` | yes | `text`, `markdown`, or `json`. | | `format` | yes | `text`, `markdown`, or `json`. |
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema`. | | `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema`. |
| `schema_path` | for `json_schema` | Path to a schema in the configured schema source. | | `schema_path` | for `json_schema` | Path to a schema in the configured schema source. |
| `repair_attempts` | no | Integer zero or greater; omitted means zero. | | `repair_attempts` | no | Integer from zero through three; omitted means zero. A positive value requires `basic`, `json`, or `json_schema` validation. |
The validation modes behave as follows: The validation modes behave as follows:
@@ -128,14 +142,31 @@ The validation modes behave as follows:
- `json_schema` requires valid JSON that satisfies the selected schema. - `json_schema` requires valid JSON that satisfies the selected schema.
`format` controls output artifact metadata. JSON Schema mode also supplies the `format` controls output artifact metadata. JSON Schema mode also supplies the
schema to compatible model clients as structured-output metadata. The public schema to compatible model clients as structured-output metadata. Plain `json`
engine does not install an output repairer, so its validation is single-pass validation accepts every valid JSON value and does not request a provider-native
even when a positive `repair_attempts` value is present. JSON-object constraint.
`repair_attempts` counts additional generation calls after a failed validation.
Zero is single-pass. With a positive eligible budget, Promptkit stops at the
first valid candidate. If the budget is exhausted, it returns the final
candidate and its complete failed validation result; generation and operational
validation failures remain errors. `none` never permits repair.
A request-level `OutputContract` replaces the complete prompt output contract. A request-level `OutputContract` replaces the complete prompt output contract.
It does not merge individual fields. If its format is empty, Promptkit uses It does not merge individual fields. If its format is empty, Promptkit uses
`text`. `text`.
## Built-In Backends
Every engine provides these reserved OpenAI-compatible backend IDs. Consumers
must not register either ID with `WithBackend`; exact registration and
reservation behavior belongs to the [`Backend` GoDoc](../backends.go).
| ID | Base endpoint | API-key environment variable | Active generation limit | Default queue capacity |
| --- | --- | --- | ---: | ---: |
| `openrouter` | `https://openrouter.ai/api/v1` | `OPENROUTER_API_KEY` | 16 | 1024 |
| `rakestrawhome` | `https://inference.ai.rakestrawhome.com/v1` | `RAKESTRAWHOME_INFERENCE_API_KEY` | 4 | 1024 |
## Profile Definitions ## Profile Definitions
A profile supplies model execution settings: A profile supplies model execution settings:
@@ -154,11 +185,21 @@ extra_params:
provider_option: enabled provider_option: enabled
``` ```
A derived profile can use a named base and override only the settings it owns:
```yaml
id: local-summary-fast
base_profile: local-summary
timeout_seconds: 30
reasoning_effort: low
```
| Field | Required | Meaning | | Field | Required | Meaning |
| --- | --- | --- | | --- | --- | --- |
| `id` | yes | Non-empty profile identifier. IDs must be unique within one source. | | `id` | yes | Profile identifier, trimmed before selection and publication. It must be non-empty after trimming and unique within one source after normalization. |
| `base_profile` | no | One optional parent profile ID. A derived profile may inherit target fields from it. |
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared or inspected. | | `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared or inspected. |
| `endpoint` | unless `backend` is present | Non-empty OpenAI-compatible base URL, including an API version path when required. When both connection fields are present, this overrides the backend endpoint without changing backend identity. | | `endpoint` | unless `backend` is present | OpenAI-compatible base URL, including an API version path when required. A nonempty value is trimmed and must be absolute HTTP or HTTPS with a host and without user information, a query, or a fragment. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
| `model` | yes | Non-empty provider model name. | | `model` | yes | Non-empty provider model name. |
| `temperature` | no | Number from 0 through 2. | | `temperature` | no | Number from 0 through 2. |
| `max_tokens` | no | Integer zero or greater. | | `max_tokens` | no | Integer zero or greater. |
@@ -166,16 +207,21 @@ extra_params:
| `timeout_seconds` | no | Per-generation deadline in whole seconds; integer zero or greater. | | `timeout_seconds` | no | Per-generation deadline in whole seconds; integer zero or greater. |
| `service_tier` | no | Provider-specific request tier. | | `service_tier` | no | Provider-specific request tier. |
| `reasoning_effort` | no | Provider-specific reasoning setting. | | `reasoning_effort` | no | Provider-specific reasoning setting. |
| `api_key_env` | no | Name of an environment variable containing the API key. | | `api_key_env` | no | Optional environment-variable lookup source for an API key. |
| `extra_params` | no | JSON-compatible provider-specific outbound fields. | | `extra_params` | no | JSON-compatible provider-specific outbound fields. |
Raw `api_key` is prohibited in profile YAML. Store only an environment Raw `api_key` is prohibited in profile YAML. Store only an environment
variable name in `api_key_env`. variable name in `api_key_env`.
A standalone profile must provide a model and at least one of `backend` or
`endpoint`. A derived profile may omit those target fields because its selected
base chain can provide them. Local parsing still validates a derived profile's
own ID, supplied endpoint, execution-setting bounds, and `extra_params`.
Promptkit does not infer a backend from a model or endpoint. Endpoint-only Promptkit does not infer a backend from a model or endpoint. Endpoint-only
profiles remain supported and have no effective backend ID. profiles remain supported and have no effective backend ID.
The engine always provides the built-in `openrouter` ID. Consumers can add The engine always provides the built-in `openrouter` and `rakestrawhome` IDs.
engine-scoped IDs with Consumers can add engine-scoped IDs with
[`WithBackend`](../backends.go); exact registration validation belongs to its [`WithBackend`](../backends.go); exact registration validation belongs to its
GoDoc. GoDoc.
@@ -183,6 +229,7 @@ GoDoc.
objects with string keys. Keys must be non-empty. With the built-in client, objects with string keys. Keys must be non-empty. With the built-in client,
they also cannot collide with the standard fields listed in the they also cannot collide with the standard fields listed in the
[outbound request contract](integrations/openai-compatible-chat.md#request-body). [outbound request contract](integrations/openai-compatible-chat.md#request-body).
Excessively deep or large JSON-shaped values are rejected for safety.
### Defaults And Overrides ### Defaults And Overrides
@@ -236,7 +283,7 @@ Profile sources resolve matching IDs in this order:
2. the ordinary configured source selected by a profile file, `fs.FS`, or 2. the ordinary configured source selected by a profile file, `fs.FS`, or
configured profile directory; configured profile directory;
3. application fallback profiles supplied with `WithFallbackProfileFS`; and 3. application fallback profiles supplied with `WithFallbackProfileFS`; and
4. embedded built-in profiles. 4. maintained external catalog profiles.
A profile source supplies a complete definition; definitions and their fields A profile source supplies a complete definition; definitions and their fields
are not merged across sources. A higher-precedence source falls back only when are not merged across sources. A higher-precedence source falls back only when
@@ -246,41 +293,66 @@ profiles. They use `APIKeyRequired` for request-scoped credentials instead of
`api_key_env`. Preparation and exact profile inspection use this same source `api_key_env`. Preparation and exact profile inspection use this same source
precedence. precedence.
When a selected definition names `base_profile`, every profile ID in that
chain is looked up through this same precedence order. A higher-precedence
definition therefore shadows a lower-precedence definition of the same base
ID, including a built-in. References are not source-qualified.
### Profile Inheritance
Promptkit resolves one linear base chain of at most 32 profiles, including the
selected profile. It merges settings from the root base to the selected leaf.
The leaf's `id` remains the selected profile identity. Nonblank string fields
(`backend`, `endpoint`, `model`, `service_tier`, `reasoning_effort`, and
`api_key_env`) and nonzero numeric fields replace inherited values. A nonempty
`extra_params` map replaces the complete inherited map rather than merging
keys, and `APIKeyRequired: true` remains true through the chain. Backend and
endpoint are independent: replacing one does not clear the other.
There is no profile-level clearing syntax. Blank strings, zero numbers, false,
and empty maps remain unspecified and inherit from a base. Use existing
presence-aware request overrides where an execution needs an explicit zero or
empty reasoning setting.
An absent directly selected profile reports the ordinary not-found error. Once
the selected profile exists, a missing base, cycle, overlong chain, or
incomplete resolved target is a profile-load failure. Ordinary operations
resolve chains afresh; prepared execution retains the fully resolved target.
## Built-In Profile Catalog ## Built-In Profile Catalog
Every built-in selects the `openrouter` backend. The engine's built-in backend Every built-in profile selects one maintained built-in backend and inherits
registry supplies `https://openrouter.ai/api/v1` and the environment-variable that backend's connection and credential metadata. Profile files do not repeat
name `OPENROUTER_API_KEY`, so individual profiles contain only model and those values. A configured, application fallback, or in-memory profile with
generation settings. Built-in profile files do not repeat those connection the same profile ID takes precedence.
values. A configured, application fallback, or in-memory profile with the same
profile ID takes precedence.
| Provider | ID | Model | | Provider | ID | Backend | Model |
| --- | --- | --- | | --- | --- | --- | --- |
| aion-labs | `aion-2` | `aion-labs/aion-2.0` | | aion-labs | `aion-2` | `openrouter` | `aion-labs/aion-2.0` |
| anthropic | `claude-fable-latest` | `~anthropic/claude-fable-latest` | | anthropic | `claude-fable-latest` | `openrouter` | `~anthropic/claude-fable-latest` |
| anthropic | `claude-haiku-latest` | `~anthropic/claude-haiku-latest` | | anthropic | `claude-haiku-latest` | `openrouter` | `~anthropic/claude-haiku-latest` |
| anthropic | `claude-opus-latest` | `~anthropic/claude-opus-latest` | | anthropic | `claude-opus-latest` | `openrouter` | `~anthropic/claude-opus-latest` |
| anthropic | `claude-sonnet-latest` | `~anthropic/claude-sonnet-latest` | | anthropic | `claude-sonnet-latest` | `openrouter` | `~anthropic/claude-sonnet-latest` |
| deepseek | `deepseek-3-2` | `deepseek/deepseek-v3.2` | | deepseek | `deepseek-3-2` | `openrouter` | `deepseek/deepseek-v3.2` |
| deepseek | `deepseek-4-flash` | `deepseek/deepseek-v4-flash` | | deepseek | `deepseek-4-flash` | `openrouter` | `deepseek/deepseek-v4-flash` |
| deepseek | `deepseek-4-pro` | `deepseek/deepseek-v4-pro` | | deepseek | `deepseek-4-pro` | `openrouter` | `deepseek/deepseek-v4-pro` |
| google | `gemini-2-flash` | `google/gemini-2.5-flash` | | google | `gemini-2-flash` | `openrouter` | `google/gemini-2.5-flash` |
| google | `gemini-2-flash-lite` | `google/gemini-2.5-flash-lite` | | google | `gemini-2-flash-lite` | `openrouter` | `google/gemini-2.5-flash-lite` |
| google | `gemini-2-pro` | `google/gemini-2.5-pro` | | google | `gemini-2-pro` | `openrouter` | `google/gemini-2.5-pro` |
| google | `gemini-3-flash-lite` | `google/gemini-3.1-flash-lite` | | google | `gemini-3-flash-lite` | `openrouter` | `google/gemini-3.1-flash-lite` |
| google | `gemini-flash-latest` | `~google/gemini-flash-latest` | | google | `gemini-flash-latest` | `openrouter` | `~google/gemini-flash-latest` |
| google | `gemini-pro-latest` | `~google/gemini-pro-latest` | | google | `gemini-pro-latest` | `openrouter` | `~google/gemini-pro-latest` |
| google | `gemma-4-31b` | `google/gemma-4-31b-it:exacto` | | google | `gemma-4-31b` | `openrouter` | `google/gemma-4-31b-it:exacto` |
| minimax | `minimax-m2` | `minimax/minimax-m2.5` | | google | `rakestrawhome-gemma-4-31b` | `rakestrawhome` | `google/gemma-4-31b-it` |
| minimax | `minimax-m3` | `minimax/minimax-m3` | | minimax | `minimax-m2` | `openrouter` | `minimax/minimax-m2.5` |
| mistral | `mistral-large-2512` | `mistralai/mistral-large-2512` | | minimax | `minimax-m3` | `openrouter` | `minimax/minimax-m3` |
| mistral | `mistral-medium-3-5` | `mistralai/mistral-medium-3-5` | | mistral | `mistral-large-2512` | `openrouter` | `mistralai/mistral-large-2512` |
| mistral | `mistral-small-3` | `mistralai/mistral-small-3.2-24b-instruct` | | mistral | `mistral-medium-3-5` | `openrouter` | `mistralai/mistral-medium-3-5` |
| mistral | `mistral-small-4` | `mistralai/mistral-small-2603` | | mistral | `mistral-small-3` | `openrouter` | `mistralai/mistral-small-3.2-24b-instruct` |
| nvidia | `nemotron-3-ultra` | `nvidia/nemotron-3-ultra-550b-a55b` | | mistral | `mistral-small-4` | `openrouter` | `mistralai/mistral-small-2603` |
| openai | `gpt-5-mini` | `openai/gpt-5.4-mini` | | nvidia | `nemotron-3-ultra` | `openrouter` | `nvidia/nemotron-3-ultra-550b-a55b` |
| openai | `gpt-5-nano` | `openai/gpt-5.4-nano` | | openai | `gpt-5-mini` | `openrouter` | `openai/gpt-5.4-mini` |
| openai | `gpt-5-nano` | `openrouter` | `openai/gpt-5.4-nano` |
## Schemas ## Schemas
@@ -299,16 +371,25 @@ schema produces a failed validation result.
Credential values belong at the request or environment boundary, never in Credential values belong at the request or environment boundary, never in
prompt, profile, schema, or example files: prompt, profile, schema, or example files:
- a file profile names an environment variable with `api_key_env`; - a backend or file profile can name an optional environment lookup source
- an in-memory profile may set `APIKeyRequired`; with `APIKeyEnv` or `api_key_env`;
- a request can provide a direct `APIKey` or override `APIKeyEnv`; and - an in-memory profile may set `APIKeyRequired` as an explicit local
requirement;
- a request can provide a direct `APIKey` or override the optional `APIKeyEnv`
source; and
- a direct request key takes precedence over environment lookup. - a direct request key takes precedence over environment lookup.
After a direct request key, the credential-source precedence is request After a direct request key, the credential-source precedence is request
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory `APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
profile with `APIKeyRequired` clears an inherited backend environment name and profile with `APIKeyRequired` clears an inherited backend environment name and
requires a direct key unless the request explicitly supplies `APIKeyEnv`. requires a direct key unless the request explicitly supplies `APIKeyEnv`.
Promptkit validates required credential availability during preparation. Named environment sources are optional: when the selected source is absent,
empty, or whitespace-only, the built-in client omits the `Authorization`
header and handles the provider response normally. `APIKeyRequired` is the
only explicit local availability requirement. Promptkit validates required
credential availability during preparation and rechecks it when a prepared
execution runs. Injected clients receive resolved source metadata but define
their own credential-resolution behavior.
Direct keys are excluded from JSON results and redacted by public string Direct keys are excluded from JSON results and redacted by public string
formatters. Environment-variable names may appear in prepared metadata, but formatters. Environment-variable names may appear in prepared metadata, but
their values do not. their values do not.

View File

@@ -14,10 +14,16 @@ that produce these outbound settings.
Generation sends an HTTP `POST` with `Content-Type: application/json`. Generation sends an HTTP `POST` with `Content-Type: application/json`.
Before the client is called, the engine resolves framework, backend, profile, Before the client is called, the engine resolves framework, backend, profile,
and request values into one execution target. A non-empty endpoint from that and request values into one execution target. Endpoint configuration is trimmed
target overrides the client's configured base URL. After trailing slashes are and must be an absolute HTTP or HTTPS URL with a host and without user
removed, `/chat/completions` is appended. Generation fails before sending when information, a query, or a fragment. A non-empty endpoint from the target
neither source supplies an endpoint. overrides the client's configured base URL. The final selected endpoint is
validated again before transport.
The completion URL is composed through parsed URL path operations. Nested base
paths are retained, repeated trailing slashes are normalized, and the result
has exactly one appended `/chat/completions` suffix. Generation fails before
sending when neither source supplies a valid endpoint.
The target's backend ID is routing metadata for prepared values, results, and The target's backend ID is routing metadata for prepared values, results, and
injected clients. The built-in client does not derive the URL from that ID and injected clients. The built-in client does not derive the URL from that ID and
@@ -25,11 +31,13 @@ does not serialize it in the provider request.
## Authentication ## Authentication
A non-empty API key supplied directly on the execution target takes A usable API key supplied directly on the execution target takes precedence.
precedence. Otherwise, when an API-key environment-variable name is supplied, Otherwise, when an API-key environment-variable name is supplied, the client
the client reads that variable and requires a non-empty value. The selected reads and trims that variable. A bearer header is sent only when the resolved
key is sent as `Authorization: Bearer <key>`. No authorization header is sent direct or environment credential is non-empty. When neither source is usable,
when neither mechanism is configured. the client omits `Authorization` and handles the provider response normally.
An explicitly required target with no usable source is rejected before
transport.
The target contains the already resolved environment-variable name: an The target contains the already resolved environment-variable name: an
explicit request override takes precedence over profile metadata, which takes explicit request override takes precedence over profile metadata, which takes
@@ -47,6 +55,12 @@ Each ordinary message contains its `role` and string `content`. A
cache-controlled message instead uses a text content block containing `type`, cache-controlled message instead uses a text content block containing `type`,
`text`, and `cache_control`; an empty cache-control TTL is omitted. `text`, and `cache_control`; an empty cache-control TTL is omitted.
Promptkit sends only `developer`, `system`, `user`, and `assistant` roles and
does so without provider-specific translation. Tool and deprecated function
payloads are outside this text-message contract. A backend or model that
rejects an otherwise supported role or context returns its ordinary provider
error, which follows the normal generation-error path.
The effective direct or prompt-rendered session ID is trimmed, limited to 256 The effective direct or prompt-rendered session ID is trimmed, limited to 256
Unicode code points, and sent when nonempty as top-level `session_id`. It is Unicode code points, and sent when nonempty as top-level `session_id`. It is
never also sent as a session header. never also sent as a session header.
@@ -59,7 +73,8 @@ The client conditionally includes:
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly - non-empty `service_tier` and effective `reasoning_effort`; an explicitly
disabled reasoning setting is empty and therefore omitted; and disabled reasoning setting is empty and therefore omitted; and
- `response_format` for JSON Schema structured output, including its name, - `response_format` for JSON Schema structured output, including its name,
strict flag, and schema document. strict flag, and schema document. Plain JSON validation does not add an
object-only response constraint.
The engine resolves backend, profile, and request extra-parameter maps by The engine resolves backend, profile, and request extra-parameter maps by
whole-map replacement rather than key merging. The resulting effective map is whole-map replacement rather than key merging. The resulting effective map is
@@ -82,13 +97,47 @@ request fields.
## Response Handling ## Response Handling
Any 2xx response is decoded as an OpenAI-compatible chat response. The client Any 2xx response body is limited to 16 MiB (16,777,216 bytes). A larger
returns the first choice's non-empty message content and maps prompt, declared `Content-Length` is rejected before the body is read, and streamed,
completion, total, cached, and cache-write token counts. chunked, or underreported bodies are read through the same bound with at most
one additional byte used to detect overflow. A body exactly at the limit is
allowed. The body is closed on every outcome and an oversized stream is not
drained.
Invalid JSON, absent choices, and empty first-choice content are malformed The bounded body must contain exactly one OpenAI-compatible JSON response
responses. For a non-2xx status, the error includes the status code but never object followed only by JSON whitespace and EOF. The client returns the first
the provider response body. choice's explicitly present string message content, including an empty or
whitespace-only string, and maps prompt, completion, total, cached, and
cache-write token counts. Invalid or truncated JSON, trailing non-whitespace
data, a second JSON value, absent choices, missing content, `null` content,
non-string content, and size overflow are malformed responses and return no
partial result.
For a non-2xx status, Promptkit recognizes one JSON document with a top-level
object-valued `error` member. Its optional `message` and `type` fields must be
strings, and `code` may be a string or JSON number. Valid supported fields are
handled independently, numeric codes retain their JSON number text, and
unknown fields are ignored. Missing, invalid, malformed, or multiply framed
envelopes contribute no provider detail.
Non-success bodies have a 65,536-byte limit. A larger declared
`Content-Length` is not read; otherwise the client reads at most one additional
byte to detect streamed or underreported overflow. Empty, unreadable,
oversized, malformed, and unrecognized bodies retain only the received status.
The body is always closed and no oversized stream is drained beyond that probe.
Extracted strings are made valid UTF-8, trimmed, and converted to one line by
collapsing Unicode whitespace, control, and format-character runs. Blank
values are omitted. Codes and types longer than 256 Unicode code points are
omitted; messages longer than 4,096 code points are truncated at a code-point
boundary with an ellipsis inside the limit. Promptkit never exposes raw bodies,
headers, endpoints, credentials, request data, schemas, generated content, or
unsupported provider metadata through this handling.
An outbound `http.Client.Do` failure retains both Promptkit's request-failure
identity and the exact transport error for `errors.Is` and `errors.As` checks.
The rendered error does not include the selected endpoint, request headers,
request content, credentials, or provider body.
## Timeout And Cancellation ## Timeout And Cancellation
@@ -103,5 +152,7 @@ Timeouts are layered:
timeout when the supplied value is not positive. timeout when the supplied value is not positive.
The earliest applicable caller, generation, or transport deadline controls the The earliest applicable caller, generation, or transport deadline controls the
request. Constructing the internal client does not mutate a supplied request. Caller cancellation retains `context.Canceled`; caller, generation,
`http.Client`. and whole-request timeout failures retain `context.DeadlineExceeded`, together
with the request-failure identity. Constructing the internal client does not
mutate a supplied `http.Client`.

View File

@@ -56,7 +56,10 @@ reserve another bounded slot.
`NewClient` wraps the engine's selected internal model client after public `NewClient` wraps the engine's selected internal model client after public
client adaptation or built-in client construction. Initial generation and the client adaptation or built-in client construction. Initial generation and the
default repairer receive the same wrapper. default repairer receive the same wrapper. Their requests retain the same
effective backend, credential, numeric-presence metadata, and structured-output
settings, so scheduling does not change provider omission semantics between
calls.
For each `Generate` call, the wrapper selects a pool from the request's For each `Generate` call, the wrapper selects a pool from the request's
effective backend ID. An unlimited call passes directly to the next client. A effective backend ID. An unlimited call passes directly to the next client. A
@@ -71,7 +74,9 @@ other backend IDs.
The wrapper passes generation requests, responses, and collaborator errors The wrapper passes generation requests, responses, and collaborator errors
through unchanged. It owns scheduling only; the concrete model client remains through unchanged. It owns scheduling only; the concrete model client remains
responsible for provider transport behavior. responsible for provider transport behavior. The runner, rather than the
capacity layer, sums all five usage fields from the initial response and every
completed repair response into the successful run result.
## Cancellation And Release ## Cancellation And Release

View File

@@ -21,20 +21,26 @@ uses internal domain values for rendered prompts, execution targets,
structured output, responses, and token usage. structured output, responses, and token usage.
The runner supplies a fully resolved target after applying backend, profile, The runner supplies a fully resolved target after applying backend, profile,
and request precedence. The client uses its endpoint, credential metadata, and request precedence, plus canonical provider-bound text messages. The
generation fields, and extra parameters. `BackendID` remains routing metadata client uses its endpoint, credential metadata, generation fields, and extra
parameters. `BackendID` remains routing metadata
for the generation boundary and is not mapped into the provider payload. for the generation boundary and is not mapped into the provider payload.
Construction validates the configured base URL and clones any supplied Construction trims and validates a nonempty configured base URL and clones any
`http.Client` so Promptkit can apply its timeout default without mutating the supplied `http.Client` so Promptkit can apply its timeout default without
caller's client. Generation then: mutating the caller's client. An empty configured base remains valid because a
resolved request target may supply the endpoint. Generation then:
1. validates request-level timeout and endpoint requirements; 1. validates shared execution-setting invariants and the final selected base
endpoint;
2. maps the internal request into the OpenAI-compatible chat payload; 2. maps the internal request into the OpenAI-compatible chat payload;
3. validates and merges extra parameters; 3. validates and merges extra parameters;
4. resolves authentication; 4. composes `/chat/completions` through parsed URL path operations;
5. performs the outbound request under the applicable deadlines; and 5. resolves authentication;
6. decodes the first response choice and token usage. 6. performs the outbound request under the applicable deadlines; and
7. decodes one strictly framed, size-bounded successful response object and
maps its first choice and token usage, or decodes bounded structured
non-success detail.
`internal/llm` owns the set of reserved OpenAI-compatible request fields used `internal/llm` owns the set of reserved OpenAI-compatible request fields used
when validating extra parameters. Backend registration consumes the same rule when validating extra parameters. Backend registration consumes the same rule
@@ -50,12 +56,14 @@ the target, rendered messages, and structured-output constraint retained by
executable preparation. Execution does not reopen or rerender consumer executable preparation. Execution does not reopen or rerender consumer
sources. sources.
Before backend admission, the runner rechecks that the frozen credential Before backend admission, the runner rechecks a frozen credential
environment-variable name is available. The handle does not retain the environment-variable name only when the target explicitly requires a
environment value; the model client resolves the value visible when generation credential. The handle does not retain the environment value; the model client
begins. A direct request key remains in private execution state only until the resolves the value visible when generation begins. For optional sources with no
claimed execution finishes or an unclaimed handle is discarded. Exact public usable value, the built-in client omits `Authorization` and continues to the
ownership and redaction semantics belong to the provider. A direct request key remains in private execution state only until
the claimed execution finishes or an unclaimed handle is discarded. Exact
public ownership and redaction semantics belong to the
[`PreparedExecution` GoDoc](../../prepared_execution.go). [`PreparedExecution` GoDoc](../../prepared_execution.go).
## Failure Categories ## Failure Categories
@@ -63,11 +71,47 @@ ownership and redaction semantics belong to the
The package preserves distinct error identities for invalid client The package preserves distinct error identities for invalid client
configuration, invalid generation requests, request execution failures, configuration, invalid generation requests, request execution failures,
non-success provider statuses, and malformed successful responses. Provider non-success provider statuses, and malformed successful responses. Provider
response bodies are not included in non-success errors. response bodies are never exposed in raw form through non-success errors.
Caller cancellation and deadline failures during the outbound request are Invalid nonempty configured endpoints are configuration failures. A missing or
reported as request execution failures. The runner classifies these identities invalid final selected endpoint is an invalid generation request and is
without depending on HTTP status mapping. rejected before transport.
Authentication resolves a trimmed direct key before a trimmed configured
environment value. Optional missing, empty, or whitespace-only sources do not
block transport and produce no `Authorization` header. An explicitly required
target with no usable source is rejected before transport with the existing
invalid-request diagnostics.
Successful response bodies have a fixed 16 MiB limit enforced by declared
length and by reading at most one byte beyond the boundary. The decoder accepts
exactly one JSON object plus trailing whitespace and EOF. Size overflow,
truncation, malformed JSON, trailing data, and a second value are malformed
responses with no partial result or provider content in the error. Every body
is closed, and an unbounded oversized stream is not drained.
After framing succeeds, the first choice must contain an explicitly present
string `message.content`. The string is returned exactly, including empty or
whitespace-only content. Missing choices, missing or `null` content, and
non-string content are malformed responses. Output validation and correction
eligibility remain outside this package.
For a non-success response, `ProviderHTTPError` retains the HTTP status and
only normalized detail from the bounded recognized envelope. It retains
`ErrUnexpectedStatus` through unwrapping. The client owns response closure;
its bounded reader and parser never close or drain a body themselves. The root
facade converts this concrete internal error into the public
[`GenerationError`](../../generation_error.go), while arbitrary injected-client
errors continue through the ordinary generation-error mapping unchanged.
An `http.Client.Do` failure is represented by a redacting multi-cause error:
the package request-failure sentinel and the exact returned transport error are
both available through `errors.Is` and `errors.As`, while the rendered text
does not expose the endpoint, headers, request content, credential, transport
detail, or provider body. Caller cancellation retains `context.Canceled`;
caller deadlines, generation deadlines, and whole-request client timeouts
retain `context.DeadlineExceeded`. The runner adds its generation category
without discarding those identities or depending on HTTP status mapping.
## Test Ownership ## Test Ownership
@@ -75,7 +119,15 @@ The
[OpenAI-compatible client tests](../../internal/llm/openai_compatible_client_test.go) [OpenAI-compatible client tests](../../internal/llm/openai_compatible_client_test.go)
own configuration, client cloning, deterministic deadline precedence, own configuration, client cloning, deterministic deadline precedence,
authentication, request and response mapping, malformed data, error identity, authentication, request and response mapping, malformed data, error identity,
cancellation, and response-body suppression. The root transport contract test cancellation, endpoint selection and composition, pre-transport rejection, and
also verifies that resolved backend settings reach this client without bounded single-document successful-response framing, closure, and
serializing backend identity. All use local test servers or test transports; response-body suppression. The focused
the default suite makes no live or paid provider requests. [provider HTTP error tests](../../internal/llm/provider_http_error_test.go)
own envelope parsing, normalization, and bounded-reader cases; their
[transport tests](../../internal/llm/provider_http_error_transport_test.go)
own non-success response closure and integration. Root transport contract tests
own public `GenerationError` conversion, while also verifying that resolved
backend settings reach this client without serializing backend identity and
that ordinary-run cancellation retains its public generation and context
identities. All use local test servers or controlled test transports; the
default suite makes no live or paid provider requests.

View File

@@ -11,23 +11,23 @@ contributor workflow and validation.
| Component | Implemented responsibility | References | | Component | Implemented responsibility | References |
| --- | --- | --- | | --- | --- | --- |
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, and profile-inspection values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, typed capacity errors, public error mapping, and engine-local profile-source assembly including application fallbacks. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) | | Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, and profile-inspection values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, typed capacity and generation error mapping, engine-local profile-source assembly including application fallbacks, and bounded output-repair assembly. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) | | `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) | | `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
| `internal/backend` | Constructs each engine's immutable registry from the built-in OpenRouter definition and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) | | `internal/backend` | Constructs each engine's immutable registry from maintained definitions and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
| `internal/catalog` | Strictly validates imported immutable maintained backend and profile catalog assets before engine assembly uses them. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) |
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) | | `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation. | [Domain declarations](../../internal/domain/domain.go) | | `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [endpoint invariant](../../internal/domain/endpoint.go) |
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) | | `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) | | `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
| `internal/jsonvalue` | Validates and deeply copies JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types. | [JSON values](../../internal/jsonvalue/jsonvalue.go) | | `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) | | `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) | | `internal/profile` | Loads strictly decoded, locally validated execution profiles from filesystem and `fs.FS` sources, overlays raw sources with error-preserving fallback, and resolves inherited profiles. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go), [internal sources](sources.md#profiles-and-built-ins) |
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) | | `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) | | `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates frozen validation plans for prepared execution. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) | | `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) | | `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, generation, validation, capacity, and optional repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) | | `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
The root package assembles these internal components without exposing their The root package assembles these internal components without exposing their
representations. Consumers depend only on the root facade. representations. Consumers depend only on the root facade.

View File

@@ -20,10 +20,13 @@ and override semantics consumed by the runner.
profiles, backend resolution, artifacts, rendering, model generation, and profiles, backend resolution, artifacts, rendering, model generation, and
validation. The root engine supplies one immutable registry containing the validation. The root engine supplies one immutable registry containing the
built-in backend and validated consumer additions, one engine-local run built-in backend and validated consumer additions, one engine-local run
admitter, and a model client wrapped by the same capacity manager. Schema admitter, and a model client wrapped by the same capacity manager. Validation
documents are loaded through the validator's optional schema-loader interface. plans and provider-facing schema metadata come from the validator's preparation
An output repairer can be injected internally, but the ordinary runner interface.
constructor does not enable one. The root engine supplies one default output repairer through the explicit
runner constructor, using the same capacity-wrapped client as initial
generation. The no-repair runner constructor remains available for focused
internal callers and tests.
Each invocation carries its state in request, prepared-run, and result values. Each invocation carries its state in request, prepared-run, and result values.
The runner has no durable run or session store. The runner has no durable run or session store.
@@ -69,17 +72,20 @@ performs only the work needed to validate routing and admission:
5. resolve application-neutral defaults, backend defaults, profile values, 5. resolve application-neutral defaults, backend defaults, profile values,
and explicit request overrides in that order; and explicit request overrides in that order;
6. validate endpoint, model, numeric overrides, and credential requirements; 6. validate endpoint, model, numeric overrides, and credential requirements;
7. resolve the effective output contract without loading its schema; and 7. resolve and validate the effective output contract without loading its
schema; and
8. retain the definition, source identities, effective settings, output 8. retain the definition, source identities, effective settings, output
contract, and preparation start time in invocation-local state. contract, and preparation start time in invocation-local state.
The completion phase consumes that state without reloading the prompt, The completion phase consumes that state without reloading the prompt,
profile, or backend: profile, or backend:
1. load structured-output schema metadata when required; 1. create one operation-local validation plan and derive structured-output
schema metadata from it when required;
2. load and hash input artifacts; 2. load and hash input artifacts;
3. render messages and the prompt-defined session; 3. render messages and the prompt-defined session;
4. apply any direct session ID; 4. apply any direct session ID, then append already-normalized request messages
after the rendered definition messages;
5. hash the effective rendered prompt; and 5. hash the effective rendered prompt; and
6. construct the prepared value and preparation timing. 6. construct the prepared value and preparation timing.
@@ -87,7 +93,10 @@ profile, or backend:
`Run` performs backend admission between the phases. This structure preserves `Run` performs backend admission between the phases. This structure preserves
one execution-precedence and error-ordering implementation while allowing a one execution-precedence and error-ordering implementation while allowing a
full backend pool to reject work before expensive schema, artifact, and full backend pool to reject work before expensive schema, artifact, and
rendering operations. rendering operations. `Prepare` discards the plan after returning its public
metadata. `Run` retains the plan through initial and repaired-output validation
and discards it when the operation ends. Prepared execution stores the same
kind of plan only in its private payload.
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
out-of-range values fail as invalid requests. Endpoint overrides do not change out-of-range values fail as invalid requests. Endpoint overrides do not change
@@ -105,7 +114,10 @@ the prompt session template, and is applied after ordinary message rendering.
A blank direct value retains prompt-template behavior. The runner clears the A blank direct value retains prompt-template behavior. The runner clears the
template only on a value copy of the definition, so the definition hash always template only on a value copy of the definition, so the definition hash always
describes the original source while the rendered-prompt hash includes the describes the original source while the rendered-prompt hash includes the
effective direct or rendered session. effective direct or rendered session and the complete effective message
sequence. The rendered-prompt hash uses a versioned, length-framed SHA-256
encoding of that session, every role and content value, and cache-control
presence and values; its hexadecimal value is opaque.
The registry is read-only after engine construction. Concurrent `Prepare` and The registry is read-only after engine construction. Concurrent `Prepare` and
`Run` calls resolve independent defensive backend values and keep all `Run` calls resolve independent defensive backend values and keep all
@@ -118,8 +130,17 @@ its `RunAdmitter` to reserve capacity for the effective backend ID. A nil
admitter is an internal unlimited fallback. After successful admission, `Run` admitter is an internal unlimited fallback. After successful admission, `Run`
immediately defers the returned release function, performs the completion immediately defers the returned release function, performs the completion
phase, makes one initial generation call, builds the named output artifact, phase, makes one initial generation call, builds the named output artifact,
and validates that artifact. Invalid generated content remains a validation and validates that artifact with the plan compiled during completion. Invalid
result; an inability to perform validation is an operational error. generated content remains a validation result; an inability to generate or
perform validation is an operational error.
Validation preparation and execution honor cancellation at every
Promptkit-controlled boundary and do not publish a partial plan or result.
Schema reads are bounded and context-checked between chunks; JSON decoding,
schema compilation, and schema execution are checked immediately before and
after their synchronous calls. Promptkit does not move arbitrary filesystem or
JSON Schema work to background goroutines, so an already-blocked dependency
method must return before cancellation can take precedence over its outcome.
The admission lease covers completion-phase preparation, initial generation, The admission lease covers completion-phase preparation, initial generation,
validation, every repair, and every exit. It bounds accepted work without validation, every repair, and every exit. It bounds accepted work without
@@ -127,22 +148,38 @@ serializing preparation or validation behind the active-generation limit.
The wrapped model client separately acquires a FIFO active permit only around The wrapped model client separately acquires a FIFO active permit only around
each actual generation call. each actual generation call.
When an internal repairer is present, a JSON or JSON Schema content failure can After a failed `basic`, JSON, or JSON Schema validation with a positive frozen
trigger bounded repair attempts. Repair receives the effective execution budget, the installed repairer can make a bounded corrective call. Each request
target and session ID, validation errors, prior output, and structured-output starts with a fresh copy of the complete effective message sequence (the
specification. The default repairer uses the same wrapped client as initial configured prefix followed by the request suffix), includes only the latest
nonempty candidate as an assistant message, and appends one corrective user
message. Empty candidates omit that assistant message. The
correction carries validation diagnostics as JSON data bounded to 64 KiB; the
full diagnostics remain in the validation result.
Repair receives the effective execution target, explicit numeric-presence bits,
credential, backend identity, session ID, and structured-output specification.
The same request constructor supplies those common fields to initial and repair
generation. The default repairer uses the same wrapped client as initial
generation, so each repair reacquires the selected backend's active permit generation, so each repair reacquires the selected backend's active permit
while remaining inside its original admission lease. Repair never performs a while remaining inside its original admission lease. Repair never performs a
second bounded admission. This capability remains internal and is not a public second bounded admission, and repaired outputs use the operation's existing
option. validation plan. The runner stops at the first valid candidate, sums completed
generation usage, reports calls actually started, and returns the final failed
validation result on exhaustion. A repair generation failure follows the
ordinary generation-error category rather than becoming a validation error.
A successful result includes the output artifact and raw output, validation A successful result includes the output artifact and raw output, validation
state, effective session ID, prompt and rendered-prompt hashes, selected state, effective session ID, prompt and rendered-prompt hashes, selected
profile and backend, effective settings, input hashes, token usage, a generated profile and backend, effective settings, input hashes, token usage, a generated
run identifier, and UTC timing. The same effective session reaches initial run identifier, and UTC timing. The same effective session reaches initial
generation and any repair attempt through the rendered prompt. The same generation and any repair attempt through the rendered prompt. The same
effective target, including backend identity, reaches generation and any effective target and presence metadata, including backend identity and direct
repair attempt. credential during execution, reaches generation and every repair attempt.
Result usage is the field-wise sum of all five usage values from the initial
response and every completed repair response. Final raw output, artifact, and
validation state still come from the last candidate. A repair error returns no
partial run result or partial usage.
## Failure Categories ## Failure Categories
@@ -165,7 +202,8 @@ category. Deferred release restores the admission lease on preparation,
generation, validation, repair, and cancellation failures. generation, validation, repair, and cancellation failures.
Other context cancellation propagates through the invoked collaborator and is Other context cancellation propagates through the invoked collaborator and is
classified by the owning operation. classified by the owning operation. In particular, cancellation observed by
validation retains the context identity through the validation error category.
An overlong direct session is an invalid request before source loading, while An overlong direct session is an invalid request before source loading, while
an invalid or overlong prompt session template remains a prompt-render failure. an invalid or overlong prompt session template remains a prompt-render failure.
An unknown selected backend, or a selected backend with no configured resolver, An unknown selected backend, or a selected backend with no configured resolver,
@@ -177,8 +215,9 @@ The [runner tests](../../internal/usecase/runner_test.go) own preparation order,
selection and override precedence, the two-phase boundary, early admission, selection and override precedence, the two-phase boundary, early admission,
lease lifetime and release, direct-session resolution, schema-before-generation lease lifetime and release, direct-session resolution, schema-before-generation
behavior, hashing, generation and validation outcomes, backend propagation, behavior, hashing, generation and validation outcomes, backend propagation,
bounded repair, shared initial/repair capacity, credentials and redaction, bounded repair progression, initial/repair request parity, cumulative usage,
error categories, artifact metadata, usage, and timing. The shared initial/repair capacity, credentials and redaction, error categories,
artifact metadata, and timing. The
[capacity subsystem document](capacity.md) identifies the focused pool, [capacity subsystem document](capacity.md) identifies the focused pool,
waiter, and wrapped-client tests. waiter, and wrapped-client tests.

View File

@@ -12,9 +12,27 @@ validation modes, built-in catalog, and source precedence.
## Prompt Definitions ## Prompt Definitions
`internal/promptdef` discovers YAML deterministically, decodes and validates `internal/promptdef` uses one source-neutral flow for prompt selection and
definitions, selects an ID and optional version, and resolves file-backed normalization. That flow scans normalized YAML ID and version metadata,
message content within the selected operating-system or `fs.FS` source. requires one strictly decoded document per file, classifies errors for the
selected definition, detects duplicates, and normalizes the exact match.
Small operating-system and `fs.FS` adapters own discovery, byte reads, display
paths, content opening, and root containment. Each lookup remains a
point-in-time scan: definitions and catalogs are not cached, and file-backed
message content is opened only for the exact selected candidate.
Message roles are normalized through the shared domain owner by trimming
Unicode whitespace and lowercasing. Only `developer`, `system`, `user`, and
`assistant` are published; invalid roles remain selected prompt-definition
failures rather than becoming request errors. Cache-control metadata uses the
same shared domain normalization and defensive-copy rule.
Operating-system sources enforce containment against canonical roots and
targets so symlinks cannot escape. Injected `fs.FS` sources enforce containment
in their clean relative path namespace. A single-file source uses the selected
prompt file's containing directory as its root. Every content path must be
relative and is opened from its exact parsed text after a separate blank check;
contained parent components and whitespace-bearing names remain valid.
Exact prompt inspection performs one point-in-time lookup through that same Exact prompt inspection performs one point-in-time lookup through that same
repository and validates referenced message content before returning declared repository and validates referenced message content before returning declared
@@ -27,39 +45,78 @@ duplicate detection, and source containment:
## Profiles And Built-Ins ## Profiles And Built-Ins
`internal/profile` loads and validates execution profiles from an `internal/profile` loads, locally validates, overlays, and resolves execution
operating-system filesystem or an `fs.FS`. Its overlay repository consults the profiles from an operating-system filesystem or an `fs.FS`. A file contains
next repository only when the higher-precedence repository reports that a exactly one YAML document and its trimmed YAML `id` is its only selection
profile is absent. Strict YAML decoding recognizes the optional `backend` identity; filenames do not confer authority. Each point lookup reads discovered
field, trims its value, and requires a model plus at least one non-blank files once for their metadata and reuses the selected file's bytes for strict
backend or endpoint. Loading does not check registry membership because the decoding; unrelated profiles are not fully decoded. Strict selected decoding
available registry belongs to the assembled engine; the runner checks recognizes `base_profile` and the optional `backend` field, trims their values,
membership during preparation and exact profile inspection. and permits inherited target fields only when a base is named. File-backed
`extra_params` values are validated and defensively copied through the shared
bounded JSON-value owner before a profile is published. OpenAI-compatible
reserved-field policy remains with the model-client and backend-registry owners.
The root engine assembles profile repositories in precedence order: in-memory `LoadFSRepository` is the eager immutable loading boundary for internal
profiles, one ordinary configured source, an application fallback source, then catalog consumers. It discovers and strictly validates every raw profile once,
the embedded built-in catalog. An explicit file or `fs.FS` profile source preserves safe source metadata including explicitly present YAML fields, and
replaces `Config.ProfileDir` within the ordinary configured-source category. publishes independently copied values from memory. It does not resolve profile
inheritance. Configured consumer sources continue to use the lazy point lookup
repositories described above.
Exact profile inspection performs one point-in-time lookup through those `internal/catalog` validates the imported immutable OpenRouter and
profile sources and checks the resolved target without reading prompt, input, Rakestrawhome asset modules as one private adapter boundary. It enforces their
manifest, layout, profile ownership, inheritance, and secret-safety rules
before returning raw catalog sources. Root assembly uses those validated
catalogs as the maintained lowest-precedence profile source.
The overlay repository consults the next repository only when the
higher-precedence repository reports that a profile is absent. A reliably
selected malformed profile stops fallback, while an unrelated malformed file
does not become authoritative through its filename. Loading does not check
backend registry membership because the available registry belongs to the
assembled engine; the runner checks membership during preparation and exact
profile inspection.
The root engine assembles one raw composite catalog in precedence order:
in-memory profiles, one ordinary configured source, an application fallback
source, then the maintained external catalog. An explicit file or `fs.FS` profile
source replaces `Config.ProfileDir` within the ordinary configured-source
category. One outer resolving repository wraps that complete raw catalog, so
each base lookup observes the same precedence and shadowing rules.
The resolving repository traverses every selected chain afresh, retains no
cache, detects cycles, limits a chain to 32 profiles, merges root-to-leaf into a
new caller-owned value, and validates the final target before publishing it. It
does not check backend registry membership. Exact `base_profile` syntax, merge
rules, and consumer-visible failure behavior belong to the [framework format
reference](../formats.md#profile-inheritance).
Exact profile inspection performs one point-in-time resolved lookup through
those profile sources and checks the final target without reading prompt, input,
or schema sources. It does not retain that lookup for a later execution. or schema sources. It does not retain that lookup for a later execution.
Prepared execution instead freezes the fully resolved target; a later ordinary
operation performs a fresh traversal.
`internal/profile/builtin` embeds the maintained built-in profile catalog. The maintained external catalog provides every built-in profile and its
Every embedded profile selects `openrouter` and inherits its endpoint and matching backend definition. Profile loading and overlay behavior are owned by the
credential environment-variable name from the built-in backend registry rather [profile repository tests](../../internal/profile/repository_test.go), while
than repeating those values. Profile loading and overlay behavior are owned by catalog completeness, backend selection, and duplicate IDs are owned by the
the [profile repository tests](../../internal/profile/repository_test.go), [catalog adapter tests](../../internal/catalog/catalog_test.go).
while catalog completeness, the backend-selection invariant, and duplicate IDs
are owned by the
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
## Ordinary Artifacts ## Ordinary Artifacts
`internal/artifact` resolves inline references and unrestricted, `internal/artifact` accepts explicitly typed inline references even when their
caller-selected file paths. It copies content into an artifact, records body is empty. It also resolves unrestricted, caller-selected paths only when
metadata and a content hash, applies a content-type fallback, and honors they identify regular operating-system files, checking that condition before
context cancellation. and after opening the file. It copies content into an artifact, records
metadata and an opaque content-equality value, and applies a content-type
fallback.
Regular files are read synchronously in bounded chunks. Cancellation is
checked before opening, before and after every read, and before publishing the
artifact, so a canceled read never publishes partial content. The ordinary
reader does not detach file reads into background goroutines.
This ordinary reader does not implement an inbound HTTP security boundary. In This ordinary reader does not implement an inbound HTTP security boundary. In
particular, it does not constrain files to an application root or impose an particular, it does not constrain files to an application root or impose an
@@ -71,7 +128,17 @@ implemented reader behavior and failures.
## Rendering ## Rendering
`internal/prompt` renders definition messages as Go templates using named `internal/prompt` renders definition messages as Go templates using named
artifacts and variables. It carries message roles, session IDs, and cache artifacts and variables. Within one render, each referenced artifact body is
converted to text lazily and cached by input name for reuse across the session
and every message; the cache is not shared across renders. Conversion uses
bounded chunks and preserves the artifact bytes exactly.
Session and message parsing and execution remain synchronous. The renderer
checks cancellation before and after each parse and execution boundary,
between artifact conversion chunks, around each message, and before publishing
the complete prompt. It cannot interrupt template work already in progress and
never publishes a partial prompt after observing cancellation. It validates and
canonicalizes message roles before carrying roles, session IDs, and cache
control into the rendered prompt. The control into the rendered prompt. The
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior. [renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
@@ -82,22 +149,35 @@ filesystem or an `fs.FS`. Invalid generated content is returned as a validation
result; inability to load, register, or compile a schema is an operational result; inability to load, register, or compile a schema is an operational
error. error.
For executable preparation, the built-in validators create a frozen validation Every preparation operation creates one operation-local validation plan. None,
plan. None, basic, and JSON modes retain the effective output contract without basic, and JSON modes retain the effective output contract without source
source access. JSON Schema mode loads the root document, resolves and compiles access. JSON Schema mode loads the root document once, resolves and compiles
every transitive reference during preparation, and retains the compiled each transitive reference, and retains the compiled validator. Schema compiler
validator. The provider-facing structured-output metadata uses that same resources use canonical escaped file or private-scheme URLs; loaders decode
captured root document. their paths once and enforce the configured source boundary. The
provider-facing structured-output metadata uses the root document captured by
the same plan.
`PrepareExecution` also completes prompt and profile selection, artifact Schema preparation and execution remain synchronous. Promptkit checks
loading and hashing, session and message rendering, and target resolution. cancellation before and after source resolution, JSON decoding, compilation,
`RunPrepared` uses the retained source-derived state and validation plan; it and validation, and between bounded schema-read chunks. Once cancellation is
does not reopen prompt, profile, input, or schema sources and does not rerender observed it returns the context error without publishing a partial plan or
the request. By contrast, ordinary `Prepare` produces a preparation value only: validation result, even when a compiler or validator has just returned a
a later `Run` performs its own source resolution and preparation. different error or a successful result. An `fs.FS` method or JSON Schema
dependency call already in progress cannot be preempted; Promptkit waits for
that call to return and then gives cancellation precedence. Validation does
not detach dependency work into background goroutines.
`Prepare` discards its validation plan after returning metadata. `Run` retains
its plan for initial and repaired-output validation, then discards it with the
operation. `PrepareExecution` retains the plan in its private frozen payload;
`RunPrepared` uses that plan without reopening prompt, profile, input, or
schema sources or rerendering the request. A later ordinary `Run` always
performs fresh source resolution and preparation.
The [validator tests](../../internal/validate/standard_validator_test.go) own The [validator tests](../../internal/validate/standard_validator_test.go) own
basic, JSON, JSON Schema, source resolution, schema loading, compilation, basic, JSON, JSON Schema, source resolution, schema loading, compilation,
frozen-reference behavior, and content-failure behavior. Prepared execution frozen-reference behavior, content-failure behavior, and the synchronous
cancellation boundary. Prepared execution
orchestration is owned by the orchestration is owned by the
[use-case tests](../../internal/usecase/prepared_execution_test.go). [use-case tests](../../internal/usecase/prepared_execution_test.go).

View File

@@ -19,24 +19,24 @@ results, public values, extension interfaces, profiles, and error sentinels.
The implemented internal components consist of: The implemented internal components consist of:
- `internal/domain`, which owns framework data values shared by later internal - `internal/domain`, which owns framework data values and source-neutral
components; invariants shared by later internal components;
- `internal/backend`, which owns validated immutable OpenAI-compatible backend - `internal/backend`, which owns validated immutable OpenAI-compatible backend
definitions and the built-in OpenRouter definition; definitions;
- `internal/capacity`, which owns engine-local bounded run admission and - `internal/capacity`, which owns engine-local bounded run admission and
model-generation scheduling for limited backends; model-generation scheduling for limited backends;
- `internal/defaults`, which owns application-neutral framework defaults and - `internal/defaults`, which owns application-neutral framework defaults and
constructs the default execution target; constructs the default execution target;
- `internal/filecatalog`, which discovers YAML files and provides source-path - `internal/filecatalog`, which discovers YAML files and provides source-path
helpers for filesystem and `fs.FS` consumers; helpers for filesystem and `fs.FS` consumers;
- `internal/catalog`, which validates immutable external maintained-catalog
assets before they are eligible for engine assembly;
- `internal/jsonvalue`, which validates and defensively copies JSON-compatible - `internal/jsonvalue`, which validates and defensively copies JSON-compatible
extra-parameter trees; extra-parameter trees;
- `internal/promptdef`, which loads and validates prompt definitions from - `internal/promptdef`, which loads and validates prompt definitions from
filesystem and `fs.FS` sources; filesystem and `fs.FS` sources;
- `internal/profile`, which loads, validates, and overlays execution profiles - `internal/profile`, which loads, validates, and overlays execution profiles
from filesystem and `fs.FS` sources; from filesystem and `fs.FS` sources;
- `internal/profile/builtin`, which embeds the built-in execution profile
catalog;
- `internal/prompt`, which renders prompt messages from Go templates; - `internal/prompt`, which renders prompt messages from Go templates;
- `internal/artifact`, which resolves ordinary inline and unrestricted - `internal/artifact`, which resolves ordinary inline and unrestricted
caller-selected file references; caller-selected file references;
@@ -54,13 +54,15 @@ packages or participate in internal assembly.
The root facade assembles one immutable backend registry, one capacity manager, The root facade assembles one immutable backend registry, one capacity manager,
the internal repositories, renderer, validator, outbound client, and use-case the internal repositories, renderer, validator, outbound client, and use-case
runner while translating public values and errors at the library boundary. The runner while translating public values and errors at the library boundary. The
registry contains built-ins plus validated engine-scoped consumer additions. registry contains validated maintained definitions plus engine-scoped consumer
additions.
The facade constructs the capacity manager from the registry's immutable The facade constructs the capacity manager from the registry's immutable
policy snapshot, wraps the selected built-in or injected model client, and policy snapshot, wraps the selected built-in or injected model client, and
supplies bounded admission to the runner. The defaults and renderer depend on supplies bounded admission to the runner. The defaults and renderer depend on
the domain model. Prompt-definition and profile repositories use the domain the domain model. Prompt-definition and profile repositories use the domain
model, file catalog, and YAML decoder. The built-in profile repository supplies model, file catalog, and YAML decoder. The catalog adapter validates imported
an embedded `fs.FS` to the profile package. Artifact reading uses the domain immutable maintained data before root assembly supplies it to the profile
package. Artifact reading uses the domain
model and application-neutral defaults. Validation uses the domain model, file model and application-neutral defaults. Validation uses the domain model, file
catalog, and JSON Schema implementation. The model client uses the domain catalog, and JSON Schema implementation. The model client uses the domain
model, application-neutral defaults, and an injected or standard-library HTTP model, application-neutral defaults, and an injected or standard-library HTTP
@@ -92,6 +94,13 @@ coordinates internal components and adapts the supported public extension
interfaces to narrow internal abstractions. Internal components must not depend interfaces to narrow internal abstractions. Internal components must not depend
on consumers or on Scriptorium. on consumers or on Scriptorium.
`internal/domain` owns source-neutral invariants for values shared across
multiple input and execution boundaries, including execution-setting bounds,
OpenAI-compatible base endpoints, session identifiers, and output-contract
legality. Callers retain source parsing, required-field rules, other
source-specific normalization, defaulting, error classification, and policy
specific to their own boundary.
## Repository And Consumer Boundary ## Repository And Consumer Boundary
Scriptorium is a downstream application that consumes Promptkit through Scriptorium is a downstream application that consumes Promptkit through

View File

@@ -50,19 +50,18 @@ Examples of appropriate seams include clocks, randomness, subprocesses, remote A
## Test execution requirements ## Test execution requirements
Promptkit currently uses maintainer-run validation rather than hosted CI. Promptkit currently uses maintainer-run validation rather than hosted CI.
Maintainers run the repository-documented test, vet, build, formatting, Maintainers run the complete local workflow in the
documentation-link, and repository-hygiene checks before accepting changes. [development guide](../development.md#maintainer-validation) before accepting
changes. That guide is the canonical owner of exact commands, formatting,
documentation-link validation, and repository-hygiene checks.
Introducing hosted CI later would supplement, not silently redefine, this Introducing hosted CI later would supplement, not silently redefine, this
documented validation model. documented validation model.
The complete test sequence includes ordinary and race-enabled package tests. Maintainer validation must include ordinary and race-enabled package tests,
The maintained offline consumer workflow is also run from the repository root: static analysis, a complete build, and execution of both maintained offline
consumer examples. The preparation example protects assembled preparation and
```sh inspection behavior. The execution example separately protects assembled
go test ./... `Run`, injected-client, validation, usage, and result behavior.
go test -race ./...
go run ./examples/go-library/prepare
```
Tests in the default suite must be deterministic, offline, and independent of Tests in the default suite must be deterministic, offline, and independent of
real credentials. They must not invoke paid APIs, use live network real credentials. They must not invoke paid APIs, use live network
@@ -88,8 +87,9 @@ Use each test type where it protects a distinct risk:
interaction, while replacing live or nondeterministic external boundaries. interaction, while replacing live or nondeterministic external boundaries.
- External-package root tests exercise the public facade as a Go consumer, - External-package root tests exercise the public facade as a Go consumer,
while internal package tests own focused implementation behavior. while internal package tests own focused implementation behavior.
- The maintained offline preparation example protects one representative - The maintained offline preparation and execution examples protect distinct
assembled consumer workflow without contacting a model provider. representative assembled consumer workflows without contacting a model
provider.
- Fixtures should be minimal, synthetic, versioned with the behavior they - Fixtures should be minimal, synthetic, versioned with the behavior they
exercise, and free of credentials or private data. exercise, and free of credentials or private data.
- Golden files are appropriate only when the complete output is intentionally - Golden files are appropriate only when the complete output is intentionally

View File

@@ -106,48 +106,12 @@ gitea.maximumdirect.net/eric/promptkit 1.25.5
promptkit gitea.maximumdirect.net/eric/promptkit promptkit gitea.maximumdirect.net/eric/promptkit
``` ```
Run the complete maintainer validation required by the As a release prerequisite, run the complete
[development guide](development.md): [maintainer validation workflow](development.md#maintainer-validation) against
the clean candidate. Do not substitute a partial command list: the development
```sh guide owns the tests, race checks, analysis, build, both offline examples,
go test ./... formatting, Markdown links, generated-output and credential review, and
go test -race ./... repository hygiene. Record the successful workflow result with the candidate.
go vet ./...
go build ./...
go run ./examples/go-library/prepare
```
Check every tracked Go file. This command must produce no output:
```sh
unformatted=$(
git ls-files '*.go' |
while IFS= read -r go_file
do
gofmt -l "$go_file"
done
)
test -z "$unformatted"
```
Follow every maintained Markdown link and confirm that its local or published
target exists. Review the repository for generated binaries, test or coverage
output, credentials, template residue, downloaded assets, and other files that
do not belong in source control.
Recheck module and repository hygiene, whitespace, and the clean checkout:
```sh
test -z "$(git ls-files go.work go.work.sum)"
test ! -e vendor
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
then
printf '%s\n' 'go.mod contains a replacement' >&2
exit 1
fi
git diff --check
test -z "$(git status --porcelain)"
```
## Write The Release Note ## Write The Release Note
@@ -182,6 +146,13 @@ grep -F 'Consumer action:' "$RELEASE_NOTES_FILE"
Inspect the complete message and confirm that it accurately records the Inspect the complete message and confirm that it accurately records the
compatibility impact, public API changes, and required consumer action. compatibility impact, public API changes, and required consumer action.
When a release introduces or updates the maintained external catalogs, state
that it selects two independently versioned data dependencies, preserves the
public API and configuration, requires no consumer migration, and guarantees
only the catalog versions selected and tested by that Promptkit release. Link
to the current architecture and format documentation rather than duplicating
catalog contracts in the release note.
## Create And Inspect The Tag ## Create And Inspect The Tag
Run the candidate guard again immediately before tag creation. This ensures Run the candidate guard again immediately before tag creation. This ensures

131
docs/releases/v0.6.0.md Normal file
View File

@@ -0,0 +1,131 @@
# Promptkit v0.6.0
This supplemental changelog and migration guide summarizes the consumer-facing
changes from `v0.5.0` to `v0.6.0`. The annotated `v0.6.0` tag is the
authoritative release record. Exact current contracts belong to the linked
GoDoc and durable documentation.
## Summary
`v0.6.0` is a broad correctness, safety, efficiency, and maintainability
release. It does not add or remove public declarations. The release:
- centralizes shared execution-setting, output-contract, endpoint, and
JSON-compatible-value rules;
- unifies prompt repository behavior and avoids unnecessary prompt and profile
decoding;
- bounds consumer-controlled JSON trees and successful provider responses;
- hardens prompt content paths, artifact files, provider URLs, JSON framing,
and error propagation;
- reuses compiled schema plans and rendered artifact text within an operation;
and
- improves cancellation behavior, prepared-value ownership, deterministic
transport testing, and maintainer validation.
## Compatibility
No public declaration was added, removed, or changed. Ordinary valid `v0.5.0`
configurations and requests should continue to compile and behave as before.
The release intentionally rejects or reports several inputs that were
previously accepted, altered, or misclassified:
- execution settings must be finite, within their documented ranges, and safe
to convert to Go durations;
- output formats, validation modes, repair counts, and JSON Schema dependencies
are validated consistently;
- file-backed prompt and profile identity comes from normalized YAML metadata,
not filenames;
- prompt `content_file` values must be exact relative paths contained by their
configured source root;
- built-in file artifacts must resolve to regular files;
- selected provider endpoints must be absolute HTTP or HTTPS URLs without user
information, query strings, or fragments;
- JSON documents and successful provider responses must contain exactly one
value, and successful provider bodies are limited to 16 MiB; and
- excessively deep or expansive JSON-compatible values fail with ordinary
validation errors.
These are compatibility corrections and safety boundaries rather than new
consumer configuration requirements. Consumers relying on an invalid or
ambiguous input should correct that input before upgrading.
## Upgrade
Update the module dependency with:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.6.0
go mod tidy
```
Run the consuming project's ordinary and race-enabled tests after upgrading.
Applications with custom prompt/profile sources, local provider endpoints,
unusual artifact paths, or assertions over provider error identities should
pay particular attention to the compatibility notes below.
## Source Loading And Identity
Prompt definitions now share one source-neutral selection and normalization
flow across operating-system and `fs.FS` sources. YAML `id` and `version`
metadata are authoritative; filenames do not create a second identity system.
Only selected content bodies are loaded, malformed unrelated definitions do
not shadow valid exact matches, and per-file read failures are reported as
prompt-load failures rather than false absence.
File-backed profiles likewise use normalized YAML IDs, reuse their metadata
read for selected strict decoding, and avoid fully decoding unrelated files.
Selected malformed definitions remain authoritative and do not silently fall
through to a lower-precedence source.
Prompt `content_file` paths are opened exactly as declared after a separate
blank check. They must remain relative to and contained by the configured
prompt source root, including across operating-system symlinks.
See the [framework source and identity reference](../formats.md) and
[internal source overview](../internal/sources.md) for the current contracts.
## Validation, Cancellation, And Efficiency
JSON Schema documents preserve exact JSON-number representations. Schema
resource URLs safely escape legal filesystem names, and each operation loads
and compiles its schema graph once. `Run` and prepared execution reuse that
operation-local plan; Promptkit does not introduce a cross-operation cache.
Artifact reading, rendering, schema loading, compilation, and validation now
check cancellation at the synchronous boundaries Promptkit controls. Rendering
memoizes each artifact's text within one render operation, while plain JSON
validation avoids materializing an unnecessary generic tree.
The shared JSON-compatible-value owner now limits nesting and produced work so
unsafe consumer-controlled structures return errors instead of risking
unbounded recursion or allocation. See the
[architecture policy](../policy/architecture.md) for invariant ownership and
the [format reference](../formats.md) for validation behavior.
## Provider Transport Hardening
OpenAI-compatible endpoints are parsed and composed structurally, including
nested base paths. Underlying transport cancellation and deadline errors remain
discoverable with `errors.Is` through Promptkit's generation error category.
Successful provider bodies are read with a fixed 16 MiB bound and must contain
exactly one JSON response object followed only by whitespace. Oversized,
truncated, malformed, or multiply framed responses fail without returning a
partial result. See the
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
for the canonical request, endpoint, error, and response behavior.
## Public API Changes
None.
## Consumer Action
- Correct any configuration or request that depends on the formerly permissive
cases described under Compatibility.
- Confirm custom local provider endpoints are absolute HTTP or HTTPS base URLs
without credentials, queries, or fragments.
- Confirm prompt content paths remain within their configured source root and
file artifacts resolve to regular files.
- Run ordinary and race-enabled consumer tests after updating the dependency.

155
docs/releases/v0.7.0.md Normal file
View File

@@ -0,0 +1,155 @@
# Promptkit v0.7.0
This supplemental changelog and migration guide summarizes the consumer-facing
changes from `v0.6.0` to `v0.7.0`. The annotated `v0.7.0` tag is the
authoritative release record. Exact current contracts belong to the linked
GoDoc and durable documentation.
## Summary
`v0.7.0` expands provider integration and profile composition while making
credential and generation-failure handling more flexible:
- Promptkit now includes the `rakestrawhome` backend and its Gemma profile;
- built-in generation failures expose bounded structured provider details;
- an unavailable optional API-key environment source no longer prevents a
request from reaching an upstream that permits unauthenticated access; and
- profiles can inherit from and selectively refine another profile.
## Compatibility
This release adds public declarations and fields but removes none. Existing
keyed configuration literals and ordinary `errors.Is` handling continue to
work.
Adding `BaseProfileID` to `Profile` and `OpenAICompatibleProfileConfig` changes
their struct shape. Consumers using positional composite literals for either
type must convert them to keyed literals. Existing keyed literals require no
change.
The `rakestrawhome` backend ID is now built in and reserved. A consumer that
previously registered that exact ID with `WithBackend` must remove its manual
registration before upgrading. Other custom backend registrations are
unchanged.
When an optional backend, profile, or request `APIKeyEnv` is unset, empty, or
whitespace-only, the built-in client now omits `Authorization` and sends the
request. Previously this condition could fail before transport. Set
`Profile.APIKeyRequired` when missing credentials must remain a local
preflight error.
Provider non-success responses continue to match `ErrLLMGenerate`. Their
rendered wording is not a compatibility contract; consumers can now use
`errors.As` with `*GenerationError` when structured status information is
needed.
## Upgrade
Update the module dependency with:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.7.0
go mod tidy
```
Remove any manual `rakestrawhome` backend registration, convert positional
profile literals to keyed literals, and run the consuming project's ordinary
and race-enabled tests.
## Rakestrawhome Built-In Backend And Profile
Every engine now includes the reserved `rakestrawhome` backend, identified by
`BackendRakestrawHome`. The built-in `rakestrawhome-gemma-4-31b` profile
selects that backend. Consumers can use the maintained endpoint, credential,
capacity, and model defaults without registering either definition themselves.
See the [built-in backend and profile catalogs](../formats.md#built-in-backends)
and the [consumer adoption example](../consumers/pkg-promptkit.md#use-the-rakestrawhome-built-in-profile)
for the current contracts.
## Structured Generation Errors
Non-2xx responses from the built-in OpenAI-compatible client now return an
immutable `*GenerationError`. Consumers can inspect the HTTP status and any
safely extracted provider code, type, or message while retaining the ordinary
generation-error category:
```go
var generationErr *promptkit.GenerationError
if errors.As(err, &generationErr) {
status := generationErr.StatusCode()
_ = status
}
```
Provider fields are bounded and normalized but remain untrusted and may
contain sensitive request or schema details. Default and Go-syntax formatting
omit those fields. Applications must apply their own disclosure policy before
logging or presenting accessor values.
See the [`GenerationError` GoDoc](../../generation_error.go), the
[consumer error-handling guide](../consumers/pkg-promptkit.md#handle-errors),
and the [OpenAI-compatible response contract](../integrations/openai-compatible-chat.md#response-handling).
## Optional Credential Sources
`APIKeyEnv` names an optional environment lookup source unless the selected
profile explicitly sets `APIKeyRequired`. When neither a direct request key nor
a usable environment value exists, the built-in client omits the bearer header
and handles the upstream response normally. This supports local and other
OpenAI-compatible providers that permit unauthenticated requests without
hiding an authentication error returned by a provider that requires one.
The [credential format reference](../formats.md#credentials), the
[`Backend` GoDoc](../../backends.go), the
[`ExecutionTargetOverride` GoDoc](../../types.go), and the
[authentication integration contract](../integrations/openai-compatible-chat.md#authentication)
define the current precedence and availability rules.
## Profile Inheritance
YAML profiles can name one parent with `base_profile`; in-memory profiles use
`Profile.BaseProfileID`, and `OpenAICompatibleProfileConfig` forwards the same
field. A profile can act as an application-owned alias of a built-in or refine
selected inherited settings:
```go
promptkit.WithProfiles(promptkit.Profile{
ID: "weather-light",
BaseProfileID: "deepseek-4-flash",
ReasoningEffort: "high",
})
```
Base lookup observes the existing source precedence. Chains are linear,
cycle-safe, and resolved afresh for ordinary operations. Prepared execution
freezes the fully resolved target. The selected leaf ID remains public while
effective execution settings reflect the resolved chain.
See the [profile inheritance format reference](../formats.md#profile-inheritance),
the [consumer alias example](../consumers/pkg-promptkit.md#alias-a-built-in-profile),
and the [`Profile` GoDoc](../../types.go) for exact merge and validation
behavior.
## Public API Changes
The release adds:
- `BackendRakestrawHome`;
- `GenerationError`, including `StatusCode`, `ProviderCode`, `ProviderType`,
`ProviderMessage`, `Error`, `GoString`, and `Unwrap`;
- `Profile.BaseProfileID`; and
- `OpenAICompatibleProfileConfig.BaseProfileID`.
No public declaration was removed.
## Consumer Action
- Remove a manual backend registration whose ID is exactly `rakestrawhome`.
- Convert positional `Profile` or `OpenAICompatibleProfileConfig` literals to
keyed literals.
- Set `Profile.APIKeyRequired` where a missing credential must fail locally
instead of reaching the provider unauthenticated.
- Treat `GenerationError` provider fields as untrusted and potentially
sensitive when adopting the new accessors.
- Run consumer ordinary and race-enabled tests after updating the module.

109
docs/releases/v0.8.0.md Normal file
View File

@@ -0,0 +1,109 @@
# Promptkit v0.8.0
This supplemental changelog and migration guide summarizes the consumer-facing
changes from `v0.7.0` to `v0.8.0`. The annotated `v0.8.0` tag is the
authoritative release record. Exact current contracts belong to the linked
GoDoc and durable documentation.
## Summary
`v0.8.0` activates Promptkit's bounded output-repair workflow:
- failed nonempty-text, JSON, and JSON Schema validation can make a limited
number of corrective model calls;
- corrective calls preserve the original rendered conversation, effective
target, session, structured-output contract, and backend capacity policy;
- results report cumulative usage and the number of corrective calls actually
made; and
- explicitly empty OpenAI-compatible response content now reaches output
validation instead of being classified as a malformed provider envelope.
## Compatibility
This release adds no public declarations or fields and removes none. Existing
source code remains source-compatible.
The behavior of the existing `OutputContract.RepairAttempts` field and prompt
YAML `repair_attempts` field has changed. A positive value now authorizes real
additional model calls after eligible validation failures; earlier releases
accepted the field but the public engine remained single-pass. Consumers that
set a positive value should expect additional latency, token usage, and
provider cost when repair is needed.
Repair budgets must now be between zero and three. A positive budget requires
`basic`, `json`, or `json_schema` validation. Values above three and a positive
budget paired with `none` are invalid contracts rather than ignored settings.
An explicitly present empty or whitespace-only string returned by the built-in
OpenAI-compatible client is now a completed generation candidate. `none`
validation permits it, while `basic`, `json`, and `json_schema` classify it
under their ordinary validation rules and may repair it when configured.
Missing, `null`, or non-string content remains a malformed provider response.
## Upgrade
Update the module dependency with:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.8.0
go mod tidy
```
Review every prompt definition and request override that sets a positive repair
budget. Use zero or omit the field to retain single-pass execution. Ensure each
positive budget is no greater than three and uses an eligible validation mode,
then run the consuming project's ordinary and race-enabled tests.
## Bounded Output Repair
`repair_attempts` counts corrective calls in addition to the initial model
call. Promptkit validates each completed candidate, stops at the first valid
one, and never exceeds the configured bound. If every candidate remains
invalid, the run completes successfully with the final candidate and its
failed validation result rather than returning an operational error.
Each correction starts from the original rendered messages and includes only
the latest invalid candidate and latest validation diagnostics. JSON Schema
mode retains the provider-native structured-output request as its first line of
defense. Promptkit performs only deterministic structural validation; a valid
response is not necessarily factual or correct for an application's domain.
Usage in the final result is cumulative across the initial response and every
completed corrective response. `ValidationResult.RepairAttempts` reports the
number of corrective calls actually made. Corrective generation failures use
the same public generation-error categories and structured provider details as
an initial generation failure.
See the [output-contract format reference](../formats.md#output-contract), the
[consumer repair example](../consumers/pkg-promptkit.md#repair-a-structured-result),
and the [`OutputContract` and `ValidationResult` GoDoc](../../types.go) for the
current contracts.
## Explicit Empty Content
The built-in OpenAI-compatible client now distinguishes an explicitly present
empty string from a missing or malformed `content` field. This aligns built-in
and injected clients by letting the selected output contract decide whether an
empty candidate is acceptable, invalid, or eligible for repair.
See the
[OpenAI-compatible response contract](../integrations/openai-compatible-chat.md#response-handling)
for the exact envelope behavior.
## Public API Changes
None. This release activates and tightens the documented behavior of existing
fields.
## Consumer Action
- Remove or set `repair_attempts` to zero where execution must remain
single-pass.
- Keep every positive repair budget at three or fewer and pair it with
`basic`, `json`, or `json_schema` validation.
- Account for additional latency, usage, and provider cost when enabling
repair.
- Continue checking the returned validation status because bounded repair can
exhaust without producing a valid candidate.
- Review workflows that previously treated explicit empty provider content as
a generation error.

125
docs/releases/v0.9.0.md Normal file
View File

@@ -0,0 +1,125 @@
# Promptkit v0.9.0
This supplemental changelog and migration guide summarizes the consumer-facing
changes from `v0.8.0` to `v0.9.0`. The annotated `v0.9.0` tag is the
authoritative release record. Exact current contracts belong to the linked
GoDoc and durable documentation.
## Summary
`v0.9.0` adds stateless request-message composition and separates maintained
provider data from Promptkit's core implementation:
- callers can append already-rendered messages to a configured prompt for
application-owned conversations and semantic correction workflows;
- the public package now publishes constants for the four supported text-chat
roles, and prompt definitions use the same normalized role vocabulary; and
- the OpenRouter and Rakestrawhome backend/profile catalogs now come from two
independently versioned Go module dependencies.
## Compatibility
This release adds one field and four constants to the public API and removes no
public declarations. Existing keyed `RunRequest` literals that omit
`AppendedMessages` retain their behavior. Adding the field changes the struct
shape, so consumers using positional `RunRequest` literals must convert them
to keyed literals.
Message roles in prompt definitions are now trimmed, lowercased, and required
to be `developer`, `system`, `user`, or `assistant`. Definitions using another
role that earlier releases accepted as an arbitrary nonblank string now fail
prompt loading. In particular, the text-only message contract does not support
`tool` or the deprecated `function` role. Otherwise valid roles with different
case or surrounding whitespace are normalized rather than rejected.
The catalog extraction preserves Promptkit's public API, built-in backend and
profile IDs, configuration, precedence, credential handling, and capacity
behavior. Consumers do not import or register either catalog themselves, and
no configuration migration is required. Promptkit now selects two
independently versioned data dependencies and guarantees only the catalog
versions selected and tested by this Promptkit release.
## Upgrade
Update the module dependency with:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.9.0
go mod tidy
```
Convert any positional `RunRequest` literals to keyed literals. Review prompt
definitions for unsupported roles, then run the consuming project's ordinary
and race-enabled tests.
## Appended Request Messages
`RunRequest.AppendedMessages` accepts caller-owned `RenderedMessage` values
that Promptkit validates, copies, and appends after every rendered
prompt-definition message in caller order. Promptkit does not template this
content, retain conversation state between calls, impose a retry policy, or
apply a message-count, byte-size, token, or context-window limit. Upstream
rejections continue through the ordinary generation-error boundary.
This primitive supports application-owned conversation continuations and
domain-aware correction loops while preserving Promptkit's existing
preparation, hashing, prepared-execution, structural repair, backend-capacity,
credential, and cancellation behavior. Appended content can include sensitive
model output or application feedback; prepared values expose the complete
effective messages by design, while default request formatting reports only
the appended-message count.
See the
[consumer appended-message example](../consumers/pkg-promptkit.md#append-already-rendered-messages),
the [`RunRequest`, `RenderedMessage`, and `CacheControl` GoDoc](../../types.go),
and the [OpenAI-compatible request contract](../integrations/openai-compatible-chat.md#request-body)
for current behavior.
## Supported Message Roles
The new `RoleDeveloper`, `RoleSystem`, `RoleUser`, and `RoleAssistant`
constants identify the complete role vocabulary accepted by Promptkit's
text-chat message model. The same validation and normalization now apply to
prompt-definition messages and request-supplied appended messages. Promptkit
does not translate between roles; provider- or model-specific rejection of an
otherwise supported role remains an upstream generation error.
See the [message format reference](../formats.md#messages-and-templates) for
the canonical prompt-definition contract.
## Independently Versioned Backend Catalogs
Promptkit imports immutable catalog data for its maintained OpenRouter and
Rakestrawhome backends and profiles. Engine construction validates and
assembles both catalogs behind the existing built-in registry and profile
precedence rules. Promptkit no longer keeps duplicate embedded profile assets
or hard-coded definitions for those maintained backends.
The module versions in Promptkit's `go.mod` identify the catalog releases
tested with this release. The [built-in backend and profile format
reference](../formats.md#built-in-backends) remains the canonical consumer
contract, while the [internal source documentation](../internal/sources.md#profiles-and-built-ins)
describes the dependency boundary.
## Public API Changes
The release adds:
- `RunRequest.AppendedMessages`;
- `RoleDeveloper`;
- `RoleSystem`;
- `RoleUser`; and
- `RoleAssistant`.
No public declaration was removed.
## Consumer Action
- Convert positional `RunRequest` literals to keyed literals.
- Replace unsupported prompt-definition roles with an appropriate supported
role, or keep richer tool-call protocols in an application-owned client.
- Treat appended messages and prepared effective messages according to the
application's sensitive-data policy.
- Do not add direct catalog imports or registration calls; existing Promptkit
construction and configuration remain correct.
- Run consumer ordinary and race-enabled tests after updating the module.

View File

@@ -1,635 +0,0 @@
# Codebase Audit Sequence
## Purpose
This document defines the staged sequence for auditing Promptkit before further
feature development. The audit is intended to identify high-confidence
opportunities to improve correctness, efficiency, duplication, implementation
clarity, and test-suite quality without changing production behavior during the
review itself.
The audit findings belong in `audit.md`. A later, separate planning pass will
translate accepted findings into a staged remediation plan in
`implementation.md`. Neither this sequence nor the findings log owns current
behavior; the canonical sources identified by the
[documentation policy](../policy/documentation.md) remain authoritative.
Each stage below is deliberately scoped for one LLM coding-agent prompt. Run
the stages in order and do not combine them. A stage may discover a concern
outside its scope, but it should record that concern for the owning later stage
rather than expanding its own review.
## Governing Policies And Boundaries
Every stage must follow:
- the [development guide](../development.md), including its task-specific
reading guide;
- the [architecture policy](../policy/architecture.md), especially the public
facade, internal-package, dependency-direction, and consumer boundaries;
- the [testing policy](../policy/testing.md), including its risk-based,
behavior-oriented standard; and
- the [documentation policy](../policy/documentation.md), including canonical
ownership and the temporary nature of roadmap documents.
This is an audit, not an implementation pass:
- Do not change production code, tests, examples, fixtures, public contracts,
or current-state documentation.
- Limit repository edits to the audit artifacts explicitly authorized for the
stage.
- Do not silently repair an issue while investigating it.
- Do not treat coverage, complexity, similarity, lint, or graph output as a
finding without confirming the underlying behavior in source and tests.
- Do not recommend centralization merely because code looks similar. The code
must implement the same semantic rule, and consolidation must improve
ownership or reduce a credible drift risk.
- Do not recommend performance work without identifying a relevant execution
path and establishing a defensible cost model, measurement, or complexity
problem.
- Preserve unrelated working-tree changes. Record the audit baseline rather
than requiring an otherwise unrelated dirty tree to be cleaned.
## Finding Standard
Record each actionable finding in `audit.md` with:
- a stable ID in the form `SNN-FNN`, where the first number is the stage;
- category: correctness, efficiency, duplication, clarity, testing, or
contract-documentation consistency;
- severity: critical, high, medium, or low;
- confidence: confirmed, high, medium, or low;
- affected packages, files, symbols, and tests;
- the contract, invariant, policy, or maintenance concern at issue;
- concrete evidence and a concise explanation of the failure mode or cost;
- the recommended direction, without implementation-level sequencing;
- the verification or regression protection that remediation would require;
and
- status: accepted, deferred, rejected, superseded, or resolved.
Use **confirmed** confidence when the problem is reproduced or follows
unavoidably from a complete trace. Use **high** confidence when direct source
and test evidence establishes the problem but a safe reproduction is not
practical. Medium- and low-confidence concerns belong in a separate
observations section until a later stage confirms or rejects them; they must
not enter the remediation plan as if they were findings.
Severity describes impact, not implementation effort:
- **Critical:** credible data disclosure, data corruption, deadlock, unbounded
resource consumption, or a broadly unusable public contract.
- **High:** violation of an important public contract or invariant, a likely
concurrency or resource-lifecycle defect, or a failure with substantial
downstream impact.
- **Medium:** a real but narrower behavioral defect, meaningful avoidable cost,
duplicated policy with credible drift risk, or a material testing gap.
- **Low:** a bounded clarity, maintainability, or testing-friction problem with
a concrete improvement and little behavioral risk.
When a reviewed area yields no finding, record the important behavior or risk
that was inspected and found adequately implemented or tested. This coverage
ledger prevents later reviewers from mistaking silence for omission.
## Per-Stage Procedure
Unless a stage says otherwise, its single agent prompt should:
1. Read the required policies, focused internal documentation, production
files, and tests for that stage.
2. Use the code knowledge graph for symbol discovery, callers, callees, and
cross-package traces; confirm important conclusions against source.
3. Trace normal, boundary, and failure paths through the narrowest relevant
public or package contract.
4. Review correctness, meaningful runtime cost, semantic duplication,
responsibility clarity, and the value and ownership of tests in scope.
5. Run the narrowest existing tests needed to validate conclusions. Use
race-enabled or repeated focused tests when concurrency or nondeterminism is
in scope. Do not add permanent tests during the audit.
6. Add the stage result to `audit.md`: accepted findings, unresolved
observations, areas verified, commands run, and any handoff to a later
stage.
7. Recheck the working tree and confirm that only the authorized audit artifact
changed.
## Stage 0: Initialize The Audit And Establish The Baseline
Create `audit.md` and establish a reproducible starting point before reviewing
individual components.
Record:
- the audited commit, branch, Go version, module identity, and working-tree
state;
- unrelated pre-existing changes that all later stages must preserve;
- the implemented package and public-facade inventory;
- the baseline validation results; and
- the finding template, status vocabulary, and coverage ledger used by later
stages.
Refresh the code knowledge graph for the recorded commit. Run the repository's
ordinary tests, race tests, vet, build, maintained offline preparation example,
Go formatting check, Markdown link check, and repository-hygiene checks. Run
package coverage once as a diagnostic and record the result without defining a
coverage target or committing generated output. Measure coarse package test
duration only if it can be done without adding tooling or changing tests.
Compare the validation requirements stated by the testing policy, development
guide, and release procedure. Record a finding if their ownership or command
sets are materially inconsistent; do not edit those documents in this stage.
**Exit condition:** `audit.md` contains the baseline, ledger structure, and
validation result, and no component-level audit has begun.
## Stage 1: Public Values, Conversion, Errors, And Formatting
Review the root facade's public request, result, inspection, prepared-run, and
error values together with public-to-internal and internal-to-public
conversion. Scope the review to `doc.go`, `types.go`, `convert.go`, `errors.go`,
`capacity_error.go`, `formatting.go`, and `prepared_execution.go`, plus the
directly relevant portions of root tests.
Focus on:
- zero-value and nil behavior;
- defensive copying, aliasing, and immutable snapshots;
- lossless conversion and field precedence;
- error identity through `errors.Is` and `errors.As`;
- containment of internal representations;
- safe `String`, `GoString`, and diagnostic formatting;
- accidental disclosure of credentials, prompt content, generated content, or
other private state; and
- conversion or copying logic that represents the same rule in multiple
places.
Review only tests that own these value and boundary contracts. Defer engine
assembly, execution coordination, and transport behavior to their later
stages.
**Exit condition:** all root value-conversion and error-formatting paths have a
recorded audit result without evaluating engine orchestration.
## Stage 2: Public Configuration And Extension Adapters
Review the smaller public construction and extension surfaces in
`backends.go`, `profiles.go`, `artifact_reader.go`, `json.go`, and
`llm_adapter.go`, together with their directly relevant root and internal
adapter tests.
Focus on:
- validation performed at the public boundary;
- ownership and copying of caller-supplied maps, slices, filesystems, readers,
and clients;
- adapter error propagation and cancellation;
- consistency between convenience constructors and general configuration;
- whether extension interfaces are as narrow as their consumers require;
- whether public helpers duplicate internal policy or merely translate it;
and
- whether tests protect consumer-visible behavior rather than private adapter
choreography.
Do not review how `NewEngine` combines these values; that belongs to Stage 3.
**Exit condition:** every non-engine public configuration helper and adapter
has a recorded result and any assembly questions are handed to Stage 3.
## Stage 3: Engine Construction, Options, And Source Assembly
Review the construction and configuration portions of `engine.go` and the
corresponding tests in `engine_test.go`. Limit the scope to `NewEngine`, option
application, dependency defaults, backend registration, profile and prompt
source composition, fallback-profile placement, validator and client
selection, capacity-manager construction, and construction-time validation.
Focus on:
- deterministic option precedence;
- required versus optional dependencies;
- isolation between engine instances;
- freezing or copying consumer configuration at the correct boundary;
- correct dependency direction and absence of process-global mutable state;
- failure atomicity and useful public errors;
- consistency between configured backends and capacity policies; and
- assembly logic that is repeated or split across unclear owners.
Do not audit the runtime behavior of `Run`, `Prepare`, or inspection methods;
that belongs to Stage 4 and the internal use-case stages.
**Exit condition:** engine construction and source assembly are fully accounted
for, including tests, without expanding into runtime orchestration.
## Stage 4: Engine Operations And Root Contract Coverage
Review the remaining public methods in `engine.go` and their directly relevant
root tests, including the external-package contracts in
`public_contract_test.go` and `prepared_execution_contract_test.go` only where
they exercise the engine boundary under review.
Focus on:
- request translation and context propagation;
- ordinary run, preparation, inspection, and prepared-execution entry points;
- public error mapping and preservation of injected dependency errors;
- result and prepared-state ownership;
- consistency between method and package-level convenience functions;
- public behavior that is asserted redundantly in root internal tests and
external-package contract tests; and
- important public behavior that is tested only through internal packages.
Treat internal runner, transport, validation, and capacity mechanics as black
boxes in this stage. Hand questions about their implementation to their owning
later stages.
**Exit condition:** the public execution boundary and its contract-test
ownership are recorded without duplicating internal component audits.
## Stage 5: Internal Domain And JSON-Compatible Values
Review `internal/domain` and `internal/jsonvalue`, including all of their tests.
Focus on:
- domain invariants and invalid states;
- session normalization;
- prepared-run and schema immutability;
- deep-copy correctness for every supported JSON-compatible shape;
- numeric-type preservation and rejection policy;
- cycles, excessive nesting, unsupported values, and nil distinctions;
- avoidable repeated copying on execution paths; and
- whether generic value machinery has a single clear owner.
Trace important callers to confirm that these packages enforce the invariants
their consumers assume, but do not audit the callers' broader behavior.
**Exit condition:** shared value semantics and their test ownership are fully
recorded.
## Stage 6: Backend Registry, Defaults, And Built-In Profiles
Review `internal/backend`, `internal/defaults`, and
`internal/profile/builtin`, including their focused tests and the relevant
backend-policy traces into engine assembly and the LLM reserved-field rule.
Focus on:
- immutable registry construction and lookup;
- built-in versus consumer ID collision rules;
- endpoint, credential-environment, header, parameter, and concurrency
validation;
- defensive copies at registry boundaries;
- application-neutral default ownership;
- built-in profile/backend consistency;
- reserved request-field ownership without dependency inversion; and
- duplicated validation or default policy across public and internal layers.
Defer scheduling mechanics to Stage 15 and actual HTTP request construction to
Stage 14.
**Exit condition:** registry and default-policy correctness are recorded, with
transport and scheduling questions handed to their owning stages.
## Stage 7: File Discovery And Prompt Definitions
Review `internal/filecatalog` and `internal/promptdef`, including their tests
and fixtures. Read the framework format reference and internal source document
before evaluating behavior.
Focus on:
- deterministic discovery and duplicate handling;
- filesystem and `fs.FS` parity;
- root and relative-path normalization;
- strict YAML decoding and version selection;
- prompt ID, message, input, cache-control, and validation declarations;
- inline versus file-backed content rules;
- containment of referenced files where promised;
- malformed input and contextual error behavior;
- unnecessary repeated directory scans or file reads; and
- fixture and case duplication that does not protect distinct parser risks.
Do not audit rendering, artifact loading, profile loading, or schema validation
in this stage.
**Exit condition:** discovery and prompt-definition parsing have complete
findings and coverage-ledger entries.
## Stage 8: Profile Sources And Repository Composition
Review `internal/profile` excluding its built-in subpackage, including all
repository tests and profile fixtures. Read the profile format contract first.
Focus on:
- strict decoding and profile validation;
- filesystem and `fs.FS` parity;
- repository overlay and fallback precedence;
- distinction between absence and a malformed authoritative source;
- preservation of useful error identity and context;
- conversion to immutable execution profiles;
- duplicate IDs and deterministic selection;
- repeated parsing, validation, or copying; and
- whether tests at repository, engine, and public-contract layers have clear,
nonduplicative ownership.
Defer resolution of a profile with runtime overrides and backend definitions to
Stage 11.
**Exit condition:** profile-source and repository-composition behavior are
fully recorded.
## Stage 9: Artifact Loading And Prompt Rendering
Review `internal/artifact` and `internal/prompt`, including all focused tests.
Read the internal source document and format reference first.
Focus on:
- inline and file artifact ownership, metadata, hashing, and error behavior;
- copied versus shared byte storage;
- caller-selected path semantics and architecture-policy boundaries;
- template parsing and execution;
- artifact, variable, session, and cache-control rendering;
- missing, extra, nil, and malformed input behavior;
- deterministic output and safe diagnostics;
- unnecessary repeated reads, hashes, parses, or allocations on common paths;
and
- tests coupled to incidental template or struct implementation.
Do not audit the runner's decision about when rendering occurs.
**Exit condition:** input materialization and rendering are accounted for
through their package boundaries.
## Stage 10: Output Validation And Frozen Validation Plans
Review `internal/validate`, including all tests, schema fixtures used by the
root contract suite, and traces from preparation into frozen validation plans.
Read the format and internal source documents first.
Focus on:
- basic, JSON, and JSON Schema mode semantics;
- schema-path resolution and filesystem/`fs.FS` parity;
- schema compilation, transitive references, and source-lifetime independence;
- output normalization and preservation;
- malformed schema and malformed model-output errors;
- thread safety of reusable validators and prepared plans;
- expensive recompilation or copying on repeated execution; and
- whether parser, validator, runner, and public tests each own distinct risks.
Do not audit repair decisions or provider request construction.
**Exit condition:** validation behavior, plan lifetime, and focused test value
are fully recorded.
## Stage 11: Inspection And Execution-Target Resolution
Review `internal/usecase/profile_inspection.go`,
`internal/usecase/prompt_inspection.go`, and the preparation and target-
resolution portions of `internal/usecase/runner.go`, together with their
focused tests. Use graph traces to define the exact helper and call-path scope
before reviewing.
Focus on:
- prompt and profile selection;
- backend lookup and endpoint overrides;
- reasoning, session, and other runtime precedence;
- merge semantics for default, profile, backend, and per-run values;
- inspection fidelity versus actual execution;
- credential-name versus credential-value handling;
- prompt-definition and schema freezing during preparation;
- stable error identity and context; and
- duplicated resolution rules across inspection, preparation, and execution.
Do not review model invocation, repair execution, or prepared-handle lifecycle;
those belong to Stages 12 and 13.
**Exit condition:** all selection, merge, inspection, and preparation rules are
traced and recorded once.
## Stage 12: Ordinary Execution, Validation, And Repair Coordination
Review `internal/usecase/runner.go`, `internal/usecase/repairer.go`, and
`internal/usecase/capacity_error.go` only for the ordinary execution path after
preparation, together with the corresponding sections of `runner_test.go`.
Use the Stage 11 resolution result as an established input rather than
reauditing it.
Focus on:
- rendering, generation, validation, and optional repair transitions;
- context cancellation and dependency-error propagation;
- partial result and usage accounting;
- exact attempt count and repair eligibility;
- avoidance of unintended retries;
- capacity-error translation;
- cleanup and failure behavior on every exit path;
- repeated orchestration or request construction; and
- oversized tests, helpers, or case matrices that obscure distinct behavior.
Treat LLM transport and capacity scheduling as injected package contracts;
their mechanics belong to Stages 14 and 15.
**Exit condition:** the ordinary execution state machine and its test ownership
are fully recorded.
## Stage 13: Prepared Execution Lifecycle
Review `internal/usecase/prepared_execution.go`, its focused tests, and the
prepared-execution portions of the root facade and external contract tests.
Do not repeat the public value review from Stages 1 and 4 or the resolution
review from Stage 11.
Focus on:
- single-attempt or other lifecycle guarantees;
- concurrent use and synchronization;
- discard behavior and resource release;
- frozen source, target, credential, capacity, timing, and schema semantics;
- independence of returned details and results;
- context and error behavior;
- consistency between ordinary and prepared execution where promised;
- private-state containment in formatting; and
- redundant assertions across internal, root, and external-package tests.
Run focused race tests and repeated tests for lifecycle behavior where useful.
**Exit condition:** prepared execution has one complete lifecycle analysis and
a clear map of which test layer owns each guarantee.
## Stage 14: OpenAI-Compatible Transport
Review `internal/llm`, including all transport tests. Read the
OpenAI-compatible integration contract and internal LLM document first.
Focus on:
- request endpoint, headers, authentication, and JSON body construction;
- omission versus explicit zero-value behavior;
- reserved-field enforcement and extra-parameter collision handling;
- session ID and reasoning encoding;
- structured-output and cache-control translation;
- client and per-generation deadlines;
- cancellation, body closure, bounded response reads, and decode failures;
- non-success HTTP response behavior;
- response choices, usage, and malformed-success handling;
- wire-visible compatibility and safe error disclosure;
- unnecessary marshaling, copying, or buffering; and
- whether the large transport test file can be simplified without losing
protocol-risk coverage.
Use `httptest`-based existing tests; do not contact a live provider.
**Exit condition:** every outbound and inbound wire path has a recorded result,
including focused test ownership.
## Stage 15: Capacity, Admission, And Concurrency
Review `internal/capacity`, its tests, `capacity_contract_test.go`, and the
integration points already identified in engine and use-case stages. Read the
internal capacity document first.
Focus on:
- bounded run admission and queue-capacity enforcement;
- per-backend limited and unlimited scheduling;
- FIFO behavior and cancellation-safe waiter removal;
- permit release on success, error, panic-relevant boundaries, and
cancellation;
- goroutine, timer, and waiter lifecycle;
- starvation, deadlock, race, and engine-isolation risks;
- lock scope and meaningful contention or allocation costs;
- preservation of injected-client concurrency where promised;
- relational testing of configured limits rather than duplicated defaults;
and
- duplication between internal concurrency tests and public contract tests.
Run focused ordinary, race-enabled, and repeated tests. Repetition must remain
bounded and diagnostic; a test that passes many times is not proof of
correctness without a source-level synchronization analysis.
**Exit condition:** concurrency invariants have both a source trace and a
test-ownership assessment.
## Stage 16: Repository-Wide Test Strategy And Maintained Examples
Perform a suite-level review after every component has been audited. Review
the testing policy, test inventory, fixtures, external-package root tests,
`architecture_test.go`, and both maintained examples. Use the component-stage
coverage ledger instead of repeating every individual test assertion.
Construct a risk-to-owner matrix for:
- public compatibility and error identity;
- parsing, validation, and serialization;
- immutability and data integrity;
- external wire behavior;
- cancellation, failure propagation, and recovery;
- concurrency and resource lifecycle; and
- representative assembled consumer workflows.
Identify only evidence-backed cases of:
- consequential behavior with no credible test owner;
- the same semantic rule asserted redundantly at several layers;
- tests coupled to private helpers, internal constants, exact noncontractual
wording, or collaborator choreography;
- low-value or obsolete cases whose lifetime cost exceeds their protection;
- missing failure, cancellation, race, or boundary coverage;
- nondeterminism, shared state, environment dependence, fixed ports, or test
ordering assumptions;
- helpers and fixtures whose complexity is not justified; and
- maintained examples that duplicate one another without protecting distinct
workflows.
Use coverage and timing only to direct attention. Do not propose tests solely
to raise percentages or remove tests solely to shorten the suite.
**Exit condition:** every important risk has a named test owner or an accepted
finding, and every proposed test deletion or consolidation states what
protection remains.
## Stage 17: Cross-Cutting Duplication, Efficiency, And Architecture Review
Review the codebase as a whole using the completed component findings, graph
traces, complexity signals, similarity signals, and package dependency map.
Do not reopen settled package behavior without new cross-cutting evidence.
Focus on:
- one semantic policy implemented by multiple packages;
- repeated public/internal transformations with credible drift risk;
- interfaces broader than their actual consumers;
- responsibilities split across packages or concentrated in the facade
contrary to the architecture policy;
- repeated parsing, copying, schema compilation, request construction, or
source traversal on important paths;
- avoidable lock contention or serial work supported by the concurrency audit;
- abstractions that add indirection without enforcing a boundary; and
- discrepancies between implemented package responsibilities and their
canonical architecture or internal documentation.
For each possible consolidation, state why the code represents one rule, which
package should own it, and why the resulting dependency direction remains
valid. For each efficiency finding, state the path frequency, input scale,
complexity or measurement evidence, and the benchmark or invariant needed to
verify a remediation.
**Exit condition:** all cross-cutting opportunities are either accepted with
high confidence, retained as explicitly lower-confidence observations, or
rejected with a short rationale.
## Stage 18: Consolidate And Close The Audit
Perform a findings-only synthesis. Do not change code and do not write the
remediation plan yet.
- Recheck every accepted finding against the final audited tree.
- Merge duplicates and mark superseded IDs without erasing their history.
- Separate shared root causes from downstream symptoms.
- Confirm that every accepted item is confirmed or high confidence.
- Confirm that severity describes impact rather than effort.
- Reject speculative cleanup, coverage-driven test work, and centralization
without a clear owner or drift risk.
- Record dependencies and a recommended remediation order.
- Distinguish behavioral fixes, safe refactors, performance work, test gaps,
test consolidation, and documentation synchronization.
- Add an audit summary stating what was reviewed, what validation ran, the
accepted finding counts by category and severity, and any residual
uncertainty.
- Re-run baseline validation if audit-only investigation could have affected
repository state, and confirm that only authorized roadmap files differ from
the recorded baseline.
The recommended ordering should place correctness, data-integrity,
resource-lifecycle, and concurrency defects first; policy duplication and
missing protection for consequential behavior next; then clarity, test
consolidation, and demonstrated efficiency improvements. Actual implementation
stages must be decided in the later `implementation.md` planning pass, where
files, dependencies, acceptance criteria, and validation can be made
decision-complete.
**Exit condition:** `audit.md` is a complete, internally consistent input to a
separate remediation-planning prompt, with no code or test changes mixed into
the audit.
## Completion Criteria
The audit is complete only when:
- every production component and public boundary appears in the coverage
ledger;
- every test file and maintained example has been reviewed at its owning stage
or in the suite-wide stage;
- important cross-package paths have been traced end to end;
- concurrency-sensitive behavior has received source and race-test review;
- every accepted finding meets the evidence and confidence standard;
- lower-confidence observations are visibly separated from remediation
candidates;
- proposed test additions, deletions, and consolidations are justified against
the testing policy;
- proposed simplifications identify a durable responsibility owner;
- proposed efficiency work has a relevant cost model or measurement plan; and
- the repository remains unchanged except for the authorized audit roadmap
artifacts.

File diff suppressed because it is too large Load Diff

View File

@@ -38,33 +38,8 @@ consumers.
## Ideas ## Ideas
### Public bounded output repair No ideas are currently awaiting selection. Active feature work belongs in its
focused roadmap rather than this catalog.
After the codebase-audit remediations are complete, Promptkit should make its
bounded output-repair capability available through the public engine. A
consumer should be able to request a limited number of corrective generation
attempts when JSON or JSON Schema output fails content validation, without
having to reproduce Promptkit's generation, validation, capacity, and result-
accounting orchestration.
- Repair is validation recovery, not a general provider retry, failover, or
backoff policy. Transport failures, cancellation, and operational schema or
validation errors must retain their ordinary error behavior.
- Repair must stop after the first valid result or the configured attempt
bound. Exhausting the bound should preserve the final invalid result and its
validation diagnostics rather than inventing success.
- Initial generation and every repair attempt must use the same resolved
backend, effective execution settings and presence semantics, session,
credential boundary, structured-output contract, and backend-capacity
policy.
- Results should report the number of repair attempts and cumulative usage for
every model call made by the run.
- Ordinary and prepared execution should expose coherent behavior, including
cancellation, frozen prepared state, error identity, and capacity lifetime.
Select this work only after the accepted audit findings affecting shared
execution invariants, validation, orchestration, transport, and repair
internals have been remediated.
## Entry Format ## Entry Format

View File

@@ -1,840 +0,0 @@
# Audit Remediation Implementation Plan
## Purpose
This document is the decision-complete implementation plan for the accepted
findings in the [codebase audit](audit.md). It is written for a
`gpt-5.6-terra` coding agent that will implement one numbered stage per prompt,
in order.
The audit remains the evidence and rationale for each finding. This plan owns
implementation order, selected policy decisions, required code and test work,
and stage gates. It does not activate the future public output-repair feature
described in the [future feature catalog](future.md); it only corrects and
protects the retained internal repair machinery on which that later feature
may build.
## Implementation Policies
Every stage must follow the [development guide](../development.md),
[architecture policy](../policy/architecture.md),
[testing policy](../policy/testing.md), and
[documentation policy](../policy/documentation.md). Before changing a
subsystem, read the focused current-state documents identified by the
development guide and inspect the exact implementation and tests named by the
stage.
Apply these rules throughout:
- Implement exactly one stage per agent prompt. Do not combine stages or begin
a later stage early.
- Inspect the working tree before editing and preserve unrelated changes.
- Use the code knowledge graph to locate symbols, callers, and dependency
paths; confirm important conclusions against source.
- Keep the root package as the public facade and implementation under
`internal/`. Do not expose internal representations or add a public package.
- Put source-neutral invariants in their assigned internal owner while
preserving source-specific normalization, error classification, and public
translation at existing boundaries.
- Add regression protection at the narrowest stable owner in the same stage
as a behavioral fix. Retain only representative integration coverage at
higher layers.
- Do not add tests to raise coverage percentages. Do not preserve tests that
assert an incidental algorithm, private constant, dormant serialization
shape, or duplicated lower-layer truth table.
- Keep all tests deterministic, offline, race-safe, credential-free, and free
of fixed-port or mutable-service assumptions.
- Do not add engine-wide caches, generic facade abstractions, scheduler
changes, provider retry policy, or new consumer configuration unless a stage
explicitly requires it.
- Update canonical GoDoc and current-state documents in the same stage as the
behavior they describe. Do not describe a later stage as already
implemented.
- Format changed Go files. Run the stage's focused commands, then at least
`go test ./...` and `go vet ./...`. Run focused race tests wherever the
stage changes ownership, cancellation, shared state, or lifecycle behavior.
- Do not commit, push, tag, or publish unless separately instructed.
## Decisions Fixed By This Plan
The implementing agent must not reopen these choices:
1. **JSON-compatible numbers:** accept every value Go can faithfully encode as
a JSON number: every signed and unsigned integer width, finite `float32` and
`float64` values, and a `json.Number` whose text is valid JSON-number syntax.
Do not impose the current IEEE-754 safe-integer restriction. Reject NaN,
infinities, and malformed `json.Number` text. Preserve supported concrete
numeric types when copying.
2. **JSON-shaped traversal bounds:** allow at most 100 JSON container levels
and 100,000 produced JSON value nodes per `Copy` or `CopyMap` operation.
Count the root, each map/slice/array container, and every produced child
value; map keys are not separate nodes. Pointer and interface indirection
do not add JSON depth or an extra node. Repeated appearances of an acyclic
shared value count each produced occurrence. Continue rejecting active-path
cycles and return deterministic, path-aware validation errors on either
bound.
3. **Execution timeout bound:** a positive `TimeoutSeconds` must fit in
`time.Duration` after multiplication by `time.Second`. Derive the maximum
from `math.MaxInt64` and `time.Second`; do not duplicate its numeric literal
in tests or documentation.
4. **Output contracts:** the only valid formats are `text`, `markdown`, and
`json`; the only valid validation modes are `none`, `basic`, `json`, and
`json_schema`; repair attempts are non-negative; and `json_schema` requires
a nonblank schema path. A non-nil request replacement defaults an empty
format to `text` before shared validation. It does not default an empty
validation mode.
5. **Prompt content paths:** every `content_file` is an exact, relative path
resolved from its prompt file and contained by the configured prompt source
root. Directory, `fs.FS`, and single-file sources all reject absolute and
escaping paths. A single-file source's root is the containing directory of
that selected prompt file. Trimming determines only whether a value is
blank; it must not change the path opened. OS containment must account for
symlinks; containment inside an injected `fs.FS` remains expressed in that
filesystem's namespace.
6. **Profile IDs:** normalize file-backed IDs with `strings.TrimSpace` once,
just as in-memory IDs are normalized. Use the normalized value for
selection, duplicate detection, results, and diagnostics. A whitespace-only
ID is invalid, and IDs that become equal after normalization are
duplicates.
7. **Ordinary artifact files:** the built-in `File` reader supports regular
files, including symlinks whose targets are regular files. It rejects
directories, FIFOs, devices, sockets, and other non-regular targets before
consuming them. It remains unrestricted by an application root and does
not introduce an application-specific byte limit.
8. **Validation cancellation:** do not return early by abandoning goroutines
around `fs.FS` or the JSON Schema dependency. Promptkit must check
cancellation before, between, and after work it controls; read opened files
in context-checked chunks; and let a canceled context win before publishing
a result after synchronous decode, compile, or validation calls. Go's
`fs.FS` and the current JSON Schema library expose no general mechanism to
preempt a blocked `Open`, `Read`, compile, or validation method, so canonical
documentation must describe this synchronous limitation rather than claim
impossible asynchronous interruption.
9. **Successful provider-response limit:** the built-in OpenAI-compatible
client accepts at most 16 MiB (`16 << 20` bytes) for the complete successful
HTTP response body, including surrounding whitespace. The limit is fixed,
internal, and application-neutral. Exactly the limit is allowed; the first
byte beyond it fails as `internal/llm.ErrMalformedResponse`. Do not add a
public setting. Non-success response parsing remains outside this audit
remediation and belongs to the separate structured-generation-error
roadmap.
10. **Repair machinery:** retain and fix the internal repairer, cumulative
usage, and bounded repair state machine. The public engine must continue to
install no repairer and remain single-pass. Do not activate public repair
in this plan.
## Stage 1: Centralize Execution-Setting And Session Invariants
**Findings:** S05-F01, S17-F01, S14-F02. This stage also resolves the
source-specific evidence in S02-F02, S08-F01, and S11-F01.
Add a source-neutral execution-setting validator to `internal/domain`. It must
validate temperature, maximum tokens, top-p, and timeout on a domain execution
target: temperature and top-p must be finite and within their closed ranges,
maximum tokens must be non-negative, and timeout must be non-negative and no
greater than the derived duration-safe maximum. Keep optional-pointer presence,
profile required fields, normalization, and error wrapping outside this
validator.
Use that owner from:
- in-memory profile validation in the root package;
- OS and `fs.FS` profile validation;
- resolved request/target validation in `internal/usecase`; and
- the built-in model client as a defensive final boundary.
Remove the duplicated scalar comparisons from those callers. Preserve
`ErrInvalidConfig` for in-memory construction, profile-load identities for file
profiles, `ErrInvalidRequest` for runtime overrides, and the LLM package's
defensive invalid-request identity. Explicit numeric zero must retain its
presence semantics.
Update `internal/domain.NormalizeSessionID` to reject invalid UTF-8 before
trimming or rune counting. Preserve the existing blank and 256-code-point
rules. Direct requests must still map failures to `ErrInvalidRequest`, while
session-template failures remain renderer failures.
Add one domain-owned table for every exact setting boundary, finite neighbors,
NaN, both infinities, negative values, and the timeout representability edge.
Retain small boundary-integration cases for in-memory profiles, both file
source forms, request overrides through `Prepare` and `PrepareExecution`, and
the model-client defense. Add malformed UTF-8 session cases before, within,
and after otherwise valid content.
Update the architecture policy and internal component overview so
`internal/domain` explicitly owns source-neutral invariants for its shared
execution values, without claiming ownership of source-specific policy.
Run focused domain, profile, use-case, root, and LLM tests, including the
affected race-enabled request and profile cases, followed by the repository
test and vet gates.
## Stage 2: Centralize Output-Contract Legality
**Finding:** S17-F02, including the request-boundary symptom S11-F02.
Add one pure `internal/domain` validator for `OutputContract`. It must enforce
the format, validation-mode, non-negative repair-attempt, and JSON-Schema path
rules fixed above. It must not load schemas or apply source/request defaults.
Make prompt-definition normalization call the shared validator after its file-
specific normalization. Keep prompt-required fields and contextual
`ErrInvalidPromptDefinition` ownership in `internal/promptdef`. Make request
resolution default an empty replacement format to `text`, then call the same
validator and translate failure to `ErrInvalidRequest` before artifact,
rendering, validation, admission, or generation work. Keep schema loading and
compilation in `internal/validate`.
Add a domain table covering every supported and unsupported enum, empty values,
negative and non-negative repair counts, and schema-path relationships. Retain
small prompt-source and use-case integration tables that prove correct error
categories and parity between `Prepare` and `PrepareExecution`; do not repeat
the entire domain table at those layers.
Update the architecture and internal overview language added in Stage 1 to
include source-neutral output-contract invariants. Run focused domain,
prompt-definition, use-case, and root tests, then repository test and vet
gates.
## Stage 3: Make JSON-Compatible Value Handling Coherent And Bounded
**Findings:** S05-F02, S05-F03, S05-F04.
Refactor `internal/jsonvalue` around the numeric and traversal decisions fixed
by this plan. Remove the safe-integer restriction and apply one numeric rule to
all supported representations. Preserve concrete named and unnamed scalar,
map, slice, and array types where the existing contract promises preservation;
keep nil versus empty container distinctions and `Copy` versus `CopyMap` empty-
key behavior.
Extend the traversal state to track JSON container depth and produced-node
work. Enforce the 100-level and 100,000-node limits before allocation or
descent would cross them. Continue using active-path identity for cycle
detection; do not use alias memoization that would make distinct JSON paths
share mutable output. Errors must identify the structural path and whether the
depth or work budget was exceeded.
Expand the focused package tables by behavior branch: signed and unsigned
integer widths, ordinary and named finite floats, `json.Number`, pointers and
interfaces, named maps/slices/arrays, nil and empty values, mixed nested trees,
arrays, mutation isolation, active cycles, alternating just-below/at/over
depth, and shared acyclic subgraphs just below and over the work budget. Tests
must derive their edges from package constants or relationships instead of
copying unexplained literals.
Retain only representative public/backend/profile/prepared integration cases
that prove error translation and ownership. Update public GoDoc only if it
currently states the narrower safe-integer behavior; otherwise the existing
finite JSON-compatible-number contract remains canonical. Update the relevant
public value GoDoc and format/internal documentation to state that excessively
deep or large JSON-shaped values are rejected for safety; keep the exact
numeric limits owned by the internal constants rather than duplicating them
throughout consumer documentation. Run focused package and caller tests,
focused race tests, repository tests, and vet.
## Stage 4: Consolidate Stable Public JSON And Remove Dormant Internal JSON
**Findings:** S02-F01, S02-F05, S05-F05.
Refactor `json.go` so each public value has one ordinary field mapping. Use
private aliases or embedded wire representations for ordinary fields and keep
only timestamp, millisecond-duration, and intentional omission exceptions
explicit. Preserve every existing JSON name and omission rule.
Before converting `duration_ms`, reject values outside the millisecond range
that can be multiplied by `time.Millisecond` without overflow. Derive both
edges from `time.Duration` bounds. Return a contextual decode error and do not
partially update the receiver on failure.
Add fully populated `PreparedRun` and `RunResult` contract cases. Verify all
ordinary fields, intentional omissions, zero and nonzero timing, complete
round trips, the largest safe positive and negative millisecond values, and
their first unsafe neighbors.
Remove unused JSON tags and serialization tests from
`internal/domain.PreparedRun` after confirming production never marshals that
type. Keep credential absence protected at preparation/clone producers and
move any useful cache-control JSON assertion to the public `PreparedRun`
contract. Do not retain a parallel internal wire format.
Run focused domain and root JSON tests, repository tests, and vet.
## Stage 5: Harden Public Ownership And Diagnostic Contracts
**Findings:** S01-F01, S02-F03, S02-F04, S13-F01.
Extend the existing run-request formatting test with distinct input URI,
input-body, variable, and API-key sentinels. Require their absence from
`String`, `GoString`, `%v`, `%+v`, and `%#v` while retaining positive structural
summary assertions.
Add one focused public-LLM-adapter ownership test. Have the injected client
mutate and retain prompt messages, cache-control pointers, nested target extra
parameters, and structured-output schema values; prove the domain/prepared
source remains unchanged and later details or execution cannot race with those
mutations.
Add one direct all-field mapping test for `OpenAICompatibleProfile`. Populate
every field distinctly and compare the complete returned `Profile`. Keep only
the existing higher-level cases that prove normal validation and nested-value
ownership.
Make copied `PreparedExecution` values format opaquely by using value-receiver
formatting behavior shared by non-nil pointers and values. A nil pointer may
use Go's normal `<nil>` formatting, but formatting must never panic or expose
internal types, field names, addresses, credentials, or content. Cover original
pointers, copied values, zero values, and nil pointers under string, Go-string,
and ordinary fmt verbs, and prove formatting does not claim or discard a
handle.
Run focused root tests and the affected prepared/adapter race tests, followed
by repository tests and vet.
## Stage 6: Correct Engine Construction Edges And Immutable Defaults
**Findings:** S03-F01, S03-F02, S06-F01.
Change the shared single-file option helper so trimming is used only for the
blank-input check. Perform `Stat`, path decomposition, storage, diagnostics,
and later access with the exact caller path for prompt, profile, and schema
files. Add one compact table covering existing leading- and trailing-whitespace
names through all three options.
Strengthen engine construction tests with three discriminating cases:
- reverse the argument order of in-memory, ordinary, fallback, and built-in
profile categories while retaining fixed category precedence;
- collide `Config.ProfileDir` with an ordinary profile option and prove the
option replaces the configuration source; and
- place a valid same-category replacement after an invalid option and prove
construction still fails at the earlier invalid option.
Convert `internal/defaults.LLMRequestTimeoutDefault` from a variable to a
constant without changing its value or adding a setter. Do not add a test that
mutates or pins a noncontractual default; existing client deadline behavior is
the verification owner.
Run focused engine construction, default-client construction, and race tests,
then repository tests and vet.
## Stage 7: Contain And Preserve Prompt Content Paths
**Finding:** S07-F01.
Refactor prompt content resolution so both repository forms receive an
explicit source-root abstraction. Enforce the path decision fixed by this plan
before any content read. Use exact parsed path text after a separate blank
check. For OS sources, canonicalize the root and resolved target sufficiently
to reject symlink escape; for injected `fs.FS`, use its clean relative path
namespace. A parent component that remains inside the root is valid. Absolute,
escaping, and symlink-escaping targets are invalid.
Apply the same behavioral table to an OS directory, `WithPromptFS`, and a
single-file source: ordinary sibling, nested parent still within root, parent
escape, absolute path, symlink escape where supported, and existing names with
leading or trailing whitespace. Prove rejected targets cause no outside read
and public operations preserve `ErrPromptLoad`.
Update the framework format reference and internal source document to make the
single-file root and absolute-path rule explicit. Run focused prompt-definition
and public source tests, including race tests, then repository tests and vet.
## Stage 8: Correct Prompt Selection, Strictness, Coverage, And Lookup Cost
**Findings:** S07-F02, S07-F03, S07-F04, S07-F06.
Correct both existing prompt repository paths before consolidating them in the
next stage:
- Recover selector metadata from YAML `id` and `version`; never use a filename
stem as an identity.
- Apply normalized ID and requested-version selection before semantic
normalization or `content_file` reads.
- Associate strict YAML, semantic, and content errors only with a reliably
matching selected definition. An unidentifiable malformed file is unrelated
to point lookup; a reliably selected malformed file remains authoritative.
- Require exactly one YAML document. Comments and trailing whitespace are
allowed; a second empty or populated document and malformed trailing YAML
are `ErrInvalidYAML`.
- Continue scanning the YAML metadata required for duplicate detection, but
open content only for selected candidates. A selected content file is opened
once; unrelated and different-version bodies are never opened.
Add paired OS and `fs.FS` regressions for same-stem/different-ID malformed
files, same-ID/different-version invalid files, selected malformed definitions,
additional YAML documents, duplicates, and counting filesystem behavior.
Add a compact normalization table for the previously uncovered missing
version, blank input name, blank message role, invalid output format, negative
repair attempts, and explicit blank default profile. Output-contract rows
should exercise the shared Stage 2 owner rather than recreate its full table.
Run focused prompt-definition, use-case inspection, and root source tests,
focused race tests, repository tests, and vet.
## Stage 9: Unify Prompt Repository Semantics
**Finding:** S07-F05.
After Stage 8 establishes correct behavior in both paths, replace their
duplicated discovery-to-selection algorithms with one source-neutral prompt
selection and normalization flow. Introduce only the small internal source
adapter needed for YAML discovery, bytes, exact content opening, display paths,
and root containment. Keep genuine OS and `fs.FS` mechanics at the adapter
edge.
Move exact selection, version filtering, strict one-document decoding,
selected-error classification, normalization, duplicate handling, and
not-found behavior into the shared flow. Preserve point-in-time source access;
do not cache catalogs or definitions across operations.
Turn the Stage 8 behavior matrix into a shared suite over both adapters and
retain source-specific tests only for distinct path and I/O failures. Delete
superseded duplicate helpers and tests only after the shared suite protects
their meaningful behavior. Use a counting filesystem and a before/after
benchmark over small and large prompt catalogs to confirm unrelated content is
not read and the refactor adds no second scan; do not enforce wall-clock
thresholds.
Update the internal source document to describe the unified semantic owner.
Run focused package, integration, race, repository test, and vet gates.
## Stage 10: Correct Profile Source Validation And Identity
**Findings:** S08-F02, S08-F03, S08-F04, S08-F05. The non-finite scalar
symptom S08-F01 is already resolved by Stage 1.
Make file profile normalization pass `extra_params` through
`internal/jsonvalue.CopyMap` before publishing a domain profile. Preserve
`ErrInvalidProfile` and source path context for empty keys, non-finite values,
nested invalid data, or traversal-budget failures. Do not move reserved
OpenAI-compatible field policy into the profile package.
Use YAML metadata ID as the only selector; never infer authority from a
filename. Normalize the decoded ID once according to this plan. Require exactly
one YAML document in both metadata and strict selected decoding, so trailing
raw credentials, unknown fields, empty documents, and malformed YAML cannot be
ignored. Reliably selected malformed definitions must stop overlay fallback;
unrelated malformed files must not.
Add shared OS and `fs.FS` tables for invalid/valid extra parameters,
same-stem/different-ID malformed files beside a valid profile, fallback
behavior, additional documents, leading/trailing/blank IDs, normalized
duplicates, and exact inspection/preparation of the normalized ID. Retain only
representative public error-translation cases.
Update the framework format and internal source documents if needed to state
ID normalization and one-document behavior. Run focused profile, use-case,
root, and race tests, followed by repository tests and vet.
## Stage 11: Eliminate Duplicate Profile Decoding
**Finding:** S08-F06.
Refactor point lookup so each file receives one metadata pass and only
canonical ID matches receive strict full decoding and normalization. Reuse
bytes already read for metadata; do not decode every unrelated full profile or
turn the repository into a cache. Preserve deterministic duplicate detection,
strict selected errors, overlay fallthrough only on not-found, and fresh
point-in-time reads on every operation.
Add counting/parser-observation tests where stable behavior can be observed,
plus benchmarks for small and large catalogs reporting time and allocations.
Exercise valid selection, unrelated malformed files, selected malformed files,
duplicates, overlay fallthrough, and repeated lookup. Do not add brittle exact
allocation thresholds to ordinary tests.
Run focused profile and root integration tests, benchmarks for diagnostic
comparison, race tests, repository tests, and vet.
## Stage 12: Correct Artifact Semantics, Cancellation, And Hash Tests
**Findings:** S09-F01, S09-F02, S09-F05.
Treat an explicitly typed empty inline reference as a valid zero-byte artifact,
including `InlineWithURI`. Keep absence at the input map/reference boundary and
compute the same metadata and opaque equality value used for other bodies.
For ordinary file references, inspect the target before opening and again
after opening; reject anything that is not a regular file under the decision
above. Replace unbounded `io.ReadAll` with a normal synchronous chunked read
that checks `ctx.Err()` before open, before and after each read, and before
publishing the artifact. Do not return partial artifacts, add a hidden size
limit, or launch an abandoned reader goroutine.
Add source-parity cases for empty and nonempty inline, inline-with-URI, and file
content. Add a platform-appropriate FIFO regression proving the known FIFO is
rejected without requiring an external writer, and cancellation cases for a
pre-canceled file and a progressing regular-file read. Run them repeatedly and
under the race detector.
Replace exact SHA-256 literals with relational assertions: nonempty and stable
for repeat reads, equal for equal inline/file bodies, unequal for changed
bodies, and propagated opaquely through preparation. Do not document or test a
specific algorithm.
Update public GoDoc and the internal source document to describe regular-file
support and cancellation checkpoints. Run focused package, use-case, root,
race, repository test, and vet gates.
## Stage 13: Make Rendering Cancellation-Aware And Reuse Artifact Text
**Findings:** S09-F03, S09-F04.
Check context before session work, before and after each template parse and
execution, before and after every message, and before returning the completed
prompt. Make the `input` helper return an error when cancellation is observed.
Do not run template execution in a detached goroutine.
Within one `Render` call, lazily convert each named artifact body to text once
and memoize that string for the session and all messages. Build the cached
string in 64 KiB chunks with one pre-grown `strings.Builder`, checking the
context between chunks. Preserve bytes exactly, including invalid UTF-8; do not
cache across render calls or mutate artifacts. Unknown and nil inputs retain
their current errors, and a canceled conversion must not publish or cache a
partial string.
Add deterministic tests for pre-cancellation, cancellation during the chunked
input conversion, and cancellation observed after final-message execution.
Require the context identity and no partial prompt while preserving active-
context template errors. Add benchmarks for one and repeated references across
session and messages; report allocations without hard-coded timing limits.
Run focused renderer/use-case/root tests, benchmarks, repeated race tests,
repository tests, and vet.
## Stage 14: Preserve Exact JSON Validation Semantics
**Findings:** S10-F01, S10-F05.
Create one helper for decoding exactly one JSON value with
`json.Decoder.UseNumber` and required EOF after trailing whitespace. Use it for
schema documents and JSON Schema instance values so large integers, precise
decimals, and exponents retain exact `json.Number` semantics through
compilation, prepared metadata, copying, and validation.
For plain `ValidationJSON`, use a non-materializing complete-document syntax
check such as `json.Valid`; do not build a generic tree. Preserve the current
result distinction: malformed generated JSON is a completed failed validation,
not an operational error, and original output bytes remain unchanged.
Add focused OS and `fs.FS` cases around `2^53`, `1e400`, precise decimals,
ordinary numbers, malformed syntax, and trailing values. Exercise schema
`const`, minimum/maximum, and `multipleOf`, and verify the public structured
schema retains exact numeric values. Add benchmarks for scalar, object, and
large-array JSON validation with allocation reporting but no wall-clock
contract.
Update format/internal validation documentation only where it currently
implies float64-limited semantics. Run focused validator/use-case/root tests,
benchmarks, race tests, repository tests, and vet.
## Stage 15: Escape Schema Resources And Compile Once Per Operation
**Findings:** S10-F02, S10-F03.
Represent schema compiler resources with `url.URL` rather than string
concatenation. Use canonical escaped file URLs for OS paths and a private
scheme URL whose path segments are escaped for `fs.FS`. Preserve separators,
decode resource paths exactly once at the loader boundary, and continue
rejecting remote and escaping references. Legal filenames containing percent,
space, `#`, `?`, or Unicode must compile, including contained relative
references.
Unify JSON Schema preparation around `validate.PreparedValidation`:
- the shared preparation pipeline must create one operation-local compiled
plan and derive provider-facing root schema metadata from that plan;
- `Prepare` may discard the plan after returning metadata;
- `Run` must retain and use the plan for its one operation so the schema graph
is not loaded or compiled again during validation; and
- `PrepareExecution` must retain the same plan in its frozen payload.
Remove the document-only `SchemaDocumentLoader` capability if it has no
remaining production caller. Do not add an engine-wide or cross-operation
schema cache. Keep a clear private preparation carrier in `internal/usecase`
if needed so public `domain.PreparedRun` remains free of validator interfaces.
Replace the existing legal-filename expected failure with valid behavior and
retain a genuine compiler-registration failure only if reachable through a
valid source. Add public parity tests for invalid keywords, malformed and
missing direct/second-level references, unsupported dialects, escapes, remote
references, and valid multi-document graphs. A counting source must show each
document read once per operation and fresh reads across separate operations.
Update internal source, validator, and runner documentation for the unified
plan lifetime. Run focused validator/use-case/root tests, race tests,
repository tests, and vet.
## Stage 16: Make Validation Cancellation Authoritative
**Finding:** S10-F04.
Apply the cancellation decision fixed above. Thread context through schema
resource loaders and all Promptkit-controlled read/decode helpers. Read opened
schema files in context-checked chunks. Check the context immediately before
and after JSON decoding, schema compilation, and schema execution; if
cancellation occurred during a synchronous dependency call, return the context
error instead of a schema or successful validation result. Do not publish a
partial plan or validation result.
Do not place arbitrary `fs.FS` calls or JSON Schema work in goroutines merely
to race them against `ctx.Done()`. Tests must therefore distinguish:
- prompt cancellation before work;
- cancellation between controlled read chunks;
- cancellation that becomes authoritative immediately after a synchronous
compile or validation call returns; and
- the documented limitation that Promptkit cannot preempt a dependency method
that never returns.
Use deterministic controlled readers/contexts rather than sleeps. Assert no
goroutine growth or leaked work and preserve operational validation and public
context identities. Update validator GoDoc and internal source/runner documents
to state the synchronous cancellation boundary accurately.
Run focused cancellation tests normally, repeatedly, and under the race
detector, followed by repository tests and vet.
## Stage 17: Repair And Protect The Retained Internal Repair Path
**Findings:** S12-F01, S12-F02, S12-F03.
Retain the internal repair architecture. Extend `RepairRequest` with
`ExecutionTargetPresence` and carry the resolved presence bits unchanged into
the default repairer's `GenerateRequest`. Factor one use-case-local constructor
for common initial/repair generation fields—effective target, presence,
credential, backend identity, session, and structured output—while keeping the
initial and repair prompts intentionally separate.
Accumulate every completed generation response's five token-usage fields into
run-level usage. The final content/raw output/artifact continues to come from
the last candidate, while usage includes initial generation and every completed
repair exactly once. A repair call that returns an error still returns no
partial public result under current error semantics.
Replace the one-attempt-only repair coverage with a compact state-machine
table for:
- initial success with no repair;
- ineligible basic validation despite a positive budget;
- explicit zero and inherited-zero presence across initial and repair calls;
- success before a larger budget is exhausted;
- exact exhaustion of a larger budget; and
- advancement of attempt number, maximum, prior output, diagnostics, final
status, cumulative usage, and collaborator call count.
Retain the distinct capacity integration proving initial and repair generation
use the same backend pool and one whole-run admission lease, but simplify it if
the new focused table makes repair-state assertions redundant.
Update the internal runner and capacity documents for presence fidelity and
cumulative usage. Do not alter `NewRunner` to install a repairer, public GoDoc
that says the engine is single-pass, or the future public repair roadmap.
Run focused use-case, prepared, capacity, and race tests, followed by repository
tests and vet.
## Stage 18: Preserve Transport Error Identities And Test Deterministically
**Findings:** S14-F01, S04-F01, S16-F02. Timeout overflow S14-F02 is already
resolved through Stage 1's shared bound.
Preserve the underlying `http.Client.Do` error in the chain while retaining
`internal/llm.ErrRequestFailed` and the public generation category. Do not add
headers, request content, endpoints, or provider bodies to error text.
Cancellation and deadline identities must survive caller cancellation, caller
deadline, generation deadline, and whole-request client timeout.
Replace the port-9999 test with a controlled round tripper or `httptest`
endpoint that records the selected URL and returns a deliberate result. It must
make no host-dependent connection and must separately prove empty configured
base acceptance and request-endpoint precedence.
Extend the existing external ordinary-run cancellation test to require both
`ErrLLMGenerate` and `context.Canceled`; retain lower-layer tests only for their
distinct error owners.
Update the OpenAI-compatible integration and internal LLM documents for error
identity behavior. Run focused LLM and root tests with repetition and the race
detector, followed by repository tests and vet.
## Stage 19: Validate And Compose Effective Provider Endpoints
**Finding:** S14-F03.
Add one source-neutral OpenAI-compatible base-endpoint validator in
`internal/domain`, alongside the effective execution target invariant. It must
trim surrounding configuration whitespace, require absolute HTTP or HTTPS with
a host, and reject user information, query, and fragment. Use it from backend
registration, in-memory and file profiles, resolved request overrides, and the
built-in client defense while preserving each boundary's existing config,
profile-load, invalid-request, or LLM error category.
An empty configured base URL remains valid for a built-in client because a
resolved request endpoint may supply it later. Validate only a nonempty
configured base at construction, and always validate the final selected
endpoint before transport. Backends and endpoint-only profiles retain their
existing nonempty endpoint requirements.
Compose the completion URL through parsed URL operations (prefer
`url.JoinPath`) so nested paths and trailing slashes reach exactly one
`/chat/completions` suffix. Never append to a raw string.
Add endpoint tables for HTTP and HTTPS, hosts, nested paths, repeated trailing
slashes, queries, fragments, user information, relative paths, missing hosts,
unsupported schemes, and request/profile/config error mapping. Require every
invalid selected endpoint to fail before transport.
Update the architecture/internal overview for domain endpoint invariants and
the OpenAI-compatible integration and internal LLM documents for URL behavior.
Run focused domain/backend/profile/use-case/LLM/root tests, repetition and race
tests where ownership crosses packages, followed by repository tests and vet.
## Stage 20: Bound And Strictly Frame Successful Provider Responses
**Findings:** S14-F04, S14-F05.
Enforce the 16 MiB successful-response decision without first copying the
entire body. Reject an over-limit `Content-Length` immediately, but also wrap
the body in a counting/limited reader that reads at most one byte beyond the
limit so chunked or dishonest responses cannot bypass it. Exactly-limit bodies
remain valid. Always close the body; do not drain an unbounded oversized
stream.
Decode exactly one response object. After the first decode, require only
trailing JSON whitespace and EOF. A second value, non-whitespace suffix,
truncated body, malformed JSON, or size overflow returns
`ErrMalformedResponse` with no partial response and no provider content in the
error.
Add streaming tests just below, at, and one byte over the limit with and
without `Content-Length`, plus a continuing oversized stream. Assert bounded
bytes read, timely return, no partial result, and closure. Add trailing
whitespace success and trailing garbage/second-value failures. Retain ordinary
response mapping and redaction cases.
Document the fixed successful-response boundary and strict one-document rule
in the integration and internal LLM documents. Explicitly leave bounded
non-success error-envelope parsing to the structured-generation-error roadmap.
Run focused transport tests normally, repeatedly, and under race, followed by
repository tests and vet.
## Stage 21: Consolidate Transport Test Scaffolding
**Finding:** S14-F06.
After transport behavior is stable, introduce one small recording-provider
fixture for common request capture and successful/error response setup.
Organize focused tables around request mapping, authentication, endpoint
composition, timeout/error identity, and response framing. Keep specialized
round trippers/readers for cancellation, deadlines, byte counts, continuing
streams, and body closure.
Retain every existing durable assertion for method, path, headers,
authentication, omission and explicit presence, reserved fields, cache
control, structured output, response mapping, usage, error redaction, and
timeout precedence. Retain all Stage 18 through 20 regressions. Delete repeated
servers, generic-map decoding, and response literals only where the fixture
makes the owning behavior clearer; do not replace wire assertions with a broad
snapshot.
Run the LLM suite normally, with shuffle/repetition, and under the race
detector. Deliberately inspect the resulting test inventory against the audit's
transport matrix before running repository tests and vet.
## Stage 22: Restore One Canonical Maintainer Validation Workflow
**Finding:** S16-F01.
Make `docs/development.md` the canonical owner of the complete local maintainer
workflow, as assigned by the documentation policy. Its validation section must
include, from the repository root:
```sh
go test ./...
go test -race ./...
go vet ./...
go build ./...
go run ./examples/go-library/prepare
go run ./examples/go-library/run
```
It must also own the Go formatting, local Markdown link, `git diff --check`,
workspace/vendor/replacement, generated-output, credential, and working-tree
hygiene checks used before accepting changes.
Change the testing policy to state the semantic requirements and link to that
canonical workflow instead of maintaining a partial competing command list.
Change the release procedure to invoke the development-guide validation as a
release prerequisite rather than presenting a separately maintained copy;
retain release-specific metadata, candidate, tag, and publication commands in
the release document.
Run both examples offline and confirm that a missing or invalid Run example
fixture makes its command fail. Validate all changed Markdown links and ensure
current-state documentation describes only the implemented workflow.
## Stage 23: Complete Traceability And Final Validation
This final stage introduces no new behavior. Review the final tree against the
finding-to-stage table below and the evidence in `audit.md`. Confirm every
canonical group is implemented and every source-specific symptom retains its
required regression and error boundary. Do not mark a finding resolved merely
because a nearby refactor landed.
Run the complete development-guide workflow, including both examples, all
formatting and link checks, and repository hygiene. Also run shuffled ordinary
tests and repeated race-enabled tests for the changed concurrency,
cancellation, prepared, validation, repair, and transport packages. Run the
accepted performance benchmarks for prompt lookup, profile lookup, rendering,
and JSON validation and record only qualitative before/after conclusions; do
not establish release timing promises.
Inspect canonical GoDoc, formats, integration, architecture, and internal
documents against the final implementation. Confirm the public engine still
performs no output repair and the future repair entry remains future work.
Confirm the structured-generation-error feature was not implemented as part of
transport remediation.
Leave `audit-sequence.md`, `audit.md`, and this plan in place for maintainer
review. Retire them only in a separately authorized roadmap-cleanup pass after
the remediation has been reviewed and accepted.
## Finding-To-Stage Traceability
| Stage | Canonical findings | Historical or source-specific records handled with the canonical owner |
| ---: | --- | --- |
| 1 | S05-F01, S17-F01, S14-F02 | S02-F02, S08-F01, S11-F01 |
| 2 | S17-F02 | S11-F02 |
| 3 | S05-F02, S05-F03, S05-F04 | None |
| 4 | S02-F01, S02-F05, S05-F05 | None |
| 5 | S01-F01, S02-F03, S02-F04, S13-F01 | None |
| 6 | S03-F01, S03-F02, S06-F01 | None |
| 7 | S07-F01 | None |
| 8 | S07-F02, S07-F03, S07-F04, S07-F06 | None |
| 9 | S07-F05 | None |
| 10 | S08-F02, S08-F03, S08-F04, S08-F05 | S08-F01 was handled in Stage 1 |
| 11 | S08-F06 | None |
| 12 | S09-F01, S09-F02, S09-F05 | None |
| 13 | S09-F03, S09-F04 | None |
| 14 | S10-F01, S10-F05 | None |
| 15 | S10-F02, S10-F03 | None |
| 16 | S10-F04 | None |
| 17 | S12-F01, S12-F02, S12-F03 | None |
| 18 | S14-F01, S04-F01, S16-F02 | S14-F02 was handled in Stage 1 |
| 19 | S14-F03 | None |
| 20 | S14-F04, S14-F05 | None |
| 21 | S14-F06 | None |
| 22 | S16-F01 | None |
The table maps all 49 canonical remediation groups exactly once. S02-F02 is
the one superseded historical finding retained as evidence under S17-F01;
S08-F01, S11-F01, and S11-F02 retain their source-specific regression
responsibilities without being double-counted as canonical groups.
## Open Questions
None. The numeric contract, resource bounds, path and identity rules,
validation-cancellation limitation, repair retention, transport response
limit, and documentation ownership required to implement these stages are
fixed above.

View File

@@ -1,71 +0,0 @@
# Structured Generation Errors
## Purpose
Promptkit should give downstream applications actionable, machine-readable
details when the built-in OpenAI-compatible client receives a non-success HTTP
response. Today the client reports only the status code and discards the
provider response body. This makes ordinary configuration failures—such as an
unsupported strict JSON Schema keyword—unnecessarily difficult to diagnose.
## Target End State
Failures from the built-in transport are available through a public typed error
that works with `errors.As` while continuing to match `ErrLLMGenerate` through
`errors.Is`. The error should expose:
- the HTTP status code;
- a normalized provider error code or type when supplied; and
- a bounded provider message extracted from a recognized OpenAI-compatible
JSON error envelope.
The ordinary `Error()` string should remain safe and concise: it should include
the status and provider code or type, but not automatically include the
provider message. Consumers that deliberately want the provider's diagnostic
text can retrieve it from the typed error and apply their own disclosure and
logging policy.
This contract should be available for both ordinary and prepared execution.
Errors returned by injected model clients must continue to preserve their own
identity and should not be converted into fabricated HTTP details.
## Safety And Compatibility Boundaries
- Never expose the raw response body, response headers, endpoint, credentials,
request messages, schema document, or generated content through this API.
- Read only a small fixed maximum response body, reject malformed or
unrecognized envelopes, normalize invalid UTF-8 and control characters, and
cap every retained diagnostic field independently.
- Treat the extracted provider message as untrusted and potentially sensitive:
its GoDoc must tell consumers not to log or display it without applying their
own policy.
- Preserve the existing generic behavior when a response is empty, non-JSON,
oversized, or does not match a recognized error envelope.
- Do not assign retryability from an HTTP status. Promptkit supplies facts;
downstream applications retain retry and presentation policy.
## Recommended API Direction
Prefer one immutable public `GenerationError` value, constructed internally and
carrying accessors for HTTP status, provider code or type, and provider message.
This keeps the exact representation evolvable while giving consumers an
idiomatic `errors.As` contract. Public Go declarations and GoDoc should own the
final exact names and semantics.
The internal OpenAI-compatible client should parse only the conventional
top-level `error` envelope and pass normalized details through the use-case and
public error-mapping layers. The integration documentation should continue to
own wire behavior; the public declarations should own the consumer contract.
## Acceptance Criteria
- A downstream consumer can distinguish a provider HTTP 400 from other
generation failures and obtain a bounded provider explanation when present.
- The typed error still satisfies `errors.Is(err, ErrLLMGenerate)`.
- Existing cancellation, capacity, validation, and injected-client error
identities remain unchanged.
- Tests cover recognized string and numeric provider codes, absent and malformed
envelopes, oversized bodies and fields, control characters, and error-chain
behavior without making live provider requests.
- Current-state GoDoc and the OpenAI-compatible integration and internal-client
documents are updated only when the implementation lands.

108
engine.go
View File

@@ -11,14 +11,16 @@ import (
"strings" "strings"
"time" "time"
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter"
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome"
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact" artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
"gitea.maximumdirect.net/eric/promptkit/internal/backend" "gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity" "gitea.maximumdirect.net/eric/promptkit/internal/capacity"
"gitea.maximumdirect.net/eric/promptkit/internal/catalog"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults" "gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm" "gitea.maximumdirect.net/eric/promptkit/internal/llm"
"gitea.maximumdirect.net/eric/promptkit/internal/profile" "gitea.maximumdirect.net/eric/promptkit/internal/profile"
"gitea.maximumdirect.net/eric/promptkit/internal/profile/builtin"
"gitea.maximumdirect.net/eric/promptkit/internal/prompt" "gitea.maximumdirect.net/eric/promptkit/internal/prompt"
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef" "gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
"gitea.maximumdirect.net/eric/promptkit/internal/usecase" "gitea.maximumdirect.net/eric/promptkit/internal/usecase"
@@ -53,9 +55,9 @@ var (
// an execution profile or resolve its backend, except for the profile // an execution profile or resolve its backend, except for the profile
// not-found case represented by ErrProfileNotFound. // not-found case represented by ErrProfileNotFound.
ErrProfileLoad = errors.New("failed to load execution profile") ErrProfileLoad = errors.New("failed to load execution profile")
// ErrAPIKeyEnvMissing identifies an APIKeyEnv whose environment variable is // ErrAPIKeyEnvMissing identifies an explicitly required APIKeyEnv whose
// unset or empty when no direct RunRequest.APIKey takes precedence. Such an // environment variable is unset or empty after direct RunRequest.APIKey
// error also matches ErrInvalidRequest. // precedence is applied. Such an error also matches ErrInvalidRequest.
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable") ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
// ErrArtifactLoad identifies a failure to resolve an input artifact. Errors // ErrArtifactLoad identifies a failure to resolve an input artifact. Errors
// returned by an injected ArtifactReader remain available through errors.Is. // returned by an injected ArtifactReader remain available through errors.Is.
@@ -69,8 +71,9 @@ var (
// request, an LLM or provider rate-limit response, or ErrLLMGenerate. // request, an LLM or provider rate-limit response, or ErrLLMGenerate.
ErrCapacityExceeded = errors.New("backend capacity exceeded") ErrCapacityExceeded = errors.New("backend capacity exceeded")
// ErrLLMGenerate identifies a model-client failure or a nil successful // ErrLLMGenerate identifies a model-client failure or a nil successful
// response. Errors returned by an injected LLMClient remain available // response. A built-in OpenAI-compatible non-2xx response is available as a
// through errors.Is. // [GenerationError]. Errors returned by an injected LLMClient remain
// available through errors.Is.
ErrLLMGenerate = errors.New("failed to generate output") ErrLLMGenerate = errors.New("failed to generate output")
// ErrValidation identifies an operational failure to load or compile a // ErrValidation identifies an operational failure to load or compile a
// schema or validate output. A completed validation whose Status is // schema or validate output. A completed validation whose Status is
@@ -99,7 +102,7 @@ type Config struct {
// prompt source. // prompt source.
PromptDir string PromptDir string
// ProfileDir is an optional ordinary configured source whose profiles take // ProfileDir is an optional ordinary configured source whose profiles take
// precedence over application fallback and embedded built-in profiles. An // precedence over application fallback and maintained catalog profiles. An
// empty value selects the lower-precedence sources unless a profile-source // empty value selects the lower-precedence sources unless a profile-source
// option supplies the ordinary source. // option supplies the ordinary source.
ProfileDir string ProfileDir string
@@ -219,7 +222,7 @@ func WithPromptFile(path string) Option {
if err != nil { if err != nil {
return err return err
} }
options.promptDefs = promptdef.NewFSRepository(fsys, root) options.promptDefs = promptdef.NewFileRepository(fsys, root, filepath.Dir(path))
options.promptSource = true options.promptSource = true
return nil return nil
}) })
@@ -272,7 +275,7 @@ func WithProfileFile(path string) Option {
// //
// Profile lookup checks, in order, profiles supplied by WithProfiles; the // Profile lookup checks, in order, profiles supplied by WithProfiles; the
// ordinary configured source selected by WithProfileFile, WithProfileFS, or // ordinary configured source selected by WithProfileFile, WithProfileFS, or
// Config.ProfileDir; this fallback source; and Promptkit's embedded built-in // Config.ProfileDir; this fallback source; and Promptkit's maintained catalog
// profiles. Each source supplies a complete profile definition; profile fields // profiles. Each source supplies a complete profile definition; profile fields
// are not merged between sources. Only an absent profile ID proceeds to the // are not merged between sources. Only an absent profile ID proceeds to the
// next source. A matching read, parse, duplicate, validation, or credential // next source. A matching read, parse, duplicate, validation, or credential
@@ -303,10 +306,12 @@ func WithFallbackProfileFS(fsys fs.FS, root string) Option {
// WithProfiles configures in-memory profiles that take precedence over // WithProfiles configures in-memory profiles that take precedence over
// ordinary configured, application fallback, and built-in profiles. // ordinary configured, application fallback, and built-in profiles.
// //
// NewEngine validates and copies every profile. IDs must be unique within one // NewEngine locally validates and copies every profile. IDs must be unique
// call. An invalid profile, duplicate ID, or unsupported ExtraParams value // within one call. An invalid local definition, duplicate ID, or unsupported
// makes construction fail with ErrInvalidConfig. Repeating WithProfiles // ExtraParams value makes construction fail with ErrInvalidConfig. A derived
// replaces the complete earlier in-memory set rather than merging it. // profile's base reference and resolved target completeness are checked when it
// is selected or inspected. Repeating WithProfiles replaces the complete
// earlier in-memory set rather than merging it.
func WithProfiles(profiles ...Profile) Option { func WithProfiles(profiles ...Profile) Option {
return optionFunc(func(options *engineOptions) error { return optionFunc(func(options *engineOptions) error {
repo, err := newMemoryProfileRepository(profiles) repo, err := newMemoryProfileRepository(profiles)
@@ -387,9 +392,17 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir) promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
} }
profiles := newProfileRepository(cfg.ProfileDir, options) maintainedCatalogs, err := catalog.Load(
catalog.Source{Name: "OpenRouter", ExpectedBackendID: backend.OpenRouterID, FS: openrouter.FS(), Root: openrouter.Root},
catalog.Source{Name: "Rakestrawhome", ExpectedBackendID: backend.RakestrawHomeID, FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
)
if err != nil {
return nil, fmt.Errorf("%w: failed to load maintained catalogs: %v", ErrInvalidConfig, err)
}
backendRegistry, err := backend.NewRegistry(options.backends) profiles := newProfileRepository(cfg.ProfileDir, options, maintainedCatalogs.Profiles)
backendRegistry, err := backend.NewRegistry(maintainedCatalogs.Backends, options.backends)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err) return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err)
} }
@@ -427,7 +440,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
} }
return &Engine{ return &Engine{
runner: usecase.NewRunner( runner: usecase.NewRunnerWithRepairer(
promptDefs, promptDefs,
profiles, profiles,
backendRegistry, backendRegistry,
@@ -435,13 +448,14 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
prompt.NewGoRenderer(), prompt.NewGoRenderer(),
llmClient, llmClient,
validator, validator,
usecase.NewDefaultOutputRepairer(llmClient),
capacityManager, capacityManager,
), ),
}, nil }, nil
} }
func newProfileRepository(profileDir string, options engineOptions) profile.Repository { func newProfileRepository(profileDir string, options engineOptions, maintained profile.Repository) profile.Repository {
repository := builtin.NewRepository() repository := maintained
if options.fallbackProfileSource { if options.fallbackProfileSource {
repository = profile.NewOverlayRepository(options.fallbackProfiles, repository) repository = profile.NewOverlayRepository(options.fallbackProfiles, repository)
@@ -457,25 +471,24 @@ func newProfileRepository(profileDir string, options engineOptions) profile.Repo
repository = profile.NewOverlayRepository(options.memoryProfiles, repository) repository = profile.NewOverlayRepository(options.memoryProfiles, repository)
} }
return repository return profile.NewResolvingRepository(repository)
} }
func fileSource(name string) (fs.FS, string, error) { func fileSource(name string) (fs.FS, string, error) {
cleanName := strings.TrimSpace(name) if strings.TrimSpace(name) == "" {
if cleanName == "" {
return nil, "", ErrInvalidConfig return nil, "", ErrInvalidConfig
} }
dir := filepath.Dir(cleanName) dir := filepath.Dir(name)
base := filepath.Base(cleanName) base := filepath.Base(name)
if base == "." || base == string(filepath.Separator) || strings.TrimSpace(base) == "" { if base == "." || base == string(filepath.Separator) {
return nil, "", ErrInvalidConfig return nil, "", ErrInvalidConfig
} }
info, err := os.Stat(cleanName) info, err := os.Stat(name)
if err != nil { if err != nil {
return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, cleanName, err) return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, name, err)
} }
if info.IsDir() { if info.IsDir() {
return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, cleanName) return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, name)
} }
return os.DirFS(dir), filepath.ToSlash(base), nil return os.DirFS(dir), filepath.ToSlash(base), nil
} }
@@ -568,6 +581,8 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
} }
// Prepare resolves and renders a prompt request without calling an LLM. // Prepare resolves and renders a prompt request without calling an LLM.
// It appends any validated RunRequest.AppendedMessages after rendered
// definition messages in the returned caller-owned snapshot.
// //
// Prepare selects the prompt and profile, resolves any selected backend and // Prepare selects the prompt and profile, resolves any selected backend and
// effective execution settings, resolves the output contract, loads and hashes // effective execution settings, resolves the output contract, loads and hashes
@@ -602,7 +617,8 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
} }
// PrepareExecution completely prepares a prompt request without calling the // PrepareExecution completely prepares a prompt request without calling the
// configured LLMClient or reserving backend admission capacity. // configured LLMClient or reserving backend admission capacity. Validated
// RunRequest.AppendedMessages are included in the frozen effective messages.
// //
// The returned opaque handle is bound to this Engine and permits one // The returned opaque handle is bound to this Engine and permits one
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources, // [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
@@ -634,24 +650,29 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
} }
// Run prepares a request, invokes the configured LLMClient, and validates the // Run prepares a request, invokes the configured LLMClient, and validates the
// generated output. // generated output. Each call resolves current sources and composes a fresh,
// stateless effective prompt with any validated RunRequest.AppendedMessages.
// //
// A content-validation failure is a successful run whose // A content-validation failure is a successful run whose
// RunResult.Validation has Status ValidationFailed. An inability to perform // RunResult.Validation has Status ValidationFailed. When its output contract
// validation returns an error matching ErrValidation and no partial result. // has a positive repair budget, a failed eligible validation can make bounded
// The public Engine does not perform output repair, so validation is // additional model calls and stops at the first valid candidate. Exhaustion
// single-pass even when OutputContract.RepairAttempts is positive. // returns the final failed validation result with cumulative usage and actual
// repair attempts. An inability to generate or validate returns an error and
// no partial result.
// //
// Run can return every error category documented by [Engine.Prepare], plus // Run can return every error category documented by [Engine.Prepare], plus
// ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is // ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is
// discoverable as [CapacityError] and still matches ErrCapacityExceeded. It // discoverable as [CapacityError] and still matches ErrCapacityExceeded. It
// occurs before artifacts, schemas, rendering, or model generation because the // occurs before artifacts, schemas, rendering, or model generation because the
// selected backend's admission capacity is full; it does not match // selected backend's admission capacity is full; it does not match
// ErrInvalidRequest or ErrLLMGenerate. Errors from injected clients remain // ErrInvalidRequest or ErrLLMGenerate. A built-in OpenAI-compatible non-2xx
// available through errors.Is. Cancellation while waiting for model-generation // response is discoverable as [GenerationError]. Errors from injected clients
// capacity matches both ErrLLMGenerate and the context error. Cancellation // remain available through errors.Is. Cancellation while waiting for
// otherwise follows the active collaborator's documented behavior. A nil // model-generation capacity matches both ErrLLMGenerate and the context error.
// Engine returns ErrInvalidConfig. Run returns no partial result on error. // Cancellation otherwise follows the active collaborator's documented
// behavior. A nil Engine returns ErrInvalidConfig. Run returns no partial
// result on error.
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) { func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
if e == nil || e.runner == nil { if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig) return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
@@ -682,16 +703,17 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
// //
// The supplied context governs this execution attempt independently of the // The supplied context governs this execution attempt independently of the
// preparation context. It covers credential revalidation, admission, // preparation context. It covers credential revalidation, admission,
// generation, validation, and any internal repair. Result timing begins after // generation, validation, and any bounded output repair. Result timing begins
// the claim and excludes preparation and consumer-held delay. // after the claim and excludes preparation and consumer-held delay.
// //
// RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing, // RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing,
// ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while // ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while
// preserving documented collaborator and context identities. An engine // preserving documented collaborator and context identities. An engine
// admission rejection is discoverable as [CapacityError] and still matches // admission rejection is discoverable as [CapacityError] and still matches
// ErrCapacityExceeded. A completed content-validation rejection is returned // ErrCapacityExceeded. A built-in OpenAI-compatible non-2xx response is
// in RunResult, not as an operational error. An operational error returns no // discoverable as [GenerationError]. A completed content-validation rejection,
// partial RunResult. // including repair exhaustion, is returned in RunResult, not as an operational
// error. An operational error returns no partial RunResult.
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) { func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
if e == nil || e.runner == nil { if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig) return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)

File diff suppressed because it is too large Load Diff

View File

@@ -6,6 +6,7 @@ import (
"strings" "strings"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity" "gitea.maximumdirect.net/eric/promptkit/internal/capacity"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
"gitea.maximumdirect.net/eric/promptkit/internal/profile" "gitea.maximumdirect.net/eric/promptkit/internal/profile"
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef" "gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
"gitea.maximumdirect.net/eric/promptkit/internal/usecase" "gitea.maximumdirect.net/eric/promptkit/internal/usecase"
@@ -21,6 +22,19 @@ func mapPublicError(err error) error {
return &CapacityError{BackendID: internalCapacityError.BackendID} return &CapacityError{BackendID: internalCapacityError.BackendID}
} }
publicErr := publicErrorFor(err) publicErr := publicErrorFor(err)
var providerHTTPError *llm.ProviderHTTPError
if errors.As(err, &providerHTTPError) && providerHTTPError != nil {
generationErr := newGenerationError(
providerHTTPError.StatusCode(),
providerHTTPError.ProviderCode(),
providerHTTPError.ProviderType(),
providerHTTPError.ProviderMessage(),
)
if publicErr != nil && !errors.Is(publicErr, ErrLLMGenerate) {
return fmt.Errorf("%w: %w", publicErr, generationErr)
}
return generationErr
}
if publicErr == nil { if publicErr == nil {
return err return err
} }

View File

@@ -6,6 +6,7 @@ import (
"fmt" "fmt"
"testing" "testing"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
"gitea.maximumdirect.net/eric/promptkit/internal/usecase" "gitea.maximumdirect.net/eric/promptkit/internal/usecase"
) )
@@ -48,3 +49,27 @@ func TestMapPublicErrorTranslatesCapacityError(t *testing.T) {
t.Fatalf("mapped backend ID changed with source error: %q", publicErr.BackendID) t.Fatalf("mapped backend ID changed with source error: %q", publicErr.BackendID)
} }
} }
func TestMapPublicErrorPreservesValidationAroundGenerationError(t *testing.T) {
internalErr := fmt.Errorf(
"%w: %w",
usecase.ErrValidation,
&llm.ProviderHTTPError{},
)
err := mapPublicError(internalErr)
if !errors.Is(err, ErrValidation) {
t.Fatalf("mapped error=%v, want ErrValidation", err)
}
if !errors.Is(err, ErrLLMGenerate) {
t.Fatalf("mapped error=%v, want ErrLLMGenerate", err)
}
var generationErr *GenerationError
if !errors.As(err, &generationErr) || generationErr == nil {
t.Fatalf("mapped error=%v, want GenerationError", err)
}
var leakedInternalErr *llm.ProviderHTTPError
if errors.As(err, &leakedInternalErr) {
t.Fatalf("mapped error exposes internal ProviderHTTPError: %v", err)
}
}

View File

@@ -18,7 +18,7 @@ func (r RunRequest) GoString() string {
func (r RunRequest) redactedString() string { func (r RunRequest) redactedString() string {
return fmt.Sprintf( return fmt.Sprintf(
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t}", "promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t AppendedMessages:%d}",
r.PromptID, r.PromptID,
r.PromptVersion, r.PromptVersion,
r.ProfileID, r.ProfileID,
@@ -27,6 +27,7 @@ func (r RunRequest) redactedString() string {
len(r.Vars), len(r.Vars),
r.Execution != nil, r.Execution != nil,
r.Validation != nil, r.Validation != nil,
len(r.AppendedMessages),
) )
} }

87
generation_error.go Normal file
View File

@@ -0,0 +1,87 @@
package promptkit
import "fmt"
// GenerationError reports a non-2xx response from Promptkit's built-in
// OpenAI-compatible client during [Engine.Run] or [Engine.RunPrepared].
//
// Engine-produced values are immutable, caller-owned values. Use errors.Is to
// match [ErrLLMGenerate] and errors.As with a *GenerationError target to obtain
// this type. The four provider accessors expose untrusted provider-controlled
// values that can contain sensitive request or schema fragments. Applications
// must apply their own disclosure policy before logging, displaying, or
// returning them to another caller.
//
// Accessors, Error, GoString, and Unwrap are safe on a nil receiver and a zero
// value. Default and Go-syntax formatting deliberately redact provider details.
// GenerationError has no stable JSON representation.
type GenerationError struct {
statusCode int
providerCode string
providerType string
providerMessage string
}
func newGenerationError(statusCode int, providerCode, providerType, providerMessage string) *GenerationError {
return &GenerationError{
statusCode: statusCode,
providerCode: providerCode,
providerType: providerType,
providerMessage: providerMessage,
}
}
// StatusCode returns the received provider HTTP status code, or zero for a nil
// receiver or zero value.
func (e *GenerationError) StatusCode() int {
if e == nil {
return 0
}
return e.statusCode
}
// ProviderCode returns the normalized provider error code, if present. Its
// value is untrusted and may contain sensitive data.
func (e *GenerationError) ProviderCode() string {
if e == nil {
return ""
}
return e.providerCode
}
// ProviderType returns the normalized provider error type, if present. Its
// value is untrusted and may contain sensitive data.
func (e *GenerationError) ProviderType() string {
if e == nil {
return ""
}
return e.providerType
}
// ProviderMessage returns the bounded normalized provider diagnostic, if
// present. Its value is untrusted and may contain sensitive data.
func (e *GenerationError) ProviderMessage() string {
if e == nil {
return ""
}
return e.providerMessage
}
// Error returns a redacted diagnostic that is not a parsing contract.
func (e *GenerationError) Error() string {
if e == nil || e.statusCode == 0 {
return ErrLLMGenerate.Error()
}
return fmt.Sprintf("%s: provider returned HTTP status %d", ErrLLMGenerate, e.statusCode)
}
// GoString returns the same redacted diagnostic as Error.
func (e *GenerationError) GoString() string {
return e.Error()
}
// Unwrap returns ErrLLMGenerate. It is safe to call on a nil receiver or zero
// value.
func (e *GenerationError) Unwrap() error {
return ErrLLMGenerate
}

View File

@@ -0,0 +1,150 @@
package promptkit_test
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestBuiltInGenerationError(t *testing.T) {
const (
codeMarker = "provider-code-marker"
typeMarker = "provider-type-marker"
messageMarker = "provider-message-marker"
)
engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity,
`{"error":{"code":"`+codeMarker+`","type":"`+typeMarker+`","message":"`+messageMarker+`"}}`)
result, err := engine.Run(context.Background(), generationErrorRunRequest())
if result != nil {
t.Fatalf("Run result = %#v, want nil", result)
}
assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker)
preparedEngine := newBuiltInGenerationErrorEngine(t, http.StatusServiceUnavailable, `{"error":{}}`)
prepared, err := preparedEngine.PrepareExecution(context.Background(), generationErrorRunRequest())
if err != nil {
t.Fatalf("PrepareExecution: %v", err)
}
result, err = preparedEngine.RunPrepared(context.Background(), prepared)
if result != nil {
t.Fatalf("RunPrepared result = %#v, want nil", result)
}
assertGenerationError(t, err, http.StatusServiceUnavailable, "", "", "")
}
func TestBuiltInGenerationErrorWithAppendedMessages(t *testing.T) {
const messageMarker = "combined-request-provider-marker"
engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity,
`{"error":{"message":"`+messageMarker+`"}}`)
request := generationErrorRunRequest()
request.AppendedMessages = []promptkit.RenderedMessage{
{Role: promptkit.RoleAssistant, Content: "previous response"},
{Role: promptkit.RoleUser, Content: "consumer correction"},
}
result, err := engine.Run(context.Background(), request)
if result != nil {
t.Fatalf("Run result = %#v, want nil", result)
}
assertGenerationError(t, err, http.StatusUnprocessableEntity, "", "", messageMarker)
}
func TestBuiltInRepairGenerationError(t *testing.T) {
const (
codeMarker = "repair-code-marker"
typeMarker = "repair-type-marker"
messageMarker = "repair-message-marker"
)
calls := 0
config := contractConfig(frameworkSchemaDir)
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
calls++
if calls == 1 {
body := `{"choices":[{"message":{"content":"not-json"}}]}`
return &http.Response{StatusCode: http.StatusOK, ContentLength: int64(len(body)), Body: io.NopCloser(strings.NewReader(body))}, nil
}
body := `{"error":{"code":"` + codeMarker + `","type":"` + typeMarker + `","message":"` + messageMarker + `"}}`
return &http.Response{StatusCode: http.StatusUnprocessableEntity, ContentLength: int64(len(body)), Body: io.NopCloser(strings.NewReader(body))}, nil
})}
engine, err := promptkit.NewEngine(config)
if err != nil {
t.Fatalf("NewEngine: %v", err)
}
req := generationErrorRunRequest()
req.Validation = &promptkit.OutputContract{
Format: promptkit.FormatJSON,
ValidationMode: promptkit.ValidationJSON,
RepairAttempts: 1,
}
result, err := engine.Run(context.Background(), req)
if result != nil {
t.Fatalf("Run result = %#v, want nil", result)
}
if calls != 2 {
t.Fatalf("provider calls = %d, want 2", calls)
}
assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker)
}
func assertGenerationError(t *testing.T, err error, statusCode int, code, providerType, message string) {
t.Helper()
if !errors.Is(err, promptkit.ErrLLMGenerate) {
t.Fatalf("errors.Is(%v, ErrLLMGenerate) = false", err)
}
var generationErr *promptkit.GenerationError
if !errors.As(err, &generationErr) || generationErr == nil {
t.Fatalf("error = %T, want *GenerationError", err)
}
if generationErr.StatusCode() != statusCode || generationErr.ProviderCode() != code || generationErr.ProviderType() != providerType || generationErr.ProviderMessage() != message {
t.Fatalf("GenerationError = %#v", generationErr)
}
wantFormatted := fmt.Sprintf("failed to generate output: provider returned HTTP status %d", statusCode)
for _, rendered := range []string{fmt.Sprintf("%v", generationErr), fmt.Sprintf("%+v", generationErr), fmt.Sprintf("%#v", generationErr)} {
if rendered != wantFormatted {
t.Fatalf("formatted error = %q, want %q", rendered, wantFormatted)
}
for _, marker := range []string{code, providerType, message} {
if marker != "" && strings.Contains(rendered, marker) {
t.Fatalf("formatted error exposed provider marker %q: %q", marker, rendered)
}
}
}
}
func newBuiltInGenerationErrorEngine(t *testing.T, statusCode int, body string) *promptkit.Engine {
t.Helper()
config := contractConfig(frameworkSchemaDir)
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: statusCode,
ContentLength: int64(len(body)),
Body: io.NopCloser(strings.NewReader(body)),
}, nil
})}
engine, err := promptkit.NewEngine(config)
if err != nil {
t.Fatalf("NewEngine: %v", err)
}
return engine
}
func generationErrorRunRequest() promptkit.RunRequest {
return promptkit.RunRequest{
PromptID: frameworkMarkdownSummaryPromptID,
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("Rin opens the gate."),
"glossary": promptkit.Inline("gate: A guarded passage."),
},
}
}

View File

@@ -0,0 +1,32 @@
package promptkit
import (
"errors"
"fmt"
"testing"
)
func TestGenerationErrorNilAndZeroValue(t *testing.T) {
var nilError *GenerationError
zeroError := &GenerationError{}
for name, err := range map[string]*GenerationError{
"nil": nilError,
"zero": zeroError,
} {
t.Run(name, func(t *testing.T) {
if err.StatusCode() != 0 || err.ProviderCode() != "" || err.ProviderType() != "" || err.ProviderMessage() != "" {
t.Fatalf("accessors returned provider details: %#v", err)
}
if err.Error() != "failed to generate output" || err.GoString() != "failed to generate output" {
t.Fatalf("redacted formatting = (%q, %q)", err.Error(), err.GoString())
}
if fmt.Sprintf("%v", err) != "failed to generate output" || fmt.Sprintf("%#v", err) != "failed to generate output" {
t.Fatalf("formatted error = (%q, %q)", fmt.Sprintf("%v", err), fmt.Sprintf("%#v", err))
}
if !errors.Is(err, ErrLLMGenerate) {
t.Fatalf("errors.Is(%v, ErrLLMGenerate) = false", err)
}
})
}
}

2
go.mod
View File

@@ -3,6 +3,8 @@ module gitea.maximumdirect.net/eric/promptkit
go 1.25.5 go 1.25.5
require ( require (
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
) )

4
go.sum
View File

@@ -1,3 +1,7 @@
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 h1:lc062euk2qseO//D762i3JaFyulDNML3eQQX7DkYTho=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0/go.mod h1:AIa7kAu2mfrRQgcspe4L+DW51WqgnALQT60lqkEywJI=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 h1:j9YY7wsTVjzke2kHH4YAzpU0oUpM+x+nXwl1IeS+2eg=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0/go.mod h1:4RNS+LILDg4JbS4Ts9Lwy1C92wauXJIbeQaalps4Koo=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=

View File

@@ -16,10 +16,12 @@ import (
var ( var (
ErrUnsupportedRefType = errors.New("unsupported artifact reference type") ErrUnsupportedRefType = errors.New("unsupported artifact reference type")
ErrMissingInlineBody = errors.New("missing body for inline artifact")
ErrMissingFilePath = errors.New("missing file path for file artifact") ErrMissingFilePath = errors.New("missing file path for file artifact")
ErrUnsupportedFile = errors.New("file artifact path is not a regular file")
) )
const fileReadChunkSize = 64 * 1024
// Reader resolves artifact references into actual artifacts. // Reader resolves artifact references into actual artifacts.
type Reader interface { type Reader interface {
Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error)
@@ -34,7 +36,7 @@ type CompositeReader struct {
func NewCompositeReader() Reader { func NewCompositeReader() Reader {
return &CompositeReader{ return &CompositeReader{
inlineReader: &inlineReader{}, inlineReader: &inlineReader{},
fileReader: &fileReader{}, fileReader: &fileReader{open: openArtifactFile},
} }
} }
@@ -64,10 +66,6 @@ func (r *inlineReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domai
default: default:
} }
if ref.Body == "" {
return nil, ErrMissingInlineBody
}
body := []byte(ref.Body) body := []byte(ref.Body)
return &domain.Artifact{ return &domain.Artifact{
ContentType: defaults.ContentTypeTextPlain, ContentType: defaults.ContentTypeTextPlain,
@@ -78,7 +76,15 @@ func (r *inlineReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domai
}, nil }, nil
} }
type fileReader struct{} type artifactFile interface {
Read([]byte) (int, error)
Stat() (os.FileInfo, error)
Close() error
}
type fileReader struct {
open func(string) (artifactFile, error)
}
func (r *fileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) { func (r *fileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
select { select {
@@ -91,25 +97,71 @@ func (r *fileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.
return nil, ErrMissingFilePath return nil, ErrMissingFilePath
} }
return readFileArtifact(ref.URI) return readFileArtifact(ctx, ref.URI, r.open)
} }
func readFileArtifact(path string) (*domain.Artifact, error) { func openArtifactFile(path string) (artifactFile, error) {
file, err := os.Open(path) return os.Open(path)
}
func readFileArtifact(ctx context.Context, path string, open func(string) (artifactFile, error)) (*domain.Artifact, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
info, err := os.Stat(path)
if err != nil {
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
}
if !info.Mode().IsRegular() {
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path)
}
if err := ctx.Err(); err != nil {
return nil, err
}
file, err := open(path)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to read file %s: %w", path, err) return nil, fmt.Errorf("failed to read file %s: %w", path, err)
} }
defer file.Close() defer file.Close()
data, err := io.ReadAll(file) openedInfo, err := file.Stat()
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to read file %s: %w", path, err) return nil, fmt.Errorf("failed to inspect opened file %s: %w", path, err)
}
if !openedInfo.Mode().IsRegular() {
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path)
}
data := make([]byte, 0)
chunk := make([]byte, fileReadChunkSize)
for {
if err := ctx.Err(); err != nil {
return nil, err
}
n, readErr := file.Read(chunk)
if n > 0 {
data = append(data, chunk[:n]...)
}
if err := ctx.Err(); err != nil {
return nil, err
}
if errors.Is(readErr, io.EOF) {
break
}
if readErr != nil {
return nil, fmt.Errorf("failed to read file %s: %w", path, readErr)
}
} }
contentType := mime.TypeByExtension(filepath.Ext(path)) contentType := mime.TypeByExtension(filepath.Ext(path))
if contentType == "" { if contentType == "" {
contentType = defaults.ContentTypeTextPlain contentType = defaults.ContentTypeTextPlain
} }
hash := fmt.Sprintf("%x", sha256.Sum256(data))
if err := ctx.Err(); err != nil {
return nil, err
}
return &domain.Artifact{ return &domain.Artifact{
Name: filepath.Base(path), Name: filepath.Base(path),
@@ -117,6 +169,6 @@ func readFileArtifact(path string) (*domain.Artifact, error) {
Body: data, Body: data,
URI: path, URI: path,
Size: int64(len(data)), Size: int64(len(data)),
Hash: fmt.Sprintf("%x", sha256.Sum256(data)), Hash: hash,
}, nil }, nil
} }

View File

@@ -0,0 +1,43 @@
//go:build linux
package artifact
import (
"context"
"errors"
"path/filepath"
"syscall"
"testing"
"time"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestFileReaderRejectsFIFOBeforeOpen(t *testing.T) {
path := filepath.Join(t.TempDir(), "artifact.fifo")
if err := syscall.Mkfifo(path, 0o600); err != nil {
t.Fatalf("create fifo: %v", err)
}
type result struct {
artifact *domain.Artifact
err error
}
done := make(chan result, 1)
go func() {
artifact, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{
Type: domain.ArtifactRefFile,
URI: path,
})
done <- result{artifact: artifact, err: err}
}()
select {
case got := <-done:
if got.artifact != nil || !errors.Is(got.err, ErrUnsupportedFile) {
t.Fatalf("artifact=%#v err=%v, want nil/ErrUnsupportedFile", got.artifact, got.err)
}
case <-time.After(time.Second):
t.Fatal("FIFO read blocked instead of rejecting the non-regular file")
}
}

View File

@@ -1,6 +1,7 @@
package artifact package artifact
import ( import (
"bytes"
"context" "context"
"errors" "errors"
"os" "os"
@@ -11,54 +12,97 @@ import (
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/domain"
) )
func TestCompositeReader_Read(t *testing.T) { func TestCompositeReaderRejectsUnsupportedReferences(t *testing.T) {
_, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{
Type: domain.ArtifactRefType("unsupported"),
URI: "unsupported://bucket/key",
})
if !errors.Is(err, ErrUnsupportedRefType) {
t.Fatalf("expected ErrUnsupportedRefType, got %v", err)
}
}
func TestCompositeReaderSourceParityAndOpaqueHashes(t *testing.T) {
reader := NewCompositeReader() reader := NewCompositeReader()
ctx := context.Background() hashes := make(map[string]string)
tests := []struct {
name string
content string
}{
{name: "empty", content: ""},
{name: "ordinary", content: "same content"},
{name: "changed", content: "changed content"},
}
t.Run("inline artifact", func(t *testing.T) { for _, tc := range tests {
ref := domain.ArtifactRef{ t.Run(tc.name, func(t *testing.T) {
Type: domain.ArtifactRefInline, filePath := filepath.Join(t.TempDir(), "artifact.txt")
Body: "hello world", if err := os.WriteFile(filePath, []byte(tc.content), 0o600); err != nil {
} t.Fatal(err)
art, err := reader.Read(ctx, ref) }
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if string(art.Body) != "hello world" {
t.Errorf("expected 'hello world', got %s", string(art.Body))
}
if art.ContentType != "text/plain" {
t.Errorf("expected text/plain content type, got %q", art.ContentType)
}
if art.Hash != "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" {
t.Errorf("unexpected hash: %s", art.Hash)
}
if art.Size != int64(len(ref.Body)) {
t.Errorf("expected size %d, got %d", len(ref.Body), art.Size)
}
})
t.Run("inline artifact missing body", func(t *testing.T) { sources := []struct {
ref := domain.ArtifactRef{ name string
Type: domain.ArtifactRefInline, ref domain.ArtifactRef
Body: "", wantURI string
} }{
_, err := reader.Read(ctx, ref) {
if !errors.Is(err, ErrMissingInlineBody) { name: "inline",
t.Errorf("expected ErrMissingInlineBody, got %v", err) ref: domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: tc.content},
} },
}) {
name: "inline with uri",
ref: domain.ArtifactRef{Type: domain.ArtifactRefInline, URI: "memory://input", Body: tc.content},
wantURI: "memory://input",
},
{
name: "file",
ref: domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: filePath},
wantURI: filePath,
},
}
t.Run("unsupported ref type", func(t *testing.T) { var sourceHash string
ref := domain.ArtifactRef{ for _, source := range sources {
Type: domain.ArtifactRefType("unsupported"), t.Run(source.name, func(t *testing.T) {
URI: "unsupported://bucket/key", first, err := reader.Read(context.Background(), source.ref)
} if err != nil {
_, err := reader.Read(ctx, ref) t.Fatalf("first read: %v", err)
if !errors.Is(err, ErrUnsupportedRefType) { }
t.Error("expected error for unsupported type") second, err := reader.Read(context.Background(), source.ref)
} if err != nil {
}) t.Fatalf("second read: %v", err)
}
if string(first.Body) != tc.content || first.Size != int64(len(tc.content)) {
t.Fatalf("body=%q size=%d, want %q/%d", first.Body, first.Size, tc.content, len(tc.content))
}
if first.URI != source.wantURI {
t.Fatalf("URI = %q, want %q", first.URI, source.wantURI)
}
if first.Hash == "" || first.Hash != second.Hash {
t.Fatalf("hashes are not non-empty and stable: %q/%q", first.Hash, second.Hash)
}
if sourceHash == "" {
sourceHash = first.Hash
} else if first.Hash != sourceHash {
t.Fatalf("equal content hashes differ: %q/%q", sourceHash, first.Hash)
}
if source.ref.Type == domain.ArtifactRefFile {
if first.Name != filepath.Base(filePath) || !strings.HasPrefix(first.ContentType, "text/plain") {
t.Fatalf("unexpected file metadata: %+v", first)
}
} else if first.ContentType != "text/plain" {
t.Fatalf("inline content type = %q", first.ContentType)
}
})
}
hashes[tc.name] = sourceHash
})
}
if hashes["empty"] == hashes["ordinary"] || hashes["ordinary"] == hashes["changed"] {
t.Fatalf("changed content did not change opaque hash: %#v", hashes)
}
} }
func TestCompositeReaderCopiesInlineData(t *testing.T) { func TestCompositeReaderCopiesInlineData(t *testing.T) {
@@ -87,94 +131,154 @@ func TestCompositeReaderCopiesInlineData(t *testing.T) {
} }
} }
func TestCompositeReaderHonorsCancellation(t *testing.T) { func TestCompositeReaderHonorsPreCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) filePath := filepath.Join(t.TempDir(), "artifact.txt")
cancel() if err := os.WriteFile(filePath, []byte("ignored"), 0o600); err != nil {
t.Fatal(err)
}
tests := []struct {
name string
ref domain.ArtifactRef
}{
{name: "inline", ref: domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: "ignored"}},
{name: "file", ref: domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: filePath}},
}
_, err := NewCompositeReader().Read(ctx, domain.ArtifactRef{ for _, tc := range tests {
Type: domain.ArtifactRefInline, t.Run(tc.name, func(t *testing.T) {
Body: "ignored", ctx, cancel := context.WithCancel(context.Background())
}) cancel()
if !errors.Is(err, context.Canceled) {
t.Fatalf("expected context cancellation, got %v", err) artifact, err := NewCompositeReader().Read(ctx, tc.ref)
if artifact != nil || !errors.Is(err, context.Canceled) {
t.Fatalf("artifact=%#v err=%v, want nil/context.Canceled", artifact, err)
}
})
} }
} }
func TestFileReader_Read(t *testing.T) { func TestFileReaderFailuresAndMetadata(t *testing.T) {
content := []byte("test file content")
filePath := filepath.Join(t.TempDir(), "artifact.txt")
if err := os.WriteFile(filePath, content, 0o600); err != nil {
t.Fatal(err)
}
reader := NewCompositeReader() reader := NewCompositeReader()
ctx := context.Background()
t.Run("file artifact loading", func(t *testing.T) {
ref := domain.ArtifactRef{
Type: domain.ArtifactRefFile,
URI: filePath,
}
art, err := reader.Read(ctx, ref)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if string(art.Body) != string(content) {
t.Errorf("expected %s, got %s", string(content), string(art.Body))
}
if art.Name != filepath.Base(filePath) {
t.Errorf("expected name %q, got %q", filepath.Base(filePath), art.Name)
}
if !strings.HasPrefix(art.ContentType, "text/plain") {
t.Errorf("expected text content type, got %q", art.ContentType)
}
if art.URI != filePath {
t.Errorf("expected URI %q, got %q", filePath, art.URI)
}
if art.Size != int64(len(content)) {
t.Errorf("expected size %d, got %d", len(content), art.Size)
}
if art.Hash != "60f5237ed4049f0382661ef009d2bc42e48c3ceb3edb6600f7024e7ab3b838f3" {
t.Errorf("unexpected hash: %s", art.Hash)
}
})
t.Run("missing file path", func(t *testing.T) { t.Run("missing file path", func(t *testing.T) {
ref := domain.ArtifactRef{ _, err := reader.Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefFile})
Type: domain.ArtifactRefFile,
URI: "",
}
_, err := reader.Read(ctx, ref)
if !errors.Is(err, ErrMissingFilePath) { if !errors.Is(err, ErrMissingFilePath) {
t.Errorf("expected ErrMissingFilePath, got %v", err) t.Fatalf("expected ErrMissingFilePath, got %v", err)
} }
}) })
t.Run("missing file", func(t *testing.T) { t.Run("missing file", func(t *testing.T) {
ref := domain.ArtifactRef{ _, err := reader.Read(context.Background(), domain.ArtifactRef{
Type: domain.ArtifactRefFile, Type: domain.ArtifactRefFile,
URI: filepath.Join(t.TempDir(), "missing.txt"), URI: filepath.Join(t.TempDir(), "missing.txt"),
} })
if _, err := reader.Read(ctx, ref); err == nil { if err == nil {
t.Fatal("expected missing file error") t.Fatal("expected missing file error")
} }
}) })
t.Run("directory rejected before open", func(t *testing.T) {
artifact, err := reader.Read(context.Background(), domain.ArtifactRef{
Type: domain.ArtifactRefFile,
URI: t.TempDir(),
})
if artifact != nil || !errors.Is(err, ErrUnsupportedFile) {
t.Fatalf("artifact=%#v err=%v, want nil/ErrUnsupportedFile", artifact, err)
}
})
t.Run("non-regular opened target rejected", func(t *testing.T) {
filePath := filepath.Join(t.TempDir(), "artifact.txt")
if err := os.WriteFile(filePath, []byte("content"), 0o600); err != nil {
t.Fatal(err)
}
directoryInfo, err := os.Stat(t.TempDir())
if err != nil {
t.Fatal(err)
}
fileReader := &fileReader{open: func(path string) (artifactFile, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
return &reportedInfoFile{artifactFile: file, info: directoryInfo}, nil
}}
artifact, err := fileReader.Read(context.Background(), domain.ArtifactRef{
Type: domain.ArtifactRefFile,
URI: filePath,
})
if artifact != nil || !errors.Is(err, ErrUnsupportedFile) {
t.Fatalf("artifact=%#v err=%v, want nil/ErrUnsupportedFile", artifact, err)
}
})
t.Run("unknown extension uses text fallback", func(t *testing.T) { t.Run("unknown extension uses text fallback", func(t *testing.T) {
path := filepath.Join(t.TempDir(), "artifact.unknownextension") filePath := filepath.Join(t.TempDir(), "artifact.unknownextension")
if err := os.WriteFile(path, content, 0o600); err != nil { if err := os.WriteFile(filePath, []byte("content"), 0o600); err != nil {
t.Fatal(err) t.Fatal(err)
} }
art, err := reader.Read(ctx, domain.ArtifactRef{ artifact, err := reader.Read(context.Background(), domain.ArtifactRef{
Type: domain.ArtifactRefFile, Type: domain.ArtifactRefFile,
URI: path, URI: filePath,
}) })
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("read artifact: %v", err)
} }
if art.ContentType != "text/plain" { if artifact.ContentType != "text/plain" {
t.Errorf("expected text/plain fallback, got %q", art.ContentType) t.Fatalf("content type = %q", artifact.ContentType)
} }
}) })
} }
func TestFileReaderCancelsAfterReadProgress(t *testing.T) {
filePath := filepath.Join(t.TempDir(), "artifact.bin")
content := bytes.Repeat([]byte("x"), fileReadChunkSize*2)
if err := os.WriteFile(filePath, content, 0o600); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
var opened *cancelAfterProgressFile
reader := &fileReader{open: func(path string) (artifactFile, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
opened = &cancelAfterProgressFile{artifactFile: file, cancel: cancel}
return opened, nil
}}
artifact, err := reader.Read(ctx, domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: filePath})
if artifact != nil || !errors.Is(err, context.Canceled) {
t.Fatalf("artifact=%#v err=%v, want nil/context.Canceled", artifact, err)
}
if opened == nil || opened.reads != 1 {
t.Fatalf("read count = %v, want one progressing read", opened)
}
}
type reportedInfoFile struct {
artifactFile
info os.FileInfo
}
func (f *reportedInfoFile) Stat() (os.FileInfo, error) {
return f.info, nil
}
type cancelAfterProgressFile struct {
artifactFile
cancel context.CancelFunc
reads int
}
func (f *cancelAfterProgressFile) Read(buffer []byte) (int, error) {
n, err := f.artifactFile.Read(buffer)
if n > 0 {
f.reads++
f.cancel()
}
return n, err
}

View File

@@ -5,7 +5,6 @@ package backend
import ( import (
"errors" "errors"
"fmt" "fmt"
"net/url"
"regexp" "regexp"
"sort" "sort"
"strings" "strings"
@@ -19,12 +18,11 @@ const (
// OpenRouterID is the reserved ID of Promptkit's built-in OpenRouter // OpenRouterID is the reserved ID of Promptkit's built-in OpenRouter
// backend. // backend.
OpenRouterID = "openrouter" OpenRouterID = "openrouter"
// RakestrawHomeID is the reserved ID of Promptkit's built-in Rakestrawhome
// backend.
RakestrawHomeID = "rakestrawhome"
openRouterEndpoint = "https://openrouter.ai/api/v1" defaultQueueCapacity = 1024
openRouterAPIKeyEnv = "OPENROUTER_API_KEY"
openRouterConcurrencyLimit = 16
defaultQueueCapacity = 1024
) )
// ErrBackendNotFound identifies a registry lookup for an unknown backend ID. // ErrBackendNotFound identifies a registry lookup for an unknown backend ID.
@@ -37,20 +35,15 @@ type Registry struct {
backends map[string]domain.Backend backends map[string]domain.Backend
} }
// NewRegistry constructs a registry containing the built-in OpenRouter // NewRegistry constructs a registry containing maintained definitions followed
// definition followed by the supplied additions. Every ID must be unique. // by consumer additions. Every ID must be unique across both groups.
func NewRegistry(additions []domain.Backend) (*Registry, error) { func NewRegistry(maintained, additions []domain.Backend) (*Registry, error) {
registry := &Registry{ registry := &Registry{
backends: make(map[string]domain.Backend, len(additions)+1), backends: make(map[string]domain.Backend, len(maintained)+len(additions)),
} }
definitions := make([]domain.Backend, 0, len(additions)+1) definitions := make([]domain.Backend, 0, len(maintained)+len(additions))
definitions = append(definitions, domain.Backend{ definitions = append(definitions, maintained...)
ID: OpenRouterID,
Endpoint: openRouterEndpoint,
APIKeyEnv: openRouterAPIKeyEnv,
ConcurrencyLimit: openRouterConcurrencyLimit,
})
definitions = append(definitions, additions...) definitions = append(definitions, additions...)
for _, definition := range definitions { for _, definition := range definitions {
@@ -62,7 +55,7 @@ func NewRegistry(additions []domain.Backend) (*Registry, error) {
return nil, fmt.Errorf("backend ID %q is already registered", definition.ID) return nil, fmt.Errorf("backend ID %q is already registered", definition.ID)
} }
normalized, err := normalizeBackend(definition) normalized, err := NormalizeDefinition(definition)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -108,11 +101,13 @@ func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy {
return policies return policies
} }
func normalizeBackend(definition domain.Backend) (domain.Backend, error) { // NormalizeDefinition validates and defensively copies one backend definition.
definition.Endpoint = strings.TrimSpace(definition.Endpoint) func NormalizeDefinition(definition domain.Backend) (domain.Backend, error) {
if err := validateEndpoint(definition.Endpoint); err != nil { endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(definition.Endpoint)
if err != nil {
return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err) return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err)
} }
definition.Endpoint = endpoint
definition.APIKeyEnv = strings.TrimSpace(definition.APIKeyEnv) definition.APIKeyEnv = strings.TrimSpace(definition.APIKeyEnv)
if definition.APIKeyEnv != "" && !environmentVariableName.MatchString(definition.APIKeyEnv) { if definition.APIKeyEnv != "" && !environmentVariableName.MatchString(definition.APIKeyEnv) {
@@ -182,31 +177,3 @@ func normalizeBackend(definition domain.Backend) (domain.Backend, error) {
definition.ExtraParams = extraParams definition.ExtraParams = extraParams
return definition, nil return definition, nil
} }
func validateEndpoint(endpoint string) error {
if endpoint == "" {
return errors.New("must not be blank")
}
if strings.Contains(endpoint, "#") {
return errors.New("must not contain a fragment")
}
parsed, err := url.Parse(endpoint)
if err != nil {
return fmt.Errorf("must be a valid URL: %w", err)
}
scheme := strings.ToLower(parsed.Scheme)
if scheme != "http" && scheme != "https" {
return errors.New("must use http or https")
}
if !parsed.IsAbs() || parsed.Hostname() == "" {
return errors.New("must be absolute and include a host")
}
if parsed.User != nil {
return errors.New("must not contain user information")
}
if parsed.RawQuery != "" || parsed.ForceQuery {
return errors.New("must not contain a query string")
}
return nil
}

View File

@@ -2,6 +2,7 @@ package backend_test
import ( import (
"errors" "errors"
"reflect"
"strings" "strings"
"testing" "testing"
@@ -11,32 +12,39 @@ import (
const validEndpoint = "https://backend.example/v1" const validEndpoint = "https://backend.example/v1"
func TestRegistryIncludesExactOpenRouterDefinition(t *testing.T) { func TestRegistryIncludesMaintainedDefinitions(t *testing.T) {
registry, err := backend.NewRegistry(nil) maintained := []domain.Backend{
{ID: backend.OpenRouterID, Endpoint: validEndpoint, ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true},
{ID: backend.RakestrawHomeID, Endpoint: "https://second.example/v1", ConcurrencyLimit: 4, QueueCapacity: 5, QueueCapacitySet: true},
}
registry, err := backend.NewRegistry(maintained, nil)
if err != nil { if err != nil {
t.Fatalf("construct registry: %v", err) t.Fatalf("construct registry: %v", err)
} }
definition, err := registry.GetBackend(backend.OpenRouterID) for _, expected := range maintained {
if err != nil { t.Run(expected.ID, func(t *testing.T) {
t.Fatalf("look up OpenRouter: %v", err) definition, err := registry.GetBackend(expected.ID)
} if err != nil {
if definition.ID != "openrouter" || t.Fatalf("look up maintained definition: %v", err)
definition.Endpoint != "https://openrouter.ai/api/v1" || }
definition.APIKeyEnv != "OPENROUTER_API_KEY" || if !reflect.DeepEqual(definition, expected) {
definition.ConcurrencyLimit != 16 || t.Fatalf("unexpected maintained definition: %#v", definition)
definition.QueueCapacity != 1024 || }
!definition.QueueCapacitySet || })
definition.ExtraParams != nil {
t.Fatalf("unexpected OpenRouter definition: %#v", definition)
} }
policies := registry.CapacityPolicies() policies := registry.CapacityPolicies()
if len(policies) != 1 || if len(policies) != 2 ||
policies["openrouter"] != (domain.BackendCapacityPolicy{ policies[backend.OpenRouterID] != (domain.BackendCapacityPolicy{
ConcurrencyLimit: 16, ConcurrencyLimit: 2,
QueueCapacity: 1024, QueueCapacity: 3,
}) ||
policies[backend.RakestrawHomeID] != (domain.BackendCapacityPolicy{
ConcurrencyLimit: 4,
QueueCapacity: 5,
}) { }) {
t.Fatalf("unexpected OpenRouter capacity policies: %#v", policies) t.Fatalf("unexpected built-in capacity policies: %#v", policies)
} }
} }
@@ -46,7 +54,7 @@ func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
"count": int64(7), "count": int64(7),
"nested": nested, "nested": nested,
} }
registry, err := backend.NewRegistry([]domain.Backend{ registry, err := backend.NewRegistry(nil, []domain.Backend{
{ {
ID: " custom ", ID: " custom ",
Endpoint: " https://custom.example/openai/v1 ", Endpoint: " https://custom.example/openai/v1 ",
@@ -109,11 +117,11 @@ func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
} }
policies := registry.CapacityPolicies() policies := registry.CapacityPolicies()
if len(policies) != 2 { if len(policies) != 1 {
t.Fatalf("unexpected capacity policy count: %#v", policies) t.Fatalf("unexpected capacity policy count: %#v", policies)
} }
policies["custom"] = domain.BackendCapacityPolicy{} policies["custom"] = domain.BackendCapacityPolicy{}
delete(policies, backend.OpenRouterID) delete(policies, "custom")
againPolicies := registry.CapacityPolicies() againPolicies := registry.CapacityPolicies()
if againPolicies["custom"] != (domain.BackendCapacityPolicy{ if againPolicies["custom"] != (domain.BackendCapacityPolicy{
ConcurrencyLimit: 3, ConcurrencyLimit: 3,
@@ -121,9 +129,6 @@ func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
}) { }) {
t.Fatalf("capacity policy map mutated registry state: %#v", againPolicies) t.Fatalf("capacity policy map mutated registry state: %#v", againPolicies)
} }
if _, ok := againPolicies[backend.OpenRouterID]; !ok {
t.Fatalf("capacity policy deletion mutated registry state: %#v", againPolicies)
}
} }
func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) { func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) {
@@ -199,7 +204,7 @@ func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
tc.definition.ID = "custom" tc.definition.ID = "custom"
tc.definition.Endpoint = validEndpoint tc.definition.Endpoint = validEndpoint
registry, err := backend.NewRegistry([]domain.Backend{tc.definition}) registry, err := backend.NewRegistry(nil, []domain.Backend{tc.definition})
if tc.wantError { if tc.wantError {
if err == nil { if err == nil {
t.Fatal("expected invalid capacity policy error") t.Fatal("expected invalid capacity policy error")
@@ -242,11 +247,14 @@ func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
wantID string wantID string
}{ }{
{ {
name: "built-in collision after normalization", name: "OpenRouter collision after normalization",
additions: []domain.Backend{{ additions: []domain.Backend{{ID: " openrouter "}},
ID: " openrouter ", wantID: backend.OpenRouterID,
}}, },
wantID: "openrouter", {
name: "Rakestrawhome collision after normalization",
additions: []domain.Backend{{ID: " rakestrawhome "}},
wantID: backend.RakestrawHomeID,
}, },
{ {
name: "consumer collision after normalization", name: "consumer collision after normalization",
@@ -260,7 +268,7 @@ func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry(tc.additions) _, err := backend.NewRegistry(nil, tc.additions)
if err == nil { if err == nil {
t.Fatal("expected duplicate ID error") t.Fatal("expected duplicate ID error")
} }
@@ -274,7 +282,7 @@ func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
func TestNewRegistryValidatesIDs(t *testing.T) { func TestNewRegistryValidatesIDs(t *testing.T) {
for _, id := range []string{"", " \t\n "} { for _, id := range []string{"", " \t\n "} {
t.Run(id, func(t *testing.T) { t.Run(id, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{ _, err := backend.NewRegistry(nil, []domain.Backend{{
ID: id, ID: id,
Endpoint: validEndpoint, Endpoint: validEndpoint,
}}) }})
@@ -303,7 +311,7 @@ func TestNewRegistryValidatesEndpoints(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{ _, err := backend.NewRegistry(nil, []domain.Backend{{
ID: "custom", ID: "custom",
Endpoint: tc.endpoint, Endpoint: tc.endpoint,
}}) }})
@@ -317,7 +325,7 @@ func TestNewRegistryValidatesEndpoints(t *testing.T) {
func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) { func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) {
for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} { for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} {
t.Run(name, func(t *testing.T) { t.Run(name, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{ _, err := backend.NewRegistry(nil, []domain.Backend{{
ID: "custom", ID: "custom",
Endpoint: validEndpoint, Endpoint: validEndpoint,
APIKeyEnv: name, APIKeyEnv: name,
@@ -340,7 +348,7 @@ func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{ _, err := backend.NewRegistry(nil, []domain.Backend{{
ID: "custom", ID: "custom",
Endpoint: validEndpoint, Endpoint: validEndpoint,
ExtraParams: tc.extraParams, ExtraParams: tc.extraParams,
@@ -353,7 +361,7 @@ func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
} }
func TestRegistryLookupReportsNotFound(t *testing.T) { func TestRegistryLookupReportsNotFound(t *testing.T) {
registry, err := backend.NewRegistry(nil) registry, err := backend.NewRegistry(nil, nil)
if err != nil { if err != nil {
t.Fatalf("construct registry: %v", err) t.Fatalf("construct registry: %v", err)
} }

246
internal/catalog/catalog.go Normal file
View File

@@ -0,0 +1,246 @@
// Package catalog validates immutable maintained backend catalog assets.
package catalog
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"path"
"sort"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
// Source identifies one immutable backend catalog asset tree.
type Source struct {
Name string
ExpectedBackendID string
FS fs.FS
Root string
}
// Set is the validated maintained backend and raw profile catalog.
type Set struct {
Backends []domain.Backend
Profiles profile.Repository
profileIDs []string
}
// Load validates and combines immutable catalog sources in source order.
func Load(sources ...Source) (Set, error) {
if len(sources) == 0 {
return Set{}, errors.New("at least one catalog source is required")
}
loaded := Set{Backends: make([]domain.Backend, 0, len(sources))}
names := map[string]bool{}
backendIDs := map[string]bool{}
profileIDs := map[string]bool{}
for _, source := range sources {
if err := validateSource(source, names); err != nil {
return Set{}, err
}
names[source.Name] = true
definition, err := loadBackend(source)
if err != nil {
return Set{}, err
}
if backendIDs[definition.ID] {
return Set{}, fmt.Errorf("catalog %s: backend ID duplicates an earlier catalog", source.Name)
}
backendIDs[definition.ID] = true
repository, metadata, err := profile.LoadFSRepository(context.Background(), source.FS, path.Join(source.Root, "profiles"))
if err != nil {
return Set{}, fmt.Errorf("catalog %s: %w", source.Name, err)
}
if len(metadata) == 0 {
return Set{}, fmt.Errorf("catalog %s: profiles must not be empty", source.Name)
}
resolvingRepository := profile.NewResolvingRepository(repository)
for _, entry := range metadata {
if profileIDs[entry.ID] {
return Set{}, fmt.Errorf("catalog %s: %s duplicates an earlier profile ID", source.Name, entry.Path)
}
if containsField(entry.ExplicitFields, "endpoint") || containsField(entry.ExplicitFields, "api_key_env") {
return Set{}, fmt.Errorf("catalog %s: %s contains connection metadata", source.Name, entry.Path)
}
value, err := repository.GetProfile(context.Background(), entry.ID)
if err != nil {
return Set{}, fmt.Errorf("catalog %s: %s: %w", source.Name, entry.Path, err)
}
if err := rejectSecretKeys(value.ExtraParams); err != nil {
return Set{}, fmt.Errorf("catalog %s: %s: prohibited extra parameter key", source.Name, entry.Path)
}
resolved, err := resolvingRepository.GetProfile(context.Background(), entry.ID)
if err != nil {
return Set{}, fmt.Errorf("catalog %s: %s has invalid profile inheritance", source.Name, entry.Path)
}
if resolved.BackendID != definition.ID {
return Set{}, fmt.Errorf("catalog %s: %s selects a different backend", source.Name, entry.Path)
}
profileIDs[entry.ID] = true
loaded.profileIDs = append(loaded.profileIDs, entry.ID)
}
if loaded.Profiles == nil {
loaded.Profiles = repository
} else {
loaded.Profiles = profile.NewOverlayRepository(loaded.Profiles, repository)
}
loaded.Backends = append(loaded.Backends, definition)
}
sort.Strings(loaded.profileIDs)
return loaded, nil
}
func validateSource(source Source, names map[string]bool) error {
if strings.TrimSpace(source.Name) == "" || source.Name != strings.TrimSpace(source.Name) {
return errors.New("catalog source name must not be blank")
}
if names[source.Name] {
return fmt.Errorf("duplicate catalog source name %q", source.Name)
}
if source.FS == nil {
return fmt.Errorf("catalog %s: filesystem is nil", source.Name)
}
if source.Root == "." || source.Root == "" || !fs.ValidPath(source.Root) {
return fmt.Errorf("catalog %s: asset root is invalid", source.Name)
}
if strings.TrimSpace(source.ExpectedBackendID) == "" {
return fmt.Errorf("catalog %s: expected backend ID is blank", source.Name)
}
return validateLayout(source)
}
func validateLayout(source Source) error {
manifestPath := path.Join(source.Root, "backend.json")
profilesRoot := path.Join(source.Root, "profiles")
return fs.WalkDir(source.FS, source.Root, func(assetPath string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
if assetPath == source.Root {
if !entry.IsDir() {
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
}
return nil
}
if entry.IsDir() {
if assetPath == profilesRoot || strings.HasPrefix(assetPath, profilesRoot+"/") {
return nil
}
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
}
if assetPath == manifestPath && entry.Type().IsRegular() {
return nil
}
if strings.HasPrefix(assetPath, profilesRoot+"/") && entry.Type().IsRegular() && strings.HasSuffix(assetPath, ".yml") {
return nil
}
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
})
}
func loadBackend(source Source) (domain.Backend, error) {
data, err := fs.ReadFile(source.FS, path.Join(source.Root, "backend.json"))
if err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: %w", source.Name, err)
}
type manifest struct {
SchemaVersion *int `json:"schema_version"`
ID *string `json:"id"`
Endpoint *string `json:"endpoint"`
APIKeyEnv *string `json:"api_key_env"`
ConcurrencyLimit *int `json:"concurrency_limit"`
QueueCapacity *int `json:"queue_capacity"`
ExtraParams json.RawMessage `json:"extra_params"`
}
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.DisallowUnknownFields()
var value manifest
if err := decoder.Decode(&value); err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
}
if value.SchemaVersion == nil || *value.SchemaVersion != 1 || value.ID == nil || value.Endpoint == nil || value.APIKeyEnv == nil || value.ConcurrencyLimit == nil || value.QueueCapacity == nil || value.ExtraParams == nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: required field is missing or unsupported", source.Name)
}
if *value.ID != source.ExpectedBackendID {
return domain.Backend{}, fmt.Errorf("catalog %s: backend ID does not match expected ID", source.Name)
}
if strings.TrimSpace(*value.APIKeyEnv) == "" {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: api key environment variable must not be blank", source.Name)
}
extraParams, err := decodeExtraParams(value.ExtraParams)
if err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid extra parameters", source.Name)
}
if err := rejectSecretKeys(extraParams); err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: prohibited extra parameter key", source.Name)
}
normalized, err := backend.NormalizeDefinition(domain.Backend{ID: *value.ID, Endpoint: *value.Endpoint, APIKeyEnv: *value.APIKeyEnv, ExtraParams: extraParams, ConcurrencyLimit: *value.ConcurrencyLimit, QueueCapacity: *value.QueueCapacity, QueueCapacitySet: true})
if err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid backend definition", source.Name)
}
return normalized, nil
}
func decodeExtraParams(data []byte) (map[string]any, error) {
if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
return nil, nil
}
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.UseNumber()
var value map[string]any
if err := decoder.Decode(&value); err != nil || value == nil {
return nil, errors.New("extra parameters must be an object or null")
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return nil, errors.New("extra parameters must contain exactly one JSON value")
}
return value, nil
}
func containsField(fields []string, target string) bool {
for _, field := range fields {
if field == target {
return true
}
}
return false
}
func rejectSecretKeys(value map[string]any) error {
return rejectSecretValue(value)
}
func rejectSecretValue(value any) error {
switch value := value.(type) {
case map[string]any:
for key, child := range value {
switch strings.ToLower(key) {
case "api_key", "apikey", "authorization", "credential", "credentials", "password", "secret", "token", "access_token":
return errors.New("prohibited key")
}
if err := rejectSecretValue(child); err != nil {
return err
}
}
case []any:
for _, child := range value {
if err := rejectSecretValue(child); err != nil {
return err
}
}
}
return nil
}

View File

@@ -0,0 +1,364 @@
package catalog
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
"testing/fstest"
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter"
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome"
)
func TestLoadPublishedCatalogsMatchCompatibilityFixture(t *testing.T) {
loaded, err := Load(
Source{Name: "OpenRouter", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
Source{Name: "Rakestrawhome", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
)
if err != nil {
t.Fatalf("load published catalogs: %v", err)
}
expected := loadCompatibilityFixture(t)
expectedIDs := fixtureProfileIDs(t, expected)
if !reflect.DeepEqual(loaded.profileIDs, expectedIDs) {
t.Fatalf("published profile IDs differ from compatibility fixture: got %q, want %q", loaded.profileIDs, expectedIDs)
}
actual := catalogValue(t, loaded)
actualJSON, err := json.Marshal(actual)
if err != nil {
t.Fatalf("encode loaded catalogs: %v", err)
}
expectedJSON, err := json.Marshal(expected)
if err != nil {
t.Fatalf("encode compatibility fixture: %v", err)
}
if !bytes.Equal(actualJSON, expectedJSON) {
t.Fatalf("published catalogs differ from compatibility fixture: got %#v, want %#v", actual, expected)
}
}
func TestLoadRejectsInvalidSources(t *testing.T) {
for name, sources := range map[string][]Source{
"none": nil,
"blank name": {{Name: " ", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root}},
"duplicate name": {
{Name: "same", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
{Name: "same", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
},
"nil filesystem": {{Name: "missing", ExpectedBackendID: "openrouter", Root: openrouter.Root}},
"invalid root": {{Name: "invalid-root", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: "."}},
"blank expected backend": {{Name: "blank-backend", ExpectedBackendID: " ", FS: openrouter.FS(), Root: openrouter.Root}},
} {
t.Run(name, func(t *testing.T) {
if _, err := Load(sources...); err == nil {
t.Fatal("expected error")
}
})
}
}
func TestLoadRejectsInvalidLayouts(t *testing.T) {
tests := map[string]func(fstest.MapFS){
"unexpected file": func(fsys fstest.MapFS) {
fsys["catalog/notes.txt"] = &fstest.MapFile{Data: []byte("unexpected")}
},
"unexpected directory": func(fsys fstest.MapFS) {
fsys["catalog/unexpected"] = &fstest.MapFile{Mode: fs.ModeDir}
},
"nonregular manifest": func(fsys fstest.MapFS) {
fsys["catalog/backend.json"].Mode = fs.ModeSymlink
},
"wrong profile extension": func(fsys fstest.MapFS) {
fsys["catalog/profiles/extra.yaml"] = &fstest.MapFile{Data: []byte(validProfile("extra", "one"))}
},
"nonregular profile": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Mode = fs.ModeSymlink
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
fsys := validCatalogFS("one")
mutate(fsys)
if _, err := Load(testSource("one", fsys)); err == nil {
t.Fatal("expected invalid layout error")
}
})
}
}
func TestLoadRejectsInvalidManifests(t *testing.T) {
tests := map[string]string{
"malformed": `{`,
"trailing value": validManifest("one", "TEST_API_KEY", "null") + `{}`,
"missing fields": `{"schema_version":1,"id":"one"}`,
"unsupported version": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"schema_version":1`, `"schema_version":2`, 1),
"unknown field": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"extra_params":null`, `"extra_params":null,"unknown":true`, 1),
"blank API key env": validManifest("one", " ", "null"),
"invalid API key env": validManifest("one", "LEAK-MARKER", "null"),
"invalid endpoint": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `https://one.example/v1`, `ftp://leak-marker.invalid/v1`, 1),
"zero concurrency": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"concurrency_limit":2`, `"concurrency_limit":0`, 1),
"non-object parameters": validManifest("one", "TEST_API_KEY", `[]`),
"secret parameter": validManifest("one", "TEST_API_KEY", `{"nested":{"token":"leak-marker"}}`),
}
for name, manifest := range tests {
t.Run(name, func(t *testing.T) {
fsys := validCatalogFS("one")
fsys["catalog/backend.json"].Data = []byte(manifest)
_, err := Load(testSource("one", fsys))
if err == nil {
t.Fatal("expected invalid manifest error")
}
if strings.Contains(strings.ToLower(err.Error()), "leak-marker") {
t.Fatalf("catalog error exposed manifest content: %v", err)
}
})
}
}
func TestLoadPreservesManifestJSONNumbers(t *testing.T) {
fsys := validCatalogFS("one")
fsys["catalog/backend.json"].Data = []byte(validManifest(
"one",
"TEST_API_KEY",
`{"large":9007199254740993,"nested":[1.25]}`,
))
loaded, err := Load(testSource("one", fsys))
if err != nil {
t.Fatalf("load catalog: %v", err)
}
if got := loaded.Backends[0].ExtraParams["large"]; got != json.Number("9007199254740993") {
t.Fatalf("large JSON integer = %#v, want preserved json.Number", got)
}
nested := loaded.Backends[0].ExtraParams["nested"].([]any)
if nested[0] != json.Number("1.25") {
t.Fatalf("nested JSON number = %#v, want preserved json.Number", nested[0])
}
}
func TestLoadRejectsInvalidCatalogProfiles(t *testing.T) {
tests := map[string]func(fstest.MapFS){
"empty": func(fsys fstest.MapFS) {
delete(fsys, "catalog/profiles/one-profile.yml")
fsys["catalog/profiles"] = &fstest.MapFile{Mode: fs.ModeDir}
},
"malformed": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: [")
},
"raw API key": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key: leak-marker\n")
},
"endpoint field": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "endpoint: ''\n")
},
"API key environment field": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key_env: ''\n")
},
"owner mismatch": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "other"))
},
"missing base": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: leak-marker\n")
},
"cyclic base": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: second\n")
fsys["catalog/profiles/second.yml"] = &fstest.MapFile{Data: []byte("id: second\nbase_profile: one-profile\n")}
},
"secret profile parameter": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "extra_params:\n nested:\n password: leak-marker\n")
},
"unknown field is redacted": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "leak_marker: leak-marker\n")
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
fsys := validCatalogFS("one")
mutate(fsys)
_, err := Load(testSource("one", fsys))
if err == nil {
t.Fatal("expected invalid profile error")
}
if strings.Contains(strings.ToLower(err.Error()), "leak-marker") {
t.Fatalf("catalog error exposed profile content: %v", err)
}
})
}
}
func TestLoadRejectsCrossCatalogConflicts(t *testing.T) {
t.Run("duplicate backend", func(t *testing.T) {
second := catalogFS("one", map[string]string{
"catalog/profiles/second.yml": validProfile("second", "one"),
}, "null")
_, err := Load(
Source{Name: "first", ExpectedBackendID: "one", FS: validCatalogFS("one"), Root: "catalog"},
Source{Name: "second", ExpectedBackendID: "one", FS: second, Root: "catalog"},
)
if err == nil {
t.Fatal("expected duplicate backend error")
}
})
t.Run("duplicate profile", func(t *testing.T) {
first := catalogFS("one", map[string]string{
"catalog/profiles/shared.yml": validProfile("shared", "one"),
}, "null")
second := catalogFS("two", map[string]string{
"catalog/profiles/shared.yml": validProfile("shared", "two"),
}, "null")
_, err := Load(testSource("one", first), testSource("two", second))
if err == nil {
t.Fatal("expected duplicate profile error")
}
})
t.Run("cross-catalog base", func(t *testing.T) {
first := catalogFS("one", map[string]string{
"catalog/profiles/base.yml": validProfile("base", "one"),
}, "null")
second := catalogFS("two", map[string]string{
"catalog/profiles/child.yml": "id: child\nbase_profile: base\n",
}, "null")
_, err := Load(testSource("one", first), testSource("two", second))
if err == nil {
t.Fatal("expected cross-catalog base error")
}
})
}
func TestLoadReturnsDefensiveCatalogValues(t *testing.T) {
fsys := catalogFS("one", map[string]string{
"catalog/profiles/one-profile.yml": validProfile("one-profile", "one") + "extra_params:\n nested:\n value: profile\n",
}, `{"nested":{"value":"backend"}}`)
loaded, err := Load(testSource("one", fsys))
if err != nil {
t.Fatalf("load catalog: %v", err)
}
loaded.Backends[0].ExtraParams["nested"].(map[string]any)["value"] = "changed"
profileValue, err := loaded.Profiles.GetProfile(context.Background(), "one-profile")
if err != nil {
t.Fatalf("load profile: %v", err)
}
profileValue.ExtraParams["nested"].(map[string]any)["value"] = "changed"
again, err := Load(testSource("one", fsys))
if err != nil {
t.Fatalf("reload catalog: %v", err)
}
if got := again.Backends[0].ExtraParams["nested"].(map[string]any)["value"]; got != "backend" {
t.Fatalf("backend mutation escaped returned set: %#v", got)
}
againProfile, err := loaded.Profiles.GetProfile(context.Background(), "one-profile")
if err != nil {
t.Fatalf("reload profile: %v", err)
}
if got := againProfile.ExtraParams["nested"].(map[string]any)["value"]; got != "profile" {
t.Fatalf("profile mutation escaped returned value: %#v", got)
}
}
func loadCompatibilityFixture(t *testing.T) map[string]any {
t.Helper()
data, err := os.ReadFile(filepath.Join("..", "..", "testdata", "builtin-catalog-v1.json"))
if err != nil {
t.Fatalf("read fixture: %v", err)
}
var value map[string]any
if err := json.Unmarshal(data, &value); err != nil {
t.Fatalf("decode fixture: %v", err)
}
return value
}
func fixtureProfileIDs(t *testing.T, fixture map[string]any) []string {
t.Helper()
profiles, ok := fixture["profiles"].([]any)
if !ok {
t.Fatal("compatibility fixture profiles are malformed")
}
ids := make([]string, 0, len(profiles))
for _, entry := range profiles {
profileValue, ok := entry.(map[string]any)
if !ok {
t.Fatal("compatibility fixture profile is malformed")
}
id, ok := profileValue["id"].(string)
if !ok {
t.Fatal("compatibility fixture profile ID is malformed")
}
ids = append(ids, id)
}
sort.Strings(ids)
return ids
}
func catalogValue(t *testing.T, loaded Set) map[string]any {
t.Helper()
backends := make([]any, 0, len(loaded.Backends))
for _, backend := range loaded.Backends {
backends = append(backends, map[string]any{"id": backend.ID, "endpoint": backend.Endpoint, "api_key_env": backend.APIKeyEnv, "extra_params": interfaceValue(backend.ExtraParams), "concurrency_limit": backend.ConcurrencyLimit, "queue_capacity": backend.QueueCapacity, "queue_capacity_set": backend.QueueCapacitySet})
}
sort.Slice(backends, func(left, right int) bool {
return backends[left].(map[string]any)["id"].(string) < backends[right].(map[string]any)["id"].(string)
})
actualProfiles := make([]any, 0, len(loaded.profileIDs))
for _, id := range loaded.profileIDs {
profile, err := loaded.Profiles.GetProfile(context.Background(), id)
if err != nil {
t.Fatalf("load profile %q: %v", id, err)
}
actualProfiles = append(actualProfiles, map[string]any{"id": profile.ID, "base_profile": profile.BaseProfileID, "backend": profile.BackendID, "endpoint": profile.Endpoint, "model": profile.Model, "temperature": profile.Temperature, "max_tokens": profile.MaxTokens, "top_p": profile.TopP, "timeout_seconds": profile.TimeoutSeconds, "service_tier": profile.ServiceTier, "reasoning_effort": profile.ReasoningEffort, "api_key_env": profile.APIKeyEnv, "api_key_required": profile.APIKeyRequired, "extra_params": interfaceValue(profile.ExtraParams)})
}
return map[string]any{"backends": backends, "profiles": actualProfiles}
}
func interfaceValue(value map[string]any) any {
if value == nil {
return nil
}
return value
}
func testSource(id string, fsys fs.FS) Source {
return Source{Name: id, ExpectedBackendID: id, FS: fsys, Root: "catalog"}
}
func validCatalogFS(id string) fstest.MapFS {
return catalogFS(id, map[string]string{
"catalog/profiles/" + id + "-profile.yml": validProfile(id+"-profile", id),
}, "null")
}
func catalogFS(id string, profiles map[string]string, extraParams string) fstest.MapFS {
fsys := fstest.MapFS{
"catalog/backend.json": &fstest.MapFile{Data: []byte(validManifest(id, "TEST_API_KEY", extraParams))},
}
for name, content := range profiles {
fsys[name] = &fstest.MapFile{Data: []byte(content)}
}
return fsys
}
func validManifest(id, apiKeyEnv, extraParams string) string {
return fmt.Sprintf(
`{"schema_version":1,"id":%q,"endpoint":%q,"api_key_env":%q,"concurrency_limit":2,"queue_capacity":3,"extra_params":%s}`,
id,
"https://"+id+".example/v1",
apiKeyEnv,
extraParams,
)
}
func validProfile(id, backendID string) string {
return fmt.Sprintf("id: %s\nbackend: %s\nmodel: test-model\n", id, backendID)
}
var _ fs.FS = openrouter.FS()

View File

@@ -15,10 +15,7 @@ const (
OpenAIChatCompletionsPath = "/chat/completions" OpenAIChatCompletionsPath = "/chat/completions"
ExecutionDefaultTimeoutSeconds = 600 ExecutionDefaultTimeoutSeconds = 600
) LLMRequestTimeoutDefault = 10 * time.Minute
var (
LLMRequestTimeoutDefault = 10 * time.Minute
) )
func ExecutionTargetDefault() domain.ExecutionTarget { func ExecutionTargetDefault() domain.ExecutionTarget {

View File

@@ -60,15 +60,16 @@ type CacheControl struct {
// RunRequest represents a request to generate a single artifact. // RunRequest represents a request to generate a single artifact.
type RunRequest struct { type RunRequest struct {
PromptID string PromptID string
PromptVersion string PromptVersion string
ProfileID string ProfileID string
SessionID string SessionID string
APIKey string `json:"-" yaml:"-"` APIKey string `json:"-" yaml:"-"`
Inputs map[string]ArtifactRef Inputs map[string]ArtifactRef
Vars map[string]string Vars map[string]string
Execution *ExecutionTargetOverride Execution *ExecutionTargetOverride
Validation *OutputContract Validation *OutputContract
AppendedMessages []RenderedMessage
} }
// RunResult represents the complete result of a prompt execution run. // RunResult represents the complete result of a prompt execution run.
@@ -97,22 +98,22 @@ type RunResult struct {
// PreparedRun contains pre-LLM execution state from the prepare/render phase. // PreparedRun contains pre-LLM execution state from the prepare/render phase.
// It must never include resolved API key values, model output, or validation data. // It must never include resolved API key values, model output, or validation data.
type PreparedRun struct { type PreparedRun struct {
PromptID string `json:"prompt_id"` PromptID string
PromptVersion string `json:"prompt_version,omitempty"` PromptVersion string
PromptHash string `json:"prompt_hash,omitempty"` PromptHash string
SelectedProfileID string `json:"selected_profile_id"` SelectedProfileID string
SelectedBackendID string `json:"selected_backend_id,omitempty"` SelectedBackendID string
EffectiveModelParams ExecutionTarget `json:"effective_model_params"` EffectiveModelParams ExecutionTarget
TargetPresence ExecutionTargetPresence `json:"-"` TargetPresence ExecutionTargetPresence
OutputContract OutputContract `json:"output_contract"` OutputContract OutputContract
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"` StructuredOutput *StructuredOutputSpec
InputHashes map[string]string `json:"input_hashes,omitempty"` InputHashes map[string]string
SessionID string `json:"session_id,omitempty"` SessionID string
RenderedPromptHash string `json:"rendered_prompt_hash"` RenderedPromptHash string
Messages []RenderedMessage `json:"messages"` Messages []RenderedMessage
StartTime time.Time `json:"start_time,omitempty"` StartTime time.Time
EndTime time.Time `json:"end_time,omitempty"` EndTime time.Time
DurationMS int64 `json:"duration_ms,omitempty"` DurationMS int64
} }
// ArtifactRef represents a reference to an input artifact. // ArtifactRef represents a reference to an input artifact.
@@ -192,6 +193,7 @@ type BackendCapacityPolicy struct {
// ExecutionProfile describes how and where to execute a model. // ExecutionProfile describes how and where to execute a model.
type ExecutionProfile struct { type ExecutionProfile struct {
ID string `yaml:"id"` ID string `yaml:"id"`
BaseProfileID string `yaml:"base_profile"`
BackendID string `yaml:"backend"` BackendID string `yaml:"backend"`
Endpoint string `yaml:"endpoint"` Endpoint string `yaml:"endpoint"`
Model string `yaml:"model"` Model string `yaml:"model"`

View File

@@ -0,0 +1,39 @@
package domain
import (
"errors"
"net/url"
"strings"
)
// NormalizeOpenAICompatibleBaseEndpoint trims and validates a source-neutral
// OpenAI-compatible provider base endpoint.
func NormalizeOpenAICompatibleBaseEndpoint(endpoint string) (string, error) {
endpoint = strings.TrimSpace(endpoint)
if endpoint == "" {
return "", errors.New("endpoint must not be blank")
}
if strings.Contains(endpoint, "#") {
return "", errors.New("endpoint must not contain a fragment")
}
parsed, err := url.Parse(endpoint)
if err != nil {
return "", errors.New("endpoint must be a valid URL")
}
parsed.Scheme = strings.ToLower(parsed.Scheme)
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return "", errors.New("endpoint must use http or https")
}
if !parsed.IsAbs() || parsed.Hostname() == "" {
return "", errors.New("endpoint must be absolute and include a host")
}
if parsed.User != nil {
return "", errors.New("endpoint must not contain user information")
}
if parsed.RawQuery != "" || parsed.ForceQuery {
return "", errors.New("endpoint must not contain a query string")
}
return parsed.String(), nil
}

View File

@@ -0,0 +1,47 @@
package domain
import "testing"
func TestNormalizeOpenAICompatibleBaseEndpoint(t *testing.T) {
tests := []struct {
name string
endpoint string
want string
wantErr bool
}{
{name: "http host", endpoint: "http://provider.example", want: "http://provider.example"},
{name: "https nested path and whitespace", endpoint: " HTTPS://provider.example/api/openai/v1 ", want: "https://provider.example/api/openai/v1"},
{name: "IPv4 host and port", endpoint: "http://127.0.0.1:8080/v1", want: "http://127.0.0.1:8080/v1"},
{name: "IPv6 host and port", endpoint: "https://[::1]:8443/v1", want: "https://[::1]:8443/v1"},
{name: "repeated trailing slashes", endpoint: "https://provider.example/v1///", want: "https://provider.example/v1///"},
{name: "blank", endpoint: " \t\n ", wantErr: true},
{name: "relative path", endpoint: "/api/v1", wantErr: true},
{name: "scheme relative", endpoint: "//provider.example/v1", wantErr: true},
{name: "missing host", endpoint: "https:///v1", wantErr: true},
{name: "unsupported scheme", endpoint: "ftp://provider.example/v1", wantErr: true},
{name: "user information", endpoint: "https://user:secret@provider.example/v1", wantErr: true},
{name: "query", endpoint: "https://provider.example/v1?mode=chat", wantErr: true},
{name: "empty query", endpoint: "https://provider.example/v1?", wantErr: true},
{name: "fragment", endpoint: "https://provider.example/v1#chat", wantErr: true},
{name: "empty fragment", endpoint: "https://provider.example/v1#", wantErr: true},
{name: "malformed URL", endpoint: "https://provider.example/%zz", wantErr: true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got, err := NormalizeOpenAICompatibleBaseEndpoint(tc.endpoint)
if tc.wantErr {
if err == nil {
t.Fatalf("expected endpoint error, got %q", got)
}
return
}
if err != nil {
t.Fatalf("normalize endpoint: %v", err)
}
if got != tc.want {
t.Fatalf("normalized endpoint = %q, want %q", got, tc.want)
}
})
}
}

View File

@@ -0,0 +1,34 @@
package domain
import (
"errors"
"math"
"time"
)
const maxExecutionTimeoutSeconds int64 = math.MaxInt64 / int64(time.Second)
// ValidateExecutionTargetSettings validates source-neutral execution-setting
// invariants on a resolved target.
func ValidateExecutionTargetSettings(target ExecutionTarget) error {
if !isFinite(target.Temperature) || target.Temperature < 0 || target.Temperature > 2 {
return errors.New("temperature must be finite and between 0 and 2")
}
if target.MaxTokens < 0 {
return errors.New("max_tokens must be greater than or equal to 0")
}
if !isFinite(target.TopP) || target.TopP < 0 || target.TopP > 1 {
return errors.New("top_p must be finite and between 0 and 1")
}
if target.TimeoutSeconds < 0 {
return errors.New("timeout_seconds must be greater than or equal to 0")
}
if int64(target.TimeoutSeconds) > maxExecutionTimeoutSeconds {
return errors.New("timeout_seconds exceeds the maximum supported duration")
}
return nil
}
func isFinite(value float64) bool {
return !math.IsNaN(value) && !math.IsInf(value, 0)
}

View File

@@ -0,0 +1,73 @@
package domain
import (
"math"
"strconv"
"strings"
"testing"
)
func TestValidateExecutionTargetSettings(t *testing.T) {
valid := ExecutionTarget{
Temperature: 1,
MaxTokens: 1,
TopP: 0.5,
TimeoutSeconds: 1,
}
type testCase struct {
name string
change func(*ExecutionTarget)
wantErr string
}
tests := []testCase{
{name: "temperature lower boundary", change: func(v *ExecutionTarget) { v.Temperature = 0 }},
{name: "temperature finite lower neighbor", change: func(v *ExecutionTarget) { v.Temperature = math.Nextafter(0, 1) }},
{name: "temperature finite upper neighbor", change: func(v *ExecutionTarget) { v.Temperature = math.Nextafter(2, 0) }},
{name: "temperature upper boundary", change: func(v *ExecutionTarget) { v.Temperature = 2 }},
{name: "temperature below lower boundary", change: func(v *ExecutionTarget) { v.Temperature = math.Nextafter(0, math.Inf(-1)) }, wantErr: "temperature"},
{name: "temperature above upper boundary", change: func(v *ExecutionTarget) { v.Temperature = math.Nextafter(2, math.Inf(1)) }, wantErr: "temperature"},
{name: "temperature NaN", change: func(v *ExecutionTarget) { v.Temperature = math.NaN() }, wantErr: "temperature"},
{name: "temperature positive infinity", change: func(v *ExecutionTarget) { v.Temperature = math.Inf(1) }, wantErr: "temperature"},
{name: "temperature negative infinity", change: func(v *ExecutionTarget) { v.Temperature = math.Inf(-1) }, wantErr: "temperature"},
{name: "max tokens lower boundary", change: func(v *ExecutionTarget) { v.MaxTokens = 0 }},
{name: "max tokens finite neighbor", change: func(v *ExecutionTarget) { v.MaxTokens = 1 }},
{name: "max tokens below lower boundary", change: func(v *ExecutionTarget) { v.MaxTokens = -1 }, wantErr: "max_tokens"},
{name: "top p lower boundary", change: func(v *ExecutionTarget) { v.TopP = 0 }},
{name: "top p finite lower neighbor", change: func(v *ExecutionTarget) { v.TopP = math.Nextafter(0, 1) }},
{name: "top p finite upper neighbor", change: func(v *ExecutionTarget) { v.TopP = math.Nextafter(1, 0) }},
{name: "top p upper boundary", change: func(v *ExecutionTarget) { v.TopP = 1 }},
{name: "top p below lower boundary", change: func(v *ExecutionTarget) { v.TopP = math.Nextafter(0, math.Inf(-1)) }, wantErr: "top_p"},
{name: "top p above upper boundary", change: func(v *ExecutionTarget) { v.TopP = math.Nextafter(1, math.Inf(1)) }, wantErr: "top_p"},
{name: "top p NaN", change: func(v *ExecutionTarget) { v.TopP = math.NaN() }, wantErr: "top_p"},
{name: "top p positive infinity", change: func(v *ExecutionTarget) { v.TopP = math.Inf(1) }, wantErr: "top_p"},
{name: "top p negative infinity", change: func(v *ExecutionTarget) { v.TopP = math.Inf(-1) }, wantErr: "top_p"},
{name: "timeout lower boundary", change: func(v *ExecutionTarget) { v.TimeoutSeconds = 0 }},
{name: "timeout finite neighbor", change: func(v *ExecutionTarget) { v.TimeoutSeconds = 1 }},
{name: "timeout below lower boundary", change: func(v *ExecutionTarget) { v.TimeoutSeconds = -1 }, wantErr: "timeout_seconds"},
}
if strconv.IntSize == 64 {
durationLimit := maxExecutionTimeoutSeconds
tests = append(tests,
testCase{name: "timeout duration boundary", change: func(v *ExecutionTarget) { v.TimeoutSeconds = int(durationLimit) }},
testCase{name: "timeout above duration boundary", change: func(v *ExecutionTarget) { v.TimeoutSeconds = int(durationLimit) + 1 }, wantErr: "timeout_seconds"},
)
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
target := valid
tt.change(&target)
err := ValidateExecutionTargetSettings(target)
if tt.wantErr == "" {
if err != nil {
t.Fatalf("validate execution settings: %v", err)
}
return
}
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("error = %v, want diagnostic containing %q", err, tt.wantErr)
}
})
}
}

View File

@@ -0,0 +1,82 @@
package domain
import (
"errors"
"strings"
"unicode/utf8"
)
const (
RoleDeveloper = "developer"
RoleSystem = "system"
RoleUser = "user"
RoleAssistant = "assistant"
)
// NormalizeMessageRole validates and canonicalizes a provider-bound chat role.
func NormalizeMessageRole(role string) (string, error) {
if !utf8.ValidString(role) {
return "", errors.New("message role must be valid UTF-8")
}
normalized := strings.ToLower(strings.TrimSpace(role))
switch normalized {
case RoleDeveloper, RoleSystem, RoleUser, RoleAssistant:
return normalized, nil
default:
return "", errors.New("message role must be developer, system, user, or assistant")
}
}
// NormalizeCacheControl validates, canonicalizes, and copies cache metadata.
func NormalizeCacheControl(control *CacheControl) (*CacheControl, error) {
if control == nil {
return nil, nil
}
if !utf8.ValidString(string(control.Type)) {
return nil, errors.New("cache control type must be valid UTF-8")
}
if !utf8.ValidString(control.TTL) {
return nil, errors.New("cache control ttl must be valid UTF-8")
}
cacheType := strings.TrimSpace(string(control.Type))
if cacheType == "" {
return nil, errors.New("cache control type is required")
}
if CacheControlType(cacheType) != CacheControlEphemeral {
return nil, errors.New("unsupported type")
}
ttl := strings.TrimSpace(control.TTL)
if ttl != "" && ttl != "1h" {
return nil, errors.New("unsupported ttl")
}
return &CacheControl{Type: CacheControlType(cacheType), TTL: ttl}, nil
}
// CloneRenderedMessages returns a deep copy of rendered messages.
func CloneRenderedMessages(messages []RenderedMessage) []RenderedMessage {
cloned := make([]RenderedMessage, len(messages))
copyRenderedMessages(cloned, messages)
return cloned
}
// ConcatRenderedMessages returns an independently owned concatenation of messages.
func ConcatRenderedMessages(prefix, suffix []RenderedMessage) []RenderedMessage {
messages := make([]RenderedMessage, len(prefix)+len(suffix))
copyRenderedMessages(messages, prefix)
copyRenderedMessages(messages[len(prefix):], suffix)
return messages
}
func copyRenderedMessages(destination, source []RenderedMessage) {
for index, message := range source {
destination[index] = message
if message.CacheControl != nil {
cacheControl := *message.CacheControl
destination[index].CacheControl = &cacheControl
}
}
}

View File

@@ -0,0 +1,121 @@
package domain
import (
"strings"
"testing"
)
func TestNormalizeMessageRole(t *testing.T) {
tests := []struct {
name string
input string
want string
wantErr bool
}{
{name: "developer", input: RoleDeveloper, want: RoleDeveloper},
{name: "system", input: RoleSystem, want: RoleSystem},
{name: "user", input: RoleUser, want: RoleUser},
{name: "assistant", input: RoleAssistant, want: RoleAssistant},
{name: "surrounding whitespace and mixed case", input: " \u2003UsEr\u2003 ", want: RoleUser},
{name: "blank", input: " \t\n ", wantErr: true},
{name: "tool", input: "tool", wantErr: true},
{name: "function", input: "function", wantErr: true},
{name: "custom", input: "custom-role", wantErr: true},
{name: "invalid UTF-8", input: string([]byte{0xff}), wantErr: true},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got, err := NormalizeMessageRole(test.input)
if test.wantErr {
if err == nil {
t.Fatal("expected an error")
}
if strings.Contains(err.Error(), test.input) {
t.Fatalf("error exposed the input: %v", err)
}
return
}
if err != nil {
t.Fatalf("NormalizeMessageRole() error = %v", err)
}
if got != test.want {
t.Fatalf("NormalizeMessageRole() = %q, want %q", got, test.want)
}
})
}
}
func TestNormalizeCacheControl(t *testing.T) {
tests := []struct {
name string
input *CacheControl
want *CacheControl
wantErr bool
}{
{name: "nil", input: nil, want: nil},
{
name: "canonical values",
input: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"},
want: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"},
},
{
name: "trims values",
input: &CacheControl{Type: " ephemeral ", TTL: " 1h\t"},
want: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"},
},
{name: "empty type", input: &CacheControl{}, wantErr: true},
{name: "unsupported type", input: &CacheControl{Type: "persistent"}, wantErr: true},
{name: "unsupported ttl", input: &CacheControl{Type: CacheControlEphemeral, TTL: "5m"}, wantErr: true},
{name: "invalid type UTF-8", input: &CacheControl{Type: CacheControlType(string([]byte{0xff}))}, wantErr: true},
{name: "invalid ttl UTF-8", input: &CacheControl{Type: CacheControlEphemeral, TTL: string([]byte{0xff})}, wantErr: true},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got, err := NormalizeCacheControl(test.input)
if test.wantErr {
if err == nil {
t.Fatal("expected an error")
}
return
}
if err != nil {
t.Fatalf("NormalizeCacheControl() error = %v", err)
}
if got == nil || test.want == nil {
if got != test.want {
t.Fatalf("NormalizeCacheControl() = %#v, want %#v", got, test.want)
}
return
}
if *got != *test.want {
t.Fatalf("NormalizeCacheControl() = %#v, want %#v", got, test.want)
}
if got == test.input {
t.Fatal("normalized cache control aliases its input")
}
})
}
}
func TestRenderedMessageCloning(t *testing.T) {
prefix := []RenderedMessage{{Role: RoleSystem, Content: "prefix", CacheControl: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"}}}
suffix := []RenderedMessage{{Role: RoleUser, Content: " suffix "}, {Role: RoleAssistant, Content: "", CacheControl: &CacheControl{Type: CacheControlEphemeral}}}
cloned := CloneRenderedMessages(prefix)
combined := ConcatRenderedMessages(prefix, suffix)
if len(combined) != 3 || combined[0].Content != "prefix" || combined[1].Content != " suffix " || combined[2].Content != "" {
t.Fatalf("unexpected combined messages: %#v", combined)
}
if cloned[0].CacheControl == prefix[0].CacheControl || combined[0].CacheControl == prefix[0].CacheControl || combined[2].CacheControl == suffix[1].CacheControl {
t.Fatal("cloned cache controls alias their inputs")
}
prefix[0].Content = "changed"
prefix[0].CacheControl.TTL = ""
suffix[1].CacheControl.Type = "changed"
if cloned[0].Content != "prefix" || cloned[0].CacheControl.TTL != "1h" || combined[0].Content != "prefix" || combined[0].CacheControl.TTL != "1h" || combined[2].CacheControl.Type != CacheControlEphemeral {
t.Fatalf("cloned messages changed with their inputs: cloned=%#v combined=%#v", cloned, combined)
}
}

View File

@@ -0,0 +1,38 @@
package domain
import (
"errors"
"fmt"
"strings"
)
const maxOutputRepairAttempts = 3
// ValidateOutputContract validates source-neutral output-contract invariants.
func ValidateOutputContract(contract OutputContract) error {
switch contract.Format {
case FormatText, FormatMarkdown, FormatJSON:
default:
return fmt.Errorf("invalid output format: %q", contract.Format)
}
switch contract.ValidationMode {
case ValidationNone, ValidationBasic, ValidationJSON, ValidationJSONSchema:
default:
return fmt.Errorf("invalid validation mode: %q", contract.ValidationMode)
}
if contract.ValidationMode == ValidationJSONSchema && strings.TrimSpace(contract.SchemaPath) == "" {
return errors.New("schema_path is required when validation_mode is json_schema")
}
if contract.RepairAttempts < 0 {
return errors.New("repair_attempts must be greater than or equal to 0")
}
if contract.RepairAttempts > maxOutputRepairAttempts {
return fmt.Errorf("repair_attempts must be less than or equal to %d", maxOutputRepairAttempts)
}
if contract.ValidationMode == ValidationNone && contract.RepairAttempts > 0 {
return errors.New("repair_attempts requires basic, json, or json_schema validation")
}
return nil
}

View File

@@ -0,0 +1,87 @@
package domain
import (
"strings"
"testing"
)
func TestValidateOutputContract(t *testing.T) {
valid := OutputContract{
Format: FormatText,
ValidationMode: ValidationNone,
}
tests := []struct {
name string
change func(*OutputContract)
wantErr string
}{
{name: "text format", change: func(c *OutputContract) { c.Format = FormatText }},
{name: "markdown format", change: func(c *OutputContract) { c.Format = FormatMarkdown }},
{name: "json format", change: func(c *OutputContract) { c.Format = FormatJSON }},
{name: "empty format", change: func(c *OutputContract) { c.Format = "" }, wantErr: "format"},
{name: "unsupported format", change: func(c *OutputContract) { c.Format = OutputFormat("binary") }, wantErr: "format"},
{name: "none validation", change: func(c *OutputContract) { c.ValidationMode = ValidationNone }},
{name: "basic validation", change: func(c *OutputContract) { c.ValidationMode = ValidationBasic }},
{name: "json validation", change: func(c *OutputContract) { c.ValidationMode = ValidationJSON }},
{name: "json schema validation", change: func(c *OutputContract) {
c.ValidationMode = ValidationJSONSchema
c.SchemaPath = "schema.json"
}},
{name: "empty validation mode", change: func(c *OutputContract) { c.ValidationMode = "" }, wantErr: "validation mode"},
{name: "unsupported validation mode", change: func(c *OutputContract) { c.ValidationMode = ValidationMode("unknown") }, wantErr: "validation mode"},
{name: "negative repair attempts", change: func(c *OutputContract) {
c.ValidationMode = ValidationBasic
c.RepairAttempts = -1
}, wantErr: "repair_attempts"},
{name: "zero repair attempts", change: func(c *OutputContract) { c.RepairAttempts = 0 }},
{name: "one repair attempt", change: func(c *OutputContract) {
c.ValidationMode = ValidationBasic
c.RepairAttempts = 1
}},
{name: "maximum repair attempts", change: func(c *OutputContract) {
c.ValidationMode = ValidationJSON
c.RepairAttempts = 3
}},
{name: "too many repair attempts", change: func(c *OutputContract) {
c.ValidationMode = ValidationJSONSchema
c.SchemaPath = "schema.json"
c.RepairAttempts = 4
}, wantErr: "repair_attempts"},
{name: "none validation with repair attempts", change: func(c *OutputContract) {
c.ValidationMode = ValidationNone
c.RepairAttempts = 1
}, wantErr: "repair_attempts"},
{name: "json schema empty path", change: func(c *OutputContract) {
c.ValidationMode = ValidationJSONSchema
c.SchemaPath = ""
}, wantErr: "schema_path"},
{name: "json schema whitespace path", change: func(c *OutputContract) {
c.ValidationMode = ValidationJSONSchema
c.SchemaPath = " \t "
}, wantErr: "schema_path"},
{name: "json schema nonblank path", change: func(c *OutputContract) {
c.ValidationMode = ValidationJSONSchema
c.SchemaPath = " schema.json "
}},
{name: "non-schema empty path", change: func(c *OutputContract) { c.SchemaPath = "" }},
{name: "non-schema populated path", change: func(c *OutputContract) { c.SchemaPath = "ignored.json" }},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
contract := valid
tt.change(&contract)
err := ValidateOutputContract(contract)
if tt.wantErr == "" {
if err != nil {
t.Fatalf("validate output contract: %v", err)
}
return
}
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("error = %v, want diagnostic containing %q", err, tt.wantErr)
}
})
}
}

View File

@@ -1,141 +0,0 @@
package domain
import (
"encoding/json"
"strings"
"testing"
)
func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
const envName = "PROMPTKIT_TEST_API_KEY"
const secret = "super-secret-value"
t.Setenv(envName, secret)
prepared := PreparedRun{
PromptID: "prompt.id",
PromptVersion: "v1",
PromptHash: "prompt-hash",
SelectedProfileID: "local-fast",
EffectiveModelParams: ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "gpt-test",
APIKeyEnv: envName,
APIKey: secret,
},
InputHashes: map[string]string{"transcript": "hash-1"},
RenderedPromptHash: "rendered-hash",
Messages: []RenderedMessage{
{Role: "system", Content: "You are helpful."},
{Role: "user", Content: "Summarize this."},
},
}
b, err := json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal failed: %v", err)
}
out := string(b)
if strings.Contains(out, secret) {
t.Fatalf("prepared run JSON unexpectedly contains secret value: %s", out)
}
if !strings.Contains(out, `"api_key_env":"`+envName+`"`) {
t.Fatalf("prepared run JSON should include api_key_env name: %s", out)
}
var top map[string]any
if err := json.Unmarshal(b, &top); err != nil {
t.Fatalf("unmarshal failed: %v", err)
}
for _, forbidden := range []string{"raw_output", "validation", "artifact"} {
if _, ok := top[forbidden]; ok {
t.Fatalf("prepared run JSON should not include %q", forbidden)
}
}
}
func TestPreparedRunJSONIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
prepared := PreparedRun{
PromptID: "prompt.id",
SelectedProfileID: "local-fast",
EffectiveModelParams: ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "gpt-test",
},
RenderedPromptHash: "rendered-hash",
Messages: []RenderedMessage{
{
Role: "system",
Content: "You are helpful.",
CacheControl: &CacheControl{
Type: CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "Summarize this."},
},
}
b, err := json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal failed: %v", err)
}
var decoded struct {
Messages []map[string]any `json:"messages"`
}
if err := json.Unmarshal(b, &decoded); err != nil {
t.Fatalf("unmarshal failed: %v", err)
}
if len(decoded.Messages) != 2 {
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
}
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
if !ok {
t.Fatalf("expected cache_control on first message, got %#v", decoded.Messages[0])
}
if cacheControl["type"] != string(CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
}
if _, ok := decoded.Messages[1]["cache_control"]; ok {
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
}
}
func TestPreparedRunJSONIncludesSessionIDOnlyWhenPresent(t *testing.T) {
prepared := PreparedRun{
PromptID: "prompt.id",
SelectedProfileID: "local-fast",
EffectiveModelParams: ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "gpt-test",
},
SessionID: "session-123",
RenderedPromptHash: "rendered-hash",
Messages: []RenderedMessage{{Role: "user", Content: "Summarize this."}},
}
b, err := json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal failed: %v", err)
}
var decoded map[string]any
if err := json.Unmarshal(b, &decoded); err != nil {
t.Fatalf("unmarshal failed: %v", err)
}
if decoded["session_id"] != "session-123" {
t.Fatalf("expected session_id in prepared run JSON, got %#v", decoded["session_id"])
}
prepared.SessionID = ""
b, err = json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal failed: %v", err)
}
if strings.Contains(string(b), "session_id") {
t.Fatalf("expected empty session_id to be omitted, got %s", b)
}
}

View File

@@ -8,6 +8,9 @@ import (
// NormalizeSessionID applies the shared session identifier rule. // NormalizeSessionID applies the shared session identifier rule.
func NormalizeSessionID(raw string) (string, error) { func NormalizeSessionID(raw string) (string, error) {
if !utf8.ValidString(raw) {
return "", fmt.Errorf("session_id must contain valid UTF-8")
}
normalized := strings.TrimSpace(raw) normalized := strings.TrimSpace(raw)
if normalized == "" { if normalized == "" {
return "", nil return "", nil

View File

@@ -7,10 +7,10 @@ import (
func TestNormalizeSessionID(t *testing.T) { func TestNormalizeSessionID(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
raw string raw string
want string want string
wantErr bool wantErrContains string
}{ }{
{ {
name: "trims surrounding Unicode whitespace", name: "trims surrounding Unicode whitespace",
@@ -28,21 +28,24 @@ func TestNormalizeSessionID(t *testing.T) {
want: strings.Repeat("界", SessionIDMaxLength), want: strings.Repeat("界", SessionIDMaxLength),
}, },
{ {
name: "one Unicode code point over maximum is rejected", name: "one Unicode code point over maximum is rejected",
raw: strings.Repeat("界", SessionIDMaxLength+1), raw: strings.Repeat("界", SessionIDMaxLength+1),
wantErr: true, wantErrContains: "exceeds maximum",
}, },
{name: "invalid UTF-8 before valid content", raw: string([]byte{0xff}) + "session", wantErrContains: "valid UTF-8"},
{name: "invalid UTF-8 within valid content", raw: "ses" + string([]byte{0xff}) + "sion", wantErrContains: "valid UTF-8"},
{name: "invalid UTF-8 after valid content", raw: "session" + string([]byte{0xff}), wantErrContains: "valid UTF-8"},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
got, err := NormalizeSessionID(tt.raw) got, err := NormalizeSessionID(tt.raw)
if tt.wantErr { if tt.wantErrContains != "" {
if err == nil { if err == nil {
t.Fatal("expected normalization error") t.Fatal("expected normalization error")
} }
if !strings.Contains(err.Error(), "exceeds maximum") { if !strings.Contains(err.Error(), tt.wantErrContains) {
t.Fatalf("expected useful length diagnostic, got %v", err) t.Fatalf("expected diagnostic containing %q, got %v", tt.wantErrContains, err)
} }
return return
} }

View File

@@ -36,7 +36,8 @@ func FindYAMLFiles(ctx context.Context, root string) ([]string, error) {
return files, err return files, err
} }
// FindFSYAMLFiles returns sorted paths for .yaml and .yml files under root in fsys. // FindFSYAMLFiles returns root itself when it names a file. For a directory
// root, it returns sorted paths for .yaml and .yml files beneath that root.
func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, error) { func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, error) {
cleanRoot := CleanFSRoot(root) cleanRoot := CleanFSRoot(root)
var files []string var files []string
@@ -52,6 +53,10 @@ func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, er
if d.IsDir() { if d.IsDir() {
return nil return nil
} }
if name == cleanRoot {
files = append(files, name)
return nil
}
if !IsYAMLFile(d.Name()) { if !IsYAMLFile(d.Name()) {
return nil return nil
} }
@@ -71,10 +76,10 @@ func RelativePath(root string, filePath string) string {
return filepath.Clean(rel) return filepath.Clean(rel)
} }
// CleanFSRoot normalizes a root path for use with fs.FS. // CleanFSRoot normalizes a root path for use with fs.FS while preserving
// nonblank leading and trailing whitespace.
func CleanFSRoot(root string) string { func CleanFSRoot(root string) string {
root = strings.TrimSpace(root) if strings.TrimSpace(root) == "" || root == "." {
if root == "" || root == "." {
return "." return "."
} }
return path.Clean(root) return path.Clean(root)
@@ -97,22 +102,21 @@ func DisplayPath(root string, name string) string {
// ResolveFSPath resolves userPath from baseDir and keeps it inside root. // ResolveFSPath resolves userPath from baseDir and keeps it inside root.
func ResolveFSPath(root string, baseDir string, userPath string) (string, string, error) { func ResolveFSPath(root string, baseDir string, userPath string) (string, string, error) {
cleanRoot := CleanFSRoot(root) cleanRoot := CleanFSRoot(root)
cleanBase := path.Clean(strings.TrimSpace(baseDir)) cleanBase := path.Clean(baseDir)
if cleanBase == "" { if strings.TrimSpace(baseDir) == "" {
cleanBase = cleanRoot cleanBase = cleanRoot
} }
if !containsFSPath(cleanRoot, cleanBase) { if !containsFSPath(cleanRoot, cleanBase) {
return "", "", fmt.Errorf("base path %q is outside source root %q", cleanBase, cleanRoot) return "", "", fmt.Errorf("base path %q is outside source root %q", cleanBase, cleanRoot)
} }
cleanUserPath := strings.TrimSpace(userPath) if strings.TrimSpace(userPath) == "" {
if cleanUserPath == "" {
return "", "", fmt.Errorf("path is required") return "", "", fmt.Errorf("path is required")
} }
cleanUserPath = path.Clean(cleanUserPath) if path.IsAbs(userPath) {
if path.IsAbs(cleanUserPath) {
return "", "", fmt.Errorf("path %q must be relative", userPath) return "", "", fmt.Errorf("path %q must be relative", userPath)
} }
cleanUserPath := path.Clean(userPath)
resolved := path.Clean(path.Join(cleanBase, cleanUserPath)) resolved := path.Clean(path.Join(cleanBase, cleanUserPath))
if !containsFSPath(cleanRoot, resolved) { if !containsFSPath(cleanRoot, resolved) {
@@ -130,13 +134,6 @@ func containsFSPath(root string, name string) bool {
return name == root || strings.HasPrefix(name, strings.TrimSuffix(root, "/")+"/") return name == root || strings.HasPrefix(name, strings.TrimSuffix(root, "/")+"/")
} }
// Stem strips .yaml or .yml from a file name.
func Stem(name string) string {
name = strings.TrimSuffix(name, ".yaml")
name = strings.TrimSuffix(name, ".yml")
return name
}
func IsYAMLFile(name string) bool { func IsYAMLFile(name string) bool {
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml") return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
} }

View File

@@ -54,7 +54,7 @@ func TestFindFSYAMLFilesNestedSortedAndFiltered(t *testing.T) {
"other/ignored.yaml": &fstest.MapFile{Data: []byte("id: ignored")}, "other/ignored.yaml": &fstest.MapFile{Data: []byte("id: ignored")},
} }
got, err := FindFSYAMLFiles(context.Background(), fsys, " prompts ") got, err := FindFSYAMLFiles(context.Background(), fsys, "prompts")
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
} }
@@ -98,8 +98,11 @@ func TestCleanFSRoot(t *testing.T) {
want string want string
}{ }{
{name: "empty", root: "", want: "."}, {name: "empty", root: "", want: "."},
{name: "whitespace only", root: " \t ", want: "."},
{name: "dot", root: ".", want: "."}, {name: "dot", root: ".", want: "."},
{name: "trimmed", root: " prompts/../profiles ", want: "profiles"}, {name: "cleaned", root: "prompts/../profiles", want: "profiles"},
{name: "leading whitespace preserved", root: " profiles", want: " profiles"},
{name: "trailing whitespace preserved", root: "profiles ", want: "profiles "},
} }
for _, tc := range tests { for _, tc := range tests {
@@ -158,6 +161,22 @@ func TestResolveFSPath(t *testing.T) {
wantPath: "prompts/shared/user.tmpl", wantPath: "prompts/shared/user.tmpl",
wantDisplay: "shared/user.tmpl", wantDisplay: "shared/user.tmpl",
}, },
{
name: "leading whitespace preserved",
root: "prompts",
baseDir: "prompts/nested",
userPath: " user.tmpl",
wantPath: "prompts/nested/ user.tmpl",
wantDisplay: "nested/ user.tmpl",
},
{
name: "trailing whitespace preserved",
root: "prompts",
baseDir: "prompts/nested",
userPath: "user.tmpl ",
wantPath: "prompts/nested/user.tmpl ",
wantDisplay: "nested/user.tmpl ",
},
{ {
name: "escape rejected", name: "escape rejected",
root: "prompts", root: "prompts",
@@ -218,26 +237,6 @@ func TestResolveFSPath(t *testing.T) {
} }
} }
func TestStemStripsYAMLExtensions(t *testing.T) {
tests := []struct {
name string
in string
want string
}{
{name: "yaml", in: "prompt.yaml", want: "prompt"},
{name: "yml", in: "profile.yml", want: "profile"},
{name: "other", in: "file.txt", want: "file.txt"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if got := Stem(tc.in); got != tc.want {
t.Fatalf("expected %q, got %q", tc.want, got)
}
})
}
}
func TestIsYAMLFile(t *testing.T) { func TestIsYAMLFile(t *testing.T) {
tests := []struct { tests := []struct {
name string name string

View File

@@ -1,5 +1,5 @@
// Package jsonvalue validates and defensively copies JSON-compatible value // Package jsonvalue validates and defensively copies bounded JSON-compatible
// trees used by configuration, request, and prepared-state boundaries. // value trees used by configuration, request, and prepared-state boundaries.
package jsonvalue package jsonvalue
import ( import (
@@ -8,29 +8,39 @@ import (
"math" "math"
"reflect" "reflect"
"sort" "sort"
"strconv"
) )
const maxSafeJSONInteger = 1<<53 - 1 const (
maxContainerDepth = 100
maxProducedNodes = 100_000
)
type visit struct { type visit struct {
typ reflect.Type typ reflect.Type
ptr uintptr ptr uintptr
} }
type traversalState struct {
active map[visit]struct{}
producedNodes int
}
// Copy validates and deeply copies a JSON-compatible value while preserving // Copy validates and deeply copies a JSON-compatible value while preserving
// compatible concrete map, slice, array, scalar, and number types. // compatible concrete map, slice, array, scalar, and number types. It rejects
// cycles and values that exceed the package's traversal limits.
func Copy(src any) (any, error) { func Copy(src any) (any, error) {
return copyValue(reflect.ValueOf(src), "value", make(map[visit]struct{}), true) return copyValue(reflect.ValueOf(src), "value", newTraversalState(), true, 0)
} }
// CopyMap validates and deeply copies an extra-parameter map while preserving // CopyMap validates and deeply copies an extra-parameter map while preserving
// compatible concrete map, slice, array, scalar, and number types. // compatible concrete map, slice, array, scalar, and number types. It rejects
// empty object keys, cycles, and values that exceed the package's traversal
// limits.
func CopyMap(src map[string]any) (map[string]any, error) { func CopyMap(src map[string]any) (map[string]any, error) {
if src == nil { if src == nil {
return nil, nil return nil, nil
} }
copied, err := copyValue(reflect.ValueOf(src), "extra_params", make(map[visit]struct{}), false) copied, err := copyValue(reflect.ValueOf(src), "extra_params", newTraversalState(), false, 0)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -44,71 +54,90 @@ func CopyMap(src map[string]any) (map[string]any, error) {
func copyValue( func copyValue(
value reflect.Value, value reflect.Value,
path string, path string,
seen map[visit]struct{}, state *traversalState,
allowEmptyMapKeys bool, allowEmptyMapKeys bool,
containerDepth int,
) (any, error) { ) (any, error) {
if !value.IsValid() { resolved, cleanup, isNull, err := state.resolveIndirection(value, path)
if err != nil {
return nil, err
}
defer cleanup()
if isNull {
if err := state.produceNode(path); err != nil {
return nil, err
}
return nil, nil return nil, nil
} }
if value.Kind() == reflect.Interface { value = resolved
if value.IsNil() {
return nil, nil
}
return copyValue(value.Elem(), path, seen, allowEmptyMapKeys)
}
if !value.CanInterface() { if !value.CanInterface() {
return nil, fmt.Errorf("%s: value cannot be copied", path) return nil, fmt.Errorf("%s: value cannot be copied", path)
} }
if number, ok := value.Interface().(json.Number); ok { if number, ok := value.Interface().(json.Number); ok {
if _, err := json.Marshal(number); err != nil { if !validJSONNumber(number) {
return nil, fmt.Errorf("%s: invalid JSON number", path) return nil, fmt.Errorf("%s: invalid JSON number", path)
} }
f, err := strconv.ParseFloat(number.String(), 64) if err := state.produceNode(path); err != nil {
if err != nil || math.IsNaN(f) || math.IsInf(f, 0) { return nil, err
return nil, fmt.Errorf("%s: invalid JSON number", path)
} }
return number, nil return number, nil
} }
switch value.Kind() { switch value.Kind() {
case reflect.Bool, reflect.String: case reflect.Bool, reflect.String:
if err := state.produceNode(path); err != nil {
return nil, err
}
return value.Interface(), nil return value.Interface(), nil
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
if value.Int() < -maxSafeJSONInteger || value.Int() > maxSafeJSONInteger { if err := state.produceNode(path); err != nil {
return nil, fmt.Errorf("%s: integer is outside the JSON-safe range", path) return nil, err
} }
return value.Interface(), nil return value.Interface(), nil
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
if value.Uint() > maxSafeJSONInteger { if err := state.produceNode(path); err != nil {
return nil, fmt.Errorf("%s: integer is outside the JSON-safe range", path) return nil, err
} }
return value.Interface(), nil return value.Interface(), nil
case reflect.Float32, reflect.Float64: case reflect.Float32, reflect.Float64:
number := value.Convert(reflect.TypeOf(float64(0))).Float() number := value.Float()
if math.IsNaN(number) || math.IsInf(number, 0) { if math.IsNaN(number) || math.IsInf(number, 0) {
return nil, fmt.Errorf("%s: floating-point value must be finite", path) return nil, fmt.Errorf("%s: floating-point value must be finite", path)
} }
if err := state.produceNode(path); err != nil {
return nil, err
}
return value.Interface(), nil return value.Interface(), nil
case reflect.Pointer: case reflect.Map:
if value.IsNil() { if value.IsNil() {
if err := state.produceNode(path); err != nil {
return nil, err
}
return nil, nil return nil, nil
} }
current := visit{typ: value.Type(), ptr: value.Pointer()} nextDepth, err := state.enterContainer(path, containerDepth)
if _, ok := seen[current]; ok { if err != nil {
return nil, fmt.Errorf("%s: cyclic value is not supported", path) return nil, err
} }
seen[current] = struct{}{} return copyMapValue(value, path, state, allowEmptyMapKeys, nextDepth)
defer delete(seen, current)
return copyValue(value.Elem(), path, seen, allowEmptyMapKeys)
case reflect.Map:
return copyMapValue(value, path, seen, allowEmptyMapKeys)
case reflect.Slice: case reflect.Slice:
if value.IsNil() { if value.IsNil() {
if err := state.produceNode(path); err != nil {
return nil, err
}
return nil, nil return nil, nil
} }
return copySequenceValue(value, path, seen, allowEmptyMapKeys) nextDepth, err := state.enterContainer(path, containerDepth)
if err != nil {
return nil, err
}
return copySequenceValue(value, path, state, allowEmptyMapKeys, nextDepth)
case reflect.Array: case reflect.Array:
return copySequenceValue(value, path, seen, allowEmptyMapKeys) nextDepth, err := state.enterContainer(path, containerDepth)
if err != nil {
return nil, err
}
return copySequenceValue(value, path, state, allowEmptyMapKeys, nextDepth)
default: default:
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type()) return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
} }
@@ -117,22 +146,26 @@ func copyValue(
func copyMapValue( func copyMapValue(
value reflect.Value, value reflect.Value,
path string, path string,
seen map[visit]struct{}, state *traversalState,
allowEmptyMapKeys bool, allowEmptyMapKeys bool,
containerDepth int,
) (any, error) { ) (any, error) {
if value.IsNil() {
return nil, nil
}
if value.Type().Key().Kind() != reflect.String { if value.Type().Key().Kind() != reflect.String {
return nil, fmt.Errorf("%s: map key type %s is not supported", path, value.Type().Key()) return nil, fmt.Errorf("%s: map key type %s is not supported", path, value.Type().Key())
} }
if err := state.produceNode(path); err != nil {
return nil, err
}
if err := state.ensureChildCapacity(path, value.Len()); err != nil {
return nil, err
}
current := visit{typ: value.Type(), ptr: value.Pointer()} current := visit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[current]; ok { if _, ok := state.active[current]; ok {
return nil, fmt.Errorf("%s: cyclic value is not supported", path) return nil, fmt.Errorf("%s: cyclic value is not supported", path)
} }
seen[current] = struct{}{} state.active[current] = struct{}{}
defer delete(seen, current) defer delete(state.active, current)
keys := value.MapKeys() keys := value.MapKeys()
sort.Slice(keys, func(i, j int) bool { sort.Slice(keys, func(i, j int) bool {
@@ -152,7 +185,13 @@ func copyMapValue(
if name == "" && !allowEmptyMapKeys { if name == "" && !allowEmptyMapKeys {
return nil, fmt.Errorf("%s: map key must not be empty", path) return nil, fmt.Errorf("%s: map key must not be empty", path)
} }
copied, err := copyValue(value.MapIndex(key), path+"."+name, seen, allowEmptyMapKeys) copied, err := copyValue(
value.MapIndex(key),
path+"."+name,
state,
allowEmptyMapKeys,
containerDepth,
)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -190,17 +229,25 @@ func copyMapValue(
func copySequenceValue( func copySequenceValue(
value reflect.Value, value reflect.Value,
path string, path string,
seen map[visit]struct{}, state *traversalState,
allowEmptyMapKeys bool, allowEmptyMapKeys bool,
containerDepth int,
) (any, error) { ) (any, error) {
if err := state.produceNode(path); err != nil {
return nil, err
}
if err := state.ensureChildCapacity(path, value.Len()); err != nil {
return nil, err
}
var current visit var current visit
if value.Kind() == reflect.Slice { if value.Kind() == reflect.Slice {
current = visit{typ: value.Type(), ptr: value.Pointer()} current = visit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[current]; ok { if _, ok := state.active[current]; ok {
return nil, fmt.Errorf("%s: cyclic value is not supported", path) return nil, fmt.Errorf("%s: cyclic value is not supported", path)
} }
seen[current] = struct{}{} state.active[current] = struct{}{}
defer delete(seen, current) defer delete(state.active, current)
} }
values := make([]any, value.Len()) values := make([]any, value.Len())
@@ -210,8 +257,9 @@ func copySequenceValue(
copied, err := copyValue( copied, err := copyValue(
value.Index(i), value.Index(i),
fmt.Sprintf("%s[%d]", path, i), fmt.Sprintf("%s[%d]", path, i),
seen, state,
allowEmptyMapKeys, allowEmptyMapKeys,
containerDepth,
) )
if err != nil { if err != nil {
return nil, err return nil, err
@@ -248,6 +296,73 @@ func copySequenceValue(
return out, nil return out, nil
} }
func validJSONNumber(number json.Number) bool {
var parsed json.Number
if err := json.Unmarshal([]byte(number.String()), &parsed); err != nil {
return false
}
return parsed.String() == number.String()
}
func newTraversalState() *traversalState {
return &traversalState{active: make(map[visit]struct{})}
}
func (state *traversalState) produceNode(path string) error {
if state.producedNodes >= maxProducedNodes {
return fmt.Errorf("%s: JSON value work limit exceeded", path)
}
state.producedNodes++
return nil
}
func (state *traversalState) enterContainer(path string, depth int) (int, error) {
depth++
if depth > maxContainerDepth {
return 0, fmt.Errorf("%s: JSON container depth limit exceeded", path)
}
return depth, nil
}
func (state *traversalState) ensureChildCapacity(path string, count int) error {
if count > maxProducedNodes-state.producedNodes {
return fmt.Errorf("%s: JSON value work limit exceeded", path)
}
return nil
}
func (state *traversalState) resolveIndirection(
value reflect.Value,
path string,
) (reflect.Value, func(), bool, error) {
var visits []visit
cleanup := func() {
for _, current := range visits {
delete(state.active, current)
}
}
for value.IsValid() && (value.Kind() == reflect.Interface || value.Kind() == reflect.Pointer) {
if value.IsNil() {
return reflect.Value{}, cleanup, true, nil
}
if value.Kind() == reflect.Pointer {
current := visit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := state.active[current]; ok {
cleanup()
return reflect.Value{}, nil, false, fmt.Errorf("%s: cyclic value is not supported", path)
}
state.active[current] = struct{}{}
visits = append(visits, current)
}
value = value.Elem()
}
if !value.IsValid() {
return reflect.Value{}, cleanup, true, nil
}
return value, cleanup, false, nil
}
func canAssignNil(typ reflect.Type) bool { func canAssignNil(typ reflect.Type) bool {
switch typ.Kind() { switch typ.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:

View File

@@ -1,112 +1,332 @@
package jsonvalue_test package jsonvalue
import ( import (
"encoding/json" "encoding/json"
"math" "math"
"reflect" "reflect"
"strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
) )
func TestCopyMapPreservesTypesAndIsolatesMutations(t *testing.T) { type (
nested := map[string]int{"limit": 2} namedBool bool
sequence := []string{"one", "two"} namedString string
input := map[string]any{ namedInt64 int64
"count": int64(7), namedUint64 uint64
"number": json.Number("-1.25e+2"), namedFloat32 float32
"nested": nested, namedFloat64 float64
"sequence": sequence, namedKey string
} namedMap map[namedKey]namedInt64
namedSlice []namedString
copied, err := jsonvalue.CopyMap(input) namedArray [1]map[string]int
if err != nil { )
t.Fatalf("copy map: %v", err)
}
nested["limit"] = 99
sequence[0] = "changed"
input["added"] = true
if got, ok := copied["count"].(int64); !ok || got != 7 {
t.Fatalf("integer type or value changed: %#v", copied["count"])
}
if got, ok := copied["number"].(json.Number); !ok || got != "-1.25e+2" {
t.Fatalf("JSON number type or value changed: %#v", copied["number"])
}
if got := copied["nested"].(map[string]int)["limit"]; got != 2 {
t.Fatalf("nested map was not isolated: %d", got)
}
if got := copied["sequence"].([]string)[0]; got != "one" {
t.Fatalf("sequence was not isolated: %q", got)
}
if _, ok := copied["added"]; ok {
t.Fatalf("top-level map was not isolated: %#v", copied)
}
}
func TestCopyAllowsEmptyObjectKeysAndIsolatesMutations(t *testing.T) {
nested := map[string]any{"": []any{"original"}}
copiedValue, err := jsonvalue.Copy(nested)
if err != nil {
t.Fatalf("copy value: %v", err)
}
nested[""].([]any)[0] = "changed"
copied := copiedValue.(map[string]any)
if got := copied[""].([]any)[0]; got != "original" {
t.Fatalf("copied value was not isolated: %v", got)
}
}
func TestCopyMapRejectsInvalidValues(t *testing.T) {
cyclicMap := map[string]any{}
cyclicMap["self"] = cyclicMap
cyclicSlice := []any{nil}
cyclicSlice[0] = cyclicSlice
func TestCopyPreservesSupportedScalarAndNumberTypes(t *testing.T) {
maxInt := int(^uint(0) >> 1)
minInt := -maxInt - 1
tests := []struct { tests := []struct {
name string name string
value any value any
}{ }{
{name: "empty nested key", value: map[string]int{"": 1}}, {name: "bool", value: true},
{name: "non-string map key", value: map[int]string{1: "one"}}, {name: "named bool", value: namedBool(true)},
{name: "unsupported value", value: make(chan int)}, {name: "string", value: "value"},
{name: "cyclic map", value: cyclicMap}, {name: "named string", value: namedString("value")},
{name: "cyclic slice", value: cyclicSlice}, {name: "int", value: minInt},
{name: "NaN", value: math.NaN()}, {name: "int8", value: int8(-1 << 7)},
{name: "positive infinity", value: math.Inf(1)}, {name: "int16", value: int16(-1 << 15)},
{name: "unsafe signed integer", value: int64(1 << 53)}, {name: "int32", value: int32(-1 << 31)},
{name: "unsafe unsigned integer", value: uint64(1 << 53)}, {name: "int64", value: int64(-1 << 63)},
{name: "named int64", value: namedInt64(1<<63 - 1)},
{name: "uint", value: ^uint(0)},
{name: "uint8", value: ^uint8(0)},
{name: "uint16", value: ^uint16(0)},
{name: "uint32", value: ^uint32(0)},
{name: "uint64", value: ^uint64(0)},
{name: "uintptr", value: ^uintptr(0)},
{name: "named uint64", value: namedUint64(^uint64(0))},
{name: "float32", value: float32(1.25)},
{name: "float64", value: float64(-2.5e100)},
{name: "named float32", value: namedFloat32(3.5)},
{name: "named float64", value: namedFloat64(-4.5e200)},
{name: "JSON number integer", value: json.Number("18446744073709551615")},
{name: "JSON number fraction", value: json.Number("-1.25e+2")},
{name: "JSON number beyond float64", value: json.Number("1e9999")},
} }
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
if _, err := jsonvalue.CopyMap(map[string]any{"value": tc.value}); err == nil { got, err := Copy(tc.value)
if err != nil {
t.Fatalf("copy value: %v", err)
}
if !reflect.DeepEqual(got, tc.value) {
t.Fatalf("value or concrete type changed: got %#v (%T), want %#v (%T)", got, got, tc.value, tc.value)
}
})
}
}
func TestCopyRejectsInvalidNumbers(t *testing.T) {
tests := []struct {
name string
value any
}{
{name: "float32 NaN", value: float32(math.NaN())},
{name: "float64 NaN", value: math.NaN()},
{name: "named float NaN", value: namedFloat64(math.NaN())},
{name: "positive infinity", value: math.Inf(1)},
{name: "negative infinity", value: math.Inf(-1)},
{name: "empty JSON number", value: json.Number("")},
{name: "leading zero JSON number", value: json.Number("01")},
{name: "leading plus JSON number", value: json.Number("+1")},
{name: "trailing decimal JSON number", value: json.Number("1.")},
{name: "leading decimal JSON number", value: json.Number(".1")},
{name: "non-number JSON number", value: json.Number("NaN")},
{name: "spaced JSON number", value: json.Number(" 1")},
{name: "quoted JSON number", value: json.Number(`"1"`)},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if _, err := Copy(tc.value); err == nil {
t.Fatal("expected validation error") t.Fatal("expected validation error")
} }
}) })
} }
} }
func TestCopyMapValidatesJSONNumberSyntaxAndRange(t *testing.T) { func TestCopyPreservesCompatibleCollectionsAndNilEmptyDistinctions(t *testing.T) {
for _, number := range []json.Number{"0", "-1", "1.25", "-1.25e+2"} { collections := []struct {
t.Run("valid "+number.String(), func(t *testing.T) { name string
got, err := jsonvalue.CopyMap(map[string]any{"value": number}) value any
}{
{name: "unnamed map", value: map[string]int{"limit": 2}},
{name: "named map", value: namedMap{"limit": 2}},
{name: "unnamed slice", value: []string{"one", "two"}},
{name: "named slice", value: namedSlice{"one", "two"}},
{name: "unnamed array", value: [2]int{1, 2}},
{name: "named array", value: namedArray{{"limit": 2}}},
{name: "empty map", value: map[string]int{}},
{name: "empty named map", value: namedMap{}},
{name: "empty slice", value: []string{}},
{name: "empty named slice", value: namedSlice{}},
{name: "empty array", value: [0]string{}},
}
for _, tc := range collections {
t.Run(tc.name, func(t *testing.T) {
got, err := Copy(tc.value)
if err != nil { if err != nil {
t.Fatalf("copy valid JSON number: %v", err) t.Fatalf("copy collection: %v", err)
} }
if !reflect.DeepEqual(got["value"], number) { if !reflect.DeepEqual(got, tc.value) || reflect.TypeOf(got) != reflect.TypeOf(tc.value) {
t.Fatalf("JSON number changed: got %#v want %#v", got["value"], number) t.Fatalf("collection changed: got %#v (%T), want %#v (%T)", got, got, tc.value, tc.value)
}
kind := reflect.ValueOf(got).Kind()
if (kind == reflect.Map || kind == reflect.Slice) && reflect.ValueOf(got).IsNil() {
t.Fatal("non-nil collection became nil")
} }
}) })
} }
for _, number := range []json.Number{"", "01", "+1", "1.", ".1", "1e9999", "not-a-number"} { var nilMap map[string]int
t.Run("invalid "+number.String(), func(t *testing.T) { var nilSlice []string
if _, err := jsonvalue.CopyMap(map[string]any{"value": number}); err == nil { var nilPointer *namedInt64
t.Fatal("expected invalid JSON number error") for _, value := range []any{nil, nilMap, nilSlice, nilPointer} {
got, err := Copy(value)
if err != nil {
t.Fatalf("copy null value: %v", err)
}
if got != nil {
t.Fatalf("null value became %#v (%T)", got, got)
}
}
gotNil, err := CopyMap(nil)
if err != nil || gotNil != nil {
t.Fatalf("nil CopyMap result = %#v, %v", gotNil, err)
}
gotEmpty, err := CopyMap(map[string]any{})
if err != nil || gotEmpty == nil || len(gotEmpty) != 0 {
t.Fatalf("empty CopyMap result = %#v, %v", gotEmpty, err)
}
}
func TestCopyHandlesIndirectionAndIsolatesNestedMutations(t *testing.T) {
integer := namedInt64(7)
nestedMap := namedMap{"limit": 2}
nestedSlice := namedSlice{"original"}
nestedArray := namedArray{{"limit": 3}}
shared := []any{map[string]int{"value": 4}}
input := map[string]any{
"integer": &integer,
"map": nestedMap,
"slice": nestedSlice,
"array": nestedArray,
"first": shared,
"second": shared,
}
copiedValue, err := Copy(input)
if err != nil {
t.Fatalf("copy mixed tree: %v", err)
}
copied := copiedValue.(map[string]any)
nestedMap["limit"] = 20
nestedSlice[0] = "changed"
nestedArray[0]["limit"] = 30
shared[0].(map[string]int)["value"] = 40
if got, ok := copied["integer"].(namedInt64); !ok || got != 7 {
t.Fatalf("pointer target changed: %#v", copied["integer"])
}
if got := copied["map"].(namedMap)["limit"]; got != 2 {
t.Fatalf("nested map aliased input: %d", got)
}
if got := copied["slice"].(namedSlice)[0]; got != "original" {
t.Fatalf("nested slice aliased input: %q", got)
}
if got := copied["array"].(namedArray)[0]["limit"]; got != 3 {
t.Fatalf("nested array aliased input: %d", got)
}
first := copied["first"].([]any)
second := copied["second"].([]any)
if got := first[0].(map[string]int)["value"]; got != 4 {
t.Fatalf("shared child aliased input: %d", got)
}
first[0].(map[string]int)["value"] = 99
if got := second[0].(map[string]int)["value"]; got != 4 {
t.Fatalf("repeated acyclic value shared copied output: %d", got)
}
}
func TestCopyAndCopyMapApplyDistinctEmptyKeyRules(t *testing.T) {
nested := map[string]any{"": []any{"original"}}
copiedValue, err := Copy(nested)
if err != nil {
t.Fatalf("Copy rejected empty schema key: %v", err)
}
nested[""].([]any)[0] = "changed"
if got := copiedValue.(map[string]any)[""].([]any)[0]; got != "original" {
t.Fatalf("copied schema value was not isolated: %v", got)
}
_, err = CopyMap(map[string]any{"nested": map[string]any{"": true}})
if err == nil || !strings.Contains(err.Error(), "extra_params.nested") {
t.Fatalf("CopyMap empty-key error = %v", err)
}
}
func TestCopyRejectsUnsupportedValuesAndActiveCycles(t *testing.T) {
cyclicMap := map[string]any{}
cyclicMap["self"] = cyclicMap
cyclicSlice := []any{nil}
cyclicSlice[0] = cyclicSlice
var cyclicPointer any
cyclicPointer = &cyclicPointer
tests := []struct {
name string
value any
wantPath string
}{
{name: "non-string map key", value: map[int]string{1: "one"}, wantPath: "value"},
{name: "unsupported channel", value: make(chan int), wantPath: "value"},
{name: "deterministic map path", value: map[string]any{"z": make(chan int), "a": make(chan int)}, wantPath: "value.a"},
{name: "cyclic map", value: cyclicMap, wantPath: "value.self"},
{name: "cyclic slice", value: cyclicSlice, wantPath: "value[0]"},
{name: "cyclic pointer", value: cyclicPointer, wantPath: "value"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := Copy(tc.value)
if err == nil || !strings.Contains(err.Error(), tc.wantPath) {
t.Fatalf("error = %v, want structural path %q", err, tc.wantPath)
} }
}) })
} }
} }
func TestCopyEnforcesContainerDepth(t *testing.T) {
tests := []struct {
name string
depth int
wantErr bool
}{
{name: "just below", depth: maxContainerDepth - 1},
{name: "at limit", depth: maxContainerDepth},
{name: "over limit", depth: maxContainerDepth + 1, wantErr: true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := Copy(alternatingContainers(tc.depth))
if tc.wantErr {
if err == nil || !strings.HasPrefix(err.Error(), "value") || !strings.Contains(err.Error(), "container depth limit") {
t.Fatalf("depth error = %v", err)
}
return
}
if err != nil {
t.Fatalf("copy depth %d: %v", tc.depth, err)
}
})
}
}
func TestCopyEnforcesProducedNodeBudgetForRepeatedAcyclicValues(t *testing.T) {
shared := []any{true}
sharedOccurrences := (maxProducedNodes - 2) / 2
justBelow := repeatedValues(shared, sharedOccurrences, 0)
atLimit := repeatedValues(shared, sharedOccurrences, 1)
overLimit := repeatedValues(shared, sharedOccurrences, 2)
for name, value := range map[string]any{
"just below": justBelow,
"at limit": atLimit,
} {
t.Run(name, func(t *testing.T) {
if _, err := Copy(value); err != nil {
t.Fatalf("copy value within work budget: %v", err)
}
})
}
_, err := Copy(overLimit)
if err == nil || !strings.HasPrefix(err.Error(), "value[") || !strings.Contains(err.Error(), "value work limit") {
t.Fatalf("work-budget error = %v", err)
}
_, err = Copy(make([]any, maxProducedNodes))
if err == nil || !strings.HasPrefix(err.Error(), "value:") || !strings.Contains(err.Error(), "value work limit") {
t.Fatalf("flat work-budget error = %v", err)
}
}
func alternatingContainers(depth int) any {
var value any = true
for level := 0; level < depth; level++ {
switch level % 3 {
case 0:
value = map[string]any{"child": value}
case 1:
value = []any{value}
default:
value = [1]any{value}
}
}
return value
}
func repeatedValues(shared []any, occurrences, leadingScalars int) []any {
values := make([]any, 0, leadingScalars+occurrences)
for i := 0; i < leadingScalars; i++ {
values = append(values, false)
}
for i := 0; i < occurrences; i++ {
values = append(values, shared)
}
return values
}

View File

@@ -25,6 +25,20 @@ var (
ErrMalformedResponse = errors.New("malformed llm response") ErrMalformedResponse = errors.New("malformed llm response")
) )
const maxOpenAIChatResponseBytes int64 = 16 << 20
type requestFailedError struct {
cause error
}
func (e *requestFailedError) Error() string {
return ErrRequestFailed.Error()
}
func (e *requestFailedError) Unwrap() []error {
return []error{ErrRequestFailed, e.cause}
}
type OpenAICompatibleConfig struct { type OpenAICompatibleConfig struct {
BaseURL string BaseURL string
Model string Model string
@@ -39,9 +53,11 @@ type OpenAICompatibleClient struct {
} }
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) { func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) {
baseURL := strings.TrimSpace(cfg.BaseURL) baseURL := ""
if baseURL != "" { if strings.TrimSpace(cfg.BaseURL) != "" {
if _, err := url.ParseRequestURI(baseURL); err != nil { var err error
baseURL, err = domain.NormalizeOpenAICompatibleBaseEndpoint(cfg.BaseURL)
if err != nil {
return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err) return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err)
} }
} }
@@ -63,25 +79,29 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
} }
return &OpenAICompatibleClient{ return &OpenAICompatibleClient{
baseURL: strings.TrimRight(baseURL, "/"), baseURL: baseURL,
defaultModel: cfg.Model, defaultModel: cfg.Model,
httpClient: client, httpClient: client,
}, nil }, nil
} }
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) { func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
if req.Target.TimeoutSeconds < 0 { if err := domain.ValidateExecutionTargetSettings(req.Target); err != nil {
return nil, fmt.Errorf("%w: timeout_seconds must be greater than or equal to 0", ErrInvalidRequest) return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
} }
endpoint := strings.TrimSpace(req.Target.Endpoint) selectedEndpoint := req.Target.Endpoint
if endpoint == "" { if strings.TrimSpace(selectedEndpoint) == "" {
endpoint = c.baseURL selectedEndpoint = c.baseURL
} }
if endpoint == "" { endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(selectedEndpoint)
return nil, fmt.Errorf("%w: endpoint is required", ErrInvalidRequest) if err != nil {
return nil, fmt.Errorf("%w: invalid endpoint: %v", ErrInvalidRequest, err)
}
endpoint, err = url.JoinPath(endpoint, defaults.OpenAIChatCompletionsPath)
if err != nil {
return nil, fmt.Errorf("%w: invalid endpoint path: %v", ErrInvalidRequest, err)
} }
endpoint = strings.TrimRight(endpoint, "/") + defaults.OpenAIChatCompletionsPath
wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel) wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel)
if err != nil { if err != nil {
@@ -113,13 +133,18 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err) return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
} }
httpReq.Header.Set("Content-Type", "application/json") httpReq.Header.Set("Content-Type", "application/json")
if apiKey := strings.TrimSpace(req.Target.APIKey); apiKey != "" { apiKey := strings.TrimSpace(req.Target.APIKey)
httpReq.Header.Set("Authorization", "Bearer "+apiKey) envName := strings.TrimSpace(req.Target.APIKeyEnv)
} else if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" { if apiKey == "" && envName != "" {
apiKey := strings.TrimSpace(os.Getenv(envName)) apiKey = strings.TrimSpace(os.Getenv(envName))
if apiKey == "" { }
if apiKey == "" && req.Target.APIKeyRequired {
if envName != "" {
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName) return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
} }
return nil, fmt.Errorf("%w: api key is required", ErrInvalidRequest)
}
if apiKey != "" {
httpReq.Header.Set("Authorization", "Bearer "+apiKey) httpReq.Header.Set("Authorization", "Bearer "+apiKey)
} }
@@ -130,30 +155,36 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
httpResp, err := httpClient.Do(httpReq) httpResp, err := httpClient.Do(httpReq)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: %v", ErrRequestFailed, err) return nil, &requestFailedError{cause: err}
} }
defer httpResp.Body.Close() defer httpResp.Body.Close()
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 { if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
_, _ = io.Copy(io.Discard, io.LimitReader(httpResp.Body, 4096)) return nil, providerHTTPErrorFromBody(
return nil, fmt.Errorf("%w: status=%d", ErrUnexpectedStatus, httpResp.StatusCode) httpResp.StatusCode,
httpResp.ContentLength,
httpResp.Body,
)
}
if httpResp.ContentLength > maxOpenAIChatResponseBytes {
return nil, openAIChatResponseTooLargeError()
} }
var wireResp openAIChatResponse wireResp, err := decodeOpenAIChatResponse(httpResp.Body)
if err := json.NewDecoder(httpResp.Body).Decode(&wireResp); err != nil { if err != nil {
return nil, fmt.Errorf("%w: failed to decode response: %v", ErrMalformedResponse, err) return nil, err
} }
if len(wireResp.Choices) == 0 { if len(wireResp.Choices) == 0 {
return nil, fmt.Errorf("%w: no choices returned", ErrMalformedResponse) return nil, fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
} }
content := wireResp.Choices[0].Message.Content content := wireResp.Choices[0].Message.Content
if content == "" { if content == nil {
return nil, fmt.Errorf("%w: first choice has empty message content", ErrMalformedResponse) return nil, fmt.Errorf("%w: first choice has missing message content", ErrMalformedResponse)
} }
return &domain.GenerateResponse{ return &domain.GenerateResponse{
Content: content, Content: *content,
Usage: domain.TokenUsage{ Usage: domain.TokenUsage{
PromptTokens: wireResp.Usage.PromptTokens, PromptTokens: wireResp.Usage.PromptTokens,
CompletionTokens: wireResp.Usage.CompletionTokens, CompletionTokens: wireResp.Usage.CompletionTokens,
@@ -164,6 +195,46 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
}, nil }, nil
} }
func decodeOpenAIChatResponse(body io.Reader) (openAIChatResponse, error) {
limited := &io.LimitedReader{
R: body,
N: maxOpenAIChatResponseBytes + 1,
}
decoder := json.NewDecoder(limited)
var response openAIChatResponse
if err := decoder.Decode(&response); err != nil {
if limited.N == 0 {
return openAIChatResponse{}, openAIChatResponseTooLargeError()
}
return openAIChatResponse{}, fmt.Errorf("%w: failed to decode response", ErrMalformedResponse)
}
if limited.N == 0 {
return openAIChatResponse{}, openAIChatResponseTooLargeError()
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
if limited.N == 0 {
return openAIChatResponse{}, openAIChatResponseTooLargeError()
}
return openAIChatResponse{}, fmt.Errorf("%w: response contains trailing data", ErrMalformedResponse)
}
if limited.N == 0 {
return openAIChatResponse{}, openAIChatResponseTooLargeError()
}
return response, nil
}
func openAIChatResponseTooLargeError() error {
return fmt.Errorf(
"%w: response exceeds %d-byte limit",
ErrMalformedResponse,
maxOpenAIChatResponseBytes,
)
}
func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) { func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) {
model := strings.TrimSpace(req.Target.Model) model := strings.TrimSpace(req.Target.Model)
if model == "" { if model == "" {
@@ -308,8 +379,8 @@ type openAICacheControl struct {
} }
type openAIChatResponseMessage struct { type openAIChatResponseMessage struct {
Role string `json:"role"` Role string `json:"role"`
Content string `json:"content"` Content *string `json:"content"`
} }
type openAIChatResponse struct { type openAIChatResponse struct {

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,189 @@
package llm
import (
"encoding/json"
"errors"
"fmt"
"io"
"strings"
"unicode"
)
const (
maxProviderErrorResponseBytes int64 = 64 << 10
maxProviderErrorIdentifierRunes = 256
maxProviderErrorMessageRunes = 4096
)
// ProviderHTTPError describes a non-success response from an LLM provider.
type ProviderHTTPError struct {
statusCode int
providerCode string
providerType string
providerMessage string
}
func (e *ProviderHTTPError) StatusCode() int {
if e == nil {
return 0
}
return e.statusCode
}
func (e *ProviderHTTPError) ProviderCode() string {
if e == nil {
return ""
}
return e.providerCode
}
func (e *ProviderHTTPError) ProviderType() string {
if e == nil {
return ""
}
return e.providerType
}
func (e *ProviderHTTPError) ProviderMessage() string {
if e == nil {
return ""
}
return e.providerMessage
}
func (e *ProviderHTTPError) Error() string {
if e == nil || e.statusCode == 0 {
return ErrUnexpectedStatus.Error()
}
return fmt.Sprintf("%s: status=%d", ErrUnexpectedStatus, e.statusCode)
}
func (e *ProviderHTTPError) GoString() string {
return e.Error()
}
func (e *ProviderHTTPError) Unwrap() error {
return ErrUnexpectedStatus
}
type providerErrorDetails struct {
providerCode string
providerType string
providerMessage string
}
func newProviderHTTPError(statusCode int, details providerErrorDetails) *ProviderHTTPError {
return &ProviderHTTPError{
statusCode: statusCode,
providerCode: details.providerCode,
providerType: details.providerType,
providerMessage: details.providerMessage,
}
}
func providerHTTPErrorFromBody(statusCode int, contentLength int64, body io.Reader) *ProviderHTTPError {
if contentLength > maxProviderErrorResponseBytes {
return newProviderHTTPError(statusCode, providerErrorDetails{})
}
limited := &io.LimitedReader{
R: body,
N: maxProviderErrorResponseBytes + 1,
}
contents, err := io.ReadAll(limited)
if err != nil || limited.N == 0 {
return newProviderHTTPError(statusCode, providerErrorDetails{})
}
return newProviderHTTPError(statusCode, parseProviderErrorEnvelope(contents))
}
func parseProviderErrorEnvelope(body []byte) providerErrorDetails {
decoder := json.NewDecoder(strings.NewReader(string(body)))
decoder.UseNumber()
var envelope map[string]json.RawMessage
if err := decoder.Decode(&envelope); err != nil {
return providerErrorDetails{}
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return providerErrorDetails{}
}
rawError, ok := envelope["error"]
if !ok {
return providerErrorDetails{}
}
var providerError map[string]json.RawMessage
if err := json.Unmarshal(rawError, &providerError); err != nil || providerError == nil {
return providerErrorDetails{}
}
var details providerErrorDetails
if raw, ok := providerError["message"]; ok {
var value string
if json.Unmarshal(raw, &value) == nil {
details.providerMessage = normalizeProviderErrorMessage(value)
}
}
if raw, ok := providerError["type"]; ok {
var value string
if json.Unmarshal(raw, &value) == nil {
details.providerType = normalizeProviderErrorIdentifier(value)
}
}
if raw, ok := providerError["code"]; ok {
var value any
fieldDecoder := json.NewDecoder(strings.NewReader(string(raw)))
fieldDecoder.UseNumber()
if fieldDecoder.Decode(&value) == nil {
switch value := value.(type) {
case string:
details.providerCode = normalizeProviderErrorIdentifier(value)
case json.Number:
details.providerCode = normalizeProviderErrorIdentifier(value.String())
}
}
}
return details
}
func normalizeProviderErrorIdentifier(value string) string {
normalized := normalizeProviderErrorText(value)
if len([]rune(normalized)) > maxProviderErrorIdentifierRunes {
return ""
}
return normalized
}
func normalizeProviderErrorMessage(value string) string {
normalized := normalizeProviderErrorText(value)
runes := []rune(normalized)
if len(runes) <= maxProviderErrorMessageRunes {
return normalized
}
return string(runes[:maxProviderErrorMessageRunes-1]) + "…"
}
func normalizeProviderErrorText(value string) string {
value = strings.ToValidUTF8(value, "<22>")
var result strings.Builder
result.Grow(len(value))
separatorPending := false
for _, r := range value {
if unicode.IsSpace(r) || unicode.IsControl(r) || unicode.In(r, unicode.Cf) {
if result.Len() > 0 {
separatorPending = true
}
continue
}
if separatorPending {
result.WriteByte(' ')
separatorPending = false
}
result.WriteRune(r)
}
return result.String()
}

View File

@@ -0,0 +1,255 @@
package llm
import (
"errors"
"fmt"
"io"
"reflect"
"strings"
"testing"
"unicode/utf8"
)
type guardedReader struct {
reader io.Reader
remaining int64
bytes int64
violated bool
}
func (r *guardedReader) Read(buffer []byte) (int, error) {
if int64(len(buffer)) > r.remaining {
r.violated = true
return 0, errors.New("reader was read past its allowed boundary")
}
n, err := r.reader.Read(buffer)
r.bytes += int64(n)
r.remaining -= int64(n)
return n, err
}
type failingReader struct {
err error
}
func (r failingReader) Read([]byte) (int, error) {
return 0, r.err
}
func TestProviderHTTPErrorEnvelopeParsing(t *testing.T) {
tests := []struct {
name string
body string
want providerErrorDetails
}{
{
name: "all supported string fields",
body: `{"error":{"message":"diagnostic","type":"invalid_request_error","code":"unsupported_parameter"}}`,
want: providerErrorDetails{providerMessage: "diagnostic", providerType: "invalid_request_error", providerCode: "unsupported_parameter"},
},
{
name: "integer code",
body: `{"error":{"code":17}}`,
want: providerErrorDetails{providerCode: "17"},
},
{
name: "fractional code",
body: `{"error":{"code":1.25}}`,
want: providerErrorDetails{providerCode: "1.25"},
},
{
name: "exponent code",
body: `{"error":{"code":6.02e+23}}`,
want: providerErrorDetails{providerCode: "6.02e+23"},
},
{
name: "invalid fields do not discard valid fields",
body: `{"error":{"message":null,"type":"invalid_request_error","code":false}}`,
want: providerErrorDetails{providerType: "invalid_request_error"},
},
{
name: "unknown fields are ignored",
body: `{"trace":"do not retain","error":{"param":"temperature","metadata":{"secret":"x"}}}`,
want: providerErrorDetails{},
},
{name: "missing error", body: `{}`, want: providerErrorDetails{}},
{name: "null error", body: `{"error":null}`, want: providerErrorDetails{}},
{name: "scalar error", body: `{"error":"nope"}`, want: providerErrorDetails{}},
{name: "empty error", body: `{"error":{}}`, want: providerErrorDetails{}},
{name: "malformed", body: `{"error":`, want: providerErrorDetails{}},
{name: "truncated", body: `{"error":{"message":"x"`, want: providerErrorDetails{}},
{name: "trailing garbage", body: `{"error":{"message":"x"}} garbage`, want: providerErrorDetails{}},
{name: "second document", body: `{"error":{"message":"x"}} {}`, want: providerErrorDetails{}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if got := parseProviderErrorEnvelope([]byte(tc.body)); !reflect.DeepEqual(got, tc.want) {
t.Fatalf("parseProviderErrorEnvelope() = %#v, want %#v", got, tc.want)
}
})
}
}
func TestProviderErrorTextNormalizationAndLimits(t *testing.T) {
validIdentifier := strings.Repeat("界", maxProviderErrorIdentifierRunes)
validMessage := strings.Repeat("界", maxProviderErrorMessageRunes)
tests := []struct {
name string
got string
want string
}{
{name: "multibyte text", got: "Grüße 世界", want: "Grüße 世界"},
{name: "invalid UTF-8", got: string([]byte{'a', 0xff, 'b'}), want: "a<>b"},
{name: "whitespace control and format runs", got: " \n\talpha\x00\u200b\u200bbeta \r ", want: "alpha beta"},
{name: "blank normalization", got: "\t\u200b\n", want: ""},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if got := normalizeProviderErrorText(tc.got); got != tc.want {
t.Fatalf("normalizeProviderErrorText() = %q, want %q", got, tc.want)
}
})
}
if got := normalizeProviderErrorIdentifier(validIdentifier); got != validIdentifier {
t.Fatalf("exact identifier boundary = %q, want retained value", got)
}
if got := normalizeProviderErrorIdentifier(validIdentifier + "界"); got != "" {
t.Fatalf("overlong identifier = %q, want empty", got)
}
if got := normalizeProviderErrorMessage(validMessage); got != validMessage {
t.Fatalf("exact message boundary = %q, want retained value", got)
}
wantTruncatedMessage := strings.Repeat("界", maxProviderErrorMessageRunes-1) + "…"
if got := normalizeProviderErrorMessage(validMessage + "界"); got != wantTruncatedMessage {
t.Fatalf("overlong message length = %d, want %d", utf8.RuneCountInString(got), maxProviderErrorMessageRunes)
}
}
func TestProviderHTTPErrorIdentityAndFormatting(t *testing.T) {
const marker = "provider-secret-marker"
err := newProviderHTTPError(429, providerErrorDetails{
providerCode: marker + "-code",
providerType: marker + "-type",
providerMessage: marker + "-message",
})
if err.StatusCode() != 429 || err.ProviderCode() != marker+"-code" || err.ProviderType() != marker+"-type" || err.ProviderMessage() != marker+"-message" {
t.Fatalf("accessors returned unexpected values: %#v", err)
}
if !errors.Is(err, ErrUnexpectedStatus) {
t.Fatalf("errors.Is(%v, ErrUnexpectedStatus) = false", err)
}
for _, rendered := range []string{fmt.Sprintf("%v", err), fmt.Sprintf("%+v", err), fmt.Sprintf("%#v", err)} {
if rendered != "llm returned non-success status: status=429" {
t.Fatalf("formatted error = %q", rendered)
}
if strings.Contains(rendered, marker) {
t.Fatalf("formatted error exposed provider marker: %q", rendered)
}
}
var nilError *ProviderHTTPError
if nilError.StatusCode() != 0 || nilError.ProviderCode() != "" || nilError.ProviderType() != "" || nilError.ProviderMessage() != "" {
t.Fatal("nil accessors returned provider values")
}
if nilError.Error() != "llm returned non-success status" || nilError.GoString() != "llm returned non-success status" || !errors.Is(nilError, ErrUnexpectedStatus) {
t.Fatalf("nil error behavior is not safe: %v", nilError)
}
zero := &ProviderHTTPError{}
if zero.Error() != "llm returned non-success status" || zero.GoString() != "llm returned non-success status" || !errors.Is(zero, ErrUnexpectedStatus) {
t.Fatalf("zero error behavior is not safe: %v", zero)
}
}
func TestProviderHTTPErrorBodyBounds(t *testing.T) {
const (
statusCode = 502
marker = "provider-body-marker"
)
ordinaryBody := `{"error":{"message":"` + marker + `"}}`
exactLimitBody := ordinaryBody + strings.Repeat(" ", int(maxProviderErrorResponseBytes)-len(ordinaryBody))
overLimitBody := ordinaryBody + strings.Repeat(" ", int(maxProviderErrorResponseBytes)+1-len(ordinaryBody))
tests := []struct {
name string
contentLength int64
reader io.Reader
wantRead int64
wantMessage string
}{
{
name: "recognized envelope",
contentLength: int64(len(ordinaryBody)),
reader: strings.NewReader(ordinaryBody),
wantRead: int64(len(ordinaryBody)),
wantMessage: marker,
},
{
name: "exact limit",
contentLength: maxProviderErrorResponseBytes,
reader: strings.NewReader(exactLimitBody),
wantRead: maxProviderErrorResponseBytes,
wantMessage: marker,
},
{
name: "declared oversize does not read",
contentLength: maxProviderErrorResponseBytes + 1,
reader: strings.NewReader(ordinaryBody),
wantRead: 0,
},
{
name: "unknown length oversize",
contentLength: -1,
reader: strings.NewReader(overLimitBody),
wantRead: maxProviderErrorResponseBytes + 1,
},
{
name: "underreported oversize",
contentLength: maxProviderErrorResponseBytes,
reader: strings.NewReader(overLimitBody),
wantRead: maxProviderErrorResponseBytes + 1,
},
{
name: "read failure",
contentLength: -1,
reader: failingReader{err: errors.New("read failure")},
wantRead: 0,
},
{
name: "empty body",
contentLength: 0,
reader: strings.NewReader(""),
wantRead: 0,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
reader := &guardedReader{
reader: tc.reader,
remaining: maxProviderErrorResponseBytes + 1,
}
err := providerHTTPErrorFromBody(statusCode, tc.contentLength, reader)
if err == nil || err.StatusCode() != statusCode {
t.Fatalf("error status = %v, want %d", err, statusCode)
}
if reader.bytes != tc.wantRead {
t.Fatalf("body bytes read = %d, want %d", reader.bytes, tc.wantRead)
}
if reader.violated {
t.Fatal("body reader was asked to read beyond the overflow probe")
}
if got := err.ProviderMessage(); got != tc.wantMessage {
t.Fatalf("provider message = %q, want %q", got, tc.wantMessage)
}
if tc.wantMessage == "" {
if err.ProviderCode() != "" || err.ProviderType() != "" || strings.Contains(err.Error(), marker) {
t.Fatalf("discarded details were retained: %#v", err)
}
}
})
}
}

View File

@@ -0,0 +1,145 @@
package llm
import (
"context"
"errors"
"io"
"net/http"
"strings"
"testing"
)
func TestOpenAICompatibleClientStructuredNonSuccessResponse(t *testing.T) {
body := `{"error":{"message":" provider\nmessage\u200b","type":"invalid\ttype","code":1.5e+4}}`
responseBody := &countingReadCloser{reader: strings.NewReader(body)}
client := newNonSuccessResponseClient(t, http.StatusBadRequest, int64(len(body)), responseBody)
response, err := client.Generate(context.Background(), ordinaryGenerateRequest())
if response != nil {
t.Fatalf("response = %#v, want nil", response)
}
if !errors.Is(err, ErrUnexpectedStatus) {
t.Fatalf("errors.Is(%v, ErrUnexpectedStatus) = false", err)
}
var providerHTTPError *ProviderHTTPError
if !errors.As(err, &providerHTTPError) {
t.Fatalf("error = %T, want *ProviderHTTPError", err)
}
if providerHTTPError.StatusCode() != http.StatusBadRequest || providerHTTPError.ProviderCode() != "1.5e+4" || providerHTTPError.ProviderType() != "invalid type" || providerHTTPError.ProviderMessage() != "provider message" {
t.Fatalf("provider error = %#v", providerHTTPError)
}
if !responseBody.closed {
t.Fatal("non-success response body was not closed")
}
}
func TestOpenAICompatibleClientNonSuccessBodyOwnership(t *testing.T) {
const marker = "provider-body-marker"
normalBody := `{"error":{"message":"` + marker + `"}}`
overLimitBody := normalBody + strings.Repeat(" ", int(maxProviderErrorResponseBytes)+1-len(normalBody))
tests := []struct {
name string
contentLength int64
reader io.Reader
wantRead int64
wantMessage string
}{
{
name: "normal",
contentLength: int64(len(normalBody)),
reader: strings.NewReader(normalBody),
wantRead: int64(len(normalBody)),
wantMessage: marker,
},
{
name: "declared oversize",
contentLength: maxProviderErrorResponseBytes + 1,
reader: strings.NewReader(normalBody),
wantRead: 0,
},
{
name: "streamed oversize",
contentLength: -1,
reader: &guardedReader{
reader: strings.NewReader(overLimitBody),
remaining: maxProviderErrorResponseBytes + 1,
},
wantRead: maxProviderErrorResponseBytes + 1,
},
{
name: "underreported oversize",
contentLength: maxProviderErrorResponseBytes,
reader: &guardedReader{
reader: strings.NewReader(overLimitBody),
remaining: maxProviderErrorResponseBytes + 1,
},
wantRead: maxProviderErrorResponseBytes + 1,
},
{
name: "malformed",
contentLength: 1,
reader: strings.NewReader("{"),
wantRead: 1,
},
{
name: "read failure",
contentLength: -1,
reader: failingReader{err: errors.New("response read failed")},
wantRead: 0,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
body := &countingReadCloser{reader: tc.reader}
client := newNonSuccessResponseClient(t, http.StatusBadGateway, tc.contentLength, body)
response, err := client.Generate(context.Background(), ordinaryGenerateRequest())
if response != nil {
t.Fatalf("response = %#v, want nil", response)
}
var providerHTTPError *ProviderHTTPError
if !errors.As(err, &providerHTTPError) {
t.Fatalf("error = %T, want *ProviderHTTPError", err)
}
if !body.closed {
t.Fatal("response body was not closed")
}
if body.bytesRead != tc.wantRead {
t.Fatalf("body bytes read = %d, want %d", body.bytesRead, tc.wantRead)
}
if body.bytesRead > maxProviderErrorResponseBytes+1 {
t.Fatalf("body bytes read = %d, exceeds overflow probe", body.bytesRead)
}
if guarded, ok := tc.reader.(*guardedReader); ok && guarded.violated {
t.Fatal("body reader was asked to read beyond the overflow probe")
}
if got := providerHTTPError.ProviderMessage(); got != tc.wantMessage {
t.Fatalf("provider message = %q, want %q", got, tc.wantMessage)
}
if tc.wantMessage == "" && (providerHTTPError.ProviderCode() != "" || providerHTTPError.ProviderType() != "" || strings.Contains(providerHTTPError.Error(), marker)) {
t.Fatalf("discarded details were retained: %#v", providerHTTPError)
}
})
}
}
func newNonSuccessResponseClient(t *testing.T, statusCode int, contentLength int64, body io.ReadCloser) *OpenAICompatibleClient {
t.Helper()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://provider.example/v1",
Model: "m",
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: statusCode,
ContentLength: contentLength,
Body: body,
}, nil
})},
})
if err != nil {
t.Fatalf("construct client: %v", err)
}
return client
}

View File

@@ -1,8 +0,0 @@
id: aion-2
backend: openrouter
model: aion-labs/aion-2.0
temperature: 0.72
reasoning_effort: high
top_p: 0.95
timeout_seconds: 180
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: claude-fable-latest
backend: openrouter
model: "~anthropic/claude-fable-latest"
reasoning_effort: high
timeout_seconds: 600
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: claude-haiku-latest
backend: openrouter
model: "~anthropic/claude-haiku-latest"
reasoning_effort: medium
timeout_seconds: 240
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: claude-opus-latest
backend: openrouter
model: "~anthropic/claude-opus-latest"
reasoning_effort: high
timeout_seconds: 240
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: claude-sonnet-latest
backend: openrouter
model: "~anthropic/claude-sonnet-latest"
reasoning_effort: high
timeout_seconds: 240
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: deepseek-3-2
backend: openrouter
model: deepseek/deepseek-v3.2
reasoning_effort: high
timeout_seconds: 180
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: deepseek-4-flash
backend: openrouter
model: deepseek/deepseek-v4-flash
#reasoning_effort: medium
timeout_seconds: 180
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: deepseek-4-pro
backend: openrouter
model: deepseek/deepseek-v4-pro
reasoning_effort: high
timeout_seconds: 180
service_tier: flex

View File

@@ -1,8 +0,0 @@
id: gemini-2-flash-lite
backend: openrouter
model: "google/gemini-2.5-flash-lite"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -1,8 +0,0 @@
id: gemini-2-flash
backend: openrouter
model: "google/gemini-2.5-flash"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -1,8 +0,0 @@
id: gemini-2-pro
backend: openrouter
model: "google/gemini-2.5-pro"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -1,8 +0,0 @@
id: gemini-3-flash-lite
backend: openrouter
model: "google/gemini-3.1-flash-lite"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -1,8 +0,0 @@
id: gemini-flash-latest
backend: openrouter
model: "~google/gemini-flash-latest"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -1,8 +0,0 @@
id: gemini-pro-latest
backend: openrouter
model: "~google/gemini-pro-latest"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -1,8 +0,0 @@
id: gemma-4-31b
backend: openrouter
model: google/gemma-4-31b-it:exacto
temperature: 0.15
reasoning_effort: high
top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -1,8 +0,0 @@
id: minimax-m2
backend: openrouter
model: minimax/minimax-m2.5
temperature: 0.5
reasoning_effort: high
top_p: 0.95
timeout_seconds: 180
service_tier: flex

View File

@@ -1,8 +0,0 @@
id: minimax-m3
backend: openrouter
model: minimax/minimax-m3
#temperature: 0.5
reasoning_effort: high
#top_p: 0.95
timeout_seconds: 180
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: mistral-large-2512
backend: openrouter
model: mistralai/mistral-large-2512
temperature: 0.15
top_p: 0.98
timeout_seconds: 180

View File

@@ -1,7 +0,0 @@
id: mistral-medium-3-5
backend: openrouter
model: mistralai/mistral-medium-3-5
temperature: 0.15
reasoning_effort: high
top_p: 0.98
timeout_seconds: 180

View File

@@ -1,6 +0,0 @@
id: mistral-small-3
backend: openrouter
model: mistralai/mistral-small-3.2-24b-instruct
temperature: 0.05
top_p: 1.0
timeout_seconds: 180

View File

@@ -1,7 +0,0 @@
id: mistral-small-4
backend: openrouter
model: mistralai/mistral-small-2603
temperature: 0.1
reasoning_effort: high
top_p: 0.98
timeout_seconds: 180

View File

@@ -1,6 +0,0 @@
id: nemotron-3-ultra
backend: openrouter
model: nvidia/nemotron-3-ultra-550b-a55b
reasoning_effort: high
timeout_seconds: 180
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: gpt-5-mini
backend: openrouter
model: "openai/gpt-5.4-mini"
reasoning_effort: high
timeout_seconds: 240
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: gpt-5-nano
backend: openrouter
model: "openai/gpt-5.4-nano"
reasoning_effort: high
timeout_seconds: 240
service_tier: flex

View File

@@ -1,16 +0,0 @@
package builtin
import (
"embed"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
const assetRoot = "assets"
//go:embed assets/**/*.yml
var assets embed.FS
func NewRepository() profile.Repository {
return profile.NewFSRepository(assets, assetRoot)
}

View File

@@ -1,90 +0,0 @@
package builtin
import (
"context"
"io/fs"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gopkg.in/yaml.v3"
)
func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
repo := NewRepository()
ids := loadBuiltInProfileIDs(t)
if len(ids) == 0 {
t.Fatal("expected built-in profiles")
}
for id := range ids {
t.Run(id, func(t *testing.T) {
p, err := repo.GetProfile(context.Background(), id)
if err != nil {
t.Fatalf("expected built-in profile %q to load, got %v", id, err)
}
if p.ID != id {
t.Fatalf("expected profile id %q, got %q", id, p.ID)
}
if p.BackendID != backend.OpenRouterID {
t.Fatalf("expected profile %q to select %q, got %q", id, backend.OpenRouterID, p.BackendID)
}
if p.Endpoint != "" || p.APIKeyEnv != "" {
t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv)
}
})
}
}
func TestBuiltInProfilesDoNotContainDuplicateIDsOrRawAPIKeys(t *testing.T) {
loadBuiltInProfileIDs(t)
}
func loadBuiltInProfileIDs(t *testing.T) map[string]string {
t.Helper()
ids := map[string]string{}
err := fs.WalkDir(assets, assetRoot, func(name string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() || !strings.HasSuffix(name, ".yml") {
return nil
}
data, err := assets.ReadFile(name)
if err != nil {
t.Fatalf("failed to read built-in profile %s: %v", name, err)
}
var raw map[string]any
if err := yaml.Unmarshal(data, &raw); err != nil {
t.Fatalf("failed to decode built-in profile %s: %v", name, err)
}
if _, ok := raw["api_key"]; ok {
t.Fatalf("built-in profile %s contains raw api_key", name)
}
if raw["backend"] != backend.OpenRouterID {
t.Fatalf("built-in profile %s does not select %q", name, backend.OpenRouterID)
}
if _, ok := raw["endpoint"]; ok {
t.Fatalf("built-in profile %s repeats endpoint", name)
}
if _, ok := raw["api_key_env"]; ok {
t.Fatalf("built-in profile %s repeats api_key_env", name)
}
id, ok := raw["id"].(string)
if !ok || strings.TrimSpace(id) == "" {
t.Fatalf("built-in profile %s has missing id", name)
}
if previous, ok := ids[id]; ok {
t.Fatalf("duplicate built-in profile id %q in %s and %s", id, previous, name)
}
ids[id] = name
return nil
})
if err != nil {
t.Fatalf("failed to walk built-in profiles: %v", err)
}
return ids
}

View File

@@ -0,0 +1,47 @@
package profile
import (
"errors"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
// NormalizeAndValidateDefinition normalizes and validates one source-local
// profile definition without resolving a base profile.
func NormalizeAndValidateDefinition(profile *domain.ExecutionProfile) error {
if profile == nil {
return errors.New("profile is required")
}
profile.ID = strings.TrimSpace(profile.ID)
profile.BaseProfileID = strings.TrimSpace(profile.BaseProfileID)
profile.BackendID = strings.TrimSpace(profile.BackendID)
profile.Endpoint = strings.TrimSpace(profile.Endpoint)
if profile.ID == "" {
return errors.New("id is required")
}
if profile.Endpoint != "" {
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(profile.Endpoint)
if err != nil {
return err
}
profile.Endpoint = endpoint
}
if profile.BaseProfileID == "" {
if profile.BackendID == "" && profile.Endpoint == "" {
return errors.New("backend or endpoint is required")
}
if strings.TrimSpace(profile.Model) == "" {
return errors.New("model is required")
}
}
return domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{
Temperature: profile.Temperature,
MaxTokens: profile.MaxTokens,
TopP: profile.TopP,
TimeoutSeconds: profile.TimeoutSeconds,
})
}

View File

@@ -0,0 +1,98 @@
package profile
import (
"context"
"fmt"
"io/fs"
"sort"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
)
// LoadedProfileMetadata identifies one profile accepted by LoadFSRepository.
type LoadedProfileMetadata struct {
// ID is the normalized profile ID.
ID string
// Path is the safe root-relative source path.
Path string
// ExplicitFields lists the sorted top-level YAML fields present in source.
ExplicitFields []string
}
// LoadFSRepository eagerly validates every profile under root and returns an
// immutable raw repository and independently owned source metadata.
func LoadFSRepository(ctx context.Context, fsys fs.FS, root string) (Repository, []LoadedProfileMetadata, error) {
if fsys == nil {
return nil, nil, fmt.Errorf("failed to read profile directory: filesystem is nil")
}
paths, err := filecatalog.FindFSYAMLFiles(ctx, fsys, root)
if err != nil {
return nil, nil, fmt.Errorf("failed to read profile directory: %w", err)
}
repository := &loadedRepository{profiles: make(map[string]domain.ExecutionProfile, len(paths))}
metadata := make([]LoadedProfileMetadata, 0, len(paths))
for _, path := range paths {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
data, err := fs.ReadFile(fsys, path)
if err != nil {
return nil, nil, fmt.Errorf("failed to read profile file %s: %w", filecatalog.DisplayPath(root, path), err)
}
fileMetadata, err := readProfileFileMetadata(data)
if err != nil {
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidYAML, filecatalog.DisplayPath(root, path))
}
if fileMetadata.hasRawAPIKey {
return nil, nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, filecatalog.DisplayPath(root, path))
}
definition, err := decodeProfile(data)
if err != nil {
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidYAML, filecatalog.DisplayPath(root, path))
}
definition.ExtraParams, err = jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidProfile, filecatalog.DisplayPath(root, path))
}
if err := NormalizeAndValidateDefinition(definition); err != nil {
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidProfile, filecatalog.DisplayPath(root, path))
}
if _, exists := repository.profiles[definition.ID]; exists {
return nil, nil, fmt.Errorf("%w: %s: duplicate profile ID", ErrInvalidProfile, filecatalog.DisplayPath(root, path))
}
repository.profiles[definition.ID] = *definition
fields := append([]string(nil), fileMetadata.explicitFields...)
sort.Strings(fields)
metadata = append(metadata, LoadedProfileMetadata{
ID: definition.ID,
Path: filecatalog.DisplayPath(root, path),
ExplicitFields: fields,
})
}
sort.Slice(metadata, func(left, right int) bool { return metadata[left].ID < metadata[right].ID })
return repository, metadata, nil
}
type loadedRepository struct {
profiles map[string]domain.ExecutionProfile
}
func (r *loadedRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
definition, found := r.profiles[strings.TrimSpace(id)]
if !found {
return nil, ErrProfileNotFound
}
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return nil, fmt.Errorf("copy loaded profile %q: %w", definition.ID, err)
}
definition.ExtraParams = extraParams
return &definition, nil
}

View File

@@ -0,0 +1,110 @@
package profile
import (
"context"
"errors"
"io/fs"
"reflect"
"testing"
"testing/fstest"
)
func TestLoadFSRepositoryLoadsSortedIndependentProfiles(t *testing.T) {
fsys := fstest.MapFS{
"profiles/z.yml": {Data: []byte("id: z\nbackend: local\nmodel: z-model\nextra_params:\n nested:\n value: one\n")},
"profiles/a.yml": {Data: []byte("id: a\nbackend: local\nmodel: a-model\nendpoint: ''\n")},
}
repository, metadata, err := LoadFSRepository(context.Background(), fsys, "profiles")
if err != nil {
t.Fatalf("load repository: %v", err)
}
if len(metadata) != 2 || metadata[0].ID != "a" || metadata[1].ID != "z" || metadata[0].Path != "a.yml" {
t.Fatalf("unexpected metadata: %#v", metadata)
}
if len(metadata[0].ExplicitFields) != 4 || metadata[0].ExplicitFields[0] != "backend" || metadata[0].ExplicitFields[1] != "endpoint" {
t.Fatalf("expected explicitly empty endpoint metadata, got %#v", metadata[0].ExplicitFields)
}
metadata[1].ExplicitFields[0] = "changed"
first, err := repository.GetProfile(context.Background(), "z")
if err != nil {
t.Fatalf("load profile: %v", err)
}
first.ExtraParams["nested"].(map[string]any)["value"] = "changed"
second, err := repository.GetProfile(context.Background(), "z")
if err != nil {
t.Fatalf("reload profile: %v", err)
}
if second.ExtraParams["nested"].(map[string]any)["value"] != "one" {
t.Fatalf("profile value was not defensively copied: %#v", second)
}
}
func TestLoadFSRepositoryRejectsInvalidProfiles(t *testing.T) {
tests := map[string]fstest.MapFS{
"duplicate IDs": {
"profiles/one.yml": {Data: []byte("id: duplicate\nbackend: local\nmodel: one\n")},
"profiles/two.yml": {Data: []byte("id: duplicate\nbackend: local\nmodel: two\n")},
},
"raw API key": {
"profiles/one.yml": {Data: []byte("id: one\nbackend: local\nmodel: one\napi_key: forbidden\n")},
},
"multiple documents": {
"profiles/one.yml": {Data: []byte("id: one\nbackend: local\nmodel: one\n---\nid: two\n")},
},
}
for name, fsys := range tests {
t.Run(name, func(t *testing.T) {
_, _, err := LoadFSRepository(context.Background(), fsys, "profiles")
if err == nil {
t.Fatal("expected load failure")
}
})
}
}
func TestLoadedRepositoryHonorsCancellationAndMissingProfiles(t *testing.T) {
repository, _, err := LoadFSRepository(context.Background(), fstest.MapFS{
"profiles/one.yml": {Data: []byte("id: one\nbackend: local\nmodel: one\n")},
}, "profiles")
if err != nil {
t.Fatalf("load repository: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := repository.GetProfile(ctx, "one"); !errors.Is(err, context.Canceled) {
t.Fatalf("expected cancellation, got %v", err)
}
if _, err := repository.GetProfile(context.Background(), "missing"); !errors.Is(err, ErrProfileNotFound) {
t.Fatalf("expected missing profile, got %v", err)
}
}
func TestLoadFSRepositoryMatchesPointLookupForRawProfiles(t *testing.T) {
fsys := fstest.MapFS{
"profiles/base.yml": {Data: []byte("id: base\nbackend: local\nmodel: base-model\n")},
"profiles/derived.yml": {Data: []byte("id: derived\nbase_profile: base\nreasoning_effort: high\n")},
}
eager, _, err := LoadFSRepository(context.Background(), fsys, "profiles")
if err != nil {
t.Fatalf("load eager repository: %v", err)
}
pointLookup := NewFSRepository(fsys, "profiles")
for _, id := range []string{"base", "derived"} {
t.Run(id, func(t *testing.T) {
got, err := eager.GetProfile(context.Background(), id)
if err != nil {
t.Fatalf("load eager profile: %v", err)
}
want, err := pointLookup.GetProfile(context.Background(), id)
if err != nil {
t.Fatalf("load point-in-time profile: %v", err)
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("raw profiles differ: got %#v, want %#v", got, want)
}
})
}
}
var _ fs.FS = fstest.MapFS{}

View File

@@ -5,13 +5,14 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"io"
"io/fs" "io/fs"
"os" "os"
"path"
"strings" "strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog" "gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
@@ -73,7 +74,8 @@ func (r *overlayRepository) GetProfile(ctx context.Context, id string) (*domain.
} }
func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*domain.ExecutionProfile, error) { func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*domain.ExecutionProfile, error) {
if strings.TrimSpace(id) == "" { id = strings.TrimSpace(id)
if id == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile) return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
} }
if fsys == nil { if fsys == nil {
@@ -94,42 +96,49 @@ func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*doma
} }
relPath := filecatalog.DisplayPath(root, fullPath) relPath := filecatalog.DisplayPath(root, fullPath)
fileMatch := filecatalog.Stem(path.Base(fullPath)) == id
data, err := fs.ReadFile(fsys, fullPath) data, err := fs.ReadFile(fsys, fullPath)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err) return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
} }
metadata := readProfileFileMetadata(data) metadata, metadataErr := readProfileFileMetadata(data)
idMatch := fileMatch || metadata.id == id idMatch := metadata.matchesID(id)
if metadataErr != nil {
if idMatch {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, metadataErr)
}
continue
}
if metadata.hasRawAPIKey { if metadata.hasRawAPIKey {
if idMatch { if idMatch {
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath) return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath)
} }
continue continue
} }
if !idMatch {
var prof domain.ExecutionProfile
decoder := yaml.NewDecoder(bytes.NewReader(data))
decoder.KnownFields(true)
if err := decoder.Decode(&prof); err != nil {
if idMatch {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
}
continue continue
} }
prof, err := decodeProfile(data)
if err != nil {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
}
prof.ID = strings.TrimSpace(prof.ID)
if prof.ID != id { if prof.ID != id {
continue continue
} }
prof.BackendID = strings.TrimSpace(prof.BackendID) prof.ExtraParams, err = jsonvalue.CopyMap(prof.ExtraParams)
if err := validateProfile(&prof); err != nil { if err != nil {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
}
if err := NormalizeAndValidateDefinition(prof); err != nil {
if errors.Is(err, ErrRawAPIKeyNotAllowed) { if errors.Is(err, ErrRawAPIKeyNotAllowed) {
return nil, fmt.Errorf("%w: %s", err, relPath) return nil, fmt.Errorf("%w: %s", err, relPath)
} }
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err) return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
} }
matches = append(matches, profileMatch{ matches = append(matches, profileMatch{
profile: &prof, profile: prof,
path: relPath, path: relPath,
}) })
} }
@@ -155,15 +164,37 @@ type profileMatch struct {
} }
type profileFileMetadata struct { type profileFileMetadata struct {
id string ids []string
hasRawAPIKey bool hasRawAPIKey bool
explicitFields []string
} }
func readProfileFileMetadata(data []byte) profileFileMetadata { func readProfileFileMetadata(data []byte) (profileFileMetadata, error) {
decoder := yaml.NewDecoder(bytes.NewReader(data))
var node yaml.Node var node yaml.Node
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&node); err != nil { if err := decoder.Decode(&node); err != nil {
return profileFileMetadata{} return profileFileMetadata{}, err
} }
metadata := profileMetadataFromNode(&node)
documentCount := 1
for {
var trailing yaml.Node
err := decoder.Decode(&trailing)
if errors.Is(err, io.EOF) {
if documentCount == 1 {
return metadata, nil
}
return metadata, errors.New("profile file must contain exactly one YAML document")
}
if err != nil {
return metadata, err
}
documentCount++
metadata.merge(profileMetadataFromNode(&trailing))
}
}
func profileMetadataFromNode(node *yaml.Node) profileFileMetadata {
if node.Kind != yaml.DocumentNode || len(node.Content) == 0 { if node.Kind != yaml.DocumentNode || len(node.Content) == 0 {
return profileFileMetadata{} return profileFileMetadata{}
} }
@@ -176,9 +207,10 @@ func readProfileFileMetadata(data []byte) profileFileMetadata {
for i := 0; i+1 < len(mapping.Content); i += 2 { for i := 0; i+1 < len(mapping.Content); i += 2 {
key := mapping.Content[i] key := mapping.Content[i]
value := mapping.Content[i+1] value := mapping.Content[i+1]
metadata.explicitFields = append(metadata.explicitFields, key.Value)
switch key.Value { switch key.Value {
case "id": case "id":
metadata.id = strings.TrimSpace(value.Value) metadata.ids = append(metadata.ids, strings.TrimSpace(value.Value))
case "api_key": case "api_key":
metadata.hasRawAPIKey = true metadata.hasRawAPIKey = true
} }
@@ -186,29 +218,42 @@ func readProfileFileMetadata(data []byte) profileFileMetadata {
return metadata return metadata
} }
func validateProfile(p *domain.ExecutionProfile) error { func (m profileFileMetadata) matchesID(id string) bool {
if strings.TrimSpace(p.ID) == "" { for _, candidate := range m.ids {
return errors.New("id is required") if candidate == id {
return true
}
} }
if strings.TrimSpace(p.BackendID) == "" && strings.TrimSpace(p.Endpoint) == "" { return false
return errors.New("backend or endpoint is required") }
}
if strings.TrimSpace(p.Model) == "" { func (m *profileFileMetadata) merge(other profileFileMetadata) {
return errors.New("model is required") m.ids = append(m.ids, other.ids...)
} m.hasRawAPIKey = m.hasRawAPIKey || other.hasRawAPIKey
m.explicitFields = append(m.explicitFields, other.explicitFields...)
if p.Temperature < 0 || p.Temperature > 2 { }
return errors.New("temperature must be between 0 and 2")
} func decodeProfile(data []byte) (*domain.ExecutionProfile, error) {
if p.MaxTokens < 0 { var prof domain.ExecutionProfile
return errors.New("max_tokens must be greater than or equal to 0") decoder := yaml.NewDecoder(bytes.NewReader(data))
} decoder.KnownFields(true)
if p.TopP < 0 || p.TopP > 1 { if err := decoder.Decode(&prof); err != nil {
return errors.New("top_p must be between 0 and 1") return nil, err
} }
if p.TimeoutSeconds < 0 { if err := requireYAMLStreamEnd(decoder); err != nil {
return errors.New("timeout_seconds must be greater than or equal to 0") return nil, err
} }
return &prof, nil
return nil }
func requireYAMLStreamEnd(decoder *yaml.Decoder) error {
var trailing yaml.Node
err := decoder.Decode(&trailing)
if errors.Is(err, io.EOF) {
return nil
}
if err != nil {
return err
}
return errors.New("profile file must contain exactly one YAML document")
} }

View File

@@ -0,0 +1,58 @@
package profile
import (
"context"
"fmt"
"testing"
"testing/fstest"
)
func BenchmarkProfileRepositoryLookup(b *testing.B) {
for _, size := range []int{10, 1000} {
b.Run(fmt.Sprintf("catalog-%d", size), func(b *testing.B) {
files := fstest.MapFS{
"target.yaml": profileMapFile(`
id: target
endpoint: http://localhost:8000/v1
model: target-model
extra_params:
selected: true
`),
}
metadataNames := []string{"target.yaml"}
for i := 1; i < size; i++ {
name := fmt.Sprintf("profile-%04d.yaml", i)
files[name] = profileMapFile(fmt.Sprintf(`
id: profile-%04d
endpoint: http://localhost:8000/v1
model: unrelated-model
temperature: 0.5
max_tokens: 500
extra_params:
provider:
order:
- first
- second
`, i))
metadataNames = append(metadataNames, name)
}
fsys := &recordingProfileFS{FS: files}
repo := NewFSRepository(fsys, ".")
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := repo.GetProfile(context.Background(), "target"); err != nil {
b.Fatal(err)
}
}
b.StopTimer()
for _, name := range metadataNames {
if got := fsys.openCount(name); got != b.N {
b.Fatalf("metadata %q opens = %d, want %d", name, got, b.N)
}
}
})
}
}

View File

@@ -2,11 +2,13 @@ package profile
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"fmt"
"io/fs"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"sync"
"testing" "testing"
"testing/fstest" "testing/fstest"
@@ -61,7 +63,7 @@ func TestFilesystemRepository_GetProfile(t *testing.T) {
wantErr bool wantErr bool
}{ }{
{name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"}, {name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"},
{name: "endpoint only", connection: "endpoint: http://localhost:8000/v1", wantEndpoint: "http://localhost:8000/v1"}, {name: "endpoint only", connection: "endpoint: ' https://localhost:8000/nested/v1 '", wantEndpoint: "https://localhost:8000/nested/v1"},
{name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"}, {name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"},
{name: "neither", wantErr: true}, {name: "neither", wantErr: true},
{name: "blank backend", connection: "backend: ' '", wantErr: true}, {name: "blank backend", connection: "backend: ' '", wantErr: true},
@@ -126,63 +128,6 @@ temperature: 0.1
} }
}) })
t.Run("valid profile with JSON-compatible extra params", func(t *testing.T) {
writeProfileTestFile(t, filepath.Join(tmpDir, "json-extra-params.yaml"), `
id: json-extra-params
endpoint: http://localhost:8000/v1
model: nested-model
extra_params:
string_value: enabled
number_value: 42
boolean_value: true
object_value:
nested: value
count: 2
array_value:
- first
- 3
- false
`)
p, err := repo.GetProfile(ctx, "json-extra-params")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
var got map[string]any
encoded, err := json.Marshal(p.ExtraParams)
if err != nil {
t.Fatalf("expected extra_params to marshal as JSON, got %v", err)
}
if err := json.Unmarshal(encoded, &got); err != nil {
t.Fatalf("expected extra_params JSON to decode, got %v", err)
}
if got["string_value"] != "enabled" {
t.Fatalf("unexpected string extra param: %#v", got["string_value"])
}
if got["number_value"] != float64(42) {
t.Fatalf("unexpected number extra param: %#v", got["number_value"])
}
if got["boolean_value"] != true {
t.Fatalf("unexpected boolean extra param: %#v", got["boolean_value"])
}
objectValue, ok := got["object_value"].(map[string]any)
if !ok {
t.Fatalf("expected object extra param, got %#v", got["object_value"])
}
if objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
t.Fatalf("unexpected object extra param: %#v", objectValue)
}
arrayValue, ok := got["array_value"].([]any)
if !ok {
t.Fatalf("expected array extra param, got %#v", got["array_value"])
}
if len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
t.Fatalf("unexpected array extra param: %#v", arrayValue)
}
})
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) { t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), ` writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
id: duplicate-profile id: duplicate-profile
@@ -245,10 +190,10 @@ api_key: secret
} }
}) })
t.Run("invalid yaml", func(t *testing.T) { t.Run("unidentifiable invalid yaml is unrelated", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "invalid_yaml") _, err := repo.GetProfile(ctx, "invalid_yaml")
if !errors.Is(err, ErrInvalidYAML) { if !errors.Is(err, ErrProfileNotFound) {
t.Fatalf("expected ErrInvalidYAML, got %v", err) t.Fatalf("expected ErrProfileNotFound, got %v", err)
} }
}) })
@@ -274,14 +219,14 @@ api_key: secret
}) })
t.Run("unknown field", func(t *testing.T) { t.Run("unknown field", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "unknown_field") _, err := repo.GetProfile(ctx, "unknown-field")
if !errors.Is(err, ErrInvalidYAML) { if !errors.Is(err, ErrInvalidYAML) {
t.Fatalf("expected ErrInvalidYAML for strict decode unknown field, got %v", err) t.Fatalf("expected ErrInvalidYAML for strict decode unknown field, got %v", err)
} }
}) })
t.Run("raw api_key rejected", func(t *testing.T) { t.Run("raw api_key rejected", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "raw_api_key") _, err := repo.GetProfile(ctx, "raw-api-key")
if !errors.Is(err, ErrRawAPIKeyNotAllowed) { if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err) t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
} }
@@ -406,6 +351,614 @@ model: second
}) })
} }
func TestProfileRepositoriesRejectInvalidEndpoints(t *testing.T) {
tests := []struct {
name string
endpoint string
withBackend bool
}{
{name: "relative", endpoint: "/v1"},
{name: "missing host", endpoint: "https:///v1"},
{name: "unsupported scheme", endpoint: "ftp://provider.example/v1"},
{name: "user information", endpoint: "https://user@provider.example/v1"},
{name: "query", endpoint: "https://provider.example/v1?mode=chat"},
{name: "fragment", endpoint: "https://provider.example/v1#chat"},
{name: "backend with invalid override", endpoint: "/v1", withBackend: true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
backend := ""
if tc.withBackend {
backend = "backend: openrouter\n"
}
repo := NewFSRepository(fstest.MapFS{
"profiles/invalid.yaml": profileMapFile(fmt.Sprintf(
"id: invalid-endpoint\nmodel: model\n%sendpoint: %q\n",
backend,
tc.endpoint,
)),
}, "profiles")
_, err := repo.GetProfile(context.Background(), "invalid-endpoint")
if !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("expected ErrInvalidProfile, got %v", err)
}
})
}
}
func TestProfileRepositoriesValidateExtraParams(t *testing.T) {
const validProfile = `
id: selected-profile
endpoint: http://localhost:8000/v1
model: model
extra_params:
string_value: enabled
object_value:
nested: true
array_value:
- first
- 3
`
tests := []struct {
name string
definition string
wantErr bool
diagnostics []string
}{
{name: "valid nested values", definition: validProfile},
{
name: "empty key",
definition: `
id: selected-profile
endpoint: http://localhost:8000/v1
model: model
extra_params:
"": value
`,
wantErr: true,
diagnostics: []string{"extra_params", "key must not be empty"},
},
{
name: "non-finite value",
definition: `
id: selected-profile
endpoint: http://localhost:8000/v1
model: model
extra_params:
invalid: .nan
`,
wantErr: true,
diagnostics: []string{"extra_params.invalid", "must be finite"},
},
{
name: "nested non-finite value",
definition: `
id: selected-profile
endpoint: http://localhost:8000/v1
model: model
extra_params:
outer:
invalid: .inf
`,
wantErr: true,
diagnostics: []string{"extra_params.outer.invalid", "must be finite"},
},
{
name: "unsupported decoded value",
definition: `
id: selected-profile
endpoint: http://localhost:8000/v1
model: model
extra_params:
timestamp: 2026-08-11T12:34:56Z
`,
wantErr: true,
diagnostics: []string{"extra_params.timestamp", "unsupported JSON value type"},
},
{
name: "excessive nesting",
definition: deeplyNestedExtraParamsProfile(101),
wantErr: true,
diagnostics: []string{"extra_params", "JSON container depth limit exceeded"},
},
}
for _, source := range profileRepositorySources() {
for _, tc := range tests {
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
repo := source.newRepository(t, map[string]string{"selected.yaml": tc.definition})
got, err := repo.GetProfile(context.Background(), "selected-profile")
if tc.wantErr {
if !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("expected ErrInvalidProfile, got %v", err)
}
if !strings.Contains(err.Error(), "selected.yaml") {
t.Fatalf("expected source path in error, got %v", err)
}
for _, diagnostic := range tc.diagnostics {
if !strings.Contains(err.Error(), diagnostic) {
t.Fatalf("expected error to contain %q, got %v", diagnostic, err)
}
}
return
}
if err != nil {
t.Fatalf("load valid profile: %v", err)
}
if got.ExtraParams["string_value"] != "enabled" {
t.Fatalf("unexpected copied extra params: %#v", got.ExtraParams)
}
objectValue, objectOK := got.ExtraParams["object_value"].(map[string]any)
arrayValue, arrayOK := got.ExtraParams["array_value"].([]any)
if !objectOK || objectValue["nested"] != true ||
!arrayOK || len(arrayValue) != 2 || arrayValue[0] != "first" || arrayValue[1] != 3 {
t.Fatalf("unexpected copied nested extra params: %#v", got.ExtraParams)
}
})
}
}
}
func TestProfileRepositoriesSelectCanonicalYAMLID(t *testing.T) {
const validProfile = `
id: selected-profile
endpoint: http://localhost:8000/v1
model: selected-model
`
tests := []struct {
name string
files map[string]string
wantErr error
diagnostics []string
}{
{
name: "same stem unknown field with different id is unrelated",
files: map[string]string{
"selected-profile.yaml": `
id: unrelated-profile
endpoint: http://localhost:8000/v1
model: unrelated
unknown: true
`,
"valid.yaml": validProfile,
},
},
{
name: "same stem unidentifiable yaml is unrelated",
files: map[string]string{
"selected-profile.yaml": "id: [",
"valid.yaml": validProfile,
},
},
{
name: "same stem raw key with different id is unrelated",
files: map[string]string{
"selected-profile.yaml": `
id: unrelated-profile
endpoint: http://localhost:8000/v1
model: unrelated
api_key: secret
`,
"valid.yaml": validProfile,
},
},
{
name: "leading and trailing whitespace is normalized",
files: map[string]string{
"padded.yaml": `
id: " selected-profile "
endpoint: http://localhost:8000/v1
model: selected-model
`,
},
},
{
name: "blank id is unrelated",
files: map[string]string{
"selected-profile.yaml": `
id: " "
endpoint: http://localhost:8000/v1
model: unrelated
`,
},
wantErr: ErrProfileNotFound,
},
{
name: "normalized duplicates are ambiguous",
files: map[string]string{
"first.yaml": validProfile,
"nested/second.yaml": `
id: " selected-profile "
endpoint: http://localhost:8000/v1
model: duplicate
`,
},
wantErr: ErrInvalidProfile,
diagnostics: []string{"duplicate execution profile id", "first.yaml", "nested/second.yaml"},
},
{
name: "selected unknown field is authoritative",
files: map[string]string{
"malformed.yaml": `
id: selected-profile
endpoint: http://localhost:8000/v1
model: selected-model
unknown: true
`,
},
wantErr: ErrInvalidYAML,
diagnostics: []string{"malformed.yaml"},
},
{
name: "selected raw key is authoritative",
files: map[string]string{
"insecure.yaml": `
id: selected-profile
endpoint: http://localhost:8000/v1
model: selected-model
api_key: secret
`,
},
wantErr: ErrRawAPIKeyNotAllowed,
diagnostics: []string{"insecure.yaml"},
},
{
name: "selected identity in an additional document is authoritative",
files: map[string]string{
"additional-document.yaml": `
---
---
id: selected-profile
endpoint: http://localhost:8000/v1
model: selected-model
`,
},
wantErr: ErrInvalidYAML,
diagnostics: []string{"additional-document.yaml", "exactly one YAML document"},
},
}
for _, source := range profileRepositorySources() {
for _, tc := range tests {
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
repo := source.newRepository(t, tc.files)
got, err := repo.GetProfile(context.Background(), " selected-profile ")
if tc.wantErr != nil {
if !errors.Is(err, tc.wantErr) {
t.Fatalf("expected %v, got %v", tc.wantErr, err)
}
for _, diagnostic := range tc.diagnostics {
if !strings.Contains(err.Error(), diagnostic) {
t.Fatalf("expected error to contain %q, got %v", diagnostic, err)
}
}
return
}
if err != nil {
t.Fatalf("load selected profile: %v", err)
}
if got.ID != "selected-profile" || got.Model != "selected-model" {
t.Fatalf("unexpected selected profile: %+v", got)
}
})
}
}
}
func TestProfileRepositoriesRequireOneYAMLDocument(t *testing.T) {
const profile = `
id: selected-profile
endpoint: http://localhost:8000/v1
model: selected-model
`
tests := []struct {
name string
suffix string
wantErr bool
}{
{name: "comments and trailing whitespace", suffix: "\n# trailing comment\n\n"},
{name: "second populated document", suffix: "\n---\nid: another\n", wantErr: true},
{name: "second empty document", suffix: "\n---\n", wantErr: true},
{name: "malformed trailing yaml", suffix: "\n---\n[", wantErr: true},
{name: "raw key in trailing document", suffix: "\n---\napi_key: secret\n", wantErr: true},
}
for _, source := range profileRepositorySources() {
for _, tc := range tests {
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
repo := source.newRepository(t, map[string]string{"definition.yaml": profile + tc.suffix})
got, err := repo.GetProfile(context.Background(), "selected-profile")
if tc.wantErr {
if !errors.Is(err, ErrInvalidYAML) {
t.Fatalf("expected ErrInvalidYAML, got %v", err)
}
if !strings.Contains(err.Error(), "definition.yaml") {
t.Fatalf("expected source path in error, got %v", err)
}
return
}
if err != nil {
t.Fatalf("load one-document profile: %v", err)
}
if got.ID != "selected-profile" {
t.Fatalf("unexpected profile: %+v", got)
}
})
}
}
}
func TestProfileRepositoriesPreserveOverlayFallbackRules(t *testing.T) {
fallback := staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"selected-profile": {ID: "selected-profile", Endpoint: "http://fallback", Model: "fallback-model"},
}}
tests := []struct {
name string
files map[string]string
wantModel string
wantErr error
}{
{
name: "same stem malformed different id falls back",
files: map[string]string{
"selected-profile.yaml": `
id: unrelated-profile
endpoint: http://localhost:8000/v1
model: unrelated
unknown: true
`,
},
wantModel: "fallback-model",
},
{
name: "blank id falls back",
files: map[string]string{
"selected-profile.yaml": `
id: " "
endpoint: http://localhost:8000/v1
model: unrelated
`,
},
wantModel: "fallback-model",
},
{
name: "selected malformed profile stops fallback",
files: map[string]string{
"other-name.yaml": `
id: selected-profile
endpoint: http://localhost:8000/v1
model: selected
unknown: true
`,
},
wantErr: ErrInvalidYAML,
},
}
for _, source := range profileRepositorySources() {
for _, tc := range tests {
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
primary := source.newRepository(t, tc.files)
got, err := NewOverlayRepository(primary, fallback).GetProfile(context.Background(), "selected-profile")
if tc.wantErr != nil {
if !errors.Is(err, tc.wantErr) {
t.Fatalf("expected %v, got %v", tc.wantErr, err)
}
return
}
if err != nil {
t.Fatalf("load fallback profile: %v", err)
}
if got.Model != tc.wantModel {
t.Fatalf("model = %q, want %q", got.Model, tc.wantModel)
}
})
}
}
}
func TestProfileRepositoryReadsSourcesFreshOnEveryLookup(t *testing.T) {
newSource := func() (*recordingProfileFS, Repository) {
fsys := &recordingProfileFS{FS: fstest.MapFS{
"target.yaml": profileMapFile(`
id: target
endpoint: http://localhost:8000/v1
model: target-model
`),
"unrelated.yaml": profileMapFile(`
id: unrelated
endpoint: http://localhost:8000/v1
model: unrelated-model
`),
}}
return fsys, NewFSRepository(fsys, ".")
}
t.Run("selected source", func(t *testing.T) {
fsys, repo := newSource()
for lookup := 1; lookup <= 2; lookup++ {
got, err := repo.GetProfile(context.Background(), "target")
if err != nil {
t.Fatalf("lookup %d: %v", lookup, err)
}
if got.Model != "target-model" {
t.Fatalf("lookup %d model = %q", lookup, got.Model)
}
for _, name := range []string{"target.yaml", "unrelated.yaml"} {
if count := fsys.openCount(name); count != lookup {
t.Fatalf("%s opens after lookup %d = %d, want %d", name, lookup, count, lookup)
}
}
}
})
t.Run("overlay fallthrough", func(t *testing.T) {
primaryFS := &recordingProfileFS{FS: fstest.MapFS{
"unrelated.yaml": profileMapFile(`
id: unrelated
endpoint: http://localhost:8000/v1
model: unrelated-model
`),
}}
fallbackFS, fallback := newSource()
repo := NewOverlayRepository(NewFSRepository(primaryFS, "."), fallback)
for lookup := 1; lookup <= 2; lookup++ {
got, err := repo.GetProfile(context.Background(), "target")
if err != nil {
t.Fatalf("lookup %d: %v", lookup, err)
}
if got.Model != "target-model" {
t.Fatalf("lookup %d model = %q", lookup, got.Model)
}
if count := primaryFS.openCount("unrelated.yaml"); count != lookup {
t.Fatalf("primary opens after lookup %d = %d, want %d", lookup, count, lookup)
}
if count := fallbackFS.openCount("target.yaml"); count != lookup {
t.Fatalf("fallback opens after lookup %d = %d, want %d", lookup, count, lookup)
}
}
})
}
func TestProfileRepositoriesRejectInvalidExecutionSettings(t *testing.T) {
ctx := context.Background()
t.Run("operating-system filesystem", func(t *testing.T) {
dir := t.TempDir()
writeProfileTestFile(t, filepath.Join(dir, "invalid.yaml"), `
id: invalid
endpoint: http://localhost:8000/v1
model: model
temperature: .nan
`)
_, err := NewFilesystemRepository(dir).GetProfile(ctx, "invalid")
if !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("expected ErrInvalidProfile, got %v", err)
}
})
t.Run("fs.FS", func(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"profiles/invalid.yaml": profileMapFile(`
id: invalid
endpoint: http://localhost:8000/v1
model: model
top_p: .inf
`),
}, "profiles")
_, err := repo.GetProfile(ctx, "invalid")
if !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("expected ErrInvalidProfile, got %v", err)
}
})
}
func TestProfileRepositoriesValidateDerivedDefinitions(t *testing.T) {
tests := []struct {
name string
files map[string]string
wantError error
wantBaseID string
wantProfile bool
}{
{
name: "alias is locally valid and normalizes base id",
files: map[string]string{"alias.yaml": `
id: selected-profile
base_profile: " base-profile "
`},
wantBaseID: "base-profile",
wantProfile: true,
},
{
name: "derived endpoint remains valid",
files: map[string]string{"invalid.yaml": `
id: selected-profile
base_profile: base-profile
endpoint: /v1
`},
wantError: ErrInvalidProfile,
},
{
name: "derived settings remain valid",
files: map[string]string{"invalid.yaml": `
id: selected-profile
base_profile: base-profile
top_p: 1.1
`},
wantError: ErrInvalidProfile,
},
{
name: "derived extra params remain valid",
files: map[string]string{"invalid.yaml": `
id: selected-profile
base_profile: base-profile
extra_params:
timestamp: 2026-08-11T12:34:56Z
`},
wantError: ErrInvalidProfile,
},
{
name: "derived raw key remains prohibited",
files: map[string]string{"invalid.yaml": `
id: selected-profile
base_profile: base-profile
api_key: secret
`},
wantError: ErrRawAPIKeyNotAllowed,
},
{
name: "derived duplicate id remains invalid",
files: map[string]string{
"first.yaml": "id: selected-profile\nbase_profile: first-base\n",
"second.yaml": "id: selected-profile\nbase_profile: second-base\n",
},
wantError: ErrInvalidProfile,
},
{
name: "derived extra document remains invalid",
files: map[string]string{"invalid.yaml": `
id: selected-profile
base_profile: base-profile
---
id: other
`},
wantError: ErrInvalidYAML,
},
{
name: "standalone profile remains complete",
files: map[string]string{"invalid.yaml": "id: selected-profile\n"},
wantError: ErrInvalidProfile,
},
}
for _, source := range profileRepositorySources() {
for _, tc := range tests {
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
repo := source.newRepository(t, tc.files)
got, err := repo.GetProfile(context.Background(), "selected-profile")
if tc.wantError != nil {
if !errors.Is(err, tc.wantError) {
t.Fatalf("error = %v, want %v", err, tc.wantError)
}
return
}
if err != nil || !tc.wantProfile {
t.Fatalf("profile = %+v, error = %v, want valid derived definition", got, err)
}
if got.BaseProfileID != tc.wantBaseID {
t.Fatalf("BaseProfileID = %q, want %q", got.BaseProfileID, tc.wantBaseID)
}
})
}
}
}
func TestOverlayRepository(t *testing.T) { func TestOverlayRepository(t *testing.T) {
ctx := context.Background() ctx := context.Background()
primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"} primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"}
@@ -495,6 +1048,77 @@ func TestOverlayRepository(t *testing.T) {
}) })
} }
type profileRepositorySource struct {
name string
newRepository func(t *testing.T, files map[string]string) Repository
}
type recordingProfileFS struct {
fs.FS
mu sync.Mutex
opened []string
}
func (f *recordingProfileFS) Open(name string) (fs.File, error) {
f.mu.Lock()
f.opened = append(f.opened, name)
f.mu.Unlock()
return f.FS.Open(name)
}
func (f *recordingProfileFS) openCount(name string) int {
f.mu.Lock()
defer f.mu.Unlock()
count := 0
for _, opened := range f.opened {
if opened == name {
count++
}
}
return count
}
func profileRepositorySources() []profileRepositorySource {
return []profileRepositorySource{
{
name: "operating system",
newRepository: func(t *testing.T, files map[string]string) Repository {
t.Helper()
root := t.TempDir()
for name, content := range files {
filePath := filepath.Join(root, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil {
t.Fatalf("create profile directory: %v", err)
}
writeProfileTestFile(t, filePath, content)
}
return NewFilesystemRepository(root)
},
},
{
name: "filesystem",
newRepository: func(t *testing.T, files map[string]string) Repository {
t.Helper()
fsys := make(fstest.MapFS, len(files))
for name, content := range files {
fsys[name] = profileMapFile(content)
}
return NewFSRepository(fsys, ".")
},
},
}
}
func deeplyNestedExtraParamsProfile(depth int) string {
var definition strings.Builder
definition.WriteString("id: selected-profile\nendpoint: http://localhost:8000/v1\nmodel: model\nextra_params:\n")
for level := 0; level < depth; level++ {
fmt.Fprintf(&definition, "%slevel_%d:\n", strings.Repeat(" ", level+1), level)
}
fmt.Fprintf(&definition, "%svalue: true\n", strings.Repeat(" ", depth+1))
return definition.String()
}
func profileMapFile(content string) *fstest.MapFile { func profileMapFile(content string) *fstest.MapFile {
return &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))} return &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))}
} }

View File

@@ -0,0 +1,160 @@
package profile
import (
"context"
"errors"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
)
const maximumProfileChainLength = 32
type resolvingRepository struct {
source Repository
}
// NewResolvingRepository resolves inherited profile definitions from source.
func NewResolvingRepository(source Repository) Repository {
return &resolvingRepository{source: source}
}
func (r *resolvingRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
if r == nil || r.source == nil {
return nil, fmt.Errorf("%w: profile repository is required", ErrInvalidProfile)
}
requestedID := strings.TrimSpace(id)
if requestedID == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
}
if err := ctx.Err(); err != nil {
return nil, err
}
profile, err := r.getRawProfile(ctx, requestedID)
if err != nil {
return nil, err
}
if profile == nil {
return nil, fmt.Errorf("%w: selected profile %q is nil", ErrInvalidProfile, requestedID)
}
chain := []*domain.ExecutionProfile{profile}
chainIDs := []string{requestedID}
visited := map[string]struct{}{requestedID: {}}
current := profile
for {
baseID := strings.TrimSpace(current.BaseProfileID)
if baseID == "" {
break
}
if err := ctx.Err(); err != nil {
return nil, err
}
if _, seen := visited[baseID]; seen {
return nil, fmt.Errorf("%w: profile inheritance cycle %s", ErrInvalidProfile, joinProfileChain(chainIDs, baseID))
}
if len(chain) >= maximumProfileChainLength {
return nil, fmt.Errorf("%w: profile inheritance chain exceeds %d profiles: %s", ErrInvalidProfile, maximumProfileChainLength, joinProfileChain(chainIDs, baseID))
}
base, err := r.getRawProfile(ctx, baseID)
if err != nil {
if errors.Is(err, ErrProfileNotFound) {
return nil, fmt.Errorf("%w: base profile %q is missing in chain %s", ErrInvalidProfile, baseID, joinProfileChain(chainIDs, baseID))
}
return nil, fmt.Errorf("%w: failed to load base profile %q in chain %s: %w", ErrInvalidProfile, baseID, joinProfileChain(chainIDs, baseID), err)
}
if base == nil {
return nil, fmt.Errorf("%w: base profile %q is nil in chain %s", ErrInvalidProfile, baseID, joinProfileChain(chainIDs, baseID))
}
chain = append(chain, base)
chainIDs = append(chainIDs, baseID)
visited[baseID] = struct{}{}
current = base
}
resolved, err := mergeProfileChain(chain)
if err != nil {
return nil, fmt.Errorf("%w: resolved profile chain %s: %w", ErrInvalidProfile, strings.Join(chainIDs, " -> "), err)
}
if err := validateResolvedProfile(resolved); err != nil {
return nil, fmt.Errorf("%w: resolved profile chain %s: %w", ErrInvalidProfile, strings.Join(chainIDs, " -> "), err)
}
return resolved, nil
}
func (r *resolvingRepository) getRawProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
profile, err := r.source.GetProfile(ctx, id)
if err != nil {
return nil, err
}
if err := ctx.Err(); err != nil {
return nil, err
}
return profile, nil
}
func joinProfileChain(chain []string, next string) string {
return strings.Join(append(append([]string(nil), chain...), next), " -> ")
}
func mergeProfileChain(chain []*domain.ExecutionProfile) (*domain.ExecutionProfile, error) {
resolved := &domain.ExecutionProfile{ID: chain[0].ID}
for index := len(chain) - 1; index >= 0; index-- {
definition := chain[index]
if strings.TrimSpace(definition.BackendID) != "" {
resolved.BackendID = definition.BackendID
}
if strings.TrimSpace(definition.Endpoint) != "" {
resolved.Endpoint = definition.Endpoint
}
if strings.TrimSpace(definition.Model) != "" {
resolved.Model = definition.Model
}
if definition.Temperature != 0 {
resolved.Temperature = definition.Temperature
}
if definition.MaxTokens != 0 {
resolved.MaxTokens = definition.MaxTokens
}
if definition.TopP != 0 {
resolved.TopP = definition.TopP
}
if definition.TimeoutSeconds != 0 {
resolved.TimeoutSeconds = definition.TimeoutSeconds
}
if strings.TrimSpace(definition.ServiceTier) != "" {
resolved.ServiceTier = definition.ServiceTier
}
if strings.TrimSpace(definition.ReasoningEffort) != "" {
resolved.ReasoningEffort = definition.ReasoningEffort
}
if strings.TrimSpace(definition.APIKeyEnv) != "" {
resolved.APIKeyEnv = definition.APIKeyEnv
}
resolved.APIKeyRequired = resolved.APIKeyRequired || definition.APIKeyRequired
if len(definition.ExtraParams) != 0 {
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return nil, err
}
resolved.ExtraParams = extraParams
}
}
resolved.ID = chain[0].ID
resolved.BaseProfileID = ""
return resolved, nil
}
func validateResolvedProfile(profile *domain.ExecutionProfile) error {
if profile == nil {
return errors.New("resolved profile is required")
}
profile.BaseProfileID = ""
return NormalizeAndValidateDefinition(profile)
}

View File

@@ -0,0 +1,418 @@
package profile
import (
"context"
"errors"
"fmt"
"io/fs"
"reflect"
"strings"
"sync"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestResolvingRepositoryMergesProfileChain(t *testing.T) {
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
"leaf": {
ID: "leaf",
BaseProfileID: "middle",
BackendID: "leaf-backend",
TopP: 0.8,
TimeoutSeconds: 45,
ReasoningEffort: "high",
},
"middle": {
ID: "middle",
BaseProfileID: "root",
Endpoint: "https://middle.example/v1",
Model: "middle-model",
MaxTokens: 256,
APIKeyEnv: "MIDDLE_API_KEY",
APIKeyRequired: true,
ExtraParams: map[string]any{"middle": map[string]any{"value": "middle"}},
},
"root": {
ID: "root",
BackendID: "root-backend",
Endpoint: "https://root.example/v1",
Model: "root-model",
Temperature: 0.3,
ServiceTier: "priority",
ExtraParams: map[string]any{"root": "value"},
},
}}
got, err := NewResolvingRepository(repo).GetProfile(context.Background(), "leaf")
if err != nil {
t.Fatalf("resolve profile: %v", err)
}
want := &domain.ExecutionProfile{
ID: "leaf",
BackendID: "leaf-backend",
Endpoint: "https://middle.example/v1",
Model: "middle-model",
Temperature: 0.3,
MaxTokens: 256,
TopP: 0.8,
TimeoutSeconds: 45,
ServiceTier: "priority",
ReasoningEffort: "high",
APIKeyEnv: "MIDDLE_API_KEY",
APIKeyRequired: true,
ExtraParams: map[string]any{"middle": map[string]any{"value": "middle"}},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("resolved profile:\n got %#v\nwant %#v", got, want)
}
}
func TestResolvingRepositoryRejectsMissingSourceAndProfileID(t *testing.T) {
if _, err := NewResolvingRepository(nil).GetProfile(context.Background(), "profile"); !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("nil source error = %v, want ErrInvalidProfile", err)
}
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{}}
if _, err := NewResolvingRepository(repo).GetProfile(context.Background(), " \t "); !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("blank id error = %v, want ErrInvalidProfile", err)
}
if got := repo.callCount(" "); got != 0 {
t.Fatalf("blank id looked up source %d times", got)
}
}
func TestResolvingRepositoryCopiesExtraParams(t *testing.T) {
baseParams := map[string]any{"nested": map[string]any{"value": "base"}}
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
"child": {ID: "child", BaseProfileID: "base"},
"base": {
ID: "base",
Endpoint: "https://base.example/v1",
Model: "model",
ExtraParams: baseParams,
},
}}
resolver := NewResolvingRepository(repo)
first, err := resolver.GetProfile(context.Background(), "child")
if err != nil {
t.Fatalf("resolve inherited map: %v", err)
}
first.ExtraParams["nested"].(map[string]any)["value"] = "mutated"
second, err := resolver.GetProfile(context.Background(), "child")
if err != nil {
t.Fatalf("resolve inherited map again: %v", err)
}
if got := second.ExtraParams["nested"].(map[string]any)["value"]; got != "base" {
t.Fatalf("later result retained mutation: %v", got)
}
if got := baseParams["nested"].(map[string]any)["value"]; got != "base" {
t.Fatalf("source map retained mutation: %v", got)
}
repo.set("child", &domain.ExecutionProfile{
ID: "child",
BaseProfileID: "base",
ExtraParams: map[string]any{"child": "replacement"},
})
replaced, err := resolver.GetProfile(context.Background(), "child")
if err != nil {
t.Fatalf("resolve replacement map: %v", err)
}
if !reflect.DeepEqual(replaced.ExtraParams, map[string]any{"child": "replacement"}) {
t.Fatalf("extra params = %#v, want complete child replacement", replaced.ExtraParams)
}
}
func TestResolvingRepositoryUsesRawOverlayForEachLookup(t *testing.T) {
leafSource := NewFSRepository(profileTestFS(map[string]string{
"leaf.yaml": "id: leaf\nbase_profile: base\n",
}), ".")
fallback := NewFSRepository(profileTestFS(map[string]string{
"base.yaml": "id: base\nendpoint: https://fallback.example/v1\nmodel: fallback-model\n",
}), ".")
overlay := NewOverlayRepository(leafSource, fallback)
resolver := NewResolvingRepository(overlay)
got, err := resolver.GetProfile(context.Background(), "leaf")
if err != nil {
t.Fatalf("resolve fallback base: %v", err)
}
if got.Model != "fallback-model" {
t.Fatalf("fallback base model = %q", got.Model)
}
shadowing := NewOverlayRepository(NewFSRepository(profileTestFS(map[string]string{
"leaf.yaml": "id: leaf\nbase_profile: base\n",
"base.yaml": "id: base\nendpoint: https://primary.example/v1\nmodel: primary-model\n",
}), "."), fallback)
got, err = NewResolvingRepository(shadowing).GetProfile(context.Background(), "leaf")
if err != nil {
t.Fatalf("resolve shadowed base: %v", err)
}
if got.Model != "primary-model" || got.Endpoint != "https://primary.example/v1" {
t.Fatalf("shadowed base = %+v", got)
}
}
func TestResolvingRepositoryReportsSafetyAndSourceErrors(t *testing.T) {
sourceErr := errors.New("source failure")
tests := []struct {
name string
repo *resolvingTestRepository
id string
want []error
wantNot error
contains []string
}{
{
name: "missing selected profile preserves not found",
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{}},
id: "missing",
want: []error{ErrProfileNotFound},
wantNot: ErrInvalidProfile,
},
{
name: "missing base is invalid but not not found",
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
"leaf": {ID: "leaf", BaseProfileID: "missing"},
}},
id: "leaf",
want: []error{ErrInvalidProfile},
wantNot: ErrProfileNotFound,
contains: []string{"missing", "leaf -> missing"},
},
{
name: "direct cycle",
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
"a": {ID: "a", BaseProfileID: "a"},
}},
id: "a",
want: []error{ErrInvalidProfile},
contains: []string{"a -> a"},
},
{
name: "indirect cycle",
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
"a": {ID: "a", BaseProfileID: "b"},
"b": {ID: "b", BaseProfileID: "c"},
"c": {ID: "c", BaseProfileID: "a"},
}},
id: "a",
want: []error{ErrInvalidProfile},
contains: []string{"a -> b -> c -> a"},
},
{
name: "nil result",
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
"leaf": nil,
}},
id: "leaf",
want: []error{ErrInvalidProfile},
},
{
name: "incomplete resolved profile",
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
"leaf": {ID: "leaf", BaseProfileID: "base"},
"base": {ID: "base", Model: "model"},
}},
id: "leaf",
want: []error{ErrInvalidProfile},
},
{
name: "base source error is retained",
repo: &resolvingTestRepository{
profiles: map[string]*domain.ExecutionProfile{"leaf": {ID: "leaf", BaseProfileID: "base"}},
errors: map[string]error{"base": sourceErr},
},
id: "leaf",
want: []error{ErrInvalidProfile, sourceErr},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := NewResolvingRepository(tc.repo).GetProfile(context.Background(), tc.id)
for _, want := range tc.want {
if !errors.Is(err, want) {
t.Fatalf("error = %v, want %v", err, want)
}
}
if tc.wantNot != nil && errors.Is(err, tc.wantNot) {
t.Fatalf("error = %v, must not match %v", err, tc.wantNot)
}
for _, fragment := range tc.contains {
if !strings.Contains(err.Error(), fragment) {
t.Fatalf("error = %v, want %q", err, fragment)
}
}
})
}
}
func TestResolvingRepositoryEnforcesChainLength(t *testing.T) {
for _, count := range []int{maximumProfileChainLength, maximumProfileChainLength + 1} {
t.Run(fmt.Sprintf("%d profiles", count), func(t *testing.T) {
profiles := make(map[string]*domain.ExecutionProfile, count)
for index := 1; index <= count; index++ {
id := fmt.Sprintf("profile-%d", index)
definition := &domain.ExecutionProfile{ID: id}
if index == count {
definition.Endpoint = "https://root.example/v1"
definition.Model = "model"
} else {
definition.BaseProfileID = fmt.Sprintf("profile-%d", index+1)
}
profiles[id] = definition
}
got, err := NewResolvingRepository(&resolvingTestRepository{profiles: profiles}).GetProfile(context.Background(), "profile-1")
if count == maximumProfileChainLength {
if err != nil || got == nil {
t.Fatalf("profile = %+v, error = %v, want accepted chain", got, err)
}
return
}
if !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("error = %v, want ErrInvalidProfile", err)
}
})
}
}
func TestResolvingRepositoryIsFreshAndCancellationAware(t *testing.T) {
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
"leaf": {ID: "leaf", BaseProfileID: "base"},
"base": {ID: "base", Endpoint: "https://base.example/v1", Model: "first", ExtraParams: map[string]any{"nested": map[string]any{"value": "first"}}},
}}
resolver := NewResolvingRepository(repo)
first, err := resolver.GetProfile(context.Background(), "leaf")
if err != nil || first.Model != "first" {
t.Fatalf("first result=(%+v, %v)", first, err)
}
repo.set("base", &domain.ExecutionProfile{ID: "base", Endpoint: "https://base.example/v1", Model: "second", ExtraParams: map[string]any{"nested": map[string]any{"value": "second"}}})
second, err := resolver.GetProfile(context.Background(), "leaf")
if err != nil || second.Model != "second" {
t.Fatalf("second result=(%+v, %v)", second, err)
}
canceled, cancel := context.WithCancel(context.Background())
cancel()
if _, err := resolver.GetProfile(canceled, "leaf"); !errors.Is(err, context.Canceled) {
t.Fatalf("canceled lookup error = %v", err)
}
if got := repo.callCount("leaf"); got != 2 {
t.Fatalf("calls after canceled lookup = %d, want 2", got)
}
duringTraversal, cancelDuringTraversal := context.WithCancel(context.Background())
repo.afterGet = func(id string) {
if id == "leaf" {
cancelDuringTraversal()
}
}
if _, err := resolver.GetProfile(duringTraversal, "leaf"); !errors.Is(err, context.Canceled) {
t.Fatalf("during traversal error = %v", err)
}
if got := repo.callCount("base"); got != 2 {
t.Fatalf("base calls after cancellation = %d, want 2", got)
}
terminalLookup, cancelTerminalLookup := context.WithCancel(context.Background())
repo.afterGet = func(id string) {
if id == "base" {
cancelTerminalLookup()
}
}
if _, err := resolver.GetProfile(terminalLookup, "leaf"); !errors.Is(err, context.Canceled) {
t.Fatalf("terminal lookup cancellation error = %v", err)
}
if got := repo.callCount("base"); got != 3 {
t.Fatalf("base calls after terminal cancellation = %d, want 3", got)
}
repo.afterGet = nil
var wg sync.WaitGroup
errors := make(chan error, 8)
for index := 0; index < cap(errors); index++ {
wg.Add(1)
go func() {
defer wg.Done()
resolved, err := resolver.GetProfile(context.Background(), "leaf")
if err != nil {
errors <- err
return
}
resolved.ExtraParams["nested"].(map[string]any)["value"] = "mutated"
}()
}
wg.Wait()
close(errors)
for err := range errors {
t.Errorf("concurrent resolution: %v", err)
}
latest, err := resolver.GetProfile(context.Background(), "leaf")
if err != nil || latest.ExtraParams["nested"].(map[string]any)["value"] != "second" {
t.Fatalf("latest result=(%+v, %v)", latest, err)
}
}
type resolvingTestRepository struct {
mu sync.Mutex
profiles map[string]*domain.ExecutionProfile
errors map[string]error
calls map[string]int
afterGet func(string)
}
func (r *resolvingTestRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
r.mu.Lock()
if r.calls == nil {
r.calls = make(map[string]int)
}
r.calls[id]++
err := r.errors[id]
profile := r.profiles[id]
afterGet := r.afterGet
r.mu.Unlock()
if afterGet != nil {
afterGet(id)
}
if err != nil {
return nil, err
}
if profile == nil {
if _, exists := r.profiles[id]; exists {
return nil, nil
}
return nil, ErrProfileNotFound
}
copy := *profile
return &copy, nil
}
func (r *resolvingTestRepository) set(id string, profile *domain.ExecutionProfile) {
r.mu.Lock()
defer r.mu.Unlock()
r.profiles[id] = profile
}
func (r *resolvingTestRepository) callCount(id string) int {
r.mu.Lock()
defer r.mu.Unlock()
return r.calls[id]
}
func profileTestFS(files map[string]string) fs.FS {
fsys := make(fstest.MapFS, len(files))
for name, content := range files {
fsys[name] = profileMapFile(content)
}
return fsys
}

Some files were not shown because too many files have changed in this diff Show More