27 Commits

Author SHA1 Message Date
a4264bf4b4 Prepare the v0.9.0 release 2026-08-26 17:31:53 +00:00
81f41564a2 Harden external catalog validation 2026-08-26 16:51:31 +00:00
6b5a2497cc Document the external catalog boundary 2026-08-26 15:06:33 +00:00
422cc6c978 Use external catalogs for maintained backends 2026-08-26 15:04:50 +00:00
f48e042565 Validate the external backend catalogs 2026-08-26 15:00:50 +00:00
d9442850ef Add immutable profile catalog loading 2026-08-26 14:56:56 +00:00
2e828157b6 Mark the Rakestrawhome catalog release complete 2026-08-26 14:54:42 +00:00
4189146536 Mark the OpenRouter catalog release complete 2026-08-26 14:53:02 +00:00
59be507d2f Freeze the built-in catalog compatibility baseline 2026-08-26 14:49:25 +00:00
b45cea4084 Refine the external catalog implementation plan 2026-08-26 14:20:22 +00:00
efe885c6b2 Plan external catalog extraction 2026-08-26 13:58:22 +00:00
404a4c331d Clarify appended messages and streamline repair 2026-08-26 02:37:47 +00:00
174516cb39 Document appended request messages 2026-08-26 02:19:44 +00:00
b0999112cc Compose appended request messages 2026-08-26 02:16:16 +00:00
ed0f7527d5 Add appended message request validation 2026-08-26 02:10:01 +00:00
5064cf833d Centralize message role invariants 2026-08-26 02:07:15 +00:00
8745d256bd Plan appended request messages 2026-08-26 01:53:22 +00:00
2e76003fd5 Prepare documentation for the v0.8.0 release 2026-08-25 12:19:17 +00:00
36ce5a5099 Fix output repair diagnostics and concurrency tests 2026-08-25 11:54:50 +00:00
465dc1389d Document bounded output repair 2026-08-25 09:58:10 +00:00
ae6f1a9865 Enable bounded output repair in the engine 2026-08-25 09:55:19 +00:00
ee99dc9478 Run bounded output repair attempts 2026-08-25 09:51:28 +00:00
00ee5893e9 Build bounded output repair requests 2026-08-25 09:46:56 +00:00
64d1cffd89 Preserve explicit empty model responses 2026-08-25 09:42:28 +00:00
f9e8afa2c3 Enforce bounded output repair contracts 2026-08-25 09:41:15 +00:00
e827631d8c Plan the public output repair implementation 2026-08-25 09:38:06 +00:00
115fe8ba58 Plan bounded output repair 2026-08-25 03:33:33 +00:00
84 changed files with 3587 additions and 710 deletions

View File

@@ -33,10 +33,16 @@ boundary and constraints that framework work must preserve.
## Release Guidance ## Release Guidance
Consumers upgrading from `v0.6.0` to `v0.7.0` should read the Consumers upgrading from `v0.8.0` to `v0.9.0` should read the
[v0.7.0 changelog and migration guide](docs/releases/v0.7.0.md). [v0.9.0 changelog and migration guide](docs/releases/v0.9.0.md).
Consumers upgrading from `v0.7.0` to `v0.8.0` can consult the
[v0.8.0 changelog and migration guide](docs/releases/v0.8.0.md).
Earlier adopters can consult the Earlier adopters can consult the
[v0.7.0 changelog and migration guide](docs/releases/v0.7.0.md).
Consumers upgrading from `v0.5.0` to `v0.6.0` can consult the
[v0.6.0 changelog and migration guide](docs/releases/v0.6.0.md). [v0.6.0 changelog and migration guide](docs/releases/v0.6.0.md).
Consumers upgrading from `v0.4.0` to `v0.5.0` can consult the Consumers upgrading from `v0.4.0` to `v0.5.0` can consult the

View File

@@ -0,0 +1,145 @@
package promptkit_test
import (
"context"
"errors"
"reflect"
"sync/atomic"
"testing"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestAppendedMessageRoleConstantsAreStrings(t *testing.T) {
var (
developer string = promptkit.RoleDeveloper
system string = promptkit.RoleSystem
user string = promptkit.RoleUser
assistant string = promptkit.RoleAssistant
)
if developer != "developer" || system != "system" || user != "user" || assistant != "assistant" {
t.Fatalf("unexpected role constants: %q %q %q %q", developer, system, user, assistant)
}
}
func TestAppendedMessagesComposeFrozenEffectivePrompt(t *testing.T) {
cacheControl := &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}
appended := []promptkit.RenderedMessage{
{Role: " \tAsSiStAnT\n", Content: "previous response"},
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: cacheControl},
}
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "output"}}
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "configured prompt"), "."),
promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
request := promptkit.RunRequest{PromptID: "prompt", AppendedMessages: appended}
plain, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("prepare plain request: %v", err)
}
prepared, err := engine.Prepare(context.Background(), request)
if err != nil {
t.Fatalf("prepare appended request: %v", err)
}
expected := []promptkit.RenderedMessage{
{Role: promptkit.RoleUser, Content: "configured prompt"},
{Role: promptkit.RoleAssistant, Content: "previous response"},
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}},
}
if !reflect.DeepEqual(prepared.Messages, expected) || prepared.PromptHash != plain.PromptHash || prepared.RenderedPromptHash == plain.RenderedPromptHash {
t.Fatalf("prepared effective prompt = %#v, plain = %#v", prepared, plain)
}
equivalent, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{
{Role: promptkit.RoleAssistant, Content: "previous response"},
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}},
}})
if err != nil || equivalent.RenderedPromptHash != prepared.RenderedPromptHash {
t.Fatalf("normalized-equivalent request = (%#v, %v), want matching rendered hash %q", equivalent, err, prepared.RenderedPromptHash)
}
for _, variant := range []promptkit.RunRequest{
{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleAssistant, Content: "changed"}, expected[2]}},
{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{expected[2], expected[1]}},
{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleAssistant, Content: "previous response"}, {Role: promptkit.RoleUser, Content: "corrective request"}}},
} {
variantPrepared, prepareErr := engine.Prepare(context.Background(), variant)
if prepareErr != nil || variantPrepared.RenderedPromptHash == prepared.RenderedPromptHash || variantPrepared.PromptHash != prepared.PromptHash {
t.Fatalf("variant preparation = (%#v, %v)", variantPrepared, prepareErr)
}
}
result, err := engine.Run(context.Background(), request)
if err != nil || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(client.requests[0].Prompt.Messages, expected) {
t.Fatalf("run result = (%#v, %v), request = %#v", result, err, client.requests)
}
execution, err := engine.PrepareExecution(context.Background(), request)
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
appended[0].Content = "changed caller content"
cacheControl.TTL = ""
firstDetails := execution.Details()
firstDetails.Messages[1].Content = "changed details content"
firstDetails.Messages[2].CacheControl.Type = "changed details cache"
secondDetails := execution.Details()
if !reflect.DeepEqual(secondDetails.Messages, expected) || secondDetails.RenderedPromptHash != prepared.RenderedPromptHash {
t.Fatalf("prepared execution details = %#v, want frozen %#v", secondDetails, expected)
}
result, err = engine.RunPrepared(context.Background(), execution)
if err != nil || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(client.requests[1].Prompt.Messages, expected) {
t.Fatalf("prepared result = (%#v, %v), requests = %#v", result, err, client.requests)
}
}
func TestInvalidAppendedMessagesFailBeforeSourceOrModelWork(t *testing.T) {
promptSource := &inspectionCountingFS{}
var modelCalls atomic.Int64
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(promptSource, "."),
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
request := promptkit.RunRequest{
PromptID: "unreached",
AppendedMessages: []promptkit.RenderedMessage{{
Role: "unsupported-role",
Content: "sensitive appended content",
}},
}
operations := []struct {
name string
run func() error
}{
{name: "Prepare", run: func() error { _, err := engine.Prepare(context.Background(), request); return err }},
{name: "PrepareExecution", run: func() error { _, err := engine.PrepareExecution(context.Background(), request); return err }},
{name: "Run", run: func() error { _, err := engine.Run(context.Background(), request); return err }},
}
for _, operation := range operations {
t.Run(operation.name, func(t *testing.T) {
err := operation.run()
if !errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("error = %v, want ErrInvalidRequest", err)
}
})
}
if promptSource.opens.Load() != 0 {
t.Fatalf("invalid appended message opened prompt sources %d times", promptSource.opens.Load())
}
if modelCalls.Load() != 0 {
t.Fatalf("invalid appended message invoked the model %d times", modelCalls.Load())
}
}

View File

@@ -1,7 +1,9 @@
package promptkit package promptkit
import ( import (
"fmt"
"reflect" "reflect"
"unicode/utf8"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue" "gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
@@ -12,6 +14,10 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
if err != nil { if err != nil {
return domain.RunRequest{}, err return domain.RunRequest{}, err
} }
appendedMessages, err := toDomainAppendedMessages(req.AppendedMessages)
if err != nil {
return domain.RunRequest{}, err
}
return domain.RunRequest{ return domain.RunRequest{
PromptID: req.PromptID, PromptID: req.PromptID,
PromptVersion: req.PromptVersion, PromptVersion: req.PromptVersion,
@@ -22,9 +28,38 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
Vars: copyStringMap(req.Vars), Vars: copyStringMap(req.Vars),
Execution: execution, Execution: execution,
Validation: toDomainOutputContractPtr(req.Validation), Validation: toDomainOutputContractPtr(req.Validation),
AppendedMessages: appendedMessages,
}, nil }, nil
} }
func toDomainAppendedMessages(messages []RenderedMessage) ([]domain.RenderedMessage, error) {
if messages == nil {
return nil, nil
}
converted := make([]domain.RenderedMessage, len(messages))
for index, message := range messages {
if !utf8.ValidString(message.Content) {
return nil, fmt.Errorf("appended message %d content must be valid UTF-8", index)
}
role, err := domain.NormalizeMessageRole(message.Role)
if err != nil {
return nil, fmt.Errorf("appended message %d role: %w", index, err)
}
cacheControl, err := domain.NormalizeCacheControl(toDomainCacheControl(message.CacheControl))
if err != nil {
return nil, fmt.Errorf("appended message %d cache_control: %w", index, err)
}
converted[index] = domain.RenderedMessage{
Role: role,
Content: message.Content,
CacheControl: cacheControl,
}
}
return converted, nil
}
func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun { func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
if prepared == nil { if prepared == nil {
return nil return nil
@@ -285,6 +320,16 @@ func fromDomainRenderedMessages(messages []domain.RenderedMessage) []RenderedMes
return out return out
} }
func toDomainCacheControl(cacheControl *CacheControl) *domain.CacheControl {
if cacheControl == nil {
return nil
}
return &domain.CacheControl{
Type: domain.CacheControlType(cacheControl.Type),
TTL: cacheControl.TTL,
}
}
func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl { func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl {
if cacheControl == nil { if cacheControl == nil {
return nil return nil

View File

@@ -0,0 +1,108 @@
package promptkit
import (
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestToDomainAppendedMessages(t *testing.T) {
tests := []struct {
name string
messages []RenderedMessage
want []domain.RenderedMessage
wantErr string
privateRole string
privateText string
}{
{
name: "normalizes supported roles",
messages: []RenderedMessage{
{Role: " DeVeLoPeR ", Content: "developer"},
{Role: "SyStEm", Content: "system"},
{Role: "\tUsEr\n", Content: "user"},
{Role: "assistant", Content: "assistant"},
},
want: []domain.RenderedMessage{
{Role: domain.RoleDeveloper, Content: "developer"},
{Role: domain.RoleSystem, Content: "system"},
{Role: domain.RoleUser, Content: "user"},
{Role: domain.RoleAssistant, Content: "assistant"},
},
},
{
name: "invalid role UTF-8",
messages: []RenderedMessage{{Role: string([]byte{0xff}), Content: "private-content"}},
wantErr: "appended message 0 role",
},
{
name: "invalid content UTF-8",
messages: []RenderedMessage{{Role: "private-role", Content: string([]byte{0xff})}},
wantErr: "appended message 0 content",
},
{
name: "unsupported role",
messages: []RenderedMessage{{Role: "private-role", Content: "private-content"}},
wantErr: "appended message 0 role",
privateRole: "private-role",
privateText: "private-content",
},
{
name: "invalid cache control",
messages: []RenderedMessage{{Role: RoleUser, Content: "private-content", CacheControl: &CacheControl{Type: "private-cache"}}},
wantErr: "appended message 0 cache_control",
privateText: "private-content",
},
{
name: "empty and whitespace content",
messages: []RenderedMessage{
{Role: RoleUser, Content: ""},
{Role: RoleAssistant, Content: " \t\n "},
},
want: []domain.RenderedMessage{
{Role: domain.RoleUser, Content: ""},
{Role: domain.RoleAssistant, Content: " \t\n "},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got, err := toDomainAppendedMessages(test.messages)
if test.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), test.wantErr) {
t.Fatalf("error = %v, want %q", err, test.wantErr)
}
for _, privateValue := range []string{test.privateRole, test.privateText} {
if privateValue != "" && strings.Contains(err.Error(), privateValue) {
t.Fatalf("error exposed appended message data %q: %v", privateValue, err)
}
}
return
}
if err != nil {
t.Fatalf("toDomainAppendedMessages() error = %v", err)
}
if !reflect.DeepEqual(got, test.want) {
t.Fatalf("toDomainAppendedMessages() = %#v, want %#v", got, test.want)
}
})
}
}
func TestToDomainAppendedMessagesCopiesCacheControl(t *testing.T) {
cacheControl := &CacheControl{Type: CacheControlEphemeral, TTL: "1h"}
messages := []RenderedMessage{{Role: RoleUser, Content: "content", CacheControl: cacheControl}}
request, err := toDomainRunRequest(RunRequest{AppendedMessages: messages})
if err != nil {
t.Fatalf("toDomainRunRequest() error = %v", err)
}
messages[0].Content = "changed"
cacheControl.TTL = ""
if request.AppendedMessages[0].Content != "content" || request.AppendedMessages[0].CacheControl.TTL != "1h" {
t.Fatalf("domain request did not retain an independent appended-message copy: %#v", request.AppendedMessages)
}
}

View File

@@ -173,6 +173,49 @@ semantics. The
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md) [OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
owns the built-in client's outbound HTTP behavior. owns the built-in client's outbound HTTP behavior.
### Repair A Structured Result
Set a small additional-call budget when a structurally invalid result can be
corrected automatically:
```go
request.Validation = &promptkit.OutputContract{
Format: promptkit.FormatJSON,
ValidationMode: promptkit.ValidationJSONSchema,
SchemaPath: "events.schema.json",
RepairAttempts: 1,
}
```
Each repair attempt is another model call, so it can increase latency and
usage; `RunResult.Usage` is cumulative and `Validation.RepairAttempts` reports
calls actually started. Exhaustion still returns the final failed validation
result. `basic` validation can also repair an empty candidate, but structural
validity is not evidence of factual or domain correctness. See the
[output-contract format reference](../formats.md#output-contract) and
[`OutputContract` GoDoc](../../types.go) for the exact budget and eligibility
rules.
### Append Already-Rendered Messages
An application can include an earlier assistant response and its own corrective
instruction in a fresh request without changing the configured prompt:
```go
request.AppendedMessages = []promptkit.RenderedMessage{
{Role: promptkit.RoleAssistant, Content: previousResponse},
{Role: promptkit.RoleUser, Content: correction},
}
result, err := engine.Run(ctx, request)
```
These messages are already rendered: Promptkit does not template or resolve
files in them, and they can contain sensitive model output or application
feedback. Promptkit remains stateless; every `Run` call re-resolves its current
sources and the application owns any semantic retry budget. When a
pre-execution equality check is required, use `PrepareExecution` and compare
its opaque rendered-prompt hash before invoking `RunPrepared`.
## Inputs, Profiles, And Overrides ## Inputs, Profiles, And Overrides
Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request

View File

@@ -34,6 +34,7 @@ Start with:
| Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. | | Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. |
| Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). | | Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). |
| Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. | | Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. |
| Maintained external catalogs or built-ins | The [internal source document](internal/sources.md), [framework format reference](formats.md), [testing policy](policy/testing.md), both catalog module repositories, and each catalog's release procedure. |
| Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. | | Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. |
| Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. | | Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. |
| Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. | | Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. |

View File

@@ -82,7 +82,13 @@ allowed.
### Messages And Templates ### Messages And Templates
Each message has a non-empty `role` and exactly one of: Each message has a `role` that Promptkit trims and lowercases. It must then be
exactly one of `developer`, `system`, `user`, or `assistant`; blank, custom,
`tool`, and `function` roles are invalid. This intentionally tightens the
previous nonblank-string rule. Consumers migrating to the next minor release
must update any nonstandard prompt-definition roles before upgrading.
Each message also has exactly one of:
- `content`, containing an inline Go template; or - `content`, containing an inline Go template; or
- `content_file`, naming a file whose contents are the Go template. - `content_file`, naming a file whose contents are the Go template.
@@ -126,7 +132,7 @@ outbound integration determines its wire representation.
| `format` | yes | `text`, `markdown`, or `json`. | | `format` | yes | `text`, `markdown`, or `json`. |
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema`. | | `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema`. |
| `schema_path` | for `json_schema` | Path to a schema in the configured schema source. | | `schema_path` | for `json_schema` | Path to a schema in the configured schema source. |
| `repair_attempts` | no | Integer zero or greater; omitted means zero. | | `repair_attempts` | no | Integer from zero through three; omitted means zero. A positive value requires `basic`, `json`, or `json_schema` validation. |
The validation modes behave as follows: The validation modes behave as follows:
@@ -136,9 +142,15 @@ The validation modes behave as follows:
- `json_schema` requires valid JSON that satisfies the selected schema. - `json_schema` requires valid JSON that satisfies the selected schema.
`format` controls output artifact metadata. JSON Schema mode also supplies the `format` controls output artifact metadata. JSON Schema mode also supplies the
schema to compatible model clients as structured-output metadata. The public schema to compatible model clients as structured-output metadata. Plain `json`
engine does not install an output repairer, so its validation is single-pass validation accepts every valid JSON value and does not request a provider-native
even when a positive `repair_attempts` value is present. JSON-object constraint.
`repair_attempts` counts additional generation calls after a failed validation.
Zero is single-pass. With a positive eligible budget, Promptkit stops at the
first valid candidate. If the budget is exhausted, it returns the final
candidate and its complete failed validation result; generation and operational
validation failures remain errors. `none` never permits repair.
A request-level `OutputContract` replaces the complete prompt output contract. A request-level `OutputContract` replaces the complete prompt output contract.
It does not merge individual fields. If its format is empty, Promptkit uses It does not merge individual fields. If its format is empty, Promptkit uses
@@ -271,7 +283,7 @@ Profile sources resolve matching IDs in this order:
2. the ordinary configured source selected by a profile file, `fs.FS`, or 2. the ordinary configured source selected by a profile file, `fs.FS`, or
configured profile directory; configured profile directory;
3. application fallback profiles supplied with `WithFallbackProfileFS`; and 3. application fallback profiles supplied with `WithFallbackProfileFS`; and
4. embedded built-in profiles. 4. maintained external catalog profiles.
A profile source supplies a complete definition; definitions and their fields A profile source supplies a complete definition; definitions and their fields
are not merged across sources. A higher-precedence source falls back only when are not merged across sources. A higher-precedence source falls back only when

View File

@@ -55,6 +55,12 @@ Each ordinary message contains its `role` and string `content`. A
cache-controlled message instead uses a text content block containing `type`, cache-controlled message instead uses a text content block containing `type`,
`text`, and `cache_control`; an empty cache-control TTL is omitted. `text`, and `cache_control`; an empty cache-control TTL is omitted.
Promptkit sends only `developer`, `system`, `user`, and `assistant` roles and
does so without provider-specific translation. Tool and deprecated function
payloads are outside this text-message contract. A backend or model that
rejects an otherwise supported role or context returns its ordinary provider
error, which follows the normal generation-error path.
The effective direct or prompt-rendered session ID is trimmed, limited to 256 The effective direct or prompt-rendered session ID is trimmed, limited to 256
Unicode code points, and sent when nonempty as top-level `session_id`. It is Unicode code points, and sent when nonempty as top-level `session_id`. It is
never also sent as a session header. never also sent as a session header.
@@ -67,7 +73,8 @@ The client conditionally includes:
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly - non-empty `service_tier` and effective `reasoning_effort`; an explicitly
disabled reasoning setting is empty and therefore omitted; and disabled reasoning setting is empty and therefore omitted; and
- `response_format` for JSON Schema structured output, including its name, - `response_format` for JSON Schema structured output, including its name,
strict flag, and schema document. strict flag, and schema document. Plain JSON validation does not add an
object-only response constraint.
The engine resolves backend, profile, and request extra-parameter maps by The engine resolves backend, profile, and request extra-parameter maps by
whole-map replacement rather than key merging. The resulting effective map is whole-map replacement rather than key merging. The resulting effective map is
@@ -99,10 +106,12 @@ drained.
The bounded body must contain exactly one OpenAI-compatible JSON response The bounded body must contain exactly one OpenAI-compatible JSON response
object followed only by JSON whitespace and EOF. The client returns the first object followed only by JSON whitespace and EOF. The client returns the first
choice's non-empty message content and maps prompt, completion, total, cached, choice's explicitly present string message content, including an empty or
and cache-write token counts. Invalid or truncated JSON, trailing non-whitespace whitespace-only string, and maps prompt, completion, total, cached, and
data, a second JSON value, absent choices, empty first-choice content, and size cache-write token counts. Invalid or truncated JSON, trailing non-whitespace
overflow are malformed responses and return no partial result. data, a second JSON value, absent choices, missing content, `null` content,
non-string content, and size overflow are malformed responses and return no
partial result.
For a non-2xx status, Promptkit recognizes one JSON document with a top-level For a non-2xx status, Promptkit recognizes one JSON document with a top-level
object-valued `error` member. Its optional `message` and `type` fields must be object-valued `error` member. Its optional `message` and `type` fields must be

View File

@@ -21,8 +21,9 @@ uses internal domain values for rendered prompts, execution targets,
structured output, responses, and token usage. structured output, responses, and token usage.
The runner supplies a fully resolved target after applying backend, profile, The runner supplies a fully resolved target after applying backend, profile,
and request precedence. The client uses its endpoint, credential metadata, and request precedence, plus canonical provider-bound text messages. The
generation fields, and extra parameters. `BackendID` remains routing metadata client uses its endpoint, credential metadata, generation fields, and extra
parameters. `BackendID` remains routing metadata
for the generation boundary and is not mapped into the provider payload. for the generation boundary and is not mapped into the provider payload.
Construction trims and validates a nonempty configured base URL and clones any Construction trims and validates a nonempty configured base URL and clones any
@@ -89,6 +90,12 @@ truncation, malformed JSON, trailing data, and a second value are malformed
responses with no partial result or provider content in the error. Every body responses with no partial result or provider content in the error. Every body
is closed, and an unbounded oversized stream is not drained. is closed, and an unbounded oversized stream is not drained.
After framing succeeds, the first choice must contain an explicitly present
string `message.content`. The string is returned exactly, including empty or
whitespace-only content. Missing choices, missing or `null` content, and
non-string content are malformed responses. Output validation and correction
eligibility remain outside this package.
For a non-success response, `ProviderHTTPError` retains the HTTP status and For a non-success response, `ProviderHTTPError` retains the HTTP status and
only normalized detail from the bounded recognized envelope. It retains only normalized detail from the bounded recognized envelope. It retains
`ErrUnexpectedStatus` through unwrapping. The client owns response closure; `ErrUnexpectedStatus` through unwrapping. The client owns response closure;

View File

@@ -11,10 +11,11 @@ contributor workflow and validation.
| Component | Implemented responsibility | References | | Component | Implemented responsibility | References |
| --- | --- | --- | | --- | --- | --- |
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, and profile-inspection values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, typed capacity and generation error mapping, and engine-local profile-source assembly including application fallbacks. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) | | Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, and profile-inspection values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, typed capacity and generation error mapping, engine-local profile-source assembly including application fallbacks, and bounded output-repair assembly. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) | | `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) | | `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
| `internal/backend` | Constructs each engine's immutable registry from the maintained built-in definitions and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) | | `internal/backend` | Constructs each engine's immutable registry from maintained definitions and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
| `internal/catalog` | Strictly validates imported immutable maintained backend and profile catalog assets before engine assembly uses them. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) |
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) | | `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [endpoint invariant](../../internal/domain/endpoint.go) | | `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [endpoint invariant](../../internal/domain/endpoint.go) |
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) | | `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
@@ -22,12 +23,11 @@ contributor workflow and validation.
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) | | `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) | | `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
| `internal/profile` | Loads strictly decoded, locally validated execution profiles from filesystem and `fs.FS` sources, overlays raw sources with error-preserving fallback, and resolves inherited profiles. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go), [internal sources](sources.md#profiles-and-built-ins) | | `internal/profile` | Loads strictly decoded, locally validated execution profiles from filesystem and `fs.FS` sources, overlays raw sources with error-preserving fallback, and resolves inherited profiles. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go), [internal sources](sources.md#profiles-and-built-ins) |
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select maintained built-in backends. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) | | `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) | | `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) | | `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) | | `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, operation-local validation plans, generation, capacity, and optional repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) | | `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
The root package assembles these internal components without exposing their The root package assembles these internal components without exposing their
representations. Consumers depend only on the root facade. representations. Consumers depend only on the root facade.

View File

@@ -23,8 +23,10 @@ built-in backend and validated consumer additions, one engine-local run
admitter, and a model client wrapped by the same capacity manager. Validation admitter, and a model client wrapped by the same capacity manager. Validation
plans and provider-facing schema metadata come from the validator's preparation plans and provider-facing schema metadata come from the validator's preparation
interface. interface.
An output repairer can be injected internally, but the ordinary runner The root engine supplies one default output repairer through the explicit
constructor does not enable one. runner constructor, using the same capacity-wrapped client as initial
generation. The no-repair runner constructor remains available for focused
internal callers and tests.
Each invocation carries its state in request, prepared-run, and result values. Each invocation carries its state in request, prepared-run, and result values.
The runner has no durable run or session store. The runner has no durable run or session store.
@@ -82,7 +84,8 @@ profile, or backend:
schema metadata from it when required; schema metadata from it when required;
2. load and hash input artifacts; 2. load and hash input artifacts;
3. render messages and the prompt-defined session; 3. render messages and the prompt-defined session;
4. apply any direct session ID; 4. apply any direct session ID, then append already-normalized request messages
after the rendered definition messages;
5. hash the effective rendered prompt; and 5. hash the effective rendered prompt; and
6. construct the prepared value and preparation timing. 6. construct the prepared value and preparation timing.
@@ -111,7 +114,10 @@ the prompt session template, and is applied after ordinary message rendering.
A blank direct value retains prompt-template behavior. The runner clears the A blank direct value retains prompt-template behavior. The runner clears the
template only on a value copy of the definition, so the definition hash always template only on a value copy of the definition, so the definition hash always
describes the original source while the rendered-prompt hash includes the describes the original source while the rendered-prompt hash includes the
effective direct or rendered session. effective direct or rendered session and the complete effective message
sequence. The rendered-prompt hash uses a versioned, length-framed SHA-256
encoding of that session, every role and content value, and cache-control
presence and values; its hexadecimal value is opaque.
The registry is read-only after engine construction. Concurrent `Prepare` and The registry is read-only after engine construction. Concurrent `Prepare` and
`Run` calls resolve independent defensive backend values and keep all `Run` calls resolve independent defensive backend values and keep all
@@ -125,8 +131,8 @@ admitter is an internal unlimited fallback. After successful admission, `Run`
immediately defers the returned release function, performs the completion immediately defers the returned release function, performs the completion
phase, makes one initial generation call, builds the named output artifact, phase, makes one initial generation call, builds the named output artifact,
and validates that artifact with the plan compiled during completion. Invalid and validates that artifact with the plan compiled during completion. Invalid
generated content remains a validation result; an inability to perform generated content remains a validation result; an inability to generate or
validation is an operational error. perform validation is an operational error.
Validation preparation and execution honor cancellation at every Validation preparation and execution honor cancellation at every
Promptkit-controlled boundary and do not publish a partial plan or result. Promptkit-controlled boundary and do not publish a partial plan or result.
@@ -142,17 +148,26 @@ serializing preparation or validation behind the active-generation limit.
The wrapped model client separately acquires a FIFO active permit only around The wrapped model client separately acquires a FIFO active permit only around
each actual generation call. each actual generation call.
When an internal repairer is present, a JSON or JSON Schema content failure can After a failed `basic`, JSON, or JSON Schema validation with a positive frozen
trigger bounded repair attempts. Repair receives the effective execution budget, the installed repairer can make a bounded corrective call. Each request
target, explicit numeric-presence bits, credential, backend identity, session starts with a fresh copy of the complete effective message sequence (the
ID, validation errors, prior output, and structured-output specification. One configured prefix followed by the request suffix), includes only the latest
request constructor supplies those common fields to initial and repair nonempty candidate as an assistant message, and appends one corrective user
generation while their rendered prompts remain intentionally distinct. The message. Empty candidates omit that assistant message. The
default repairer uses the same wrapped client as initial generation, so each correction carries validation diagnostics as JSON data bounded to 64 KiB; the
repair reacquires the selected backend's active permit while remaining inside full diagnostics remain in the validation result.
its original admission lease. Repair never performs a second bounded
admission, and repaired outputs use the operation's existing validation plan. Repair receives the effective execution target, explicit numeric-presence bits,
This capability remains internal and is not a public option. credential, backend identity, session ID, and structured-output specification.
The same request constructor supplies those common fields to initial and repair
generation. The default repairer uses the same wrapped client as initial
generation, so each repair reacquires the selected backend's active permit
while remaining inside its original admission lease. Repair never performs a
second bounded admission, and repaired outputs use the operation's existing
validation plan. The runner stops at the first valid candidate, sums completed
generation usage, reports calls actually started, and returns the final failed
validation result on exhaustion. A repair generation failure follows the
ordinary generation-error category rather than becoming a validation error.
A successful result includes the output artifact and raw output, validation A successful result includes the output artifact and raw output, validation
state, effective session ID, prompt and rendered-prompt hashes, selected state, effective session ID, prompt and rendered-prompt hashes, selected

View File

@@ -21,6 +21,12 @@ paths, content opening, and root containment. Each lookup remains a
point-in-time scan: definitions and catalogs are not cached, and file-backed point-in-time scan: definitions and catalogs are not cached, and file-backed
message content is opened only for the exact selected candidate. message content is opened only for the exact selected candidate.
Message roles are normalized through the shared domain owner by trimming
Unicode whitespace and lowercasing. Only `developer`, `system`, `user`, and
`assistant` are published; invalid roles remain selected prompt-definition
failures rather than becoming request errors. Cache-control metadata uses the
same shared domain normalization and defensive-copy rule.
Operating-system sources enforce containment against canonical roots and Operating-system sources enforce containment against canonical roots and
targets so symlinks cannot escape. Injected `fs.FS` sources enforce containment targets so symlinks cannot escape. Injected `fs.FS` sources enforce containment
in their clean relative path namespace. A single-file source uses the selected in their clean relative path namespace. A single-file source uses the selected
@@ -51,6 +57,19 @@ and permits inherited target fields only when a base is named. File-backed
bounded JSON-value owner before a profile is published. OpenAI-compatible bounded JSON-value owner before a profile is published. OpenAI-compatible
reserved-field policy remains with the model-client and backend-registry owners. reserved-field policy remains with the model-client and backend-registry owners.
`LoadFSRepository` is the eager immutable loading boundary for internal
catalog consumers. It discovers and strictly validates every raw profile once,
preserves safe source metadata including explicitly present YAML fields, and
publishes independently copied values from memory. It does not resolve profile
inheritance. Configured consumer sources continue to use the lazy point lookup
repositories described above.
`internal/catalog` validates the imported immutable OpenRouter and
Rakestrawhome asset modules as one private adapter boundary. It enforces their
manifest, layout, profile ownership, inheritance, and secret-safety rules
before returning raw catalog sources. Root assembly uses those validated
catalogs as the maintained lowest-precedence profile source.
The overlay repository consults the next repository only when the The overlay repository consults the next repository only when the
higher-precedence repository reports that a profile is absent. A reliably higher-precedence repository reports that a profile is absent. A reliably
selected malformed profile stops fallback, while an unrelated malformed file selected malformed profile stops fallback, while an unrelated malformed file
@@ -61,7 +80,7 @@ profile inspection.
The root engine assembles one raw composite catalog in precedence order: The root engine assembles one raw composite catalog in precedence order:
in-memory profiles, one ordinary configured source, an application fallback in-memory profiles, one ordinary configured source, an application fallback
source, then the embedded built-in catalog. An explicit file or `fs.FS` profile source, then the maintained external catalog. An explicit file or `fs.FS` profile
source replaces `Config.ProfileDir` within the ordinary configured-source source replaces `Config.ProfileDir` within the ordinary configured-source
category. One outer resolving repository wraps that complete raw catalog, so category. One outer resolving repository wraps that complete raw catalog, so
each base lookup observes the same precedence and shadowing rules. each base lookup observes the same precedence and shadowing rules.
@@ -79,15 +98,11 @@ or schema sources. It does not retain that lookup for a later execution.
Prepared execution instead freezes the fully resolved target; a later ordinary Prepared execution instead freezes the fully resolved target; a later ordinary
operation performs a fresh traversal. operation performs a fresh traversal.
`internal/profile/builtin` embeds the maintained built-in profile catalog. The maintained external catalog provides every built-in profile and its
Every embedded profile selects a maintained built-in backend and inherits that matching backend definition. Profile loading and overlay behavior are owned by the
backend's endpoint and credential environment-variable name from the built-in
backend registry rather than repeating those values. Profile loading and
overlay behavior are owned by the
[profile repository tests](../../internal/profile/repository_test.go), while [profile repository tests](../../internal/profile/repository_test.go), while
catalog completeness, the backend-selection invariant, and duplicate IDs are catalog completeness, backend selection, and duplicate IDs are owned by the
owned by the [catalog adapter tests](../../internal/catalog/catalog_test.go).
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
## Ordinary Artifacts ## Ordinary Artifacts
@@ -122,8 +137,9 @@ Session and message parsing and execution remain synchronous. The renderer
checks cancellation before and after each parse and execution boundary, checks cancellation before and after each parse and execution boundary,
between artifact conversion chunks, around each message, and before publishing between artifact conversion chunks, around each message, and before publishing
the complete prompt. It cannot interrupt template work already in progress and the complete prompt. It cannot interrupt template work already in progress and
never publishes a partial prompt after observing cancellation. It carries never publishes a partial prompt after observing cancellation. It validates and
message roles, session IDs, and cache control into the rendered prompt. The canonicalizes message roles before carrying roles, session IDs, and cache
control into the rendered prompt. The
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior. [renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
## Schemas And Output Validation ## Schemas And Output Validation

View File

@@ -22,21 +22,21 @@ The implemented internal components consist of:
- `internal/domain`, which owns framework data values and source-neutral - `internal/domain`, which owns framework data values and source-neutral
invariants shared by later internal components; invariants shared by later internal components;
- `internal/backend`, which owns validated immutable OpenAI-compatible backend - `internal/backend`, which owns validated immutable OpenAI-compatible backend
definitions and the maintained built-in definitions; definitions;
- `internal/capacity`, which owns engine-local bounded run admission and - `internal/capacity`, which owns engine-local bounded run admission and
model-generation scheduling for limited backends; model-generation scheduling for limited backends;
- `internal/defaults`, which owns application-neutral framework defaults and - `internal/defaults`, which owns application-neutral framework defaults and
constructs the default execution target; constructs the default execution target;
- `internal/filecatalog`, which discovers YAML files and provides source-path - `internal/filecatalog`, which discovers YAML files and provides source-path
helpers for filesystem and `fs.FS` consumers; helpers for filesystem and `fs.FS` consumers;
- `internal/catalog`, which validates immutable external maintained-catalog
assets before they are eligible for engine assembly;
- `internal/jsonvalue`, which validates and defensively copies JSON-compatible - `internal/jsonvalue`, which validates and defensively copies JSON-compatible
extra-parameter trees; extra-parameter trees;
- `internal/promptdef`, which loads and validates prompt definitions from - `internal/promptdef`, which loads and validates prompt definitions from
filesystem and `fs.FS` sources; filesystem and `fs.FS` sources;
- `internal/profile`, which loads, validates, and overlays execution profiles - `internal/profile`, which loads, validates, and overlays execution profiles
from filesystem and `fs.FS` sources; from filesystem and `fs.FS` sources;
- `internal/profile/builtin`, which embeds the built-in execution profile
catalog;
- `internal/prompt`, which renders prompt messages from Go templates; - `internal/prompt`, which renders prompt messages from Go templates;
- `internal/artifact`, which resolves ordinary inline and unrestricted - `internal/artifact`, which resolves ordinary inline and unrestricted
caller-selected file references; caller-selected file references;
@@ -54,13 +54,15 @@ packages or participate in internal assembly.
The root facade assembles one immutable backend registry, one capacity manager, The root facade assembles one immutable backend registry, one capacity manager,
the internal repositories, renderer, validator, outbound client, and use-case the internal repositories, renderer, validator, outbound client, and use-case
runner while translating public values and errors at the library boundary. The runner while translating public values and errors at the library boundary. The
registry contains built-ins plus validated engine-scoped consumer additions. registry contains validated maintained definitions plus engine-scoped consumer
additions.
The facade constructs the capacity manager from the registry's immutable The facade constructs the capacity manager from the registry's immutable
policy snapshot, wraps the selected built-in or injected model client, and policy snapshot, wraps the selected built-in or injected model client, and
supplies bounded admission to the runner. The defaults and renderer depend on supplies bounded admission to the runner. The defaults and renderer depend on
the domain model. Prompt-definition and profile repositories use the domain the domain model. Prompt-definition and profile repositories use the domain
model, file catalog, and YAML decoder. The built-in profile repository supplies model, file catalog, and YAML decoder. The catalog adapter validates imported
an embedded `fs.FS` to the profile package. Artifact reading uses the domain immutable maintained data before root assembly supplies it to the profile
package. Artifact reading uses the domain
model and application-neutral defaults. Validation uses the domain model, file model and application-neutral defaults. Validation uses the domain model, file
catalog, and JSON Schema implementation. The model client uses the domain catalog, and JSON Schema implementation. The model client uses the domain
model, application-neutral defaults, and an injected or standard-library HTTP model, application-neutral defaults, and an injected or standard-library HTTP

View File

@@ -146,6 +146,13 @@ grep -F 'Consumer action:' "$RELEASE_NOTES_FILE"
Inspect the complete message and confirm that it accurately records the Inspect the complete message and confirm that it accurately records the
compatibility impact, public API changes, and required consumer action. compatibility impact, public API changes, and required consumer action.
When a release introduces or updates the maintained external catalogs, state
that it selects two independently versioned data dependencies, preserves the
public API and configuration, requires no consumer migration, and guarantees
only the catalog versions selected and tested by that Promptkit release. Link
to the current architecture and format documentation rather than duplicating
catalog contracts in the release note.
## Create And Inspect The Tag ## Create And Inspect The Tag
Run the candidate guard again immediately before tag creation. This ensures Run the candidate guard again immediately before tag creation. This ensures

109
docs/releases/v0.8.0.md Normal file
View File

@@ -0,0 +1,109 @@
# Promptkit v0.8.0
This supplemental changelog and migration guide summarizes the consumer-facing
changes from `v0.7.0` to `v0.8.0`. The annotated `v0.8.0` tag is the
authoritative release record. Exact current contracts belong to the linked
GoDoc and durable documentation.
## Summary
`v0.8.0` activates Promptkit's bounded output-repair workflow:
- failed nonempty-text, JSON, and JSON Schema validation can make a limited
number of corrective model calls;
- corrective calls preserve the original rendered conversation, effective
target, session, structured-output contract, and backend capacity policy;
- results report cumulative usage and the number of corrective calls actually
made; and
- explicitly empty OpenAI-compatible response content now reaches output
validation instead of being classified as a malformed provider envelope.
## Compatibility
This release adds no public declarations or fields and removes none. Existing
source code remains source-compatible.
The behavior of the existing `OutputContract.RepairAttempts` field and prompt
YAML `repair_attempts` field has changed. A positive value now authorizes real
additional model calls after eligible validation failures; earlier releases
accepted the field but the public engine remained single-pass. Consumers that
set a positive value should expect additional latency, token usage, and
provider cost when repair is needed.
Repair budgets must now be between zero and three. A positive budget requires
`basic`, `json`, or `json_schema` validation. Values above three and a positive
budget paired with `none` are invalid contracts rather than ignored settings.
An explicitly present empty or whitespace-only string returned by the built-in
OpenAI-compatible client is now a completed generation candidate. `none`
validation permits it, while `basic`, `json`, and `json_schema` classify it
under their ordinary validation rules and may repair it when configured.
Missing, `null`, or non-string content remains a malformed provider response.
## Upgrade
Update the module dependency with:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.8.0
go mod tidy
```
Review every prompt definition and request override that sets a positive repair
budget. Use zero or omit the field to retain single-pass execution. Ensure each
positive budget is no greater than three and uses an eligible validation mode,
then run the consuming project's ordinary and race-enabled tests.
## Bounded Output Repair
`repair_attempts` counts corrective calls in addition to the initial model
call. Promptkit validates each completed candidate, stops at the first valid
one, and never exceeds the configured bound. If every candidate remains
invalid, the run completes successfully with the final candidate and its
failed validation result rather than returning an operational error.
Each correction starts from the original rendered messages and includes only
the latest invalid candidate and latest validation diagnostics. JSON Schema
mode retains the provider-native structured-output request as its first line of
defense. Promptkit performs only deterministic structural validation; a valid
response is not necessarily factual or correct for an application's domain.
Usage in the final result is cumulative across the initial response and every
completed corrective response. `ValidationResult.RepairAttempts` reports the
number of corrective calls actually made. Corrective generation failures use
the same public generation-error categories and structured provider details as
an initial generation failure.
See the [output-contract format reference](../formats.md#output-contract), the
[consumer repair example](../consumers/pkg-promptkit.md#repair-a-structured-result),
and the [`OutputContract` and `ValidationResult` GoDoc](../../types.go) for the
current contracts.
## Explicit Empty Content
The built-in OpenAI-compatible client now distinguishes an explicitly present
empty string from a missing or malformed `content` field. This aligns built-in
and injected clients by letting the selected output contract decide whether an
empty candidate is acceptable, invalid, or eligible for repair.
See the
[OpenAI-compatible response contract](../integrations/openai-compatible-chat.md#response-handling)
for the exact envelope behavior.
## Public API Changes
None. This release activates and tightens the documented behavior of existing
fields.
## Consumer Action
- Remove or set `repair_attempts` to zero where execution must remain
single-pass.
- Keep every positive repair budget at three or fewer and pair it with
`basic`, `json`, or `json_schema` validation.
- Account for additional latency, usage, and provider cost when enabling
repair.
- Continue checking the returned validation status because bounded repair can
exhaust without producing a valid candidate.
- Review workflows that previously treated explicit empty provider content as
a generation error.

125
docs/releases/v0.9.0.md Normal file
View File

@@ -0,0 +1,125 @@
# Promptkit v0.9.0
This supplemental changelog and migration guide summarizes the consumer-facing
changes from `v0.8.0` to `v0.9.0`. The annotated `v0.9.0` tag is the
authoritative release record. Exact current contracts belong to the linked
GoDoc and durable documentation.
## Summary
`v0.9.0` adds stateless request-message composition and separates maintained
provider data from Promptkit's core implementation:
- callers can append already-rendered messages to a configured prompt for
application-owned conversations and semantic correction workflows;
- the public package now publishes constants for the four supported text-chat
roles, and prompt definitions use the same normalized role vocabulary; and
- the OpenRouter and Rakestrawhome backend/profile catalogs now come from two
independently versioned Go module dependencies.
## Compatibility
This release adds one field and four constants to the public API and removes no
public declarations. Existing keyed `RunRequest` literals that omit
`AppendedMessages` retain their behavior. Adding the field changes the struct
shape, so consumers using positional `RunRequest` literals must convert them
to keyed literals.
Message roles in prompt definitions are now trimmed, lowercased, and required
to be `developer`, `system`, `user`, or `assistant`. Definitions using another
role that earlier releases accepted as an arbitrary nonblank string now fail
prompt loading. In particular, the text-only message contract does not support
`tool` or the deprecated `function` role. Otherwise valid roles with different
case or surrounding whitespace are normalized rather than rejected.
The catalog extraction preserves Promptkit's public API, built-in backend and
profile IDs, configuration, precedence, credential handling, and capacity
behavior. Consumers do not import or register either catalog themselves, and
no configuration migration is required. Promptkit now selects two
independently versioned data dependencies and guarantees only the catalog
versions selected and tested by this Promptkit release.
## Upgrade
Update the module dependency with:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.9.0
go mod tidy
```
Convert any positional `RunRequest` literals to keyed literals. Review prompt
definitions for unsupported roles, then run the consuming project's ordinary
and race-enabled tests.
## Appended Request Messages
`RunRequest.AppendedMessages` accepts caller-owned `RenderedMessage` values
that Promptkit validates, copies, and appends after every rendered
prompt-definition message in caller order. Promptkit does not template this
content, retain conversation state between calls, impose a retry policy, or
apply a message-count, byte-size, token, or context-window limit. Upstream
rejections continue through the ordinary generation-error boundary.
This primitive supports application-owned conversation continuations and
domain-aware correction loops while preserving Promptkit's existing
preparation, hashing, prepared-execution, structural repair, backend-capacity,
credential, and cancellation behavior. Appended content can include sensitive
model output or application feedback; prepared values expose the complete
effective messages by design, while default request formatting reports only
the appended-message count.
See the
[consumer appended-message example](../consumers/pkg-promptkit.md#append-already-rendered-messages),
the [`RunRequest`, `RenderedMessage`, and `CacheControl` GoDoc](../../types.go),
and the [OpenAI-compatible request contract](../integrations/openai-compatible-chat.md#request-body)
for current behavior.
## Supported Message Roles
The new `RoleDeveloper`, `RoleSystem`, `RoleUser`, and `RoleAssistant`
constants identify the complete role vocabulary accepted by Promptkit's
text-chat message model. The same validation and normalization now apply to
prompt-definition messages and request-supplied appended messages. Promptkit
does not translate between roles; provider- or model-specific rejection of an
otherwise supported role remains an upstream generation error.
See the [message format reference](../formats.md#messages-and-templates) for
the canonical prompt-definition contract.
## Independently Versioned Backend Catalogs
Promptkit imports immutable catalog data for its maintained OpenRouter and
Rakestrawhome backends and profiles. Engine construction validates and
assembles both catalogs behind the existing built-in registry and profile
precedence rules. Promptkit no longer keeps duplicate embedded profile assets
or hard-coded definitions for those maintained backends.
The module versions in Promptkit's `go.mod` identify the catalog releases
tested with this release. The [built-in backend and profile format
reference](../formats.md#built-in-backends) remains the canonical consumer
contract, while the [internal source documentation](../internal/sources.md#profiles-and-built-ins)
describes the dependency boundary.
## Public API Changes
The release adds:
- `RunRequest.AppendedMessages`;
- `RoleDeveloper`;
- `RoleSystem`;
- `RoleUser`; and
- `RoleAssistant`.
No public declaration was removed.
## Consumer Action
- Convert positional `RunRequest` literals to keyed literals.
- Replace unsupported prompt-definition roles with an appropriate supported
role, or keep richer tool-call protocols in an application-owned client.
- Treat appended messages and prepared effective messages according to the
application's sensitive-data policy.
- Do not add direct catalog imports or registration calls; existing Promptkit
construction and configuration remain correct.
- Run consumer ordinary and race-enabled tests after updating the module.

View File

@@ -38,28 +38,8 @@ consumers.
## Ideas ## Ideas
### Public bounded output repair No ideas are currently awaiting selection. Active feature work belongs in its
focused roadmap rather than this catalog.
Promptkit should make its bounded output-repair capability available through
the public engine. A consumer should be able to request a limited number of
corrective generation attempts when JSON or JSON Schema output fails content
validation, without having to reproduce Promptkit's generation, validation,
capacity, and result-accounting orchestration.
- Repair is validation recovery, not a general provider retry, failover, or
backoff policy. Transport failures, cancellation, and operational schema or
validation errors must retain their ordinary error behavior.
- Repair must stop after the first valid result or the configured attempt
bound. Exhausting the bound should preserve the final invalid result and its
validation diagnostics rather than inventing success.
- Initial generation and every repair attempt must use the same resolved
backend, effective execution settings and presence semantics, session,
credential boundary, structured-output contract, and backend-capacity
policy.
- Results should report the number of repair attempts and cumulative usage for
every model call made by the run.
- Ordinary and prepared execution should expose coherent behavior, including
cancellation, frozen prepared state, error identity, and capacity lifetime.
## Entry Format ## Entry Format

View File

@@ -11,14 +11,16 @@ import (
"strings" "strings"
"time" "time"
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter"
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome"
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact" artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
"gitea.maximumdirect.net/eric/promptkit/internal/backend" "gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity" "gitea.maximumdirect.net/eric/promptkit/internal/capacity"
"gitea.maximumdirect.net/eric/promptkit/internal/catalog"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults" "gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm" "gitea.maximumdirect.net/eric/promptkit/internal/llm"
"gitea.maximumdirect.net/eric/promptkit/internal/profile" "gitea.maximumdirect.net/eric/promptkit/internal/profile"
"gitea.maximumdirect.net/eric/promptkit/internal/profile/builtin"
"gitea.maximumdirect.net/eric/promptkit/internal/prompt" "gitea.maximumdirect.net/eric/promptkit/internal/prompt"
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef" "gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
"gitea.maximumdirect.net/eric/promptkit/internal/usecase" "gitea.maximumdirect.net/eric/promptkit/internal/usecase"
@@ -100,7 +102,7 @@ type Config struct {
// prompt source. // prompt source.
PromptDir string PromptDir string
// ProfileDir is an optional ordinary configured source whose profiles take // ProfileDir is an optional ordinary configured source whose profiles take
// precedence over application fallback and embedded built-in profiles. An // precedence over application fallback and maintained catalog profiles. An
// empty value selects the lower-precedence sources unless a profile-source // empty value selects the lower-precedence sources unless a profile-source
// option supplies the ordinary source. // option supplies the ordinary source.
ProfileDir string ProfileDir string
@@ -273,7 +275,7 @@ func WithProfileFile(path string) Option {
// //
// Profile lookup checks, in order, profiles supplied by WithProfiles; the // Profile lookup checks, in order, profiles supplied by WithProfiles; the
// ordinary configured source selected by WithProfileFile, WithProfileFS, or // ordinary configured source selected by WithProfileFile, WithProfileFS, or
// Config.ProfileDir; this fallback source; and Promptkit's embedded built-in // Config.ProfileDir; this fallback source; and Promptkit's maintained catalog
// profiles. Each source supplies a complete profile definition; profile fields // profiles. Each source supplies a complete profile definition; profile fields
// are not merged between sources. Only an absent profile ID proceeds to the // are not merged between sources. Only an absent profile ID proceeds to the
// next source. A matching read, parse, duplicate, validation, or credential // next source. A matching read, parse, duplicate, validation, or credential
@@ -390,9 +392,17 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir) promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
} }
profiles := newProfileRepository(cfg.ProfileDir, options) maintainedCatalogs, err := catalog.Load(
catalog.Source{Name: "OpenRouter", ExpectedBackendID: backend.OpenRouterID, FS: openrouter.FS(), Root: openrouter.Root},
catalog.Source{Name: "Rakestrawhome", ExpectedBackendID: backend.RakestrawHomeID, FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
)
if err != nil {
return nil, fmt.Errorf("%w: failed to load maintained catalogs: %v", ErrInvalidConfig, err)
}
backendRegistry, err := backend.NewRegistry(options.backends) profiles := newProfileRepository(cfg.ProfileDir, options, maintainedCatalogs.Profiles)
backendRegistry, err := backend.NewRegistry(maintainedCatalogs.Backends, options.backends)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err) return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err)
} }
@@ -430,7 +440,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
} }
return &Engine{ return &Engine{
runner: usecase.NewRunner( runner: usecase.NewRunnerWithRepairer(
promptDefs, promptDefs,
profiles, profiles,
backendRegistry, backendRegistry,
@@ -438,13 +448,14 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
prompt.NewGoRenderer(), prompt.NewGoRenderer(),
llmClient, llmClient,
validator, validator,
usecase.NewDefaultOutputRepairer(llmClient),
capacityManager, capacityManager,
), ),
}, nil }, nil
} }
func newProfileRepository(profileDir string, options engineOptions) profile.Repository { func newProfileRepository(profileDir string, options engineOptions, maintained profile.Repository) profile.Repository {
repository := builtin.NewRepository() repository := maintained
if options.fallbackProfileSource { if options.fallbackProfileSource {
repository = profile.NewOverlayRepository(options.fallbackProfiles, repository) repository = profile.NewOverlayRepository(options.fallbackProfiles, repository)
@@ -570,6 +581,8 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
} }
// Prepare resolves and renders a prompt request without calling an LLM. // Prepare resolves and renders a prompt request without calling an LLM.
// It appends any validated RunRequest.AppendedMessages after rendered
// definition messages in the returned caller-owned snapshot.
// //
// Prepare selects the prompt and profile, resolves any selected backend and // Prepare selects the prompt and profile, resolves any selected backend and
// effective execution settings, resolves the output contract, loads and hashes // effective execution settings, resolves the output contract, loads and hashes
@@ -604,7 +617,8 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
} }
// PrepareExecution completely prepares a prompt request without calling the // PrepareExecution completely prepares a prompt request without calling the
// configured LLMClient or reserving backend admission capacity. // configured LLMClient or reserving backend admission capacity. Validated
// RunRequest.AppendedMessages are included in the frozen effective messages.
// //
// The returned opaque handle is bound to this Engine and permits one // The returned opaque handle is bound to this Engine and permits one
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources, // [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
@@ -636,13 +650,16 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
} }
// Run prepares a request, invokes the configured LLMClient, and validates the // Run prepares a request, invokes the configured LLMClient, and validates the
// generated output. // generated output. Each call resolves current sources and composes a fresh,
// stateless effective prompt with any validated RunRequest.AppendedMessages.
// //
// A content-validation failure is a successful run whose // A content-validation failure is a successful run whose
// RunResult.Validation has Status ValidationFailed. An inability to perform // RunResult.Validation has Status ValidationFailed. When its output contract
// validation returns an error matching ErrValidation and no partial result. // has a positive repair budget, a failed eligible validation can make bounded
// The public Engine does not perform output repair, so validation is // additional model calls and stops at the first valid candidate. Exhaustion
// single-pass even when OutputContract.RepairAttempts is positive. // returns the final failed validation result with cumulative usage and actual
// repair attempts. An inability to generate or validate returns an error and
// no partial result.
// //
// Run can return every error category documented by [Engine.Prepare], plus // Run can return every error category documented by [Engine.Prepare], plus
// ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is // ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is
@@ -686,17 +703,17 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
// //
// The supplied context governs this execution attempt independently of the // The supplied context governs this execution attempt independently of the
// preparation context. It covers credential revalidation, admission, // preparation context. It covers credential revalidation, admission,
// generation, validation, and any internal repair. Result timing begins after // generation, validation, and any bounded output repair. Result timing begins
// the claim and excludes preparation and consumer-held delay. // after the claim and excludes preparation and consumer-held delay.
// //
// RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing, // RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing,
// ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while // ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while
// preserving documented collaborator and context identities. An engine // preserving documented collaborator and context identities. An engine
// admission rejection is discoverable as [CapacityError] and still matches // admission rejection is discoverable as [CapacityError] and still matches
// ErrCapacityExceeded. A built-in OpenAI-compatible non-2xx response is // ErrCapacityExceeded. A built-in OpenAI-compatible non-2xx response is
// discoverable as [GenerationError]. A completed content-validation rejection // discoverable as [GenerationError]. A completed content-validation rejection,
// is returned in RunResult, not as an operational error. An operational error // including repair exhaustion, is returned in RunResult, not as an operational
// returns no partial RunResult. // error. An operational error returns no partial RunResult.
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) { func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
if e == nil || e.runner == nil { if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig) return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)

View File

@@ -220,6 +220,14 @@ func TestRunRequestFormattingRedactsDirectAPIKey(t *testing.T) {
"transcript": promptkit.InlineWithURI(inputURI, inputBody), "transcript": promptkit.InlineWithURI(inputURI, inputBody),
}, },
Vars: map[string]string{"audience": variableValue}, Vars: map[string]string{"audience": variableValue},
AppendedMessages: []promptkit.RenderedMessage{{
Role: "private-role-sentinel",
Content: "private-appended-content-sentinel",
CacheControl: &promptkit.CacheControl{
Type: "private-cache-sentinel",
TTL: "private-ttl-sentinel",
},
}},
} }
for _, formatted := range []string{ for _, formatted := range []string{
@@ -229,7 +237,7 @@ func TestRunRequestFormattingRedactsDirectAPIKey(t *testing.T) {
fmt.Sprintf("%+v", req), fmt.Sprintf("%+v", req),
fmt.Sprintf("%#v", req), fmt.Sprintf("%#v", req),
} { } {
for _, privateValue := range []string{secret, inputURI, inputBody, variableValue} { for _, privateValue := range []string{secret, inputURI, inputBody, variableValue, "private-role-sentinel", "private-appended-content-sentinel", "private-cache-sentinel", "private-ttl-sentinel"} {
if strings.Contains(formatted, privateValue) { if strings.Contains(formatted, privateValue) {
t.Fatalf("formatted RunRequest leaked private value %q: %s", privateValue, formatted) t.Fatalf("formatted RunRequest leaked private value %q: %s", privateValue, formatted)
} }
@@ -240,6 +248,7 @@ func TestRunRequestFormattingRedactsDirectAPIKey(t *testing.T) {
"APIKeySet:true", "APIKeySet:true",
"Inputs:1", "Inputs:1",
"Vars:1", "Vars:1",
"AppendedMessages:1",
} { } {
if !strings.Contains(formatted, summary) { if !strings.Contains(formatted, summary) {
t.Fatalf("formatted RunRequest omitted structural summary %q: %s", summary, formatted) t.Fatalf("formatted RunRequest omitted structural summary %q: %s", summary, formatted)
@@ -1536,31 +1545,21 @@ func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
name string name string
profileID string profileID string
backendID string backendID string
endpoint string
apiKeyEnv string
model string
}{ }{
{ {
name: "OpenRouter", name: "OpenRouter",
profileID: "mistral-small-3", profileID: "mistral-small-3",
backendID: promptkit.BackendOpenRouter, backendID: promptkit.BackendOpenRouter,
endpoint: "https://openrouter.ai/api/v1",
apiKeyEnv: "OPENROUTER_API_KEY",
model: "mistralai/mistral-small-3.2-24b-instruct",
}, },
{ {
name: "Rakestrawhome", name: "Rakestrawhome",
profileID: "rakestrawhome-gemma-4-31b", profileID: "rakestrawhome-gemma-4-31b",
backendID: promptkit.BackendRakestrawHome, backendID: promptkit.BackendRakestrawHome,
endpoint: "https://inference.ai.rakestrawhome.com/v1",
apiKeyEnv: "RAKESTRAWHOME_INFERENCE_API_KEY",
model: "google/gemma-4-31b-it",
}, },
} }
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
t.Setenv(tc.apiKeyEnv, "test-key")
engine, err := promptkit.NewEngine(promptkit.Config{ engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: frameworkPromptDir, PromptDir: frameworkPromptDir,
SchemaDir: frameworkSchemaDir, SchemaDir: frameworkSchemaDir,
@@ -1572,6 +1571,7 @@ func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: frameworkMarkdownSummaryPromptID, PromptID: frameworkMarkdownSummaryPromptID,
ProfileID: tc.profileID, ProfileID: tc.profileID,
APIKey: "test-key",
Inputs: map[string]promptkit.ArtifactRef{ Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("Rin opens the gate."), "transcript": promptkit.Inline("Rin opens the gate."),
"glossary": promptkit.Inline("gate: A guarded passage."), "glossary": promptkit.Inline("gate: A guarded passage."),
@@ -1582,10 +1582,7 @@ func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
} }
if prepared.SelectedProfileID != tc.profileID || if prepared.SelectedProfileID != tc.profileID ||
prepared.SelectedBackendID != tc.backendID || prepared.SelectedBackendID != tc.backendID ||
prepared.EffectiveModelParams.BackendID != tc.backendID || prepared.EffectiveModelParams.BackendID != tc.backendID {
prepared.EffectiveModelParams.Endpoint != tc.endpoint ||
prepared.EffectiveModelParams.APIKeyEnv != tc.apiKeyEnv ||
prepared.EffectiveModelParams.Model != tc.model {
t.Fatalf("unexpected built-in preparation: %#v", prepared) t.Fatalf("unexpected built-in preparation: %#v", prepared)
} }
}) })
@@ -3487,6 +3484,7 @@ extra_params:
type fakeLLMClient struct { type fakeLLMClient struct {
response *promptkit.GenerateResponse response *promptkit.GenerateResponse
responses []*promptkit.GenerateResponse
err error err error
requests []promptkit.GenerateRequest requests []promptkit.GenerateRequest
} }
@@ -3664,5 +3662,12 @@ func (f *fakeLLMClient) Generate(_ context.Context, req promptkit.GenerateReques
if f.err != nil { if f.err != nil {
return nil, f.err return nil, f.err
} }
if len(f.responses) > 0 {
index := len(f.requests) - 1
if index >= len(f.responses) {
return nil, fmt.Errorf("no response configured for generation %d", index+1)
}
return f.responses[index], nil
}
return f.response, nil return f.response, nil
} }

View File

@@ -18,7 +18,7 @@ func (r RunRequest) GoString() string {
func (r RunRequest) redactedString() string { func (r RunRequest) redactedString() string {
return fmt.Sprintf( return fmt.Sprintf(
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t}", "promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t AppendedMessages:%d}",
r.PromptID, r.PromptID,
r.PromptVersion, r.PromptVersion,
r.ProfileID, r.ProfileID,
@@ -27,6 +27,7 @@ func (r RunRequest) redactedString() string {
len(r.Vars), len(r.Vars),
r.Execution != nil, r.Execution != nil,
r.Validation != nil, r.Validation != nil,
len(r.AppendedMessages),
) )
} }

View File

@@ -39,6 +39,61 @@ func TestBuiltInGenerationError(t *testing.T) {
assertGenerationError(t, err, http.StatusServiceUnavailable, "", "", "") assertGenerationError(t, err, http.StatusServiceUnavailable, "", "", "")
} }
func TestBuiltInGenerationErrorWithAppendedMessages(t *testing.T) {
const messageMarker = "combined-request-provider-marker"
engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity,
`{"error":{"message":"`+messageMarker+`"}}`)
request := generationErrorRunRequest()
request.AppendedMessages = []promptkit.RenderedMessage{
{Role: promptkit.RoleAssistant, Content: "previous response"},
{Role: promptkit.RoleUser, Content: "consumer correction"},
}
result, err := engine.Run(context.Background(), request)
if result != nil {
t.Fatalf("Run result = %#v, want nil", result)
}
assertGenerationError(t, err, http.StatusUnprocessableEntity, "", "", messageMarker)
}
func TestBuiltInRepairGenerationError(t *testing.T) {
const (
codeMarker = "repair-code-marker"
typeMarker = "repair-type-marker"
messageMarker = "repair-message-marker"
)
calls := 0
config := contractConfig(frameworkSchemaDir)
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
calls++
if calls == 1 {
body := `{"choices":[{"message":{"content":"not-json"}}]}`
return &http.Response{StatusCode: http.StatusOK, ContentLength: int64(len(body)), Body: io.NopCloser(strings.NewReader(body))}, nil
}
body := `{"error":{"code":"` + codeMarker + `","type":"` + typeMarker + `","message":"` + messageMarker + `"}}`
return &http.Response{StatusCode: http.StatusUnprocessableEntity, ContentLength: int64(len(body)), Body: io.NopCloser(strings.NewReader(body))}, nil
})}
engine, err := promptkit.NewEngine(config)
if err != nil {
t.Fatalf("NewEngine: %v", err)
}
req := generationErrorRunRequest()
req.Validation = &promptkit.OutputContract{
Format: promptkit.FormatJSON,
ValidationMode: promptkit.ValidationJSON,
RepairAttempts: 1,
}
result, err := engine.Run(context.Background(), req)
if result != nil {
t.Fatalf("Run result = %#v, want nil", result)
}
if calls != 2 {
t.Fatalf("provider calls = %d, want 2", calls)
}
assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker)
}
func assertGenerationError(t *testing.T, err error, statusCode int, code, providerType, message string) { func assertGenerationError(t *testing.T, err error, statusCode int, code, providerType, message string) {
t.Helper() t.Helper()

2
go.mod
View File

@@ -3,6 +3,8 @@ module gitea.maximumdirect.net/eric/promptkit
go 1.25.5 go 1.25.5
require ( require (
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
) )

4
go.sum
View File

@@ -1,3 +1,7 @@
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 h1:lc062euk2qseO//D762i3JaFyulDNML3eQQX7DkYTho=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0/go.mod h1:AIa7kAu2mfrRQgcspe4L+DW51WqgnALQT60lqkEywJI=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 h1:j9YY7wsTVjzke2kHH4YAzpU0oUpM+x+nXwl1IeS+2eg=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0/go.mod h1:4RNS+LILDg4JbS4Ts9Lwy1C92wauXJIbeQaalps4Koo=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=

View File

@@ -22,15 +22,6 @@ const (
// backend. // backend.
RakestrawHomeID = "rakestrawhome" RakestrawHomeID = "rakestrawhome"
openRouterEndpoint = "https://openrouter.ai/api/v1"
openRouterAPIKeyEnv = "OPENROUTER_API_KEY"
openRouterConcurrencyLimit = 16
rakestrawHomeEndpoint = "https://inference.ai.rakestrawhome.com/v1"
rakestrawHomeAPIKeyEnv = "RAKESTRAWHOME_INFERENCE_API_KEY"
rakestrawHomeConcurrencyLimit = 4
defaultQueueCapacity = 1024 defaultQueueCapacity = 1024
) )
@@ -44,16 +35,15 @@ type Registry struct {
backends map[string]domain.Backend backends map[string]domain.Backend
} }
// NewRegistry constructs a registry containing the built-in definitions // NewRegistry constructs a registry containing maintained definitions followed
// followed by the supplied additions. Every ID must be unique. // by consumer additions. Every ID must be unique across both groups.
func NewRegistry(additions []domain.Backend) (*Registry, error) { func NewRegistry(maintained, additions []domain.Backend) (*Registry, error) {
builtIns := builtInBackends()
registry := &Registry{ registry := &Registry{
backends: make(map[string]domain.Backend, len(builtIns)+len(additions)), backends: make(map[string]domain.Backend, len(maintained)+len(additions)),
} }
definitions := make([]domain.Backend, 0, len(builtIns)+len(additions)) definitions := make([]domain.Backend, 0, len(maintained)+len(additions))
definitions = append(definitions, builtIns...) definitions = append(definitions, maintained...)
definitions = append(definitions, additions...) definitions = append(definitions, additions...)
for _, definition := range definitions { for _, definition := range definitions {
@@ -65,7 +55,7 @@ func NewRegistry(additions []domain.Backend) (*Registry, error) {
return nil, fmt.Errorf("backend ID %q is already registered", definition.ID) return nil, fmt.Errorf("backend ID %q is already registered", definition.ID)
} }
normalized, err := normalizeBackend(definition) normalized, err := NormalizeDefinition(definition)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -75,23 +65,6 @@ func NewRegistry(additions []domain.Backend) (*Registry, error) {
return registry, nil return registry, nil
} }
func builtInBackends() []domain.Backend {
return []domain.Backend{
{
ID: OpenRouterID,
Endpoint: openRouterEndpoint,
APIKeyEnv: openRouterAPIKeyEnv,
ConcurrencyLimit: openRouterConcurrencyLimit,
},
{
ID: RakestrawHomeID,
Endpoint: rakestrawHomeEndpoint,
APIKeyEnv: rakestrawHomeAPIKeyEnv,
ConcurrencyLimit: rakestrawHomeConcurrencyLimit,
},
}
}
// GetBackend returns a defensive copy of the backend registered with id. // GetBackend returns a defensive copy of the backend registered with id.
func (r *Registry) GetBackend(id string) (domain.Backend, error) { func (r *Registry) GetBackend(id string) (domain.Backend, error) {
if r == nil { if r == nil {
@@ -128,7 +101,8 @@ func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy {
return policies return policies
} }
func normalizeBackend(definition domain.Backend) (domain.Backend, error) { // NormalizeDefinition validates and defensively copies one backend definition.
func NormalizeDefinition(definition domain.Backend) (domain.Backend, error) {
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(definition.Endpoint) endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(definition.Endpoint)
if err != nil { if err != nil {
return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err) return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err)

View File

@@ -2,6 +2,7 @@ package backend_test
import ( import (
"errors" "errors"
"reflect"
"strings" "strings"
"testing" "testing"
@@ -11,48 +12,24 @@ import (
const validEndpoint = "https://backend.example/v1" const validEndpoint = "https://backend.example/v1"
func TestRegistryIncludesExactBuiltInDefinitions(t *testing.T) { func TestRegistryIncludesMaintainedDefinitions(t *testing.T) {
registry, err := backend.NewRegistry(nil) maintained := []domain.Backend{
{ID: backend.OpenRouterID, Endpoint: validEndpoint, ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true},
{ID: backend.RakestrawHomeID, Endpoint: "https://second.example/v1", ConcurrencyLimit: 4, QueueCapacity: 5, QueueCapacitySet: true},
}
registry, err := backend.NewRegistry(maintained, nil)
if err != nil { if err != nil {
t.Fatalf("construct registry: %v", err) t.Fatalf("construct registry: %v", err)
} }
tests := []struct { for _, expected := range maintained {
name string t.Run(expected.ID, func(t *testing.T) {
id string definition, err := registry.GetBackend(expected.ID)
endpoint string
apiKeyEnv string
concurrent int
}{
{
name: "OpenRouter",
id: backend.OpenRouterID,
endpoint: "https://openrouter.ai/api/v1",
apiKeyEnv: "OPENROUTER_API_KEY",
concurrent: 16,
},
{
name: "Rakestrawhome",
id: backend.RakestrawHomeID,
endpoint: "https://inference.ai.rakestrawhome.com/v1",
apiKeyEnv: "RAKESTRAWHOME_INFERENCE_API_KEY",
concurrent: 4,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
definition, err := registry.GetBackend(tc.id)
if err != nil { if err != nil {
t.Fatalf("look up built-in: %v", err) t.Fatalf("look up maintained definition: %v", err)
} }
if definition.ID != tc.id || if !reflect.DeepEqual(definition, expected) {
definition.Endpoint != tc.endpoint || t.Fatalf("unexpected maintained definition: %#v", definition)
definition.APIKeyEnv != tc.apiKeyEnv ||
definition.ConcurrencyLimit != tc.concurrent ||
definition.QueueCapacity != 1024 ||
!definition.QueueCapacitySet ||
definition.ExtraParams != nil {
t.Fatalf("unexpected built-in definition: %#v", definition)
} }
}) })
} }
@@ -60,12 +37,12 @@ func TestRegistryIncludesExactBuiltInDefinitions(t *testing.T) {
policies := registry.CapacityPolicies() policies := registry.CapacityPolicies()
if len(policies) != 2 || if len(policies) != 2 ||
policies[backend.OpenRouterID] != (domain.BackendCapacityPolicy{ policies[backend.OpenRouterID] != (domain.BackendCapacityPolicy{
ConcurrencyLimit: 16, ConcurrencyLimit: 2,
QueueCapacity: 1024, QueueCapacity: 3,
}) || }) ||
policies[backend.RakestrawHomeID] != (domain.BackendCapacityPolicy{ policies[backend.RakestrawHomeID] != (domain.BackendCapacityPolicy{
ConcurrencyLimit: 4, ConcurrencyLimit: 4,
QueueCapacity: 1024, QueueCapacity: 5,
}) { }) {
t.Fatalf("unexpected built-in capacity policies: %#v", policies) t.Fatalf("unexpected built-in capacity policies: %#v", policies)
} }
@@ -77,7 +54,7 @@ func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
"count": int64(7), "count": int64(7),
"nested": nested, "nested": nested,
} }
registry, err := backend.NewRegistry([]domain.Backend{ registry, err := backend.NewRegistry(nil, []domain.Backend{
{ {
ID: " custom ", ID: " custom ",
Endpoint: " https://custom.example/openai/v1 ", Endpoint: " https://custom.example/openai/v1 ",
@@ -140,12 +117,11 @@ func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
} }
policies := registry.CapacityPolicies() policies := registry.CapacityPolicies()
if len(policies) != 3 { if len(policies) != 1 {
t.Fatalf("unexpected capacity policy count: %#v", policies) t.Fatalf("unexpected capacity policy count: %#v", policies)
} }
policies["custom"] = domain.BackendCapacityPolicy{} policies["custom"] = domain.BackendCapacityPolicy{}
delete(policies, backend.OpenRouterID) delete(policies, "custom")
delete(policies, backend.RakestrawHomeID)
againPolicies := registry.CapacityPolicies() againPolicies := registry.CapacityPolicies()
if againPolicies["custom"] != (domain.BackendCapacityPolicy{ if againPolicies["custom"] != (domain.BackendCapacityPolicy{
ConcurrencyLimit: 3, ConcurrencyLimit: 3,
@@ -153,12 +129,6 @@ func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
}) { }) {
t.Fatalf("capacity policy map mutated registry state: %#v", againPolicies) t.Fatalf("capacity policy map mutated registry state: %#v", againPolicies)
} }
if _, ok := againPolicies[backend.OpenRouterID]; !ok {
t.Fatalf("OpenRouter capacity policy deletion mutated registry state: %#v", againPolicies)
}
if _, ok := againPolicies[backend.RakestrawHomeID]; !ok {
t.Fatalf("Rakestrawhome capacity policy deletion mutated registry state: %#v", againPolicies)
}
} }
func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) { func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) {
@@ -234,7 +204,7 @@ func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
tc.definition.ID = "custom" tc.definition.ID = "custom"
tc.definition.Endpoint = validEndpoint tc.definition.Endpoint = validEndpoint
registry, err := backend.NewRegistry([]domain.Backend{tc.definition}) registry, err := backend.NewRegistry(nil, []domain.Backend{tc.definition})
if tc.wantError { if tc.wantError {
if err == nil { if err == nil {
t.Fatal("expected invalid capacity policy error") t.Fatal("expected invalid capacity policy error")
@@ -298,7 +268,7 @@ func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry(tc.additions) _, err := backend.NewRegistry(nil, tc.additions)
if err == nil { if err == nil {
t.Fatal("expected duplicate ID error") t.Fatal("expected duplicate ID error")
} }
@@ -312,7 +282,7 @@ func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
func TestNewRegistryValidatesIDs(t *testing.T) { func TestNewRegistryValidatesIDs(t *testing.T) {
for _, id := range []string{"", " \t\n "} { for _, id := range []string{"", " \t\n "} {
t.Run(id, func(t *testing.T) { t.Run(id, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{ _, err := backend.NewRegistry(nil, []domain.Backend{{
ID: id, ID: id,
Endpoint: validEndpoint, Endpoint: validEndpoint,
}}) }})
@@ -341,7 +311,7 @@ func TestNewRegistryValidatesEndpoints(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{ _, err := backend.NewRegistry(nil, []domain.Backend{{
ID: "custom", ID: "custom",
Endpoint: tc.endpoint, Endpoint: tc.endpoint,
}}) }})
@@ -355,7 +325,7 @@ func TestNewRegistryValidatesEndpoints(t *testing.T) {
func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) { func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) {
for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} { for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} {
t.Run(name, func(t *testing.T) { t.Run(name, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{ _, err := backend.NewRegistry(nil, []domain.Backend{{
ID: "custom", ID: "custom",
Endpoint: validEndpoint, Endpoint: validEndpoint,
APIKeyEnv: name, APIKeyEnv: name,
@@ -378,7 +348,7 @@ func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{ _, err := backend.NewRegistry(nil, []domain.Backend{{
ID: "custom", ID: "custom",
Endpoint: validEndpoint, Endpoint: validEndpoint,
ExtraParams: tc.extraParams, ExtraParams: tc.extraParams,
@@ -391,7 +361,7 @@ func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
} }
func TestRegistryLookupReportsNotFound(t *testing.T) { func TestRegistryLookupReportsNotFound(t *testing.T) {
registry, err := backend.NewRegistry(nil) registry, err := backend.NewRegistry(nil, nil)
if err != nil { if err != nil {
t.Fatalf("construct registry: %v", err) t.Fatalf("construct registry: %v", err)
} }

246
internal/catalog/catalog.go Normal file
View File

@@ -0,0 +1,246 @@
// Package catalog validates immutable maintained backend catalog assets.
package catalog
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"path"
"sort"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
// Source identifies one immutable backend catalog asset tree.
type Source struct {
Name string
ExpectedBackendID string
FS fs.FS
Root string
}
// Set is the validated maintained backend and raw profile catalog.
type Set struct {
Backends []domain.Backend
Profiles profile.Repository
profileIDs []string
}
// Load validates and combines immutable catalog sources in source order.
func Load(sources ...Source) (Set, error) {
if len(sources) == 0 {
return Set{}, errors.New("at least one catalog source is required")
}
loaded := Set{Backends: make([]domain.Backend, 0, len(sources))}
names := map[string]bool{}
backendIDs := map[string]bool{}
profileIDs := map[string]bool{}
for _, source := range sources {
if err := validateSource(source, names); err != nil {
return Set{}, err
}
names[source.Name] = true
definition, err := loadBackend(source)
if err != nil {
return Set{}, err
}
if backendIDs[definition.ID] {
return Set{}, fmt.Errorf("catalog %s: backend ID duplicates an earlier catalog", source.Name)
}
backendIDs[definition.ID] = true
repository, metadata, err := profile.LoadFSRepository(context.Background(), source.FS, path.Join(source.Root, "profiles"))
if err != nil {
return Set{}, fmt.Errorf("catalog %s: %w", source.Name, err)
}
if len(metadata) == 0 {
return Set{}, fmt.Errorf("catalog %s: profiles must not be empty", source.Name)
}
resolvingRepository := profile.NewResolvingRepository(repository)
for _, entry := range metadata {
if profileIDs[entry.ID] {
return Set{}, fmt.Errorf("catalog %s: %s duplicates an earlier profile ID", source.Name, entry.Path)
}
if containsField(entry.ExplicitFields, "endpoint") || containsField(entry.ExplicitFields, "api_key_env") {
return Set{}, fmt.Errorf("catalog %s: %s contains connection metadata", source.Name, entry.Path)
}
value, err := repository.GetProfile(context.Background(), entry.ID)
if err != nil {
return Set{}, fmt.Errorf("catalog %s: %s: %w", source.Name, entry.Path, err)
}
if err := rejectSecretKeys(value.ExtraParams); err != nil {
return Set{}, fmt.Errorf("catalog %s: %s: prohibited extra parameter key", source.Name, entry.Path)
}
resolved, err := resolvingRepository.GetProfile(context.Background(), entry.ID)
if err != nil {
return Set{}, fmt.Errorf("catalog %s: %s has invalid profile inheritance", source.Name, entry.Path)
}
if resolved.BackendID != definition.ID {
return Set{}, fmt.Errorf("catalog %s: %s selects a different backend", source.Name, entry.Path)
}
profileIDs[entry.ID] = true
loaded.profileIDs = append(loaded.profileIDs, entry.ID)
}
if loaded.Profiles == nil {
loaded.Profiles = repository
} else {
loaded.Profiles = profile.NewOverlayRepository(loaded.Profiles, repository)
}
loaded.Backends = append(loaded.Backends, definition)
}
sort.Strings(loaded.profileIDs)
return loaded, nil
}
func validateSource(source Source, names map[string]bool) error {
if strings.TrimSpace(source.Name) == "" || source.Name != strings.TrimSpace(source.Name) {
return errors.New("catalog source name must not be blank")
}
if names[source.Name] {
return fmt.Errorf("duplicate catalog source name %q", source.Name)
}
if source.FS == nil {
return fmt.Errorf("catalog %s: filesystem is nil", source.Name)
}
if source.Root == "." || source.Root == "" || !fs.ValidPath(source.Root) {
return fmt.Errorf("catalog %s: asset root is invalid", source.Name)
}
if strings.TrimSpace(source.ExpectedBackendID) == "" {
return fmt.Errorf("catalog %s: expected backend ID is blank", source.Name)
}
return validateLayout(source)
}
func validateLayout(source Source) error {
manifestPath := path.Join(source.Root, "backend.json")
profilesRoot := path.Join(source.Root, "profiles")
return fs.WalkDir(source.FS, source.Root, func(assetPath string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
if assetPath == source.Root {
if !entry.IsDir() {
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
}
return nil
}
if entry.IsDir() {
if assetPath == profilesRoot || strings.HasPrefix(assetPath, profilesRoot+"/") {
return nil
}
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
}
if assetPath == manifestPath && entry.Type().IsRegular() {
return nil
}
if strings.HasPrefix(assetPath, profilesRoot+"/") && entry.Type().IsRegular() && strings.HasSuffix(assetPath, ".yml") {
return nil
}
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
})
}
func loadBackend(source Source) (domain.Backend, error) {
data, err := fs.ReadFile(source.FS, path.Join(source.Root, "backend.json"))
if err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: %w", source.Name, err)
}
type manifest struct {
SchemaVersion *int `json:"schema_version"`
ID *string `json:"id"`
Endpoint *string `json:"endpoint"`
APIKeyEnv *string `json:"api_key_env"`
ConcurrencyLimit *int `json:"concurrency_limit"`
QueueCapacity *int `json:"queue_capacity"`
ExtraParams json.RawMessage `json:"extra_params"`
}
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.DisallowUnknownFields()
var value manifest
if err := decoder.Decode(&value); err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
}
if value.SchemaVersion == nil || *value.SchemaVersion != 1 || value.ID == nil || value.Endpoint == nil || value.APIKeyEnv == nil || value.ConcurrencyLimit == nil || value.QueueCapacity == nil || value.ExtraParams == nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: required field is missing or unsupported", source.Name)
}
if *value.ID != source.ExpectedBackendID {
return domain.Backend{}, fmt.Errorf("catalog %s: backend ID does not match expected ID", source.Name)
}
if strings.TrimSpace(*value.APIKeyEnv) == "" {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: api key environment variable must not be blank", source.Name)
}
extraParams, err := decodeExtraParams(value.ExtraParams)
if err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid extra parameters", source.Name)
}
if err := rejectSecretKeys(extraParams); err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: prohibited extra parameter key", source.Name)
}
normalized, err := backend.NormalizeDefinition(domain.Backend{ID: *value.ID, Endpoint: *value.Endpoint, APIKeyEnv: *value.APIKeyEnv, ExtraParams: extraParams, ConcurrencyLimit: *value.ConcurrencyLimit, QueueCapacity: *value.QueueCapacity, QueueCapacitySet: true})
if err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid backend definition", source.Name)
}
return normalized, nil
}
func decodeExtraParams(data []byte) (map[string]any, error) {
if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
return nil, nil
}
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.UseNumber()
var value map[string]any
if err := decoder.Decode(&value); err != nil || value == nil {
return nil, errors.New("extra parameters must be an object or null")
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return nil, errors.New("extra parameters must contain exactly one JSON value")
}
return value, nil
}
func containsField(fields []string, target string) bool {
for _, field := range fields {
if field == target {
return true
}
}
return false
}
func rejectSecretKeys(value map[string]any) error {
return rejectSecretValue(value)
}
func rejectSecretValue(value any) error {
switch value := value.(type) {
case map[string]any:
for key, child := range value {
switch strings.ToLower(key) {
case "api_key", "apikey", "authorization", "credential", "credentials", "password", "secret", "token", "access_token":
return errors.New("prohibited key")
}
if err := rejectSecretValue(child); err != nil {
return err
}
}
case []any:
for _, child := range value {
if err := rejectSecretValue(child); err != nil {
return err
}
}
}
return nil
}

View File

@@ -0,0 +1,364 @@
package catalog
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
"testing/fstest"
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter"
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome"
)
func TestLoadPublishedCatalogsMatchCompatibilityFixture(t *testing.T) {
loaded, err := Load(
Source{Name: "OpenRouter", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
Source{Name: "Rakestrawhome", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
)
if err != nil {
t.Fatalf("load published catalogs: %v", err)
}
expected := loadCompatibilityFixture(t)
expectedIDs := fixtureProfileIDs(t, expected)
if !reflect.DeepEqual(loaded.profileIDs, expectedIDs) {
t.Fatalf("published profile IDs differ from compatibility fixture: got %q, want %q", loaded.profileIDs, expectedIDs)
}
actual := catalogValue(t, loaded)
actualJSON, err := json.Marshal(actual)
if err != nil {
t.Fatalf("encode loaded catalogs: %v", err)
}
expectedJSON, err := json.Marshal(expected)
if err != nil {
t.Fatalf("encode compatibility fixture: %v", err)
}
if !bytes.Equal(actualJSON, expectedJSON) {
t.Fatalf("published catalogs differ from compatibility fixture: got %#v, want %#v", actual, expected)
}
}
func TestLoadRejectsInvalidSources(t *testing.T) {
for name, sources := range map[string][]Source{
"none": nil,
"blank name": {{Name: " ", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root}},
"duplicate name": {
{Name: "same", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
{Name: "same", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
},
"nil filesystem": {{Name: "missing", ExpectedBackendID: "openrouter", Root: openrouter.Root}},
"invalid root": {{Name: "invalid-root", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: "."}},
"blank expected backend": {{Name: "blank-backend", ExpectedBackendID: " ", FS: openrouter.FS(), Root: openrouter.Root}},
} {
t.Run(name, func(t *testing.T) {
if _, err := Load(sources...); err == nil {
t.Fatal("expected error")
}
})
}
}
func TestLoadRejectsInvalidLayouts(t *testing.T) {
tests := map[string]func(fstest.MapFS){
"unexpected file": func(fsys fstest.MapFS) {
fsys["catalog/notes.txt"] = &fstest.MapFile{Data: []byte("unexpected")}
},
"unexpected directory": func(fsys fstest.MapFS) {
fsys["catalog/unexpected"] = &fstest.MapFile{Mode: fs.ModeDir}
},
"nonregular manifest": func(fsys fstest.MapFS) {
fsys["catalog/backend.json"].Mode = fs.ModeSymlink
},
"wrong profile extension": func(fsys fstest.MapFS) {
fsys["catalog/profiles/extra.yaml"] = &fstest.MapFile{Data: []byte(validProfile("extra", "one"))}
},
"nonregular profile": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Mode = fs.ModeSymlink
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
fsys := validCatalogFS("one")
mutate(fsys)
if _, err := Load(testSource("one", fsys)); err == nil {
t.Fatal("expected invalid layout error")
}
})
}
}
func TestLoadRejectsInvalidManifests(t *testing.T) {
tests := map[string]string{
"malformed": `{`,
"trailing value": validManifest("one", "TEST_API_KEY", "null") + `{}`,
"missing fields": `{"schema_version":1,"id":"one"}`,
"unsupported version": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"schema_version":1`, `"schema_version":2`, 1),
"unknown field": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"extra_params":null`, `"extra_params":null,"unknown":true`, 1),
"blank API key env": validManifest("one", " ", "null"),
"invalid API key env": validManifest("one", "LEAK-MARKER", "null"),
"invalid endpoint": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `https://one.example/v1`, `ftp://leak-marker.invalid/v1`, 1),
"zero concurrency": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"concurrency_limit":2`, `"concurrency_limit":0`, 1),
"non-object parameters": validManifest("one", "TEST_API_KEY", `[]`),
"secret parameter": validManifest("one", "TEST_API_KEY", `{"nested":{"token":"leak-marker"}}`),
}
for name, manifest := range tests {
t.Run(name, func(t *testing.T) {
fsys := validCatalogFS("one")
fsys["catalog/backend.json"].Data = []byte(manifest)
_, err := Load(testSource("one", fsys))
if err == nil {
t.Fatal("expected invalid manifest error")
}
if strings.Contains(strings.ToLower(err.Error()), "leak-marker") {
t.Fatalf("catalog error exposed manifest content: %v", err)
}
})
}
}
func TestLoadPreservesManifestJSONNumbers(t *testing.T) {
fsys := validCatalogFS("one")
fsys["catalog/backend.json"].Data = []byte(validManifest(
"one",
"TEST_API_KEY",
`{"large":9007199254740993,"nested":[1.25]}`,
))
loaded, err := Load(testSource("one", fsys))
if err != nil {
t.Fatalf("load catalog: %v", err)
}
if got := loaded.Backends[0].ExtraParams["large"]; got != json.Number("9007199254740993") {
t.Fatalf("large JSON integer = %#v, want preserved json.Number", got)
}
nested := loaded.Backends[0].ExtraParams["nested"].([]any)
if nested[0] != json.Number("1.25") {
t.Fatalf("nested JSON number = %#v, want preserved json.Number", nested[0])
}
}
func TestLoadRejectsInvalidCatalogProfiles(t *testing.T) {
tests := map[string]func(fstest.MapFS){
"empty": func(fsys fstest.MapFS) {
delete(fsys, "catalog/profiles/one-profile.yml")
fsys["catalog/profiles"] = &fstest.MapFile{Mode: fs.ModeDir}
},
"malformed": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: [")
},
"raw API key": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key: leak-marker\n")
},
"endpoint field": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "endpoint: ''\n")
},
"API key environment field": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key_env: ''\n")
},
"owner mismatch": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "other"))
},
"missing base": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: leak-marker\n")
},
"cyclic base": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: second\n")
fsys["catalog/profiles/second.yml"] = &fstest.MapFile{Data: []byte("id: second\nbase_profile: one-profile\n")}
},
"secret profile parameter": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "extra_params:\n nested:\n password: leak-marker\n")
},
"unknown field is redacted": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "leak_marker: leak-marker\n")
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
fsys := validCatalogFS("one")
mutate(fsys)
_, err := Load(testSource("one", fsys))
if err == nil {
t.Fatal("expected invalid profile error")
}
if strings.Contains(strings.ToLower(err.Error()), "leak-marker") {
t.Fatalf("catalog error exposed profile content: %v", err)
}
})
}
}
func TestLoadRejectsCrossCatalogConflicts(t *testing.T) {
t.Run("duplicate backend", func(t *testing.T) {
second := catalogFS("one", map[string]string{
"catalog/profiles/second.yml": validProfile("second", "one"),
}, "null")
_, err := Load(
Source{Name: "first", ExpectedBackendID: "one", FS: validCatalogFS("one"), Root: "catalog"},
Source{Name: "second", ExpectedBackendID: "one", FS: second, Root: "catalog"},
)
if err == nil {
t.Fatal("expected duplicate backend error")
}
})
t.Run("duplicate profile", func(t *testing.T) {
first := catalogFS("one", map[string]string{
"catalog/profiles/shared.yml": validProfile("shared", "one"),
}, "null")
second := catalogFS("two", map[string]string{
"catalog/profiles/shared.yml": validProfile("shared", "two"),
}, "null")
_, err := Load(testSource("one", first), testSource("two", second))
if err == nil {
t.Fatal("expected duplicate profile error")
}
})
t.Run("cross-catalog base", func(t *testing.T) {
first := catalogFS("one", map[string]string{
"catalog/profiles/base.yml": validProfile("base", "one"),
}, "null")
second := catalogFS("two", map[string]string{
"catalog/profiles/child.yml": "id: child\nbase_profile: base\n",
}, "null")
_, err := Load(testSource("one", first), testSource("two", second))
if err == nil {
t.Fatal("expected cross-catalog base error")
}
})
}
func TestLoadReturnsDefensiveCatalogValues(t *testing.T) {
fsys := catalogFS("one", map[string]string{
"catalog/profiles/one-profile.yml": validProfile("one-profile", "one") + "extra_params:\n nested:\n value: profile\n",
}, `{"nested":{"value":"backend"}}`)
loaded, err := Load(testSource("one", fsys))
if err != nil {
t.Fatalf("load catalog: %v", err)
}
loaded.Backends[0].ExtraParams["nested"].(map[string]any)["value"] = "changed"
profileValue, err := loaded.Profiles.GetProfile(context.Background(), "one-profile")
if err != nil {
t.Fatalf("load profile: %v", err)
}
profileValue.ExtraParams["nested"].(map[string]any)["value"] = "changed"
again, err := Load(testSource("one", fsys))
if err != nil {
t.Fatalf("reload catalog: %v", err)
}
if got := again.Backends[0].ExtraParams["nested"].(map[string]any)["value"]; got != "backend" {
t.Fatalf("backend mutation escaped returned set: %#v", got)
}
againProfile, err := loaded.Profiles.GetProfile(context.Background(), "one-profile")
if err != nil {
t.Fatalf("reload profile: %v", err)
}
if got := againProfile.ExtraParams["nested"].(map[string]any)["value"]; got != "profile" {
t.Fatalf("profile mutation escaped returned value: %#v", got)
}
}
func loadCompatibilityFixture(t *testing.T) map[string]any {
t.Helper()
data, err := os.ReadFile(filepath.Join("..", "..", "testdata", "builtin-catalog-v1.json"))
if err != nil {
t.Fatalf("read fixture: %v", err)
}
var value map[string]any
if err := json.Unmarshal(data, &value); err != nil {
t.Fatalf("decode fixture: %v", err)
}
return value
}
func fixtureProfileIDs(t *testing.T, fixture map[string]any) []string {
t.Helper()
profiles, ok := fixture["profiles"].([]any)
if !ok {
t.Fatal("compatibility fixture profiles are malformed")
}
ids := make([]string, 0, len(profiles))
for _, entry := range profiles {
profileValue, ok := entry.(map[string]any)
if !ok {
t.Fatal("compatibility fixture profile is malformed")
}
id, ok := profileValue["id"].(string)
if !ok {
t.Fatal("compatibility fixture profile ID is malformed")
}
ids = append(ids, id)
}
sort.Strings(ids)
return ids
}
func catalogValue(t *testing.T, loaded Set) map[string]any {
t.Helper()
backends := make([]any, 0, len(loaded.Backends))
for _, backend := range loaded.Backends {
backends = append(backends, map[string]any{"id": backend.ID, "endpoint": backend.Endpoint, "api_key_env": backend.APIKeyEnv, "extra_params": interfaceValue(backend.ExtraParams), "concurrency_limit": backend.ConcurrencyLimit, "queue_capacity": backend.QueueCapacity, "queue_capacity_set": backend.QueueCapacitySet})
}
sort.Slice(backends, func(left, right int) bool {
return backends[left].(map[string]any)["id"].(string) < backends[right].(map[string]any)["id"].(string)
})
actualProfiles := make([]any, 0, len(loaded.profileIDs))
for _, id := range loaded.profileIDs {
profile, err := loaded.Profiles.GetProfile(context.Background(), id)
if err != nil {
t.Fatalf("load profile %q: %v", id, err)
}
actualProfiles = append(actualProfiles, map[string]any{"id": profile.ID, "base_profile": profile.BaseProfileID, "backend": profile.BackendID, "endpoint": profile.Endpoint, "model": profile.Model, "temperature": profile.Temperature, "max_tokens": profile.MaxTokens, "top_p": profile.TopP, "timeout_seconds": profile.TimeoutSeconds, "service_tier": profile.ServiceTier, "reasoning_effort": profile.ReasoningEffort, "api_key_env": profile.APIKeyEnv, "api_key_required": profile.APIKeyRequired, "extra_params": interfaceValue(profile.ExtraParams)})
}
return map[string]any{"backends": backends, "profiles": actualProfiles}
}
func interfaceValue(value map[string]any) any {
if value == nil {
return nil
}
return value
}
func testSource(id string, fsys fs.FS) Source {
return Source{Name: id, ExpectedBackendID: id, FS: fsys, Root: "catalog"}
}
func validCatalogFS(id string) fstest.MapFS {
return catalogFS(id, map[string]string{
"catalog/profiles/" + id + "-profile.yml": validProfile(id+"-profile", id),
}, "null")
}
func catalogFS(id string, profiles map[string]string, extraParams string) fstest.MapFS {
fsys := fstest.MapFS{
"catalog/backend.json": &fstest.MapFile{Data: []byte(validManifest(id, "TEST_API_KEY", extraParams))},
}
for name, content := range profiles {
fsys[name] = &fstest.MapFile{Data: []byte(content)}
}
return fsys
}
func validManifest(id, apiKeyEnv, extraParams string) string {
return fmt.Sprintf(
`{"schema_version":1,"id":%q,"endpoint":%q,"api_key_env":%q,"concurrency_limit":2,"queue_capacity":3,"extra_params":%s}`,
id,
"https://"+id+".example/v1",
apiKeyEnv,
extraParams,
)
}
func validProfile(id, backendID string) string {
return fmt.Sprintf("id: %s\nbackend: %s\nmodel: test-model\n", id, backendID)
}
var _ fs.FS = openrouter.FS()

View File

@@ -69,6 +69,7 @@ type RunRequest struct {
Vars map[string]string Vars map[string]string
Execution *ExecutionTargetOverride Execution *ExecutionTargetOverride
Validation *OutputContract Validation *OutputContract
AppendedMessages []RenderedMessage
} }
// RunResult represents the complete result of a prompt execution run. // RunResult represents the complete result of a prompt execution run.

View File

@@ -0,0 +1,82 @@
package domain
import (
"errors"
"strings"
"unicode/utf8"
)
const (
RoleDeveloper = "developer"
RoleSystem = "system"
RoleUser = "user"
RoleAssistant = "assistant"
)
// NormalizeMessageRole validates and canonicalizes a provider-bound chat role.
func NormalizeMessageRole(role string) (string, error) {
if !utf8.ValidString(role) {
return "", errors.New("message role must be valid UTF-8")
}
normalized := strings.ToLower(strings.TrimSpace(role))
switch normalized {
case RoleDeveloper, RoleSystem, RoleUser, RoleAssistant:
return normalized, nil
default:
return "", errors.New("message role must be developer, system, user, or assistant")
}
}
// NormalizeCacheControl validates, canonicalizes, and copies cache metadata.
func NormalizeCacheControl(control *CacheControl) (*CacheControl, error) {
if control == nil {
return nil, nil
}
if !utf8.ValidString(string(control.Type)) {
return nil, errors.New("cache control type must be valid UTF-8")
}
if !utf8.ValidString(control.TTL) {
return nil, errors.New("cache control ttl must be valid UTF-8")
}
cacheType := strings.TrimSpace(string(control.Type))
if cacheType == "" {
return nil, errors.New("cache control type is required")
}
if CacheControlType(cacheType) != CacheControlEphemeral {
return nil, errors.New("unsupported type")
}
ttl := strings.TrimSpace(control.TTL)
if ttl != "" && ttl != "1h" {
return nil, errors.New("unsupported ttl")
}
return &CacheControl{Type: CacheControlType(cacheType), TTL: ttl}, nil
}
// CloneRenderedMessages returns a deep copy of rendered messages.
func CloneRenderedMessages(messages []RenderedMessage) []RenderedMessage {
cloned := make([]RenderedMessage, len(messages))
copyRenderedMessages(cloned, messages)
return cloned
}
// ConcatRenderedMessages returns an independently owned concatenation of messages.
func ConcatRenderedMessages(prefix, suffix []RenderedMessage) []RenderedMessage {
messages := make([]RenderedMessage, len(prefix)+len(suffix))
copyRenderedMessages(messages, prefix)
copyRenderedMessages(messages[len(prefix):], suffix)
return messages
}
func copyRenderedMessages(destination, source []RenderedMessage) {
for index, message := range source {
destination[index] = message
if message.CacheControl != nil {
cacheControl := *message.CacheControl
destination[index].CacheControl = &cacheControl
}
}
}

View File

@@ -0,0 +1,121 @@
package domain
import (
"strings"
"testing"
)
func TestNormalizeMessageRole(t *testing.T) {
tests := []struct {
name string
input string
want string
wantErr bool
}{
{name: "developer", input: RoleDeveloper, want: RoleDeveloper},
{name: "system", input: RoleSystem, want: RoleSystem},
{name: "user", input: RoleUser, want: RoleUser},
{name: "assistant", input: RoleAssistant, want: RoleAssistant},
{name: "surrounding whitespace and mixed case", input: " \u2003UsEr\u2003 ", want: RoleUser},
{name: "blank", input: " \t\n ", wantErr: true},
{name: "tool", input: "tool", wantErr: true},
{name: "function", input: "function", wantErr: true},
{name: "custom", input: "custom-role", wantErr: true},
{name: "invalid UTF-8", input: string([]byte{0xff}), wantErr: true},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got, err := NormalizeMessageRole(test.input)
if test.wantErr {
if err == nil {
t.Fatal("expected an error")
}
if strings.Contains(err.Error(), test.input) {
t.Fatalf("error exposed the input: %v", err)
}
return
}
if err != nil {
t.Fatalf("NormalizeMessageRole() error = %v", err)
}
if got != test.want {
t.Fatalf("NormalizeMessageRole() = %q, want %q", got, test.want)
}
})
}
}
func TestNormalizeCacheControl(t *testing.T) {
tests := []struct {
name string
input *CacheControl
want *CacheControl
wantErr bool
}{
{name: "nil", input: nil, want: nil},
{
name: "canonical values",
input: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"},
want: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"},
},
{
name: "trims values",
input: &CacheControl{Type: " ephemeral ", TTL: " 1h\t"},
want: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"},
},
{name: "empty type", input: &CacheControl{}, wantErr: true},
{name: "unsupported type", input: &CacheControl{Type: "persistent"}, wantErr: true},
{name: "unsupported ttl", input: &CacheControl{Type: CacheControlEphemeral, TTL: "5m"}, wantErr: true},
{name: "invalid type UTF-8", input: &CacheControl{Type: CacheControlType(string([]byte{0xff}))}, wantErr: true},
{name: "invalid ttl UTF-8", input: &CacheControl{Type: CacheControlEphemeral, TTL: string([]byte{0xff})}, wantErr: true},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got, err := NormalizeCacheControl(test.input)
if test.wantErr {
if err == nil {
t.Fatal("expected an error")
}
return
}
if err != nil {
t.Fatalf("NormalizeCacheControl() error = %v", err)
}
if got == nil || test.want == nil {
if got != test.want {
t.Fatalf("NormalizeCacheControl() = %#v, want %#v", got, test.want)
}
return
}
if *got != *test.want {
t.Fatalf("NormalizeCacheControl() = %#v, want %#v", got, test.want)
}
if got == test.input {
t.Fatal("normalized cache control aliases its input")
}
})
}
}
func TestRenderedMessageCloning(t *testing.T) {
prefix := []RenderedMessage{{Role: RoleSystem, Content: "prefix", CacheControl: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"}}}
suffix := []RenderedMessage{{Role: RoleUser, Content: " suffix "}, {Role: RoleAssistant, Content: "", CacheControl: &CacheControl{Type: CacheControlEphemeral}}}
cloned := CloneRenderedMessages(prefix)
combined := ConcatRenderedMessages(prefix, suffix)
if len(combined) != 3 || combined[0].Content != "prefix" || combined[1].Content != " suffix " || combined[2].Content != "" {
t.Fatalf("unexpected combined messages: %#v", combined)
}
if cloned[0].CacheControl == prefix[0].CacheControl || combined[0].CacheControl == prefix[0].CacheControl || combined[2].CacheControl == suffix[1].CacheControl {
t.Fatal("cloned cache controls alias their inputs")
}
prefix[0].Content = "changed"
prefix[0].CacheControl.TTL = ""
suffix[1].CacheControl.Type = "changed"
if cloned[0].Content != "prefix" || cloned[0].CacheControl.TTL != "1h" || combined[0].Content != "prefix" || combined[0].CacheControl.TTL != "1h" || combined[2].CacheControl.Type != CacheControlEphemeral {
t.Fatalf("cloned messages changed with their inputs: cloned=%#v combined=%#v", cloned, combined)
}
}

View File

@@ -6,6 +6,8 @@ import (
"strings" "strings"
) )
const maxOutputRepairAttempts = 3
// ValidateOutputContract validates source-neutral output-contract invariants. // ValidateOutputContract validates source-neutral output-contract invariants.
func ValidateOutputContract(contract OutputContract) error { func ValidateOutputContract(contract OutputContract) error {
switch contract.Format { switch contract.Format {
@@ -26,5 +28,11 @@ func ValidateOutputContract(contract OutputContract) error {
if contract.RepairAttempts < 0 { if contract.RepairAttempts < 0 {
return errors.New("repair_attempts must be greater than or equal to 0") return errors.New("repair_attempts must be greater than or equal to 0")
} }
if contract.RepairAttempts > maxOutputRepairAttempts {
return fmt.Errorf("repair_attempts must be less than or equal to %d", maxOutputRepairAttempts)
}
if contract.ValidationMode == ValidationNone && contract.RepairAttempts > 0 {
return errors.New("repair_attempts requires basic, json, or json_schema validation")
}
return nil return nil
} }

View File

@@ -30,9 +30,28 @@ func TestValidateOutputContract(t *testing.T) {
}}, }},
{name: "empty validation mode", change: func(c *OutputContract) { c.ValidationMode = "" }, wantErr: "validation mode"}, {name: "empty validation mode", change: func(c *OutputContract) { c.ValidationMode = "" }, wantErr: "validation mode"},
{name: "unsupported validation mode", change: func(c *OutputContract) { c.ValidationMode = ValidationMode("unknown") }, wantErr: "validation mode"}, {name: "unsupported validation mode", change: func(c *OutputContract) { c.ValidationMode = ValidationMode("unknown") }, wantErr: "validation mode"},
{name: "negative repair attempts", change: func(c *OutputContract) { c.RepairAttempts = -1 }, wantErr: "repair_attempts"}, {name: "negative repair attempts", change: func(c *OutputContract) {
c.ValidationMode = ValidationBasic
c.RepairAttempts = -1
}, wantErr: "repair_attempts"},
{name: "zero repair attempts", change: func(c *OutputContract) { c.RepairAttempts = 0 }}, {name: "zero repair attempts", change: func(c *OutputContract) { c.RepairAttempts = 0 }},
{name: "positive repair attempts", change: func(c *OutputContract) { c.RepairAttempts = 1 }}, {name: "one repair attempt", change: func(c *OutputContract) {
c.ValidationMode = ValidationBasic
c.RepairAttempts = 1
}},
{name: "maximum repair attempts", change: func(c *OutputContract) {
c.ValidationMode = ValidationJSON
c.RepairAttempts = 3
}},
{name: "too many repair attempts", change: func(c *OutputContract) {
c.ValidationMode = ValidationJSONSchema
c.SchemaPath = "schema.json"
c.RepairAttempts = 4
}, wantErr: "repair_attempts"},
{name: "none validation with repair attempts", change: func(c *OutputContract) {
c.ValidationMode = ValidationNone
c.RepairAttempts = 1
}, wantErr: "repair_attempts"},
{name: "json schema empty path", change: func(c *OutputContract) { {name: "json schema empty path", change: func(c *OutputContract) {
c.ValidationMode = ValidationJSONSchema c.ValidationMode = ValidationJSONSchema
c.SchemaPath = "" c.SchemaPath = ""

View File

@@ -179,12 +179,12 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
return nil, fmt.Errorf("%w: no choices returned", ErrMalformedResponse) return nil, fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
} }
content := wireResp.Choices[0].Message.Content content := wireResp.Choices[0].Message.Content
if content == "" { if content == nil {
return nil, fmt.Errorf("%w: first choice has empty message content", ErrMalformedResponse) return nil, fmt.Errorf("%w: first choice has missing message content", ErrMalformedResponse)
} }
return &domain.GenerateResponse{ return &domain.GenerateResponse{
Content: content, Content: *content,
Usage: domain.TokenUsage{ Usage: domain.TokenUsage{
PromptTokens: wireResp.Usage.PromptTokens, PromptTokens: wireResp.Usage.PromptTokens,
CompletionTokens: wireResp.Usage.CompletionTokens, CompletionTokens: wireResp.Usage.CompletionTokens,
@@ -380,7 +380,7 @@ type openAICacheControl struct {
type openAIChatResponseMessage struct { type openAIChatResponseMessage struct {
Role string `json:"role"` Role string `json:"role"`
Content string `json:"content"` Content *string `json:"content"`
} }
type openAIChatResponse struct { type openAIChatResponse struct {

View File

@@ -363,6 +363,7 @@ func TestOpenAICompatibleClientRequestMapping(t *testing.T) {
run func(*testing.T) run func(*testing.T)
}{ }{
{name: "complete request and response mapping", run: checkCompleteRequestAndResponseMapping}, {name: "complete request and response mapping", run: checkCompleteRequestAndResponseMapping},
{name: "canonical role and exact content", run: checkCanonicalRoleAndExactContentMapping},
{name: "cache-controlled message", run: checkCacheControlledMessageMapping}, {name: "cache-controlled message", run: checkCacheControlledMessageMapping},
{name: "empty cache-control TTL", run: checkEmptyCacheControlTTLOmission}, {name: "empty cache-control TTL", run: checkEmptyCacheControlTTLOmission},
{name: "session ID", run: checkSessionIDMapping}, {name: "session ID", run: checkSessionIDMapping},
@@ -377,6 +378,34 @@ func TestOpenAICompatibleClientRequestMapping(t *testing.T) {
} }
} }
func checkCanonicalRoleAndExactContentMapping(t *testing.T) {
provider := newRecordingProvider(t)
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
const content = " exact content\nwith whitespace "
_, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{
Role: domain.RoleDeveloper,
Content: content,
}}},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("Generate() error = %v", err)
}
var messages []struct {
Role string `json:"role"`
Content string `json:"content"`
}
request := provider.lastRequest(t)
if !request.decodeField(t, "messages", &messages) || len(messages) != 1 {
t.Fatalf("messages = %#v, want one message", messages)
}
if messages[0].Role != domain.RoleDeveloper || messages[0].Content != content {
t.Fatalf("message = %#v, want role %q and exact content %q", messages[0], domain.RoleDeveloper, content)
}
}
func checkCompleteRequestAndResponseMapping(t *testing.T) { func checkCompleteRequestAndResponseMapping(t *testing.T) {
provider := newRecordingProvider(t) provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{ provider.respond(http.StatusOK, `{
@@ -765,6 +794,7 @@ func TestOpenAICompatibleClientResponseFraming(t *testing.T) {
run func(*testing.T) run func(*testing.T)
}{ }{
{name: "usage mapping", run: checkCacheUsageMapping}, {name: "usage mapping", run: checkCacheUsageMapping},
{name: "content presence", run: checkContentPresence},
{name: "common response failures", run: checkCommonResponseFailures}, {name: "common response failures", run: checkCommonResponseFailures},
{name: "successful response byte boundary", run: checkSuccessfulResponseByteBoundary}, {name: "successful response byte boundary", run: checkSuccessfulResponseByteBoundary},
{name: "continuing oversized response", run: checkContinuingOversizedResponse}, {name: "continuing oversized response", run: checkContinuingOversizedResponse},
@@ -776,6 +806,53 @@ func TestOpenAICompatibleClientResponseFraming(t *testing.T) {
} }
} }
func checkContentPresence(t *testing.T) {
tests := []struct {
name string
content string
}{
{name: "explicit empty string", content: ""},
{name: "whitespace string", content: " \n\t "},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{
"choices": [{"message": {"content": `+strconv.Quote(tc.content)+`}}],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 20,
"total_tokens": 30,
"prompt_tokens_details": {"cached_tokens": 4},
"cache_write_tokens": 5
}
}`)
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
response, err := client.Generate(context.Background(), ordinaryGenerateRequest())
if err != nil {
t.Fatalf("generate: %v", err)
}
if response == nil {
t.Fatal("expected response")
}
if response.Content != tc.content {
t.Fatalf("content = %q, want %q", response.Content, tc.content)
}
if response.Usage != (domain.TokenUsage{
PromptTokens: 10,
CompletionTokens: 20,
TotalTokens: 30,
CachedTokens: 4,
CacheWriteTokens: 5,
}) {
t.Fatalf("usage = %+v", response.Usage)
}
})
}
}
func checkCacheUsageMapping(t *testing.T) { func checkCacheUsageMapping(t *testing.T) {
provider := newRecordingProvider(t) provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{ provider.respond(http.StatusOK, `{
@@ -1129,6 +1206,9 @@ func checkCommonResponseFailures(t *testing.T) {
}, },
{name: "invalid JSON", statusCode: http.StatusOK, body: `{not valid json`, wantErr: ErrMalformedResponse}, {name: "invalid JSON", statusCode: http.StatusOK, body: `{not valid json`, wantErr: ErrMalformedResponse},
{name: "missing choices", statusCode: http.StatusOK, body: `{"choices": []}`, wantErr: ErrMalformedResponse}, {name: "missing choices", statusCode: http.StatusOK, body: `{"choices": []}`, wantErr: ErrMalformedResponse},
{name: "missing content", statusCode: http.StatusOK, body: `{"choices": [{"message": {}}]}`, wantErr: ErrMalformedResponse},
{name: "null content", statusCode: http.StatusOK, body: `{"choices": [{"message": {"content": null}}]}`, wantErr: ErrMalformedResponse},
{name: "non-string content", statusCode: http.StatusOK, body: `{"choices": [{"message": {"content": 1}}]}`, wantErr: ErrMalformedResponse},
} }
for _, tc := range tests { for _, tc := range tests {

View File

@@ -1,8 +0,0 @@
id: aion-2
backend: openrouter
model: aion-labs/aion-2.0
temperature: 0.72
reasoning_effort: high
top_p: 0.95
timeout_seconds: 180
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: claude-fable-latest
backend: openrouter
model: "~anthropic/claude-fable-latest"
reasoning_effort: high
timeout_seconds: 600
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: claude-haiku-latest
backend: openrouter
model: "~anthropic/claude-haiku-latest"
reasoning_effort: medium
timeout_seconds: 240
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: claude-opus-latest
backend: openrouter
model: "~anthropic/claude-opus-latest"
reasoning_effort: high
timeout_seconds: 240
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: claude-sonnet-latest
backend: openrouter
model: "~anthropic/claude-sonnet-latest"
reasoning_effort: high
timeout_seconds: 240
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: deepseek-3-2
backend: openrouter
model: deepseek/deepseek-v3.2
reasoning_effort: high
timeout_seconds: 180
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: deepseek-4-flash
backend: openrouter
model: deepseek/deepseek-v4-flash
#reasoning_effort: medium
timeout_seconds: 180
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: deepseek-4-pro
backend: openrouter
model: deepseek/deepseek-v4-pro
reasoning_effort: high
timeout_seconds: 180
service_tier: flex

View File

@@ -1,8 +0,0 @@
id: gemini-2-flash-lite
backend: openrouter
model: "google/gemini-2.5-flash-lite"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -1,8 +0,0 @@
id: gemini-2-flash
backend: openrouter
model: "google/gemini-2.5-flash"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -1,8 +0,0 @@
id: gemini-2-pro
backend: openrouter
model: "google/gemini-2.5-pro"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -1,8 +0,0 @@
id: gemini-3-flash-lite
backend: openrouter
model: "google/gemini-3.1-flash-lite"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -1,8 +0,0 @@
id: gemini-flash-latest
backend: openrouter
model: "~google/gemini-flash-latest"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -1,8 +0,0 @@
id: gemini-pro-latest
backend: openrouter
model: "~google/gemini-pro-latest"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -1,8 +0,0 @@
id: gemma-4-31b
backend: openrouter
model: google/gemma-4-31b-it:exacto
temperature: 0.15
reasoning_effort: high
top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -1,3 +0,0 @@
id: rakestrawhome-gemma-4-31b
backend: rakestrawhome
model: google/gemma-4-31b-it

View File

@@ -1,8 +0,0 @@
id: minimax-m2
backend: openrouter
model: minimax/minimax-m2.5
temperature: 0.5
reasoning_effort: high
top_p: 0.95
timeout_seconds: 180
service_tier: flex

View File

@@ -1,8 +0,0 @@
id: minimax-m3
backend: openrouter
model: minimax/minimax-m3
#temperature: 0.5
reasoning_effort: high
#top_p: 0.95
timeout_seconds: 180
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: mistral-large-2512
backend: openrouter
model: mistralai/mistral-large-2512
temperature: 0.15
top_p: 0.98
timeout_seconds: 180

View File

@@ -1,7 +0,0 @@
id: mistral-medium-3-5
backend: openrouter
model: mistralai/mistral-medium-3-5
temperature: 0.15
reasoning_effort: high
top_p: 0.98
timeout_seconds: 180

View File

@@ -1,6 +0,0 @@
id: mistral-small-3
backend: openrouter
model: mistralai/mistral-small-3.2-24b-instruct
temperature: 0.05
top_p: 1.0
timeout_seconds: 180

View File

@@ -1,7 +0,0 @@
id: mistral-small-4
backend: openrouter
model: mistralai/mistral-small-2603
temperature: 0.1
reasoning_effort: high
top_p: 0.98
timeout_seconds: 180

View File

@@ -1,6 +0,0 @@
id: nemotron-3-ultra
backend: openrouter
model: nvidia/nemotron-3-ultra-550b-a55b
reasoning_effort: high
timeout_seconds: 180
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: gpt-5-mini
backend: openrouter
model: "openai/gpt-5.4-mini"
reasoning_effort: high
timeout_seconds: 240
service_tier: flex

View File

@@ -1,6 +0,0 @@
id: gpt-5-nano
backend: openrouter
model: "openai/gpt-5.4-nano"
reasoning_effort: high
timeout_seconds: 240
service_tier: flex

View File

@@ -1,16 +0,0 @@
package builtin
import (
"embed"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
const assetRoot = "assets"
//go:embed assets/**/*.yml
var assets embed.FS
func NewRepository() profile.Repository {
return profile.NewFSRepository(assets, assetRoot)
}

View File

@@ -1,118 +0,0 @@
package builtin
import (
"context"
"io/fs"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gopkg.in/yaml.v3"
)
func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
repo := NewRepository()
ids := loadBuiltInProfileIDs(t)
if len(ids) == 0 {
t.Fatal("expected built-in profiles")
}
for id := range ids {
t.Run(id, func(t *testing.T) {
p, err := repo.GetProfile(context.Background(), id)
if err != nil {
t.Fatalf("expected built-in profile %q to load, got %v", id, err)
}
if p.ID != id {
t.Fatalf("expected profile id %q, got %q", id, p.ID)
}
if !builtInBackendIDs[p.BackendID] {
t.Fatalf("expected profile %q to select a maintained built-in, got %q", id, p.BackendID)
}
if p.Endpoint != "" || p.APIKeyEnv != "" {
t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv)
}
})
}
}
func TestBuiltInProfilesDoNotContainDuplicateIDsOrRawAPIKeys(t *testing.T) {
loadBuiltInProfileIDs(t)
}
func TestRakestrawhomeGemmaProfileUsesNativeDefaults(t *testing.T) {
p, err := NewRepository().GetProfile(context.Background(), "rakestrawhome-gemma-4-31b")
if err != nil {
t.Fatalf("load Rakestrawhome Gemma profile: %v", err)
}
if p.ID != "rakestrawhome-gemma-4-31b" ||
p.BackendID != backend.RakestrawHomeID ||
p.Model != "google/gemma-4-31b-it" ||
p.Endpoint != "" ||
p.Temperature != 0 ||
p.MaxTokens != 0 ||
p.TopP != 0 ||
p.TimeoutSeconds != 0 ||
p.ServiceTier != "" ||
p.ReasoningEffort != "" ||
p.APIKeyEnv != "" ||
p.APIKeyRequired ||
p.ExtraParams != nil {
t.Fatalf("unexpected Rakestrawhome Gemma profile: %#v", p)
}
}
var builtInBackendIDs = map[string]bool{
backend.OpenRouterID: true,
backend.RakestrawHomeID: true,
}
func loadBuiltInProfileIDs(t *testing.T) map[string]string {
t.Helper()
ids := map[string]string{}
err := fs.WalkDir(assets, assetRoot, func(name string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() || !strings.HasSuffix(name, ".yml") {
return nil
}
data, err := assets.ReadFile(name)
if err != nil {
t.Fatalf("failed to read built-in profile %s: %v", name, err)
}
var raw map[string]any
if err := yaml.Unmarshal(data, &raw); err != nil {
t.Fatalf("failed to decode built-in profile %s: %v", name, err)
}
if _, ok := raw["api_key"]; ok {
t.Fatalf("built-in profile %s contains raw api_key", name)
}
backendID, ok := raw["backend"].(string)
if !ok || !builtInBackendIDs[backendID] {
t.Fatalf("built-in profile %s does not select a maintained built-in: %#v", name, raw["backend"])
}
if _, ok := raw["endpoint"]; ok {
t.Fatalf("built-in profile %s repeats endpoint", name)
}
if _, ok := raw["api_key_env"]; ok {
t.Fatalf("built-in profile %s repeats api_key_env", name)
}
id, ok := raw["id"].(string)
if !ok || strings.TrimSpace(id) == "" {
t.Fatalf("built-in profile %s has missing id", name)
}
if previous, ok := ids[id]; ok {
t.Fatalf("duplicate built-in profile id %q in %s and %s", id, previous, name)
}
ids[id] = name
return nil
})
if err != nil {
t.Fatalf("failed to walk built-in profiles: %v", err)
}
return ids
}

View File

@@ -0,0 +1,98 @@
package profile
import (
"context"
"fmt"
"io/fs"
"sort"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
)
// LoadedProfileMetadata identifies one profile accepted by LoadFSRepository.
type LoadedProfileMetadata struct {
// ID is the normalized profile ID.
ID string
// Path is the safe root-relative source path.
Path string
// ExplicitFields lists the sorted top-level YAML fields present in source.
ExplicitFields []string
}
// LoadFSRepository eagerly validates every profile under root and returns an
// immutable raw repository and independently owned source metadata.
func LoadFSRepository(ctx context.Context, fsys fs.FS, root string) (Repository, []LoadedProfileMetadata, error) {
if fsys == nil {
return nil, nil, fmt.Errorf("failed to read profile directory: filesystem is nil")
}
paths, err := filecatalog.FindFSYAMLFiles(ctx, fsys, root)
if err != nil {
return nil, nil, fmt.Errorf("failed to read profile directory: %w", err)
}
repository := &loadedRepository{profiles: make(map[string]domain.ExecutionProfile, len(paths))}
metadata := make([]LoadedProfileMetadata, 0, len(paths))
for _, path := range paths {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
data, err := fs.ReadFile(fsys, path)
if err != nil {
return nil, nil, fmt.Errorf("failed to read profile file %s: %w", filecatalog.DisplayPath(root, path), err)
}
fileMetadata, err := readProfileFileMetadata(data)
if err != nil {
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidYAML, filecatalog.DisplayPath(root, path))
}
if fileMetadata.hasRawAPIKey {
return nil, nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, filecatalog.DisplayPath(root, path))
}
definition, err := decodeProfile(data)
if err != nil {
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidYAML, filecatalog.DisplayPath(root, path))
}
definition.ExtraParams, err = jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidProfile, filecatalog.DisplayPath(root, path))
}
if err := NormalizeAndValidateDefinition(definition); err != nil {
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidProfile, filecatalog.DisplayPath(root, path))
}
if _, exists := repository.profiles[definition.ID]; exists {
return nil, nil, fmt.Errorf("%w: %s: duplicate profile ID", ErrInvalidProfile, filecatalog.DisplayPath(root, path))
}
repository.profiles[definition.ID] = *definition
fields := append([]string(nil), fileMetadata.explicitFields...)
sort.Strings(fields)
metadata = append(metadata, LoadedProfileMetadata{
ID: definition.ID,
Path: filecatalog.DisplayPath(root, path),
ExplicitFields: fields,
})
}
sort.Slice(metadata, func(left, right int) bool { return metadata[left].ID < metadata[right].ID })
return repository, metadata, nil
}
type loadedRepository struct {
profiles map[string]domain.ExecutionProfile
}
func (r *loadedRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
definition, found := r.profiles[strings.TrimSpace(id)]
if !found {
return nil, ErrProfileNotFound
}
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return nil, fmt.Errorf("copy loaded profile %q: %w", definition.ID, err)
}
definition.ExtraParams = extraParams
return &definition, nil
}

View File

@@ -0,0 +1,110 @@
package profile
import (
"context"
"errors"
"io/fs"
"reflect"
"testing"
"testing/fstest"
)
func TestLoadFSRepositoryLoadsSortedIndependentProfiles(t *testing.T) {
fsys := fstest.MapFS{
"profiles/z.yml": {Data: []byte("id: z\nbackend: local\nmodel: z-model\nextra_params:\n nested:\n value: one\n")},
"profiles/a.yml": {Data: []byte("id: a\nbackend: local\nmodel: a-model\nendpoint: ''\n")},
}
repository, metadata, err := LoadFSRepository(context.Background(), fsys, "profiles")
if err != nil {
t.Fatalf("load repository: %v", err)
}
if len(metadata) != 2 || metadata[0].ID != "a" || metadata[1].ID != "z" || metadata[0].Path != "a.yml" {
t.Fatalf("unexpected metadata: %#v", metadata)
}
if len(metadata[0].ExplicitFields) != 4 || metadata[0].ExplicitFields[0] != "backend" || metadata[0].ExplicitFields[1] != "endpoint" {
t.Fatalf("expected explicitly empty endpoint metadata, got %#v", metadata[0].ExplicitFields)
}
metadata[1].ExplicitFields[0] = "changed"
first, err := repository.GetProfile(context.Background(), "z")
if err != nil {
t.Fatalf("load profile: %v", err)
}
first.ExtraParams["nested"].(map[string]any)["value"] = "changed"
second, err := repository.GetProfile(context.Background(), "z")
if err != nil {
t.Fatalf("reload profile: %v", err)
}
if second.ExtraParams["nested"].(map[string]any)["value"] != "one" {
t.Fatalf("profile value was not defensively copied: %#v", second)
}
}
func TestLoadFSRepositoryRejectsInvalidProfiles(t *testing.T) {
tests := map[string]fstest.MapFS{
"duplicate IDs": {
"profiles/one.yml": {Data: []byte("id: duplicate\nbackend: local\nmodel: one\n")},
"profiles/two.yml": {Data: []byte("id: duplicate\nbackend: local\nmodel: two\n")},
},
"raw API key": {
"profiles/one.yml": {Data: []byte("id: one\nbackend: local\nmodel: one\napi_key: forbidden\n")},
},
"multiple documents": {
"profiles/one.yml": {Data: []byte("id: one\nbackend: local\nmodel: one\n---\nid: two\n")},
},
}
for name, fsys := range tests {
t.Run(name, func(t *testing.T) {
_, _, err := LoadFSRepository(context.Background(), fsys, "profiles")
if err == nil {
t.Fatal("expected load failure")
}
})
}
}
func TestLoadedRepositoryHonorsCancellationAndMissingProfiles(t *testing.T) {
repository, _, err := LoadFSRepository(context.Background(), fstest.MapFS{
"profiles/one.yml": {Data: []byte("id: one\nbackend: local\nmodel: one\n")},
}, "profiles")
if err != nil {
t.Fatalf("load repository: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := repository.GetProfile(ctx, "one"); !errors.Is(err, context.Canceled) {
t.Fatalf("expected cancellation, got %v", err)
}
if _, err := repository.GetProfile(context.Background(), "missing"); !errors.Is(err, ErrProfileNotFound) {
t.Fatalf("expected missing profile, got %v", err)
}
}
func TestLoadFSRepositoryMatchesPointLookupForRawProfiles(t *testing.T) {
fsys := fstest.MapFS{
"profiles/base.yml": {Data: []byte("id: base\nbackend: local\nmodel: base-model\n")},
"profiles/derived.yml": {Data: []byte("id: derived\nbase_profile: base\nreasoning_effort: high\n")},
}
eager, _, err := LoadFSRepository(context.Background(), fsys, "profiles")
if err != nil {
t.Fatalf("load eager repository: %v", err)
}
pointLookup := NewFSRepository(fsys, "profiles")
for _, id := range []string{"base", "derived"} {
t.Run(id, func(t *testing.T) {
got, err := eager.GetProfile(context.Background(), id)
if err != nil {
t.Fatalf("load eager profile: %v", err)
}
want, err := pointLookup.GetProfile(context.Background(), id)
if err != nil {
t.Fatalf("load point-in-time profile: %v", err)
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("raw profiles differ: got %#v, want %#v", got, want)
}
})
}
}
var _ fs.FS = fstest.MapFS{}

View File

@@ -166,6 +166,7 @@ type profileMatch struct {
type profileFileMetadata struct { type profileFileMetadata struct {
ids []string ids []string
hasRawAPIKey bool hasRawAPIKey bool
explicitFields []string
} }
func readProfileFileMetadata(data []byte) (profileFileMetadata, error) { func readProfileFileMetadata(data []byte) (profileFileMetadata, error) {
@@ -206,6 +207,7 @@ func profileMetadataFromNode(node *yaml.Node) profileFileMetadata {
for i := 0; i+1 < len(mapping.Content); i += 2 { for i := 0; i+1 < len(mapping.Content); i += 2 {
key := mapping.Content[i] key := mapping.Content[i]
value := mapping.Content[i+1] value := mapping.Content[i+1]
metadata.explicitFields = append(metadata.explicitFields, key.Value)
switch key.Value { switch key.Value {
case "id": case "id":
metadata.ids = append(metadata.ids, strings.TrimSpace(value.Value)) metadata.ids = append(metadata.ids, strings.TrimSpace(value.Value))
@@ -228,6 +230,7 @@ func (m profileFileMetadata) matchesID(id string) bool {
func (m *profileFileMetadata) merge(other profileFileMetadata) { func (m *profileFileMetadata) merge(other profileFileMetadata) {
m.ids = append(m.ids, other.ids...) m.ids = append(m.ids, other.ids...)
m.hasRawAPIKey = m.hasRawAPIKey || other.hasRawAPIKey m.hasRawAPIKey = m.hasRawAPIKey || other.hasRawAPIKey
m.explicitFields = append(m.explicitFields, other.explicitFields...)
} }
func decodeProfile(data []byte) (*domain.ExecutionProfile, error) { func decodeProfile(data []byte) (*domain.ExecutionProfile, error) {

View File

@@ -68,8 +68,9 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
return nil, err return nil, err
} }
if tmplMsg.Role == "" { role, err := domain.NormalizeMessageRole(tmplMsg.Role)
return nil, fmt.Errorf("%w: message %d", ErrInvalidMessageRole, i) if err != nil {
return nil, fmt.Errorf("%w: message %d: %v", ErrInvalidMessageRole, i, err)
} }
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
@@ -96,7 +97,7 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
} }
renderedMessages = append(renderedMessages, domain.RenderedMessage{ renderedMessages = append(renderedMessages, domain.RenderedMessage{
Role: tmplMsg.Role, Role: role,
Content: buf.String(), Content: buf.String(),
CacheControl: cloneCacheControl(tmplMsg.CacheControl), CacheControl: cloneCacheControl(tmplMsg.CacheControl),
}) })

View File

@@ -361,6 +361,35 @@ func TestGoRenderer_Render(t *testing.T) {
t.Fatalf("expected ErrInvalidMessageRole, got %v", err) t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
} }
}) })
t.Run("canonicalizes directly supplied message roles", func(t *testing.T) {
def := &domain.PromptDefinition{
Templates: []domain.PromptMessageTemplate{
{Role: " \u2003DeVeLoPeR\u2003 ", Content: "Hello"},
},
}
res, err := renderer.Render(ctx, def, nil, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := res.Messages[0].Role; got != domain.RoleDeveloper {
t.Fatalf("rendered role = %q, want %q", got, domain.RoleDeveloper)
}
})
t.Run("rejects unsupported directly supplied message roles", func(t *testing.T) {
const unsupportedRole = "consumer-private-role"
def := &domain.PromptDefinition{
Templates: []domain.PromptMessageTemplate{{Role: unsupportedRole, Content: "Hello"}},
}
_, err := renderer.Render(ctx, def, nil, vars)
if !errors.Is(err, ErrInvalidMessageRole) {
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
}
if strings.Contains(err.Error(), unsupportedRole) {
t.Fatalf("renderer error exposed unsupported role: %v", err)
}
})
} }
func TestGoRendererCancellation(t *testing.T) { func TestGoRendererCancellation(t *testing.T) {

View File

@@ -254,20 +254,27 @@ func normalizePromptDefinitionWithContent(raw *promptDefinitionFile, readContent
templates := make([]domain.PromptMessageTemplate, 0, len(raw.Messages)) templates := make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
for i, msg := range raw.Messages { for i, msg := range raw.Messages {
role := strings.TrimSpace(msg.Role) role, err := domain.NormalizeMessageRole(msg.Role)
if role == "" { if err != nil {
return nil, fmt.Errorf("message %d role is required", i) return nil, fmt.Errorf("message %d role: %w", i, err)
} }
hasContent := strings.TrimSpace(msg.Content) != "" hasContent := strings.TrimSpace(msg.Content) != ""
hasContentFile := strings.TrimSpace(msg.ContentFile) != "" hasContentFile := strings.TrimSpace(msg.ContentFile) != ""
if hasContent == hasContentFile { if hasContent == hasContentFile {
return nil, fmt.Errorf("message %d (%s) must set exactly one of content or content_file", i, role) return nil, fmt.Errorf("message %d must set exactly one of content or content_file", i)
} }
cacheControl, err := normalizeCacheControl(msg.CacheControl) var rawCacheControl *domain.CacheControl
if msg.CacheControl != nil {
rawCacheControl = &domain.CacheControl{
Type: domain.CacheControlType(msg.CacheControl.Type),
TTL: msg.CacheControl.TTL,
}
}
cacheControl, err := domain.NormalizeCacheControl(rawCacheControl)
if err != nil { if err != nil {
return nil, fmt.Errorf("message %d (%s) cache_control: %w", i, role, err) return nil, fmt.Errorf("message %d cache_control: %w", i, err)
} }
templateContent := msg.Content templateContent := msg.Content
@@ -275,7 +282,7 @@ func normalizePromptDefinitionWithContent(raw *promptDefinitionFile, readContent
if hasContentFile { if hasContentFile {
body, resolvedPath, err := readContentFile(msg.ContentFile) body, resolvedPath, err := readContentFile(msg.ContentFile)
if err != nil { if err != nil {
return nil, fmt.Errorf("prompt %q message %d (%s): failed to read content_file %q: %w", id, i, role, msg.ContentFile, err) return nil, fmt.Errorf("prompt %q message %d: failed to read content_file %q: %w", id, i, msg.ContentFile, err)
} }
templateContent = body templateContent = body
resolvedContentFile = resolvedPath resolvedContentFile = resolvedPath
@@ -319,27 +326,3 @@ func normalizePromptDefinitionWithContent(raw *promptDefinitionFile, readContent
Validation: outputContract, Validation: outputContract,
}, nil }, nil
} }
func normalizeCacheControl(raw *cacheControlFile) (*domain.CacheControl, error) {
if raw == nil {
return nil, nil
}
cacheType := strings.TrimSpace(raw.Type)
if cacheType == "" {
return nil, errors.New("type is required")
}
if domain.CacheControlType(cacheType) != domain.CacheControlEphemeral {
return nil, fmt.Errorf("unsupported type %q", cacheType)
}
ttl := strings.TrimSpace(raw.TTL)
if ttl != "" && ttl != "1h" {
return nil, fmt.Errorf("unsupported ttl %q", ttl)
}
return &domain.CacheControl{
Type: domain.CacheControlType(cacheType),
TTL: ttl,
}, nil
}

View File

@@ -892,6 +892,38 @@ output:
format: text format: text
validation_mode: none validation_mode: none
repair_attempts: -1 repair_attempts: -1
`,
wantErr: true,
wantDiagnostic: "repair_attempts",
},
{
name: "repair attempts above maximum",
definition: `
id: normalization-rule
version: "1"
messages:
- role: user
content: test
output:
format: text
validation_mode: basic
repair_attempts: 4
`,
wantErr: true,
wantDiagnostic: "repair_attempts",
},
{
name: "none validation with repair attempts",
definition: `
id: normalization-rule
version: "1"
messages:
- role: user
content: test
output:
format: text
validation_mode: none
repair_attempts: 1
`, `,
wantErr: true, wantErr: true,
wantDiagnostic: "repair_attempts", wantDiagnostic: "repair_attempts",
@@ -1055,6 +1087,48 @@ output:
} }
} }
func TestPromptDefinitionMessageRoleNormalization(t *testing.T) {
const invalidRole = "consumer-private-role"
repo := NewFSRepository(fstest.MapFS{
"canonical.yaml": {Data: []byte(`
id: canonical
version: "1"
messages:
- role: " \u2003SyStEm\u2003 "
content: test
output:
format: text
validation_mode: none
`)},
"invalid.yaml": {Data: []byte(`
id: invalid-role
version: "1"
messages:
- role: consumer-private-role
content: test
output:
format: text
validation_mode: none
`)},
}, ".")
definition, err := repo.GetPromptDefinition(context.Background(), "canonical", "")
if err != nil {
t.Fatalf("GetPromptDefinition() error = %v", err)
}
if got := definition.Templates[0].Role; got != domain.RoleSystem {
t.Fatalf("normalized role = %q, want %q", got, domain.RoleSystem)
}
_, err = repo.GetPromptDefinition(context.Background(), "invalid-role", "")
if !errors.Is(err, ErrInvalidPromptDefinition) {
t.Fatalf("expected ErrInvalidPromptDefinition, got %v", err)
}
if strings.Contains(err.Error(), invalidRole) {
t.Fatalf("invalid prompt error exposed the supplied role: %v", err)
}
}
func assertCacheControl(t *testing.T, got *domain.CacheControl, wantType domain.CacheControlType, wantTTL string) { func assertCacheControl(t *testing.T, got *domain.CacheControl, wantType domain.CacheControlType, wantTTL string) {
t.Helper() t.Helper()
if got == nil { if got == nil {

View File

@@ -106,6 +106,8 @@ func TestRunnerPreparationRejectsInvalidOutputContractsBeforeCompletion(t *testi
{name: "unsupported format", override: domain.OutputContract{Format: "binary", ValidationMode: domain.ValidationNone}}, {name: "unsupported format", override: domain.OutputContract{Format: "binary", ValidationMode: domain.ValidationNone}},
{name: "empty validation mode", override: domain.OutputContract{Format: domain.FormatText}}, {name: "empty validation mode", override: domain.OutputContract{Format: domain.FormatText}},
{name: "negative repair attempts", override: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: -1}}, {name: "negative repair attempts", override: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: -1}},
{name: "repair attempts above maximum", override: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationBasic, RepairAttempts: 4}},
{name: "none validation with repair attempts", override: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: 1}},
{name: "json schema without path", override: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema}}, {name: "json schema without path", override: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema}},
} }

View File

@@ -213,16 +213,7 @@ func clonePreparedRun(source *domain.PreparedRun) (*domain.PreparedRun, error) {
copied.InputHashes[name] = hash copied.InputHashes[name] = hash
} }
} }
if source.Messages != nil { copied.Messages = domain.CloneRenderedMessages(source.Messages)
copied.Messages = make([]domain.RenderedMessage, len(source.Messages))
for i, message := range source.Messages {
copied.Messages[i] = message
if message.CacheControl != nil {
cacheControl := *message.CacheControl
copied.Messages[i].CacheControl = &cacheControl
}
}
}
if source.StructuredOutput != nil { if source.StructuredOutput != nil {
structuredOutput := *source.StructuredOutput structuredOutput := *source.StructuredOutput

View File

@@ -3,10 +3,12 @@ package usecase
import ( import (
"context" "context"
"errors" "errors"
"fmt"
"reflect" "reflect"
"testing" "testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
"gitea.maximumdirect.net/eric/promptkit/internal/validate" "gitea.maximumdirect.net/eric/promptkit/internal/validate"
) )
@@ -385,19 +387,20 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
admitter := &fakeRunAdmitter{} admitter := &fakeRunAdmitter{}
reader := defaultArtifactReader() reader := defaultArtifactReader()
renderer := defaultRenderer() renderer := defaultRenderer()
client := &sequenceLLM{responses: []*domain.GenerateResponse{{
Content: `{"broken":true}`,
Usage: domain.TokenUsage{
PromptTokens: 13, CompletionTokens: 17, TotalTokens: 19,
CachedTokens: 23, CacheWriteTokens: 29,
},
}}}
runner := NewRunnerWithRepairer( runner := NewRunnerWithRepairer(
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)}, &fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil, nil,
reader, reader,
renderer, renderer,
&fakeLLM{resp: &domain.GenerateResponse{ client,
Content: `{"broken":true}`,
Usage: domain.TokenUsage{
PromptTokens: 13, CompletionTokens: 17, TotalTokens: 19,
CachedTokens: 23, CacheWriteTokens: 29,
},
}},
validator, validator,
repairer, repairer,
admitter, admitter,
@@ -421,6 +424,10 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
if validator.directValidateCalls != 0 || repairer.calls != 1 { if validator.directValidateCalls != 0 || repairer.calls != 1 {
t.Fatalf("validation/repair calls=(direct=%d repair=%d), want (0, 1)", validator.directValidateCalls, repairer.calls) t.Fatalf("validation/repair calls=(direct=%d repair=%d), want (0, 1)", validator.directValidateCalls, repairer.calls)
} }
if len(client.requests) != 1 || len(repairer.reqs) != 1 ||
!reflect.DeepEqual(repairer.reqs[0].OriginalMessages, client.requests[0].Prompt.Messages) {
t.Fatalf("initial and repair messages = (%#v, %#v)", client.requests, repairer.reqs)
}
if result.Validation.Status != domain.ValidationPassed || result.Validation.RepairAttempts != 1 { if result.Validation.Status != domain.ValidationPassed || result.Validation.RepairAttempts != 1 {
t.Fatalf("unexpected repaired validation result: %+v", result.Validation) t.Fatalf("unexpected repaired validation result: %+v", result.Validation)
} }
@@ -443,6 +450,7 @@ func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
generationFailure := errors.New("generation failed") generationFailure := errors.New("generation failed")
validationFailure := errors.New("validation failed") validationFailure := errors.New("validation failed")
repairFailure := errors.New("repair failed") repairFailure := errors.New("repair failed")
repairInvalidRequest := fmt.Errorf("repair request: %w", llm.ErrInvalidRequest)
tests := []struct { tests := []struct {
name string name string
@@ -450,6 +458,7 @@ func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
validation *recordingPreparedValidation validation *recordingPreparedValidation
repairer *fakeRepairer repairer *fakeRepairer
wantError error wantError error
wantSource error
}{ }{
{ {
name: "generation failure", name: "generation failure",
@@ -475,7 +484,50 @@ func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
}}, }},
}, },
repairer: &fakeRepairer{err: repairFailure}, repairer: &fakeRepairer{err: repairFailure},
wantError: ErrValidation, wantError: ErrLLMGenerate,
wantSource: repairFailure,
},
{
name: "repair invalid request",
validation: &recordingPreparedValidation{
results: []domain.ValidationResult{{
Status: domain.ValidationFailed,
Mode: domain.ValidationJSON,
Errors: []string{"invalid"},
IsValid: false,
}},
},
repairer: &fakeRepairer{err: repairInvalidRequest},
wantError: ErrInvalidRequest,
wantSource: repairInvalidRequest,
},
{
name: "repair cancellation",
validation: &recordingPreparedValidation{
results: []domain.ValidationResult{{
Status: domain.ValidationFailed,
Mode: domain.ValidationJSON,
Errors: []string{"invalid"},
IsValid: false,
}},
},
repairer: &fakeRepairer{err: context.Canceled},
wantError: ErrLLMGenerate,
wantSource: context.Canceled,
},
{
name: "repair deadline",
validation: &recordingPreparedValidation{
results: []domain.ValidationResult{{
Status: domain.ValidationFailed,
Mode: domain.ValidationJSON,
Errors: []string{"invalid"},
IsValid: false,
}},
},
repairer: &fakeRepairer{err: context.DeadlineExceeded},
wantError: ErrLLMGenerate,
wantSource: context.DeadlineExceeded,
}, },
} }
@@ -514,6 +566,9 @@ func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
if result != nil || !errors.Is(err, test.wantError) { if result != nil || !errors.Is(err, test.wantError) {
t.Fatalf("run prepared=(%+v, %v), want %v", result, err, test.wantError) t.Fatalf("run prepared=(%+v, %v), want %v", result, err, test.wantError)
} }
if test.wantSource != nil && !errors.Is(err, test.wantSource) {
t.Fatalf("run prepared error = %v, want source %v", err, test.wantSource)
}
if len(admitter.backendIDs) != 1 || admitter.releaseCalls != 1 { if len(admitter.backendIDs) != 1 || admitter.releaseCalls != 1 {
t.Fatalf( t.Fatalf(
"admission calls=%#v releases=%d, want one each", "admission calls=%#v releases=%d, want one each",

View File

@@ -2,19 +2,29 @@ package usecase
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"strings" "strings"
"unicode/utf8"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm" "gitea.maximumdirect.net/eric/promptkit/internal/llm"
) )
const (
maxRepairDiagnosticBytes = 64 * 1024
omittedRepairDiagnostics = "additional validation diagnostics were omitted"
)
// OutputRepairer generates a corrected candidate after validation fails.
type OutputRepairer interface { type OutputRepairer interface {
Repair(ctx context.Context, req RepairRequest) (*domain.GenerateResponse, error) Repair(ctx context.Context, req RepairRequest) (*domain.GenerateResponse, error)
} }
// RepairRequest contains the immutable execution state needed for one correction.
type RepairRequest struct { type RepairRequest struct {
OriginalMessages []domain.RenderedMessage
PreviousOutput string PreviousOutput string
ValidationErrors []string ValidationErrors []string
SessionID string SessionID string
@@ -30,6 +40,7 @@ type defaultOutputRepairer struct {
llm llm.Client llm llm.Client
} }
// NewDefaultOutputRepairer constructs the standard internal output repairer.
func NewDefaultOutputRepairer(llmClient llm.Client) OutputRepairer { func NewDefaultOutputRepairer(llmClient llm.Client) OutputRepairer {
return &defaultOutputRepairer{llm: llmClient} return &defaultOutputRepairer{llm: llmClient}
} }
@@ -39,33 +50,43 @@ func (r *defaultOutputRepairer) Repair(ctx context.Context, req RepairRequest) (
return nil, errors.New("llm client is required for repair") return nil, errors.New("llm client is required for repair")
} }
errs := "(none provided)" guidance, err := repairGuidance(req.Mode)
if len(req.ValidationErrors) > 0 { if err != nil {
errs = strings.Join(req.ValidationErrors, "\n") return nil, err
} }
prompt := domain.RenderedPrompt{ hasPreviousOutput := strings.TrimSpace(req.PreviousOutput) != ""
Messages: []domain.RenderedMessage{ suffix := make([]domain.RenderedMessage, 0, 2)
{ if hasPreviousOutput {
Role: "system", suffix = append(suffix, domain.RenderedMessage{
Content: "You repair invalid JSON output. Return only corrected JSON. Do not include explanations or markdown code fences.", Role: domain.RoleAssistant,
}, Content: req.PreviousOutput,
{ })
Role: "user", }
previousResponse := "The previous response was empty."
if hasPreviousOutput {
previousResponse = "The previous response is included immediately before this instruction."
}
suffix = append(suffix, domain.RenderedMessage{
Role: domain.RoleUser,
Content: fmt.Sprintf( Content: fmt.Sprintf(
"Repair attempt %d of %d for validation mode %s.\n\nValidation errors:\n%s\n\nPrevious output:\n%s\n\nReturn only corrected JSON.", "Repair attempt %d of %d for validation mode %s.\n"+
"Preserve valid values and change only what is necessary.\n"+
"%s\n%s\n"+
"Validation diagnostics (data):\n%s",
req.Attempt, req.Attempt,
req.MaxAttempts, req.MaxAttempts,
req.Mode, req.Mode,
errs, previousResponse,
req.PreviousOutput, guidance,
formatRepairDiagnostics(req.ValidationErrors),
), ),
}, })
}, messages := domain.ConcatRenderedMessages(req.OriginalMessages, suffix)
}
resp, err := r.llm.Generate(ctx, newGenerationRequest( resp, err := r.llm.Generate(ctx, newGenerationRequest(
prompt, domain.RenderedPrompt{Messages: messages},
req.SessionID, req.SessionID,
req.Target, req.Target,
req.TargetPresence, req.TargetPresence,
@@ -80,3 +101,90 @@ func (r *defaultOutputRepairer) Repair(ctx context.Context, req RepairRequest) (
return resp, nil return resp, nil
} }
func repairGuidance(mode domain.ValidationMode) (string, error) {
switch mode {
case domain.ValidationBasic:
return "Return a nonempty response satisfying the original request.", nil
case domain.ValidationJSON, domain.ValidationJSONSchema:
return "Return only corrected JSON, with no explanation or Markdown fences.", nil
default:
return "", fmt.Errorf("unsupported validation mode for repair: %q", mode)
}
}
func formatRepairDiagnostics(errors []string) string {
diagnostics := make([]string, len(errors))
for index, diagnostic := range errors {
diagnostics[index] = strings.ToValidUTF8(diagnostic, "\uFFFD")
}
complete, _ := json.Marshal(diagnostics)
if len(complete) <= maxRepairDiagnosticBytes {
return string(complete)
}
omission, _ := json.Marshal(omittedRepairDiagnostics)
encoded := make([]byte, 0, maxRepairDiagnosticBytes)
encoded = append(encoded, '[')
for _, diagnostic := range diagnostics {
entry, _ := json.Marshal(diagnostic)
separator := 0
if len(encoded) > 1 {
separator = 1
}
available := maxRepairDiagnosticBytes - len(encoded) - separator - 1 - len(omission) - 1
if len(entry) <= available {
if separator != 0 {
encoded = append(encoded, ',')
}
encoded = append(encoded, entry...)
continue
}
if available < len(`""`) {
break
}
if separator != 0 {
encoded = append(encoded, ',')
}
encoded = append(encoded, truncateDiagnosticJSONValue(diagnostic, available)...)
break
}
if len(encoded) > 1 {
encoded = append(encoded, ',')
}
encoded = append(encoded, omission...)
encoded = append(encoded, ']')
return string(encoded)
}
func truncateDiagnosticJSONValue(value string, maxBytes int) []byte {
if maxBytes < len(`""`) {
return nil
}
boundaries := []int{0}
for end := 0; end < len(value); {
_, size := utf8.DecodeRuneInString(value[end:])
end += size
if end+len(`""`) > maxBytes {
break
}
boundaries = append(boundaries, end)
}
low, high := 0, len(boundaries)-1
best := []byte(`""`)
for low <= high {
mid := low + (high-low)/2
candidate, _ := json.Marshal(value[:boundaries[mid]])
if len(candidate) <= maxBytes {
best = candidate
low = mid + 1
continue
}
high = mid - 1
}
return best
}

View File

@@ -0,0 +1,351 @@
package usecase
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"strings"
"testing"
"unicode/utf8"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
type recordingRepairClient struct {
requests []domain.GenerateRequest
response *domain.GenerateResponse
err error
}
func (c *recordingRepairClient) Generate(_ context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
c.requests = append(c.requests, req)
return c.response, c.err
}
func TestDefaultOutputRepairerBuildsFullContextRequest(t *testing.T) {
client := &recordingRepairClient{response: &domain.GenerateResponse{Content: "corrected"}}
repairer := NewDefaultOutputRepairer(client)
original := []domain.RenderedMessage{
{Role: "system", Content: "Follow the task.", CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}},
{Role: "user", Content: "Summarize the report."},
{Role: "assistant", Content: "Consumer-supplied previous response."},
{Role: "user", Content: "Consumer-supplied correction."},
}
before := append([]domain.RenderedMessage(nil), original...)
previous := strings.Repeat("candidate ", 12_000)
target := domain.ExecutionTarget{BackendID: "backend", Endpoint: "https://provider.example/v1", Model: "model"}
presence := domain.ExecutionTargetPresence{Temperature: true, TopP: true}
structured := &domain.StructuredOutputSpec{}
response, err := repairer.Repair(context.Background(), RepairRequest{
OriginalMessages: original,
PreviousOutput: previous,
ValidationErrors: []string{"invalid JSON"},
SessionID: "session",
Target: target,
TargetPresence: presence,
StructuredOutput: structured,
Attempt: 1,
MaxAttempts: 3,
Mode: domain.ValidationJSONSchema,
})
if err != nil {
t.Fatalf("repair: %v", err)
}
if response == nil || response.Content != "corrected" {
t.Fatalf("response = %+v", response)
}
if !reflect.DeepEqual(original, before) {
t.Fatalf("original messages changed: got %#v, want %#v", original, before)
}
if len(client.requests) != 1 {
t.Fatalf("generation requests = %d, want 1", len(client.requests))
}
request := client.requests[0]
if request.Prompt.SessionID != "session" || !reflect.DeepEqual(request.Target, target) || request.TargetPresence != presence || request.StructuredOutput != structured {
t.Fatalf("generation request fields = %+v", request)
}
if len(request.Prompt.Messages) != len(original)+2 {
t.Fatalf("message count = %d, want %d", len(request.Prompt.Messages), len(original)+2)
}
if !reflect.DeepEqual(request.Prompt.Messages[:len(original)], original) {
t.Fatalf("original messages = %#v, want %#v", request.Prompt.Messages[:len(original)], original)
}
assistant := request.Prompt.Messages[len(original)]
if assistant.Role != "assistant" || assistant.Content != previous {
t.Fatalf("assistant candidate = %+v", assistant)
}
correction := request.Prompt.Messages[len(original)+1]
if correction.Role != "user" || !strings.Contains(correction.Content, "Repair attempt 1 of 3") ||
!strings.Contains(correction.Content, "Preserve valid values") ||
!strings.Contains(correction.Content, "Return only corrected JSON") {
t.Fatalf("correction message = %q", correction.Content)
}
if diagnostics := repairDiagnosticsFromMessage(t, correction.Content); !reflect.DeepEqual(diagnostics, []string{"invalid JSON"}) {
t.Fatalf("diagnostics = %#v", diagnostics)
}
}
func TestDefaultOutputRepairerDoesNotAccumulateCandidates(t *testing.T) {
client := &recordingRepairClient{response: &domain.GenerateResponse{Content: "corrected"}}
repairer := NewDefaultOutputRepairer(client)
backing := make([]domain.RenderedMessage, 1, 4)
backing[0] = domain.RenderedMessage{Role: "user", Content: "Original task"}
before := append([]domain.RenderedMessage(nil), backing...)
for _, candidate := range []string{"first invalid", "second invalid"} {
_, err := repairer.Repair(context.Background(), RepairRequest{
OriginalMessages: backing,
PreviousOutput: candidate,
Attempt: 1,
MaxAttempts: 3,
Mode: domain.ValidationJSON,
})
if err != nil {
t.Fatalf("repair %q: %v", candidate, err)
}
}
if !reflect.DeepEqual(backing, before) {
t.Fatalf("caller messages changed: got %#v, want %#v", backing, before)
}
if len(client.requests) != 2 {
t.Fatalf("generation requests = %d, want 2", len(client.requests))
}
for index, request := range client.requests {
messages := request.Prompt.Messages
if len(messages) != 3 || messages[0] != backing[0] || messages[1].Role != "assistant" || messages[1].Content != []string{"first invalid", "second invalid"}[index] {
t.Fatalf("request %d messages = %#v", index, messages)
}
}
}
func TestDefaultOutputRepairerOmitsEmptyCandidateMessage(t *testing.T) {
for _, tc := range []struct {
name string
candidate string
}{
{name: "empty", candidate: ""},
{name: "whitespace", candidate: " \n\t "},
} {
t.Run(tc.name, func(t *testing.T) {
client := &recordingRepairClient{response: &domain.GenerateResponse{Content: "corrected"}}
repairer := NewDefaultOutputRepairer(client)
_, err := repairer.Repair(context.Background(), RepairRequest{
OriginalMessages: []domain.RenderedMessage{{Role: "user", Content: "Original task"}},
PreviousOutput: tc.candidate,
Attempt: 1,
MaxAttempts: 1,
Mode: domain.ValidationBasic,
})
if err != nil {
t.Fatalf("repair: %v", err)
}
messages := client.requests[0].Prompt.Messages
if len(messages) != 2 || messages[1].Role != "user" || !strings.Contains(messages[1].Content, "previous response was empty") {
t.Fatalf("messages = %#v", messages)
}
if strings.Contains(messages[1].Content, tc.candidate) && tc.candidate != "" {
t.Fatalf("correction message repeated whitespace candidate: %q", messages[1].Content)
}
})
}
}
func TestDefaultOutputRepairerUsesModeSpecificGuidance(t *testing.T) {
tests := []struct {
mode domain.ValidationMode
want string
}{
{mode: domain.ValidationBasic, want: "Return a nonempty response"},
{mode: domain.ValidationJSON, want: "Return only corrected JSON"},
{mode: domain.ValidationJSONSchema, want: "Return only corrected JSON"},
}
for _, tc := range tests {
t.Run(string(tc.mode), func(t *testing.T) {
client := &recordingRepairClient{response: &domain.GenerateResponse{Content: "corrected"}}
_, err := NewDefaultOutputRepairer(client).Repair(context.Background(), RepairRequest{Mode: tc.mode, Attempt: 1, MaxAttempts: 1})
if err != nil {
t.Fatalf("repair: %v", err)
}
if !strings.Contains(client.requests[0].Prompt.Messages[0].Content, tc.want) {
t.Fatalf("correction message = %q", client.requests[0].Prompt.Messages[0].Content)
}
})
}
client := &recordingRepairClient{response: &domain.GenerateResponse{Content: "unexpected"}}
_, err := NewDefaultOutputRepairer(client).Repair(context.Background(), RepairRequest{Mode: domain.ValidationNone})
if err == nil || !strings.Contains(err.Error(), "unsupported validation mode") {
t.Fatalf("unsupported mode error = %v", err)
}
if len(client.requests) != 0 {
t.Fatalf("generation requests = %d, want 0", len(client.requests))
}
}
func TestFormatRepairDiagnosticsBoundsAndPreservesData(t *testing.T) {
tests := []struct {
name string
errors []string
wantOmission bool
check func(*testing.T, []string)
}{
{
name: "below limit",
errors: []string{"first", "second"},
check: func(t *testing.T, got []string) {
t.Helper()
if !reflect.DeepEqual(got, []string{"first", "second"}) {
t.Fatalf("diagnostics = %#v", got)
}
},
},
{
name: "at limit",
errors: []string{strings.Repeat("x", maxRepairDiagnosticBytes-4)},
check: func(t *testing.T, got []string) {
t.Helper()
if len(got) != 1 || len(got[0]) != maxRepairDiagnosticBytes-4 {
t.Fatalf("diagnostics lengths = %#v", got)
}
},
},
{
name: "multibyte truncation preserves prior entries",
errors: []string{"first", strings.Repeat("界", maxRepairDiagnosticBytes)},
wantOmission: true,
check: func(t *testing.T, got []string) {
t.Helper()
if len(got) != 3 || got[0] != "first" || !strings.HasPrefix(strings.Repeat("界", maxRepairDiagnosticBytes), got[1]) {
t.Fatalf("diagnostics = %#v", got)
}
},
},
{
name: "many diagnostics",
errors: manyRepairDiagnostics(),
wantOmission: true,
check: func(t *testing.T, got []string) {
t.Helper()
if len(got) < 2 || got[0] != manyRepairDiagnostics()[0] {
t.Fatalf("diagnostics = %#v", got)
}
},
},
{
name: "no room for partial diagnostic",
errors: func() []string {
omission, _ := json.Marshal(omittedRepairDiagnostics)
return []string{
strings.Repeat("x", maxRepairDiagnosticBytes-len(omission)-5),
strings.Repeat("y", 128),
}
}(),
wantOmission: true,
check: func(t *testing.T, got []string) {
t.Helper()
if len(got) != 2 || got[1] != omittedRepairDiagnostics {
t.Fatalf("diagnostics = %#v", got)
}
},
},
{
name: "invalid UTF-8",
errors: []string{"broken\xffinput"},
check: func(t *testing.T, got []string) {
t.Helper()
if !reflect.DeepEqual(got, []string{"broken\uFFFDinput"}) {
t.Fatalf("diagnostics = %#v", got)
}
},
},
{
name: "one huge diagnostic",
errors: []string{strings.Repeat("x", maxRepairDiagnosticBytes*2)},
wantOmission: true,
check: func(t *testing.T, got []string) {
t.Helper()
if len(got) != 2 || !strings.HasPrefix(strings.Repeat("x", maxRepairDiagnosticBytes*2), got[0]) {
t.Fatalf("diagnostics = %#v", got)
}
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
before := append([]string(nil), tc.errors...)
encoded := formatRepairDiagnostics(tc.errors)
if len(encoded) > maxRepairDiagnosticBytes || !utf8.ValidString(encoded) {
t.Fatalf("encoded diagnostic length/UTF-8 = (%d, %v)", len(encoded), utf8.ValidString(encoded))
}
var got []string
if err := json.Unmarshal([]byte(encoded), &got); err != nil {
t.Fatalf("decode diagnostics: %v; encoded=%q", err, encoded)
}
if !reflect.DeepEqual(tc.errors, before) {
t.Fatalf("input diagnostics changed: got %#v, want %#v", tc.errors, before)
}
if hasOmission := len(got) > 0 && got[len(got)-1] == omittedRepairDiagnostics; hasOmission != tc.wantOmission {
t.Fatalf("omission = %v, want %v; diagnostics=%#v", hasOmission, tc.wantOmission, got)
}
tc.check(t, got)
})
}
}
func manyRepairDiagnostics() []string {
diagnostics := make([]string, 1_000)
for index := range diagnostics {
diagnostics[index] = fmt.Sprintf("diagnostic %04d %s", index, strings.Repeat("x", 128))
}
return diagnostics
}
func TestDefaultOutputRepairerPropagatesGenerationFailures(t *testing.T) {
expected := errors.New("generation failed")
for _, tc := range []struct {
name string
client *recordingRepairClient
want error
}{
{name: "nil client", want: nil},
{name: "generation error", client: &recordingRepairClient{err: expected}, want: expected},
{name: "nil response", client: &recordingRepairClient{}, want: nil},
} {
t.Run(tc.name, func(t *testing.T) {
var repairer OutputRepairer
if tc.client == nil {
repairer = NewDefaultOutputRepairer(nil)
} else {
repairer = NewDefaultOutputRepairer(tc.client)
}
response, err := repairer.Repair(context.Background(), RepairRequest{Mode: domain.ValidationJSON})
if response != nil || err == nil {
t.Fatalf("response/error = (%+v, %v)", response, err)
}
if tc.want != nil && !errors.Is(err, tc.want) {
t.Fatalf("error = %v, want %v", err, tc.want)
}
})
}
}
func repairDiagnosticsFromMessage(t *testing.T, message string) []string {
t.Helper()
const marker = "Validation diagnostics (data):\n"
index := strings.Index(message, marker)
if index < 0 {
t.Fatalf("missing diagnostics marker in %q", message)
}
encoded := message[index+len(marker):]
var diagnostics []string
if err := json.Unmarshal([]byte(encoded), &diagnostics); err != nil {
t.Fatalf("decode diagnostics: %v", err)
}
return diagnostics
}

View File

@@ -4,10 +4,13 @@ import (
"context" "context"
"crypto/rand" "crypto/rand"
"crypto/sha256" "crypto/sha256"
"encoding/binary"
"encoding/hex" "encoding/hex"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"hash"
"io"
"os" "os"
"strings" "strings"
"time" "time"
@@ -184,10 +187,10 @@ func (r *Runner) executePreparedRun(
prepared.StructuredOutput, prepared.StructuredOutput,
)) ))
if err != nil { if err != nil {
if errors.Is(err, llm.ErrInvalidRequest) { return nil, wrapGenerationError(err)
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
} }
return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err) if genResp == nil {
return nil, fmt.Errorf("%w: model returned nil response", ErrLLMGenerate)
} }
usage := genResp.Usage usage := genResp.Usage
@@ -203,6 +206,7 @@ func (r *Runner) executePreparedRun(
attemptsUsed++ attemptsUsed++
repairResp, repairErr := r.repairer.Repair(ctx, RepairRequest{ repairResp, repairErr := r.repairer.Repair(ctx, RepairRequest{
OriginalMessages: prepared.Messages,
PreviousOutput: genResp.Content, PreviousOutput: genResp.Content,
ValidationErrors: validationResult.Errors, ValidationErrors: validationResult.Errors,
SessionID: prepared.SessionID, SessionID: prepared.SessionID,
@@ -214,10 +218,10 @@ func (r *Runner) executePreparedRun(
Mode: prepared.OutputContract.ValidationMode, Mode: prepared.OutputContract.ValidationMode,
}) })
if repairErr != nil { if repairErr != nil {
return nil, fmt.Errorf("%w: %w", ErrValidation, repairErr) return nil, wrapGenerationError(repairErr)
} }
if repairResp == nil { if repairResp == nil {
return nil, fmt.Errorf("%w: repairer returned nil response", ErrValidation) return nil, fmt.Errorf("%w: repairer returned nil response", ErrLLMGenerate)
} }
genResp = repairResp genResp = repairResp
@@ -257,6 +261,13 @@ func (r *Runner) executePreparedRun(
}, nil }, nil
} }
func wrapGenerationError(err error) error {
if errors.Is(err, llm.ErrInvalidRequest) {
return fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
return fmt.Errorf("%w: %w", ErrLLMGenerate, err)
}
func addTokenUsage(total, next domain.TokenUsage) domain.TokenUsage { func addTokenUsage(total, next domain.TokenUsage) domain.TokenUsage {
return domain.TokenUsage{ return domain.TokenUsage{
PromptTokens: total.PromptTokens + next.PromptTokens, PromptTokens: total.PromptTokens + next.PromptTokens,
@@ -436,9 +447,11 @@ func (r *Runner) completePreparationWithStructuredOutput(
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptRender, err) return nil, fmt.Errorf("%w: %w", ErrPromptRender, err)
} }
effectivePrompt := *renderedPrompt
if state.directSessionID != "" { if state.directSessionID != "" {
renderedPrompt.SessionID = state.directSessionID effectivePrompt.SessionID = state.directSessionID
} }
effectivePrompt.Messages = domain.ConcatRenderedMessages(renderedPrompt.Messages, req.AppendedMessages)
end := time.Now().UTC() end := time.Now().UTC()
effectiveModel := state.effectiveModel effectiveModel := state.effectiveModel
@@ -454,9 +467,9 @@ func (r *Runner) completePreparationWithStructuredOutput(
OutputContract: state.effectiveContract, OutputContract: state.effectiveContract,
StructuredOutput: structuredOutput, StructuredOutput: structuredOutput,
InputHashes: inputHashes, InputHashes: inputHashes,
SessionID: renderedPrompt.SessionID, SessionID: effectivePrompt.SessionID,
RenderedPromptHash: hashRenderedPrompt(*renderedPrompt), RenderedPromptHash: hashRenderedPrompt(effectivePrompt),
Messages: renderedPrompt.Messages, Messages: effectivePrompt.Messages,
StartTime: state.start, StartTime: state.start,
EndTime: end, EndTime: end,
DurationMS: end.Sub(state.start).Milliseconds(), DurationMS: end.Sub(state.start).Milliseconds(),
@@ -524,7 +537,12 @@ func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationR
if validationResult.Status != domain.ValidationFailed { if validationResult.Status != domain.ValidationFailed {
return false return false
} }
return contract.ValidationMode == domain.ValidationJSON || contract.ValidationMode == domain.ValidationJSONSchema switch contract.ValidationMode {
case domain.ValidationBasic, domain.ValidationJSON, domain.ValidationJSONSchema:
return true
default:
return false
}
} }
func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget { func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget {
@@ -701,29 +719,38 @@ func resolveOutputContract(def *domain.PromptDefinition, override *domain.Output
return contract, nil return contract, nil
} }
const renderedPromptHashVersion = "promptkit/rendered-prompt/v2\x00"
func hashRenderedPrompt(p domain.RenderedPrompt) string { func hashRenderedPrompt(p domain.RenderedPrompt) string {
var b strings.Builder hasher := sha256.New()
if p.SessionID != "" { var lengthBuffer [8]byte
b.WriteString("session_id=") _, _ = io.WriteString(hasher, renderedPromptHashVersion)
b.WriteString(p.SessionID) writeRenderedPromptHashString(hasher, &lengthBuffer, p.SessionID)
b.WriteString("\n---\n") writeRenderedPromptHashLength(hasher, &lengthBuffer, len(p.Messages))
for _, message := range p.Messages {
writeRenderedPromptHashString(hasher, &lengthBuffer, message.Role)
writeRenderedPromptHashString(hasher, &lengthBuffer, message.Content)
if message.CacheControl == nil {
lengthBuffer[0] = 0
_, _ = hasher.Write(lengthBuffer[:1])
continue
} }
for _, msg := range p.Messages { lengthBuffer[0] = 1
b.WriteString(msg.Role) _, _ = hasher.Write(lengthBuffer[:1])
b.WriteByte('\n') writeRenderedPromptHashString(hasher, &lengthBuffer, string(message.CacheControl.Type))
b.WriteString(msg.Content) writeRenderedPromptHashString(hasher, &lengthBuffer, message.CacheControl.TTL)
if msg.CacheControl != nil {
b.WriteString("\ncache_control.type=")
b.WriteString(string(msg.CacheControl.Type))
if msg.CacheControl.TTL != "" {
b.WriteString("\ncache_control.ttl=")
b.WriteString(msg.CacheControl.TTL)
} }
return hex.EncodeToString(hasher.Sum(nil))
} }
b.WriteString("\n---\n")
func writeRenderedPromptHashLength(hasher hash.Hash, buffer *[8]byte, length int) {
binary.BigEndian.PutUint64(buffer[:], uint64(length))
_, _ = hasher.Write(buffer[:])
} }
h := sha256.Sum256([]byte(b.String()))
return hex.EncodeToString(h[:]) func writeRenderedPromptHashString(hasher hash.Hash, buffer *[8]byte, value string) {
writeRenderedPromptHashLength(hasher, buffer, len(value))
_, _ = io.WriteString(hasher, value)
} }
func buildOutputArtifact(content string, format domain.OutputFormat) domain.Artifact { func buildOutputArtifact(content string, format domain.OutputFormat) domain.Artifact {

View File

@@ -295,8 +295,10 @@ func (c *controlledRepairLLM) Generate(
ctx context.Context, ctx context.Context,
req domain.GenerateRequest, req domain.GenerateRequest,
) (*domain.GenerateResponse, error) { ) (*domain.GenerateResponse, error) {
isRepair := len(req.Prompt.Messages) > 0 && messages := req.Prompt.Messages
strings.HasPrefix(req.Prompt.Messages[0].Content, "You repair invalid JSON") isRepair := len(messages) >= 2 &&
messages[len(messages)-2].Role == "assistant" &&
messages[len(messages)-1].Role == "user"
c.mu.Lock() c.mu.Lock()
c.calls++ c.calls++
c.active++ c.active++
@@ -1108,14 +1110,18 @@ func TestDeriveStructuredSchemaName(t *testing.T) {
} }
} }
func TestHashRenderedPromptIncludesCacheControlWhenPresent(t *testing.T) { func TestHashRenderedPromptIncludesEveryContractedField(t *testing.T) {
uncached := domain.RenderedPrompt{Messages: []domain.RenderedMessage{ base := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{Role: "system", Content: "sys"}, {Role: domain.RoleSystem, Content: "sys"},
{Role: "user", Content: "usr"}, {Role: domain.RoleUser, Content: "usr"},
}} }}
wantLegacyHash := hashString("system\nsys\n---\nuser\nusr\n---\n") identical := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
if got := hashRenderedPrompt(uncached); got != wantLegacyHash { {Role: domain.RoleSystem, Content: "sys"},
t.Fatalf("expected no-cache hash to preserve legacy input, got %q want %q", got, wantLegacyHash) {Role: domain.RoleUser, Content: "usr"},
}}
baseHash := hashRenderedPrompt(base)
if baseHash != hashRenderedPrompt(identical) {
t.Fatal("identical rendered prompts must have the same hash")
} }
withCache := domain.RenderedPrompt{Messages: []domain.RenderedMessage{ withCache := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
@@ -1127,7 +1133,7 @@ func TestHashRenderedPromptIncludesCacheControlWhenPresent(t *testing.T) {
TTL: "1h", TTL: "1h",
}, },
}, },
{Role: "user", Content: "usr"}, {Role: domain.RoleUser, Content: "usr"},
}} }}
alsoWithCache := domain.RenderedPrompt{Messages: []domain.RenderedMessage{ alsoWithCache := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{ {
@@ -1138,7 +1144,7 @@ func TestHashRenderedPromptIncludesCacheControlWhenPresent(t *testing.T) {
TTL: "1h", TTL: "1h",
}, },
}, },
{Role: "user", Content: "usr"}, {Role: domain.RoleUser, Content: "usr"},
}} }}
withoutTTL := domain.RenderedPrompt{Messages: []domain.RenderedMessage{ withoutTTL := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{ {
@@ -1148,11 +1154,11 @@ func TestHashRenderedPromptIncludesCacheControlWhenPresent(t *testing.T) {
Type: domain.CacheControlEphemeral, Type: domain.CacheControlEphemeral,
}, },
}, },
{Role: "user", Content: "usr"}, {Role: domain.RoleUser, Content: "usr"},
}} }}
cachedHash := hashRenderedPrompt(withCache) cachedHash := hashRenderedPrompt(withCache)
if cachedHash == hashRenderedPrompt(uncached) { if cachedHash == baseHash {
t.Fatal("expected cache control to change rendered prompt hash") t.Fatal("expected cache control to change rendered prompt hash")
} }
if cachedHash != hashRenderedPrompt(alsoWithCache) { if cachedHash != hashRenderedPrompt(alsoWithCache) {
@@ -1161,6 +1167,29 @@ func TestHashRenderedPromptIncludesCacheControlWhenPresent(t *testing.T) {
if cachedHash == hashRenderedPrompt(withoutTTL) { if cachedHash == hashRenderedPrompt(withoutTTL) {
t.Fatal("expected ttl changes to affect rendered prompt hash") t.Fatal("expected ttl changes to affect rendered prompt hash")
} }
variants := []struct {
name string
prompt domain.RenderedPrompt
}{
{name: "role", prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: domain.RoleDeveloper, Content: "sys"}, {Role: domain.RoleUser, Content: "usr"}}}},
{name: "content", prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: domain.RoleSystem, Content: "changed"}, {Role: domain.RoleUser, Content: "usr"}}}},
{name: "order", prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: domain.RoleUser, Content: "usr"}, {Role: domain.RoleSystem, Content: "sys"}}}},
{name: "session", prompt: domain.RenderedPrompt{SessionID: "session", Messages: base.Messages}},
}
for _, variant := range variants {
t.Run(variant.name, func(t *testing.T) {
if hashRenderedPrompt(variant.prompt) == baseHash {
t.Fatalf("%s did not change the rendered-prompt hash", variant.name)
}
})
}
oneMessage := domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: domain.RoleUser, Content: "x\n---\nassistant\ny"}}}
twoMessages := domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: domain.RoleUser, Content: "x"}, {Role: domain.RoleAssistant, Content: "y"}}}
if hashRenderedPrompt(oneMessage) == hashRenderedPrompt(twoMessages) {
t.Fatal("length-framed hashing did not distinguish legacy separator collision")
}
} }
func TestHashRenderedPromptIncludesSessionIDWhenPresent(t *testing.T) { func TestHashRenderedPromptIncludesSessionIDWhenPresent(t *testing.T) {
@@ -1202,6 +1231,75 @@ func TestHashRenderedPromptIncludesSessionIDWhenPresent(t *testing.T) {
} }
} }
func TestRunnerComposesAppendedMessagesIntoEveryRenderedPrompt(t *testing.T) {
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{
SessionID: "rendered-session",
Messages: []domain.RenderedMessage{
{Role: domain.RoleSystem, Content: "ordinary prefix", CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}},
{Role: domain.RoleUser, Content: "ordinary request"},
},
}}
client := &fakeLLM{resp: &domain.GenerateResponse{Content: "output"}}
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
renderer,
client,
nil,
nil,
)
appended := []domain.RenderedMessage{
{Role: domain.RoleAssistant, Content: "consumer response"},
{Role: domain.RoleUser, Content: "consumer correction", CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral}},
}
request := domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), AppendedMessages: appended}
plain, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if err != nil {
t.Fatalf("prepare without appended messages: %v", err)
}
empty, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), AppendedMessages: []domain.RenderedMessage{}})
if err != nil {
t.Fatalf("prepare with empty appended messages: %v", err)
}
if !reflect.DeepEqual(empty.Messages, plain.Messages) || empty.RenderedPromptHash != plain.RenderedPromptHash {
t.Fatalf("empty appended messages changed preparation: empty=%#v plain=%#v", empty, plain)
}
prepared, err := runner.Prepare(context.Background(), request)
if err != nil {
t.Fatalf("prepare with appended messages: %v", err)
}
if prepared.PromptHash != plain.PromptHash || prepared.RenderedPromptHash == plain.RenderedPromptHash {
t.Fatalf("hashes with appended messages = (%q, %q), plain = (%q, %q)", prepared.PromptHash, prepared.RenderedPromptHash, plain.PromptHash, plain.RenderedPromptHash)
}
expected := []domain.RenderedMessage{
{Role: domain.RoleSystem, Content: "ordinary prefix", CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}},
{Role: domain.RoleUser, Content: "ordinary request"},
{Role: domain.RoleAssistant, Content: "consumer response"},
{Role: domain.RoleUser, Content: "consumer correction", CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral}},
}
if !reflect.DeepEqual(prepared.Messages, expected) {
t.Fatalf("prepared messages = %#v, want %#v", prepared.Messages, expected)
}
result, err := runner.Run(context.Background(), request)
if err != nil {
t.Fatalf("run with appended messages: %v", err)
}
if result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(client.lastReq.Prompt.Messages, expected) {
t.Fatalf("run did not use the prepared effective prompt: result=%#v request=%#v prepared=%#v", result, client.lastReq.Prompt.Messages, prepared)
}
renderer.rendered.Messages[0].Content = "changed source"
appended[1].CacheControl.Type = "changed caller value"
if !reflect.DeepEqual(prepared.Messages, expected) {
t.Fatalf("prepared messages changed after source or caller mutation: %#v", prepared.Messages)
}
}
func TestRunnerRunSuccessful(t *testing.T) { func TestRunnerRunSuccessful(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)} promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}
@@ -2143,6 +2241,8 @@ func TestRunnerRepairStateMachine(t *testing.T) {
TopP: true, TopP: true,
TimeoutSeconds: true, TimeoutSeconds: true,
} }
emptyThenValid := responses(2)
emptyThenValid[0].Content = " \t "
tests := []struct { tests := []struct {
name string name string
@@ -2165,12 +2265,13 @@ func TestRunnerRepairStateMachine(t *testing.T) {
wantStatus: domain.ValidationPassed, wantStatus: domain.ValidationPassed,
}, },
{ {
name: "basic failure is ineligible despite budget", name: "empty basic output repairs successfully",
mode: domain.ValidationBasic, mode: domain.ValidationBasic,
budget: 3, budget: 3,
validationResults: []domain.ValidationResult{failed(domain.ValidationBasic, "empty output")}, validationResults: []domain.ValidationResult{failed(domain.ValidationBasic, "empty output"), passed(domain.ValidationBasic)},
responses: responses(1), responses: emptyThenValid,
wantStatus: domain.ValidationFailed, wantRepairs: 1,
wantStatus: domain.ValidationPassed,
}, },
{ {
name: "inherited numeric values remain absent", name: "inherited numeric values remain absent",
@@ -2202,7 +2303,7 @@ func TestRunnerRepairStateMachine(t *testing.T) {
{ {
name: "successful repair stops below larger budget", name: "successful repair stops below larger budget",
mode: domain.ValidationJSON, mode: domain.ValidationJSON,
budget: 4, budget: 3,
validationResults: []domain.ValidationResult{ validationResults: []domain.ValidationResult{
failed(domain.ValidationJSON, "candidate zero"), failed(domain.ValidationJSON, "candidate zero"),
failed(domain.ValidationJSON, "candidate one"), failed(domain.ValidationJSON, "candidate one"),
@@ -2305,6 +2406,9 @@ func TestRunnerRepairStateMachine(t *testing.T) {
!reflect.DeepEqual(req.ValidationErrors, tc.validationResults[index].Errors) { !reflect.DeepEqual(req.ValidationErrors, tc.validationResults[index].Errors) {
t.Fatalf("repair request %d prior state = %+v", index, req) t.Fatalf("repair request %d prior state = %+v", index, req)
} }
if !reflect.DeepEqual(req.OriginalMessages, initialRequest.Prompt.Messages) {
t.Fatalf("repair request %d original messages drifted: %#v", index, req.OriginalMessages)
}
if req.TargetPresence != tc.wantPresence || !reflect.DeepEqual(req.Target, initialRequest.Target) || if req.TargetPresence != tc.wantPresence || !reflect.DeepEqual(req.Target, initialRequest.Target) ||
req.SessionID != initialRequest.Prompt.SessionID || req.SessionID != initialRequest.Prompt.SessionID ||
!reflect.DeepEqual(req.StructuredOutput, initialRequest.StructuredOutput) { !reflect.DeepEqual(req.StructuredOutput, initialRequest.StructuredOutput) {
@@ -2318,8 +2422,19 @@ func TestRunnerRepairStateMachine(t *testing.T) {
!reflect.DeepEqual(generated.StructuredOutput, initialRequest.StructuredOutput) { !reflect.DeepEqual(generated.StructuredOutput, initialRequest.StructuredOutput) {
t.Fatalf("repair generation request %d common fields drifted: %+v", index, generated) t.Fatalf("repair generation request %d common fields drifted: %+v", index, generated)
} }
if reflect.DeepEqual(generated.Prompt.Messages, initialRequest.Prompt.Messages) { expectedMessages := len(initialRequest.Prompt.Messages) + 1
t.Fatalf("repair generation request %d reused the initial prompt", index) if strings.TrimSpace(tc.responses[index].Content) != "" {
expectedMessages++
}
if len(generated.Prompt.Messages) != expectedMessages ||
generated.Prompt.Messages[len(generated.Prompt.Messages)-1].Role != "user" {
t.Fatalf("repair generation request %d messages = %#v", index, generated.Prompt.Messages)
}
if strings.TrimSpace(tc.responses[index].Content) != "" {
assistant := generated.Prompt.Messages[len(generated.Prompt.Messages)-2]
if assistant.Role != "assistant" || assistant.Content != tc.responses[index].Content {
t.Fatalf("repair generation request %d candidate = %+v", index, assistant)
}
} }
} }

View File

@@ -176,7 +176,6 @@ func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract d
res := domain.ValidationResult{ res := domain.ValidationResult{
Mode: contract.ValidationMode, Mode: contract.ValidationMode,
SchemaPath: contract.SchemaPath, SchemaPath: contract.SchemaPath,
RepairAttempts: contract.RepairAttempts,
} }
if artifact == nil { if artifact == nil {

View File

@@ -64,6 +64,43 @@ func TestStandardValidatorBasicFailureEmpty(t *testing.T) {
} }
} }
func TestStandardValidatorReportsZeroRepairAttempts(t *testing.T) {
v := NewStandardValidator("")
for _, tc := range []struct {
name string
body string
contract domain.OutputContract
}{
{
name: "passed basic validation",
body: "answer",
contract: domain.OutputContract{
ValidationMode: domain.ValidationBasic,
RepairAttempts: 3,
},
},
{
name: "failed JSON validation",
body: `{"answer":`,
contract: domain.OutputContract{
ValidationMode: domain.ValidationJSON,
RepairAttempts: 3,
},
},
} {
t.Run(tc.name, func(t *testing.T) {
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(tc.body)}, tc.contract)
if err != nil {
t.Fatalf("validate: %v", err)
}
if res.RepairAttempts != 0 {
t.Fatalf("repair attempts = %d, want 0", res.RepairAttempts)
}
})
}
}
func TestStandardValidatorJSONSuccess(t *testing.T) { func TestStandardValidatorJSONSuccess(t *testing.T) {
v := NewStandardValidator("") v := NewStandardValidator("")

14
message_roles.go Normal file
View File

@@ -0,0 +1,14 @@
package promptkit
import "gitea.maximumdirect.net/eric/promptkit/internal/domain"
const (
// RoleDeveloper identifies a provider-bound developer instruction message.
RoleDeveloper = domain.RoleDeveloper
// RoleSystem identifies a provider-bound system instruction message.
RoleSystem = domain.RoleSystem
// RoleUser identifies a provider-bound user message.
RoleUser = domain.RoleUser
// RoleAssistant identifies a provider-bound assistant message.
RoleAssistant = domain.RoleAssistant
)

View File

@@ -22,13 +22,36 @@ func TestPreparationRejectsInvalidOutputContractWithPublicError(t *testing.T) {
t.Fatalf("construct engine: %v", err) t.Fatalf("construct engine: %v", err)
} }
req := promptkit.RunRequest{ for _, tc := range []struct {
PromptID: "prompt", name string
Validation: &promptkit.OutputContract{ contract promptkit.OutputContract
}{
{
name: "unsupported format",
contract: promptkit.OutputContract{
Format: promptkit.OutputFormat("binary"), Format: promptkit.OutputFormat("binary"),
ValidationMode: promptkit.ValidationNone, ValidationMode: promptkit.ValidationNone,
}, },
} },
{
name: "repair attempts above maximum",
contract: promptkit.OutputContract{
Format: promptkit.FormatText,
ValidationMode: promptkit.ValidationBasic,
RepairAttempts: 4,
},
},
{
name: "none validation with repair attempts",
contract: promptkit.OutputContract{
Format: promptkit.FormatText,
ValidationMode: promptkit.ValidationNone,
RepairAttempts: 1,
},
},
} {
t.Run(tc.name, func(t *testing.T) {
req := promptkit.RunRequest{PromptID: "prompt", Validation: &tc.contract}
prepared, err := engine.Prepare(context.Background(), req) prepared, err := engine.Prepare(context.Background(), req)
if prepared != nil { if prepared != nil {
@@ -45,4 +68,6 @@ func TestPreparationRejectsInvalidOutputContractWithPublicError(t *testing.T) {
if !errors.Is(err, promptkit.ErrInvalidRequest) { if !errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("prepare execution error = %v, want ErrInvalidRequest", err) t.Fatalf("prepare execution error = %v, want ErrInvalidRequest", err)
} }
})
}
} }

View File

@@ -17,7 +17,9 @@ type PreparedExecution struct {
// Details returns a fresh caller-owned, credential-redacted copy of the // Details returns a fresh caller-owned, credential-redacted copy of the
// prepared request details. Mutating the result cannot affect execution or a // prepared request details. Mutating the result cannot affect execution or a
// later Details call. Details remains available after execution or discard. // later Details call. Its complete effective message content, including any
// appended request messages, remains subject to the caller's data-handling
// policy. Details remains available after execution or discard.
// //
// A nil receiver or zero-value PreparedExecution returns a zero [PreparedRun]. // A nil receiver or zero-value PreparedExecution returns a zero [PreparedRun].
func (p *PreparedExecution) Details() PreparedRun { func (p *PreparedExecution) Details() PreparedRun {

View File

@@ -256,6 +256,50 @@ func TestPreparedExecutionLifecycleAndEngineBinding(t *testing.T) {
} }
} }
func TestPreparedExecutionRepairsEmptyBasicOutput(t *testing.T) {
client := &preparedRecordingClient{responses: []*promptkit.GenerateResponse{
{
Content: "",
Usage: promptkit.TokenUsage{PromptTokens: 3, CompletionTokens: 5, TotalTokens: 8},
},
{
Content: "Corrected summary.",
Usage: promptkit.TokenUsage{PromptTokens: 7, CompletionTokens: 11, TotalTokens: 18},
},
}}
engine := newPreparedContractEngine(t, client, "Summarize the source.")
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
PromptID: "prepared",
Validation: &promptkit.OutputContract{
Format: promptkit.FormatMarkdown,
ValidationMode: promptkit.ValidationBasic,
RepairAttempts: 1,
},
})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
details := prepared.Details()
result, err := engine.RunPrepared(context.Background(), prepared)
if err != nil {
t.Fatalf("run prepared: %v", err)
}
if result.RawOutput != "Corrected summary." || result.Validation.Status != promptkit.ValidationPassed ||
result.Validation.RepairAttempts != 1 || result.Usage != (promptkit.TokenUsage{PromptTokens: 10, CompletionTokens: 16, TotalTokens: 26}) {
t.Fatalf("repaired result = %+v", result)
}
requests := client.snapshot()
if len(requests) != 2 || len(requests[1].Prompt.Messages) != len(details.Messages)+1 ||
!reflect.DeepEqual(requests[1].Prompt.Messages[:len(details.Messages)], details.Messages) ||
requests[1].Prompt.Messages[len(requests[1].Prompt.Messages)-1].Role != "user" {
t.Fatalf("prepared repair requests = %#v", requests)
}
if _, err := engine.RunPrepared(context.Background(), prepared); !errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("second RunPrepared error = %v, want ErrInvalidRequest", err)
}
}
func TestPreparedExecutionConcurrentClaimAllowsOneGeneration(t *testing.T) { func TestPreparedExecutionConcurrentClaimAllowsOneGeneration(t *testing.T) {
release := make(chan struct{}) release := make(chan struct{})
client := &preparedRecordingClient{ client := &preparedRecordingClient{
@@ -673,6 +717,7 @@ func (r *mutablePreparedArtifactReader) callCount() int {
type preparedRecordingClient struct { type preparedRecordingClient struct {
mu sync.Mutex mu sync.Mutex
response *promptkit.GenerateResponse response *promptkit.GenerateResponse
responses []*promptkit.GenerateResponse
err error err error
requests []promptkit.GenerateRequest requests []promptkit.GenerateRequest
started chan struct{} started chan struct{}
@@ -700,6 +745,12 @@ func (c *preparedRecordingClient) Generate(
if c.err != nil { if c.err != nil {
return nil, c.err return nil, c.err
} }
if len(c.responses) > 0 {
if index := len(c.requests) - 1; index < len(c.responses) {
return c.responses[index], nil
}
return nil, fmt.Errorf("no response configured for generation %d", len(c.requests))
}
return c.response, nil return c.response, nil
} }

View File

@@ -844,7 +844,7 @@ func TestBackendExtraParamsAreDeeplyCopiedAtConstructionAndLookup(t *testing.T)
} }
} }
func TestEngineValidationIsSinglePass(t *testing.T) { func TestEngineValidationWithZeroRepairBudgetIsSinglePass(t *testing.T) {
client := &fakeLLMClient{ client := &fakeLLMClient{
response: &promptkit.GenerateResponse{Content: "not-json"}, response: &promptkit.GenerateResponse{Content: "not-json"},
} }
@@ -859,7 +859,7 @@ func TestEngineValidationIsSinglePass(t *testing.T) {
Validation: &promptkit.OutputContract{ Validation: &promptkit.OutputContract{
Format: promptkit.FormatJSON, Format: promptkit.FormatJSON,
ValidationMode: promptkit.ValidationJSON, ValidationMode: promptkit.ValidationJSON,
RepairAttempts: 3, RepairAttempts: 0,
}, },
}) })
if err != nil { if err != nil {
@@ -874,6 +874,87 @@ func TestEngineValidationIsSinglePass(t *testing.T) {
} }
} }
func TestEngineRunRepairsJSONSchemaOutput(t *testing.T) {
client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{
{
Content: "{}",
Usage: promptkit.TokenUsage{PromptTokens: 2, CompletionTokens: 3, TotalTokens: 5, CachedTokens: 7, CacheWriteTokens: 11},
},
{
Content: `{"events":[{"title":"Repaired event"}]}`,
Usage: promptkit.TokenUsage{PromptTokens: 13, CompletionTokens: 17, TotalTokens: 19, CachedTokens: 23, CacheWriteTokens: 29},
},
}}
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(client))
result, err := engine.Run(context.Background(), promptkit.RunRequest{
PromptID: frameworkStructuredEventsPromptID,
SessionID: " repair-session ",
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("Rin opens the gate."),
"glossary": promptkit.Inline("gate: A guarded passage."),
},
Validation: &promptkit.OutputContract{
Format: promptkit.FormatJSON,
ValidationMode: promptkit.ValidationJSONSchema,
SchemaPath: "structured_events.schema.json",
RepairAttempts: 1,
},
})
if err != nil {
t.Fatalf("run: %v", err)
}
if result.RawOutput != client.responses[1].Content || result.Validation.Status != promptkit.ValidationPassed ||
result.Validation.RepairAttempts != 1 {
t.Fatalf("repaired result = %+v", result)
}
wantUsage := promptkit.TokenUsage{PromptTokens: 15, CompletionTokens: 20, TotalTokens: 24, CachedTokens: 30, CacheWriteTokens: 40}
if result.Usage != wantUsage {
t.Fatalf("usage = %+v, want %+v", result.Usage, wantUsage)
}
if len(client.requests) != 2 {
t.Fatalf("generation calls = %d, want 2", len(client.requests))
}
initial, repaired := client.requests[0], client.requests[1]
if initial.Prompt.SessionID != "repair-session" || repaired.Prompt.SessionID != initial.Prompt.SessionID ||
!reflect.DeepEqual(repaired.Target, initial.Target) || repaired.TargetPresence != initial.TargetPresence ||
!reflect.DeepEqual(repaired.StructuredOutput, initial.StructuredOutput) {
t.Fatalf("generation request state drifted: initial=%+v repaired=%+v", initial, repaired)
}
if initial.StructuredOutput == nil || initial.StructuredOutput.JSONSchema == nil {
t.Fatalf("expected structured output on initial request: %+v", initial)
}
}
func TestEngineRunReturnsFinalResultAfterRepairExhaustion(t *testing.T) {
client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{
{Content: "not-json", Usage: promptkit.TokenUsage{PromptTokens: 2, CompletionTokens: 3, TotalTokens: 5}},
{Content: "still-not-json", Usage: promptkit.TokenUsage{PromptTokens: 7, CompletionTokens: 11, TotalTokens: 18}},
}}
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(client))
result, err := engine.Run(context.Background(), promptkit.RunRequest{
PromptID: frameworkMarkdownSummaryPromptID,
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("Rin opens the gate."),
"glossary": promptkit.Inline("gate: A guarded passage."),
},
Validation: &promptkit.OutputContract{
Format: promptkit.FormatJSON,
ValidationMode: promptkit.ValidationJSON,
RepairAttempts: 1,
},
})
if err != nil || result == nil {
t.Fatalf("run = (%+v, %v), want exhausted result", result, err)
}
if result.RawOutput != "still-not-json" || result.Validation.Status != promptkit.ValidationFailed ||
result.Validation.RepairAttempts != 1 || len(result.Validation.Errors) == 0 ||
result.Usage != (promptkit.TokenUsage{PromptTokens: 9, CompletionTokens: 14, TotalTokens: 23}) {
t.Fatalf("exhausted result = %+v", result)
}
}
func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) { func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
profile := promptkit.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"} profile := promptkit.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}

424
testdata/builtin-catalog-v1.json vendored Normal file
View File

@@ -0,0 +1,424 @@
{
"backends": [
{
"id": "openrouter",
"endpoint": "https://openrouter.ai/api/v1",
"api_key_env": "OPENROUTER_API_KEY",
"extra_params": null,
"concurrency_limit": 16,
"queue_capacity": 1024,
"queue_capacity_set": true
},
{
"id": "rakestrawhome",
"endpoint": "https://inference.ai.rakestrawhome.com/v1",
"api_key_env": "RAKESTRAWHOME_INFERENCE_API_KEY",
"extra_params": null,
"concurrency_limit": 4,
"queue_capacity": 1024,
"queue_capacity_set": true
}
],
"profiles": [
{
"id": "aion-2",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "aion-labs/aion-2.0",
"temperature": 0.72,
"max_tokens": 0,
"top_p": 0.95,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "claude-fable-latest",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "~anthropic/claude-fable-latest",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 600,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "claude-haiku-latest",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "~anthropic/claude-haiku-latest",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "medium",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "claude-opus-latest",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "~anthropic/claude-opus-latest",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "claude-sonnet-latest",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "~anthropic/claude-sonnet-latest",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "deepseek-3-2",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "deepseek/deepseek-v3.2",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "deepseek-4-flash",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "deepseek/deepseek-v4-flash",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "deepseek-4-pro",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "deepseek/deepseek-v4-pro",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemini-2-flash",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "google/gemini-2.5-flash",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemini-2-flash-lite",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "google/gemini-2.5-flash-lite",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemini-2-pro",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "google/gemini-2.5-pro",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemini-3-flash-lite",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "google/gemini-3.1-flash-lite",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemini-flash-latest",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "~google/gemini-flash-latest",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemini-pro-latest",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "~google/gemini-pro-latest",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemma-4-31b",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "google/gemma-4-31b-it:exacto",
"temperature": 0.15,
"max_tokens": 0,
"top_p": 0.98,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gpt-5-mini",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "openai/gpt-5.4-mini",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gpt-5-nano",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "openai/gpt-5.4-nano",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "minimax-m2",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "minimax/minimax-m2.5",
"temperature": 0.5,
"max_tokens": 0,
"top_p": 0.95,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "minimax-m3",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "minimax/minimax-m3",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "mistral-large-2512",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "mistralai/mistral-large-2512",
"temperature": 0.15,
"max_tokens": 0,
"top_p": 0.98,
"timeout_seconds": 180,
"service_tier": "",
"reasoning_effort": "",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "mistral-medium-3-5",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "mistralai/mistral-medium-3-5",
"temperature": 0.15,
"max_tokens": 0,
"top_p": 0.98,
"timeout_seconds": 180,
"service_tier": "",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "mistral-small-3",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "mistralai/mistral-small-3.2-24b-instruct",
"temperature": 0.05,
"max_tokens": 0,
"top_p": 1,
"timeout_seconds": 180,
"service_tier": "",
"reasoning_effort": "",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "mistral-small-4",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "mistralai/mistral-small-2603",
"temperature": 0.1,
"max_tokens": 0,
"top_p": 0.98,
"timeout_seconds": 180,
"service_tier": "",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "nemotron-3-ultra",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "nvidia/nemotron-3-ultra-550b-a55b",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "rakestrawhome-gemma-4-31b",
"base_profile": "",
"backend": "rakestrawhome",
"endpoint": "",
"model": "google/gemma-4-31b-it",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 0,
"service_tier": "",
"reasoning_effort": "",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
}
]
}

View File

@@ -123,6 +123,21 @@ type RunRequest struct {
// Validation optionally replaces the prompt's complete output contract. It // Validation optionally replaces the prompt's complete output contract. It
// does not merge individual fields. Nil uses the prompt contract. // does not merge individual fields. Nil uses the prompt contract.
Validation *OutputContract Validation *OutputContract
// AppendedMessages are already-rendered messages appended in caller order
// after every prompt definition message. Promptkit neither templates nor
// resolves files in them. Content must be valid UTF-8 and is preserved
// exactly, including empty or whitespace-only content. Roles are trimmed and
// lowercased, then must be [RoleDeveloper], [RoleSystem], [RoleUser], or
// [RoleAssistant]. Any CacheControl is normalized as documented on that
// type.
//
// Nil and empty slices are equivalent. Promptkit imposes no message-count,
// byte-size, token, or context-window limit and does not truncate content;
// an upstream rejection follows the ordinary generation-error contract.
// Prepare, PrepareExecution, and Run validate and copy the messages before
// source or model work. Malformed values return an error matching
// ErrInvalidRequest.
AppendedMessages []RenderedMessage
} }
// PreparedRun contains prepared prompt execution state returned by // PreparedRun contains prepared prompt execution state returned by
@@ -162,8 +177,9 @@ type PreparedRun struct {
SessionID string `json:"session_id,omitempty"` SessionID string `json:"session_id,omitempty"`
// RenderedPromptHash is an opaque equality value for SessionID and Messages. // RenderedPromptHash is an opaque equality value for SessionID and Messages.
RenderedPromptHash string `json:"rendered_prompt_hash"` RenderedPromptHash string `json:"rendered_prompt_hash"`
// Messages are the rendered messages that Run or RunPrepared passes to the // Messages are the complete effective messages that Run or RunPrepared passes
// LLM client. // to the LLM client: definition messages in rendered order followed by any
// RunRequest.AppendedMessages in caller order.
Messages []RenderedMessage `json:"messages"` Messages []RenderedMessage `json:"messages"`
// StartTime is the UTC time at which preparation began. // StartTime is the UTC time at which preparation began.
StartTime time.Time `json:"start_time,omitempty"` StartTime time.Time `json:"start_time,omitempty"`
@@ -565,8 +581,8 @@ type ExecutionTargetPresence struct {
// JSON representation. // JSON representation.
// //
// A non-nil RunRequest.Validation replaces the complete prompt contract. It // A non-nil RunRequest.Validation replaces the complete prompt contract. It
// does not merge fields. The public Engine validates generated output once and // does not merge fields. The public Engine performs bounded correction after a
// does not install an output repairer. // failed eligible validation when RepairAttempts is positive.
type OutputContract struct { type OutputContract struct {
// Format selects generated artifact metadata. An empty value in a non-nil // Format selects generated artifact metadata. An empty value in a non-nil
// request replacement defaults to FormatText. // request replacement defaults to FormatText.
@@ -577,9 +593,9 @@ type OutputContract struct {
// SchemaPath is required when ValidationMode is ValidationJSONSchema and is // SchemaPath is required when ValidationMode is ValidationJSONSchema and is
// ignored by other modes. // ignored by other modes.
SchemaPath string `json:"schema_path"` SchemaPath string `json:"schema_path"`
// RepairAttempts is a non-negative requested repair limit. Zero requests no // RepairAttempts is an additional generation-call budget from zero through
// repairs. The public Engine performs no repairs even when this value is // three. Zero is single-pass. A positive value is valid only with basic,
// positive, so its runs report zero attempts used. // json, or json_schema validation.
RepairAttempts int `json:"repair_attempts"` RepairAttempts int `json:"repair_attempts"`
} }
@@ -595,8 +611,8 @@ type ValidationResult struct {
Errors []string `json:"errors,omitempty"` Errors []string `json:"errors,omitempty"`
// SchemaPath is the effective schema path for JSON Schema validation. // SchemaPath is the effective schema path for JSON Schema validation.
SchemaPath string `json:"schema_path,omitempty"` SchemaPath string `json:"schema_path,omitempty"`
// RepairAttempts is the number of repairs actually attempted. It is always // RepairAttempts is the number of corrective generation calls actually
// zero for the public Engine. // started for this result.
RepairAttempts int `json:"repair_attempts"` RepairAttempts int `json:"repair_attempts"`
// IsValid is true for ValidationPassed and ValidationSkipped and false for // IsValid is true for ValidationPassed and ValidationSkipped and false for
// ValidationFailed. // ValidationFailed.
@@ -625,23 +641,29 @@ type RenderedPrompt struct {
// SessionID is the optional effective direct or rendered session // SessionID is the optional effective direct or rendered session
// identifier supplied to the model client. // identifier supplied to the model client.
SessionID string `json:"session_id,omitempty"` SessionID string `json:"session_id,omitempty"`
// Messages contains rendered messages in definition order. // Messages contains the effective messages in provider order: definition
// messages in rendered order followed by any RunRequest.AppendedMessages.
Messages []RenderedMessage `json:"messages"` Messages []RenderedMessage `json:"messages"`
} }
// RenderedMessage is a rendered chat message and has a stable JSON // RenderedMessage is a prepared, provider-bound text chat message and has a
// representation. // stable JSON representation. At a request boundary its role is trimmed and
// lowercased, then must be one of [RoleDeveloper], [RoleSystem], [RoleUser],
// or [RoleAssistant].
type RenderedMessage struct { type RenderedMessage struct {
// Role is the definition-supplied chat role. // Role is the provider-bound chat role.
Role string `json:"role"` Role string `json:"role"`
// Content is the rendered message text. // Content is the provider-bound message text. Request-supplied content must
// be valid UTF-8 and may be empty or whitespace-only.
Content string `json:"content"` Content string `json:"content"`
// CacheControl is optional provider cache metadata. // CacheControl is optional provider cache metadata.
CacheControl *CacheControl `json:"cache_control,omitempty"` CacheControl *CacheControl `json:"cache_control,omitempty"`
} }
// CacheControl describes provider cache metadata attached to prompt content // CacheControl describes provider cache metadata attached to prompt content
// and has a stable JSON representation. // and has a stable JSON representation. At a request boundary Type and TTL
// must be valid UTF-8 and are trimmed. Type must be [CacheControlEphemeral],
// and TTL must be empty or "1h"; other values make the request invalid.
type CacheControl struct { type CacheControl struct {
// Type identifies the cache behavior. // Type identifies the cache behavior.
Type CacheControlType `json:"type"` Type CacheControlType `json:"type"`
@@ -714,9 +736,8 @@ type GenerateRequest struct {
// GenerateResponse is returned by an injected LLM client and has a stable JSON // GenerateResponse is returned by an injected LLM client and has a stable JSON
// representation. // representation.
type GenerateResponse struct { type GenerateResponse struct {
// Content is the generated output. It must be non-empty when using the // Content is the generated output. It may be explicitly empty; Promptkit
// built-in client; injected clients may return empty content for Promptkit // applies the effective output contract to classify it.
// validation to classify.
Content string `json:"content"` Content string `json:"content"`
// Usage is the client's token accounting. // Usage is the client's token accounting.
Usage TokenUsage `json:"usage"` Usage TokenUsage `json:"usage"`