Compare commits
62 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 15e4067954 | |||
| 6e34fb9de5 | |||
| 5843ceef82 | |||
| 1cd0c611f2 | |||
| 6ed582f4b5 | |||
| db05b56166 | |||
| 0c08038b4f | |||
| 48378a3269 | |||
| cef8524b31 | |||
| 1cc754c29e | |||
| 93e53d1199 | |||
| 63bedf2a97 | |||
| 0e1ad784d1 | |||
| be07f6d873 | |||
| 836919ec4b | |||
| 07a131bd33 | |||
| ae4540822e | |||
| 65d5894648 | |||
| d31a563ca1 | |||
| ea94d66fc2 | |||
| 056bfd0289 | |||
| bd975f17f2 | |||
| 4431f34bc9 | |||
| 1580b25ea6 | |||
| 6dfd6bd719 | |||
| 9f69da716b | |||
| 5dd5b2f3cc | |||
| d79f1e9562 | |||
| c8e4abcdbd | |||
| 6ce54c1d79 | |||
| 2356f7c73d | |||
| ba1e85a805 | |||
| 4c3828857c | |||
| 02fd09955d | |||
| 7389410966 | |||
| 3bc624ec3f | |||
| e5870453f9 | |||
| 06251e385e | |||
| 722b5ddf59 | |||
| 47bc9a6655 | |||
| bd2b7b1862 | |||
| c892eacb65 | |||
| ff73104f7e | |||
| 7a858c2365 | |||
| e4540aa619 | |||
| a4264bf4b4 | |||
| 81f41564a2 | |||
| 6b5a2497cc | |||
| 422cc6c978 | |||
| f48e042565 | |||
| d9442850ef | |||
| 2e828157b6 | |||
| 4189146536 | |||
| 59be507d2f | |||
| b45cea4084 | |||
| efe885c6b2 | |||
| 404a4c331d | |||
| 174516cb39 | |||
| b0999112cc | |||
| ed0f7527d5 | |||
| 5064cf833d | |||
| 8745d256bd |
@@ -1,2 +1,5 @@
|
||||
Please review `docs/development.md` for initial orientation in this repository
|
||||
and follow its task-specific reading guide.
|
||||
|
||||
Review and follow `docs/policy/agents.md` before recommending, planning, or
|
||||
implementing any changes to this repository.
|
||||
|
||||
19
COPYRIGHT
Normal file
19
COPYRIGHT
Normal file
@@ -0,0 +1,19 @@
|
||||
PromptKit
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
This library is free software: you can redistribute it and/or modify it
|
||||
under the terms of the GNU Lesser General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
The GNU Lesser General Public License version 3 is provided in LICENSE.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
Third-party code and assets are subject to their respective license terms.
|
||||
21
README.md
21
README.md
@@ -33,7 +33,20 @@ boundary and constraints that framework work must preserve.
|
||||
|
||||
## Release Guidance
|
||||
|
||||
Consumers upgrading from `v0.7.0` to `v0.8.0` should read the
|
||||
The [v0.11.1 release notes](docs/releases/v0.11.1.md) cover the MIT-licensed
|
||||
catalog updates and licensing documentation improvements.
|
||||
|
||||
Consumers upgrading to `v0.11.0` should read the
|
||||
[v0.11.0 changelog and migration guide](docs/releases/v0.11.0.md), particularly
|
||||
the default retry behavior and license change.
|
||||
|
||||
Consumers upgrading to the `v0.10.0` profile-policy transition
|
||||
should read the [v0.10.0 transition guide](docs/releases/v0.10.0.md).
|
||||
|
||||
Consumers upgrading from `v0.8.0` to `v0.9.0` should read the
|
||||
[v0.9.0 changelog and migration guide](docs/releases/v0.9.0.md).
|
||||
|
||||
Consumers upgrading from `v0.7.0` to `v0.8.0` can consult the
|
||||
[v0.8.0 changelog and migration guide](docs/releases/v0.8.0.md).
|
||||
|
||||
Earlier adopters can consult the
|
||||
@@ -59,4 +72,8 @@ Consumers moving from `v0.1.0` to `v0.2.0` should read the
|
||||
[Scriptorium](https://gitea.maximumdirect.net/eric/scriptorium) is the CLI and
|
||||
HTTP application built on Promptkit.
|
||||
|
||||
Promptkit is licensed under the [GNU General Public License version 3](LICENSE).
|
||||
## License
|
||||
|
||||
Promptkit is licensed under the [LGPL v3.0 or later](COPYRIGHT).
|
||||
|
||||
[Open Source attribution notices](docs/oss/attribution.md) are available.
|
||||
|
||||
145
appended_messages_contract_test.go
Normal file
145
appended_messages_contract_test.go
Normal file
@@ -0,0 +1,145 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestAppendedMessageRoleConstantsAreStrings(t *testing.T) {
|
||||
var (
|
||||
developer string = promptkit.RoleDeveloper
|
||||
system string = promptkit.RoleSystem
|
||||
user string = promptkit.RoleUser
|
||||
assistant string = promptkit.RoleAssistant
|
||||
)
|
||||
if developer != "developer" || system != "system" || user != "user" || assistant != "assistant" {
|
||||
t.Fatalf("unexpected role constants: %q %q %q %q", developer, system, user, assistant)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendedMessagesComposeFrozenEffectivePrompt(t *testing.T) {
|
||||
cacheControl := &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}
|
||||
appended := []promptkit.RenderedMessage{
|
||||
{Role: " \tAsSiStAnT\n", Content: "previous response"},
|
||||
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: cacheControl},
|
||||
}
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "output"}}
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "configured prompt"), "."),
|
||||
promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
request := promptkit.RunRequest{PromptID: "prompt", AppendedMessages: appended}
|
||||
|
||||
plain, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare plain request: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare appended request: %v", err)
|
||||
}
|
||||
expected := []promptkit.RenderedMessage{
|
||||
{Role: promptkit.RoleUser, Content: "configured prompt"},
|
||||
{Role: promptkit.RoleAssistant, Content: "previous response"},
|
||||
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}},
|
||||
}
|
||||
if !reflect.DeepEqual(prepared.Messages, expected) || prepared.PromptHash != plain.PromptHash || prepared.RenderedPromptHash == plain.RenderedPromptHash {
|
||||
t.Fatalf("prepared effective prompt = %#v, plain = %#v", prepared, plain)
|
||||
}
|
||||
|
||||
equivalent, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{
|
||||
{Role: promptkit.RoleAssistant, Content: "previous response"},
|
||||
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}},
|
||||
}})
|
||||
if err != nil || equivalent.RenderedPromptHash != prepared.RenderedPromptHash {
|
||||
t.Fatalf("normalized-equivalent request = (%#v, %v), want matching rendered hash %q", equivalent, err, prepared.RenderedPromptHash)
|
||||
}
|
||||
|
||||
for _, variant := range []promptkit.RunRequest{
|
||||
{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleAssistant, Content: "changed"}, expected[2]}},
|
||||
{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{expected[2], expected[1]}},
|
||||
{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleAssistant, Content: "previous response"}, {Role: promptkit.RoleUser, Content: "corrective request"}}},
|
||||
} {
|
||||
variantPrepared, prepareErr := engine.Prepare(context.Background(), variant)
|
||||
if prepareErr != nil || variantPrepared.RenderedPromptHash == prepared.RenderedPromptHash || variantPrepared.PromptHash != prepared.PromptHash {
|
||||
t.Fatalf("variant preparation = (%#v, %v)", variantPrepared, prepareErr)
|
||||
}
|
||||
}
|
||||
|
||||
result, err := engine.Run(context.Background(), request)
|
||||
if err != nil || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(client.requests[0].Prompt.Messages, expected) {
|
||||
t.Fatalf("run result = (%#v, %v), request = %#v", result, err, client.requests)
|
||||
}
|
||||
|
||||
execution, err := engine.PrepareExecution(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
appended[0].Content = "changed caller content"
|
||||
cacheControl.TTL = ""
|
||||
firstDetails := execution.Details()
|
||||
firstDetails.Messages[1].Content = "changed details content"
|
||||
firstDetails.Messages[2].CacheControl.Type = "changed details cache"
|
||||
secondDetails := execution.Details()
|
||||
if !reflect.DeepEqual(secondDetails.Messages, expected) || secondDetails.RenderedPromptHash != prepared.RenderedPromptHash {
|
||||
t.Fatalf("prepared execution details = %#v, want frozen %#v", secondDetails, expected)
|
||||
}
|
||||
|
||||
result, err = engine.RunPrepared(context.Background(), execution)
|
||||
if err != nil || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(client.requests[1].Prompt.Messages, expected) {
|
||||
t.Fatalf("prepared result = (%#v, %v), requests = %#v", result, err, client.requests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidAppendedMessagesFailBeforeSourceOrModelWork(t *testing.T) {
|
||||
promptSource := &inspectionCountingFS{}
|
||||
var modelCalls atomic.Int64
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFS(promptSource, "."),
|
||||
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
request := promptkit.RunRequest{
|
||||
PromptID: "unreached",
|
||||
AppendedMessages: []promptkit.RenderedMessage{{
|
||||
Role: "unsupported-role",
|
||||
Content: "sensitive appended content",
|
||||
}},
|
||||
}
|
||||
|
||||
operations := []struct {
|
||||
name string
|
||||
run func() error
|
||||
}{
|
||||
{name: "Prepare", run: func() error { _, err := engine.Prepare(context.Background(), request); return err }},
|
||||
{name: "PrepareExecution", run: func() error { _, err := engine.PrepareExecution(context.Background(), request); return err }},
|
||||
{name: "Run", run: func() error { _, err := engine.Run(context.Background(), request); return err }},
|
||||
}
|
||||
for _, operation := range operations {
|
||||
t.Run(operation.name, func(t *testing.T) {
|
||||
err := operation.run()
|
||||
if !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("error = %v, want ErrInvalidRequest", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
if promptSource.opens.Load() != 0 {
|
||||
t.Fatalf("invalid appended message opened prompt sources %d times", promptSource.opens.Load())
|
||||
}
|
||||
if modelCalls.Load() != 0 {
|
||||
t.Fatalf("invalid appended message invoked the model %d times", modelCalls.Load())
|
||||
}
|
||||
}
|
||||
@@ -93,7 +93,7 @@ func WithBackend(backend Backend) Option {
|
||||
queueCapacity = *backend.QueueCapacity
|
||||
}
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
options.backends = append(options.backends, domain.Backend{
|
||||
options.internal.AddBackend(domain.Backend{
|
||||
ID: backend.ID,
|
||||
Endpoint: backend.Endpoint,
|
||||
APIKeyEnv: backend.APIKeyEnv,
|
||||
|
||||
88
convert.go
88
convert.go
@@ -1,7 +1,9 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
@@ -12,19 +14,59 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
||||
if err != nil {
|
||||
return domain.RunRequest{}, err
|
||||
}
|
||||
appendedMessages, err := toDomainAppendedMessages(req.AppendedMessages)
|
||||
if err != nil {
|
||||
return domain.RunRequest{}, err
|
||||
}
|
||||
var observer domain.ExecutionObserver
|
||||
if req.Observer != nil {
|
||||
callback := req.Observer
|
||||
observer = func(event domain.ExecutionEvent) { callback(fromDomainExecutionEvent(event)) }
|
||||
}
|
||||
return domain.RunRequest{
|
||||
PromptID: req.PromptID,
|
||||
PromptVersion: req.PromptVersion,
|
||||
ProfileID: req.ProfileID,
|
||||
SessionID: req.SessionID,
|
||||
APIKey: req.APIKey,
|
||||
Inputs: toDomainArtifactRefMap(req.Inputs),
|
||||
Vars: copyStringMap(req.Vars),
|
||||
Execution: execution,
|
||||
Validation: toDomainOutputContractPtr(req.Validation),
|
||||
PromptID: req.PromptID,
|
||||
PromptVersion: req.PromptVersion,
|
||||
ProfileID: req.ProfileID,
|
||||
SessionID: req.SessionID,
|
||||
APIKey: req.APIKey,
|
||||
Inputs: toDomainArtifactRefMap(req.Inputs),
|
||||
Vars: copyStringMap(req.Vars),
|
||||
Execution: execution,
|
||||
Validation: toDomainOutputContractPtr(req.Validation),
|
||||
AppendedMessages: appendedMessages,
|
||||
TransientRetries: copyIntPtr(req.TransientRetries),
|
||||
Observer: observer,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func toDomainAppendedMessages(messages []RenderedMessage) ([]domain.RenderedMessage, error) {
|
||||
if messages == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
converted := make([]domain.RenderedMessage, len(messages))
|
||||
for index, message := range messages {
|
||||
if !utf8.ValidString(message.Content) {
|
||||
return nil, fmt.Errorf("appended message %d content must be valid UTF-8", index)
|
||||
}
|
||||
role, err := domain.NormalizeMessageRole(message.Role)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("appended message %d role: %w", index, err)
|
||||
}
|
||||
|
||||
cacheControl, err := domain.NormalizeCacheControl(toDomainCacheControl(message.CacheControl))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("appended message %d cache_control: %w", index, err)
|
||||
}
|
||||
converted[index] = domain.RenderedMessage{
|
||||
Role: role,
|
||||
Content: message.Content,
|
||||
CacheControl: cacheControl,
|
||||
}
|
||||
}
|
||||
return converted, nil
|
||||
}
|
||||
|
||||
func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
||||
if prepared == nil {
|
||||
return nil
|
||||
@@ -90,8 +132,22 @@ func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse {
|
||||
return nil
|
||||
}
|
||||
return &domain.GenerateResponse{
|
||||
Content: resp.Content,
|
||||
Usage: toDomainTokenUsage(resp.Usage),
|
||||
Content: resp.Content,
|
||||
Usage: toDomainTokenUsage(resp.Usage),
|
||||
UsagePresence: toDomainTokenUsagePresence(resp.UsagePresence),
|
||||
}
|
||||
}
|
||||
|
||||
func toDomainTokenUsagePresence(presence *TokenUsagePresence) *domain.TokenUsagePresence {
|
||||
if presence == nil {
|
||||
return nil
|
||||
}
|
||||
return &domain.TokenUsagePresence{
|
||||
PromptTokens: presence.PromptTokens,
|
||||
CompletionTokens: presence.CompletionTokens,
|
||||
TotalTokens: presence.TotalTokens,
|
||||
CachedTokens: presence.CachedTokens,
|
||||
CacheWriteTokens: presence.CacheWriteTokens,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -285,6 +341,16 @@ func fromDomainRenderedMessages(messages []domain.RenderedMessage) []RenderedMes
|
||||
return out
|
||||
}
|
||||
|
||||
func toDomainCacheControl(cacheControl *CacheControl) *domain.CacheControl {
|
||||
if cacheControl == nil {
|
||||
return nil
|
||||
}
|
||||
return &domain.CacheControl{
|
||||
Type: domain.CacheControlType(cacheControl.Type),
|
||||
TTL: cacheControl.TTL,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl {
|
||||
if cacheControl == nil {
|
||||
return nil
|
||||
|
||||
108
convert_appended_messages_test.go
Normal file
108
convert_appended_messages_test.go
Normal file
@@ -0,0 +1,108 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestToDomainAppendedMessages(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
messages []RenderedMessage
|
||||
want []domain.RenderedMessage
|
||||
wantErr string
|
||||
privateRole string
|
||||
privateText string
|
||||
}{
|
||||
{
|
||||
name: "normalizes supported roles",
|
||||
messages: []RenderedMessage{
|
||||
{Role: " DeVeLoPeR ", Content: "developer"},
|
||||
{Role: "SyStEm", Content: "system"},
|
||||
{Role: "\tUsEr\n", Content: "user"},
|
||||
{Role: "assistant", Content: "assistant"},
|
||||
},
|
||||
want: []domain.RenderedMessage{
|
||||
{Role: domain.RoleDeveloper, Content: "developer"},
|
||||
{Role: domain.RoleSystem, Content: "system"},
|
||||
{Role: domain.RoleUser, Content: "user"},
|
||||
{Role: domain.RoleAssistant, Content: "assistant"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid role UTF-8",
|
||||
messages: []RenderedMessage{{Role: string([]byte{0xff}), Content: "private-content"}},
|
||||
wantErr: "appended message 0 role",
|
||||
},
|
||||
{
|
||||
name: "invalid content UTF-8",
|
||||
messages: []RenderedMessage{{Role: "private-role", Content: string([]byte{0xff})}},
|
||||
wantErr: "appended message 0 content",
|
||||
},
|
||||
{
|
||||
name: "unsupported role",
|
||||
messages: []RenderedMessage{{Role: "private-role", Content: "private-content"}},
|
||||
wantErr: "appended message 0 role",
|
||||
privateRole: "private-role",
|
||||
privateText: "private-content",
|
||||
},
|
||||
{
|
||||
name: "invalid cache control",
|
||||
messages: []RenderedMessage{{Role: RoleUser, Content: "private-content", CacheControl: &CacheControl{Type: "private-cache"}}},
|
||||
wantErr: "appended message 0 cache_control",
|
||||
privateText: "private-content",
|
||||
},
|
||||
{
|
||||
name: "empty and whitespace content",
|
||||
messages: []RenderedMessage{
|
||||
{Role: RoleUser, Content: ""},
|
||||
{Role: RoleAssistant, Content: " \t\n "},
|
||||
},
|
||||
want: []domain.RenderedMessage{
|
||||
{Role: domain.RoleUser, Content: ""},
|
||||
{Role: domain.RoleAssistant, Content: " \t\n "},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, err := toDomainAppendedMessages(test.messages)
|
||||
if test.wantErr != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), test.wantErr) {
|
||||
t.Fatalf("error = %v, want %q", err, test.wantErr)
|
||||
}
|
||||
for _, privateValue := range []string{test.privateRole, test.privateText} {
|
||||
if privateValue != "" && strings.Contains(err.Error(), privateValue) {
|
||||
t.Fatalf("error exposed appended message data %q: %v", privateValue, err)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("toDomainAppendedMessages() error = %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, test.want) {
|
||||
t.Fatalf("toDomainAppendedMessages() = %#v, want %#v", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestToDomainAppendedMessagesCopiesCacheControl(t *testing.T) {
|
||||
cacheControl := &CacheControl{Type: CacheControlEphemeral, TTL: "1h"}
|
||||
messages := []RenderedMessage{{Role: RoleUser, Content: "content", CacheControl: cacheControl}}
|
||||
request, err := toDomainRunRequest(RunRequest{AppendedMessages: messages})
|
||||
if err != nil {
|
||||
t.Fatalf("toDomainRunRequest() error = %v", err)
|
||||
}
|
||||
|
||||
messages[0].Content = "changed"
|
||||
cacheControl.TTL = ""
|
||||
if request.AppendedMessages[0].Content != "content" || request.AppendedMessages[0].CacheControl.TTL != "1h" {
|
||||
t.Fatalf("domain request did not retain an independent appended-message copy: %#v", request.AppendedMessages)
|
||||
}
|
||||
}
|
||||
3
doc.go
3
doc.go
@@ -55,7 +55,8 @@
|
||||
// [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile],
|
||||
// [OpenAICompatibleProfileConfig], [ProfileInspection],
|
||||
// [PromptInputDefinition], [PromptInspection], [PreparedExecution],
|
||||
// [CapacityError], and [GenerationError], do not have stable JSON
|
||||
// [CapacityError], [GenerationError], [RetryExhaustedError], and
|
||||
// [ExecutionEvent], do not have stable JSON
|
||||
// representations. Direct API keys are nevertheless excluded from JSON for
|
||||
// every public value.
|
||||
|
||||
|
||||
@@ -151,6 +151,14 @@ handling policy. The
|
||||
[engine operation GoDoc](../../engine.go) own exact lifecycle, engine-binding,
|
||||
credential, cancellation, timing, and error semantics.
|
||||
|
||||
If the request supplied `Observer`, successful preparation retains that
|
||||
function for `RunPrepared`; preparation itself emits nothing. Observation
|
||||
starts only when the owning engine successfully claims the handle. Reused,
|
||||
discarded, foreign, and invalid handles emit nothing. `Discard` clears the
|
||||
retained callback without invoking it. The function's closure is retained by
|
||||
identity, so synchronize any shared state it uses. Prepared start events can
|
||||
include identities already resolved in the frozen snapshot.
|
||||
|
||||
## Execute And Validate
|
||||
|
||||
[`Engine.Run`](../../engine.go) performs the same preparation, invokes the
|
||||
@@ -173,6 +181,41 @@ semantics. The
|
||||
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
|
||||
owns the built-in client's outbound HTTP behavior.
|
||||
|
||||
### Observe An Ordinary Run
|
||||
|
||||
Set `RunRequest.Observer` to receive synchronous metadata events from `Run`:
|
||||
|
||||
```go
|
||||
request.Observer = func(event promptkit.ExecutionEvent) {
|
||||
// Translate selected metadata into an application-owned record.
|
||||
}
|
||||
result, err := engine.Run(ctx, request)
|
||||
```
|
||||
|
||||
An ordinary run emits `started` before request conversion, followed by any
|
||||
attempt and retry-delay completions, then one `finished` event on a normal
|
||||
return. `Prepare` and `PrepareExecution` emit no events. A start event has no
|
||||
resolved prompt, profile, or model; later events gain identities only as
|
||||
selection succeeds. The terminal event has dispatched-attempt, transient-retry,
|
||||
repair-call, duration, and presence-aware usage totals. Its usage is an
|
||||
aggregate, not an additional billable attempt. A known zero differs from an
|
||||
unknown count, and incomplete totals must not be treated as complete cost
|
||||
accounting. A failed operation still returns no partial `RunResult`.
|
||||
An injected client that leaves `UsagePresence` nil can report known nonzero
|
||||
counts, but its zero counts remain unknown to observations; supply explicit
|
||||
presence when measured zeros matter. Existing `RunResult.Usage` sums returned
|
||||
values independently of observation presence.
|
||||
|
||||
Callbacks run synchronously, in order for one operation, and can overlap for
|
||||
different operations. Synchronize shared callback state and avoid blocking on
|
||||
work that needs the same run's admission slot. Callback panics propagate;
|
||||
there is no durable event history or promise of a terminal event after panic.
|
||||
Events omit content and error objects, but selected identifiers may still be
|
||||
sensitive. See the [observation GoDoc](../../observations.go) for exact fields.
|
||||
The [offline proposal and assessment example](../../examples/go-library/observe/main.go)
|
||||
shows application-owned event collection and a separate semantic acceptance
|
||||
decision using the built-in HTTP path without network access.
|
||||
|
||||
### Repair A Structured Result
|
||||
|
||||
Set a small additional-call budget when a structurally invalid result can be
|
||||
@@ -187,15 +230,49 @@ request.Validation = &promptkit.OutputContract{
|
||||
}
|
||||
```
|
||||
|
||||
Each repair attempt is another model call, so it can increase latency and
|
||||
Each repair attempt is another model generation, so it can increase latency and
|
||||
usage; `RunResult.Usage` is cumulative and `Validation.RepairAttempts` reports
|
||||
calls actually started. Exhaustion still returns the final failed validation
|
||||
result. `basic` validation can also repair an empty candidate, but structural
|
||||
repair generations rather than transient HTTP attempts. Exhaustion still
|
||||
returns the final failed validation result. `basic` validation can also repair
|
||||
an empty candidate, but structural
|
||||
validity is not evidence of factual or domain correctness. See the
|
||||
[output-contract format reference](../formats.md#output-contract) and
|
||||
[`OutputContract` GoDoc](../../types.go) for the exact budget and eligibility
|
||||
rules.
|
||||
|
||||
The built-in client also retries eligible temporary HTTP statuses. By default,
|
||||
each generation can make two additional attempts; a repair generation receives
|
||||
its own budget. Set `Config.TransientRetries` to a pointer to zero for the
|
||||
previous single-attempt behavior, or set `RunRequest.TransientRetries` to
|
||||
override one execution. A custom HTTP transport still uses this policy;
|
||||
injected `LLMClient` implementations retain their own retry policy. Application
|
||||
semantic retries are separate runs and can multiply provider traffic further.
|
||||
The caller context covers capacity and retry waiting as well as attempts;
|
||||
without a deadline a valid provider delay can be long. Promptkit does not
|
||||
infer exactly-once or complete cost accounting from a retry outcome.
|
||||
See the [provider integration contract](../integrations/openai-compatible-chat.md#transient-status-retries)
|
||||
for eligible statuses, waits, timeouts, and provider timing.
|
||||
|
||||
### Append Already-Rendered Messages
|
||||
|
||||
An application can include an earlier assistant response and its own corrective
|
||||
instruction in a fresh request without changing the configured prompt:
|
||||
|
||||
```go
|
||||
request.AppendedMessages = []promptkit.RenderedMessage{
|
||||
{Role: promptkit.RoleAssistant, Content: previousResponse},
|
||||
{Role: promptkit.RoleUser, Content: correction},
|
||||
}
|
||||
result, err := engine.Run(ctx, request)
|
||||
```
|
||||
|
||||
These messages are already rendered: Promptkit does not template or resolve
|
||||
files in them, and they can contain sensitive model output or application
|
||||
feedback. Promptkit remains stateless; every `Run` call re-resolves its current
|
||||
sources and the application owns any semantic retry budget. When a
|
||||
pre-execution equality check is required, use `PrepareExecution` and compare
|
||||
its opaque rendered-prompt hash before invoking `RunPrepared`.
|
||||
|
||||
## Inputs, Profiles, And Overrides
|
||||
|
||||
Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request
|
||||
@@ -212,6 +289,29 @@ For programmatic profiles,
|
||||
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
|
||||
OpenAI-compatible settings into a value accepted by `WithProfiles`.
|
||||
|
||||
### Choose A Profile Ownership Model
|
||||
|
||||
Choose the profile source that matches the application's continuity needs:
|
||||
|
||||
- Select `promptkit-openrouter-standard` or `promptkit-openrouter-auto` when
|
||||
the maintained literal name is the desired consumer contract. Their current
|
||||
selections are in the [format reference](../formats.md#built-in-profile-catalog).
|
||||
- Select an external catalog profile directly when the application accepts that
|
||||
profile's catalog volatility. The selected external inventories are linked
|
||||
from the [format reference](../formats.md#external-profile-inventories).
|
||||
- Define a complete application-owned profile with an explicit backend or
|
||||
endpoint, model, and settings when the application owns those choices.
|
||||
|
||||
An application profile that inherits an external base follows later changes to
|
||||
that base and fails if the base disappears; it is not a frozen copy. Promptkit
|
||||
tests the catalog versions selected in its module graph. Independently advancing
|
||||
a catalog dependency can remove an inherited base and produce an untested
|
||||
combination. Pinning the resolved module graph preserves the embedded data for
|
||||
a build, but cannot preserve provider availability. Running binaries neither
|
||||
download catalog updates nor automatically replace withdrawn models. See the
|
||||
[release policy](../policy/release.md#catalog-profile-compatibility) for the exact
|
||||
cross-version guarantees.
|
||||
|
||||
### Alias A Built-In Profile
|
||||
|
||||
Give an application-owned profile ID a built-in base when prompts should select
|
||||
@@ -221,7 +321,7 @@ only the setting it owns:
|
||||
```go
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "weather-light",
|
||||
BaseProfileID: "deepseek-4-flash",
|
||||
BaseProfileID: "promptkit-openrouter-standard",
|
||||
ReasoningEffort: "high",
|
||||
})
|
||||
```
|
||||
@@ -230,7 +330,9 @@ Select `weather-light` in a prompt or `RunRequest.ProfileID`; it remains the
|
||||
reported selected profile. See the [profile inheritance format
|
||||
reference](../formats.md#profile-inheritance) and the
|
||||
[`Profile` GoDoc](../../types.go) for exact lookup, merging, and validation
|
||||
behavior.
|
||||
behavior. If the base is an external profile, apply the volatility guidance
|
||||
above when deciding whether to use an application-owned complete definition
|
||||
instead.
|
||||
|
||||
### Use The Rakestrawhome Built-In Profile
|
||||
|
||||
@@ -503,16 +605,30 @@ var generationErr *promptkit.GenerationError
|
||||
if errors.As(err, &generationErr) {
|
||||
status := generationErr.StatusCode()
|
||||
message := generationErr.ProviderMessage()
|
||||
_, _ = status, message // Apply application retry and presentation policy.
|
||||
_, _ = status, message // Apply application presentation policy.
|
||||
}
|
||||
```
|
||||
|
||||
All provider fields are untrusted and can contain sensitive request or schema
|
||||
fragments. Do not log, display, or return them without an application-specific
|
||||
disclosure policy. Promptkit does not assign retry or presentation behavior.
|
||||
disclosure policy. Promptkit does not assign presentation behavior.
|
||||
The [`GenerationError` GoDoc](../../generation_error.go) owns the exact typed
|
||||
error contract.
|
||||
|
||||
Generation failures also match `ErrTransport` when the built-in HTTP request
|
||||
fails, or `ErrMalformedResponse` when a successful provider response cannot be
|
||||
decoded or an injected client returns a nil successful response. Both retain
|
||||
`ErrLLMGenerate`; transport failures retain their original causes for
|
||||
`errors.Is` and `errors.As`. Caller cancellation and deadlines retain their
|
||||
standard context identities. Arbitrary injected-client errors retain their own
|
||||
identities without Promptkit guessing a failure category from their text.
|
||||
An eligible built-in status that consumes a positive retry budget additionally
|
||||
matches `ErrRetriesExhausted`; use `errors.As` to read
|
||||
`RetryExhaustedError.Attempts()` and the final `GenerationError` status. With
|
||||
explicit zero retries, the first eligible status is an ordinary generation
|
||||
error, not exhaustion. Completed structural rejection remains a normal result
|
||||
with failed validation, while an operational error returns no partial result.
|
||||
|
||||
## Application Boundary
|
||||
|
||||
Promptkit is an importable library. It does not own a command, inbound HTTP
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
|
||||
This is the contributor entry point for Promptkit, a reusable Go library. All
|
||||
contributors must read the
|
||||
[architecture policy](policy/architecture.md) before making changes.
|
||||
[architecture policy](policy/architecture.md) before making changes. Agents must
|
||||
also follow the [agent policy](policy/agents.md) before recommendations or work.
|
||||
|
||||
## Initial Orientation
|
||||
|
||||
@@ -25,7 +26,8 @@ Start with:
|
||||
documentation;
|
||||
- the [testing policy](policy/testing.md) before adding, rewriting, or deleting
|
||||
tests; and
|
||||
- the [release procedure](release.md) for version and publication work.
|
||||
- the [release policy](policy/release.md) and [release procedure](release.md)
|
||||
for version and publication work.
|
||||
|
||||
## Task-Specific Reading Guide
|
||||
|
||||
@@ -34,12 +36,14 @@ Start with:
|
||||
| Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. |
|
||||
| Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). |
|
||||
| Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. |
|
||||
| Maintained external catalogs or built-ins | The [internal source document](internal/sources.md), [framework format reference](formats.md), [testing policy](policy/testing.md), both catalog module repositories, each catalog's release procedure, and the selected module versions in `go.mod`. |
|
||||
| Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. |
|
||||
| Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. |
|
||||
| Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. |
|
||||
| Maintained example | The [example](../examples/go-library/prepare/main.go), [consumer guide](consumers/pkg-promptkit.md), [framework format reference](formats.md), and [documentation policy](policy/documentation.md). |
|
||||
| Maintained example | The [preparation](../examples/go-library/prepare/main.go), [execution](../examples/go-library/run/main.go), and [observation](../examples/go-library/observe/main.go) examples, [consumer guide](consumers/pkg-promptkit.md), [framework format reference](formats.md), and [documentation policy](policy/documentation.md). |
|
||||
| Documentation | The [documentation policy](policy/documentation.md) and canonical owner of every affected contract. |
|
||||
| Release preparation or publication | The [release procedure](release.md). |
|
||||
| Licensing, dependencies/catalog updates, copied material, or attribution | The [OSS policy](policy/oss.md), [project grant](../COPYRIGHT), [license texts](../LICENSE), and [attribution entries](oss/attribution.md); inspect the selected upstream source's notices. |
|
||||
| Release preparation or publication, including release notes | The [release policy](policy/release.md), [release procedure](release.md), and [OSS discoverability rules](policy/oss.md#discoverability). |
|
||||
|
||||
For cross-cutting changes, follow every applicable row. Do not create
|
||||
placeholder documents for packages, APIs, or integrations that do not yet
|
||||
@@ -47,187 +51,40 @@ exist.
|
||||
|
||||
## Maintainer Validation
|
||||
|
||||
This section is the canonical local validation workflow for Promptkit. Run
|
||||
every command from the repository root before accepting a change. The test
|
||||
suite and maintained examples are deterministic, offline, and require no real
|
||||
provider credentials.
|
||||
|
||||
### Tests, Analysis, Build, And Examples
|
||||
|
||||
Run the ordinary and race-enabled suites, static analysis, the build, and both
|
||||
maintained consumer examples:
|
||||
For ordinary work, select validation proportionate to the change. Do not repeat
|
||||
successful checks for unchanged content merely to create a commit. The full
|
||||
release-candidate checks have one executable entry point from the repository root:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go test -race ./...
|
||||
go vet ./...
|
||||
go build ./...
|
||||
go run ./examples/go-library/prepare
|
||||
go run ./examples/go-library/run
|
||||
./scripts/release-check.sh
|
||||
```
|
||||
|
||||
Both examples must exit successfully. Review their JSON output: preparation
|
||||
must report the selected offline prompt, profile, model, and message count;
|
||||
execution must report the deterministic generated output, successful
|
||||
validation, selected offline model, and usage. Neither command may contact a
|
||||
provider or require credentials.
|
||||
The [script](../scripts/release-check.sh) owns the executable checks:
|
||||
race-enabled tests, vet, the three offline consumer examples with semantic output
|
||||
assertions, tracked Go formatting, local Markdown links, module/workspace
|
||||
hygiene, whitespace, and the credential-pattern scan. It does not publish,
|
||||
change files, validate release-note structure, or retire roadmaps. It requires Go,
|
||||
Git, a POSIX shell, and Python 3. Tests and examples require no provider
|
||||
credentials or live inference calls.
|
||||
|
||||
### Go Formatting
|
||||
The script can run on intended uncommitted changes during development.
|
||||
Formatting and documentation checks use tracked files; stage intended new
|
||||
files before final validation so those checks include them. The release
|
||||
procedure separately enforces a clean, committed candidate.
|
||||
|
||||
Check every tracked Go file. The final command must succeed and the captured
|
||||
list must be empty:
|
||||
|
||||
```sh
|
||||
unformatted=$(
|
||||
git ls-files '*.go' |
|
||||
while IFS= read -r go_file
|
||||
do
|
||||
gofmt -l "$go_file"
|
||||
done
|
||||
)
|
||||
test -z "$unformatted"
|
||||
```
|
||||
|
||||
### Local Markdown Links
|
||||
|
||||
Use the Python standard library to verify every repository-relative Markdown
|
||||
target and local heading fragment. The check is offline and prints nothing on
|
||||
success:
|
||||
|
||||
```sh
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from urllib.parse import unquote
|
||||
|
||||
root = Path.cwd().resolve()
|
||||
markdown_files = [
|
||||
root / name
|
||||
for name in subprocess.check_output(
|
||||
["git", "ls-files", "*.md"], text=True
|
||||
).splitlines()
|
||||
]
|
||||
link_pattern = re.compile(r"!?\[[^]]*\]\(([^)]+)\)")
|
||||
heading_pattern = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$")
|
||||
scheme_pattern = re.compile(r"^[a-z][a-z0-9+.-]*:", re.IGNORECASE)
|
||||
|
||||
|
||||
def markdown_lines(path):
|
||||
in_fence = False
|
||||
fence = ""
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
stripped = line.lstrip()
|
||||
marker = stripped[:3]
|
||||
if marker in {"```", "~~~"}:
|
||||
if not in_fence:
|
||||
in_fence = True
|
||||
fence = marker
|
||||
elif marker == fence:
|
||||
in_fence = False
|
||||
fence = ""
|
||||
continue
|
||||
if not in_fence:
|
||||
yield line
|
||||
|
||||
|
||||
anchor_cache = {}
|
||||
|
||||
|
||||
def anchors(path):
|
||||
if path in anchor_cache:
|
||||
return anchor_cache[path]
|
||||
found = set()
|
||||
counts = {}
|
||||
for line in markdown_lines(path):
|
||||
match = heading_pattern.match(line)
|
||||
if not match:
|
||||
continue
|
||||
heading = re.sub(r"<[^>]+>", "", match.group(1)).replace("`", "")
|
||||
base = re.sub(r"[^\w\- ]", "", heading.lower()).replace(" ", "-")
|
||||
count = counts.get(base, 0)
|
||||
counts[base] = count + 1
|
||||
found.add(base if count == 0 else f"{base}-{count}")
|
||||
anchor_cache[path] = found
|
||||
return found
|
||||
|
||||
|
||||
failures = []
|
||||
for source in markdown_files:
|
||||
text = "\n".join(markdown_lines(source))
|
||||
for match in link_pattern.finditer(text):
|
||||
target = match.group(1).strip()
|
||||
if target.startswith("<") and target.endswith(">"):
|
||||
target = target[1:-1]
|
||||
if scheme_pattern.match(target) or target.startswith("//"):
|
||||
continue
|
||||
path_text, separator, fragment = target.partition("#")
|
||||
destination = source if not path_text else source.parent / unquote(path_text)
|
||||
try:
|
||||
destination = destination.resolve()
|
||||
destination.relative_to(root)
|
||||
except ValueError:
|
||||
failures.append(f"{source.relative_to(root)}: escapes repository: {target}")
|
||||
continue
|
||||
if not destination.exists():
|
||||
failures.append(f"{source.relative_to(root)}: missing target: {target}")
|
||||
continue
|
||||
if separator and destination.suffix.lower() == ".md":
|
||||
fragment = unquote(fragment).lower()
|
||||
if fragment not in anchors(destination):
|
||||
failures.append(f"{source.relative_to(root)}: missing anchor: {target}")
|
||||
|
||||
if failures:
|
||||
print("\n".join(failures), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
```
|
||||
|
||||
### Repository Hygiene And Review
|
||||
|
||||
Reject an active Go workspace, tracked workspace files, a vendor tree, or a
|
||||
module replacement:
|
||||
|
||||
```sh
|
||||
case "$(go env GOWORK)" in
|
||||
''|off) ;;
|
||||
*) printf '%s\n' 'an active Go workspace is not allowed' >&2; exit 1 ;;
|
||||
esac
|
||||
test -z "$(git ls-files go.work go.work.sum)"
|
||||
test ! -e vendor
|
||||
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||
then
|
||||
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
Check whitespace in both unstaged and staged changes. List ignored files and
|
||||
scan tracked content for common credential forms:
|
||||
|
||||
```sh
|
||||
git diff --check
|
||||
git diff --cached --check
|
||||
test -z "$(git ls-files --others --ignored --exclude-standard)"
|
||||
credential_pattern='-----BEGIN ([A-Z0-9]+ )?PRIV''ATE KEY-----|AKI''A[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{36,}|sk-[A-Za-z0-9]{32,}'
|
||||
if git grep -nEI -e "$credential_pattern" -- .
|
||||
then
|
||||
printf '%s\n' 'possible credential found' >&2
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
### Change Review
|
||||
|
||||
Inspect `git status --short --untracked-files=all` and the complete diff before
|
||||
accepting a change. The status may contain only the intended source changes
|
||||
during development. Reject credentials, private keys, environment files,
|
||||
generated binaries, test or coverage output, downloaded assets, template
|
||||
residue, and any other artifact that does not belong in source control. The
|
||||
credential scan catches common forms but does not replace inspection of the
|
||||
actual change.
|
||||
accepting a change. Reject unintended files, credentials, private data,
|
||||
generated binaries, test output, downloaded assets, and template residue.
|
||||
The automated credential scan does not replace review of the actual change.
|
||||
|
||||
After committing the accepted change, require a clean candidate:
|
||||
For releases, follow the [release procedure](release.md). Its publication
|
||||
command runs the full local checks once on the final committed candidate and
|
||||
automates release-note structure and Git checks. Do not repeat ordinary change
|
||||
review as a release sign-off. Roadmap retirement remains preparation housekeeping.
|
||||
|
||||
```sh
|
||||
test -z "$(git status --porcelain)"
|
||||
```
|
||||
Follow the [agent commit policy](policy/agents.md#git-commit-policy) for local
|
||||
commits. Ordinary commits may leave unrelated work in the workspace; preserve
|
||||
and report it. A clean worktree and index are required for release candidates,
|
||||
as enforced by the publication command, not for every ordinary commit.
|
||||
|
||||
@@ -82,7 +82,13 @@ allowed.
|
||||
|
||||
### Messages And Templates
|
||||
|
||||
Each message has a non-empty `role` and exactly one of:
|
||||
Each message has a `role` that Promptkit trims and lowercases. It must then be
|
||||
exactly one of `developer`, `system`, `user`, or `assistant`; blank, custom,
|
||||
`tool`, and `function` roles are invalid. This intentionally tightens the
|
||||
previous nonblank-string rule. Consumers migrating to the next minor release
|
||||
must update any nonstandard prompt-definition roles before upgrading.
|
||||
|
||||
Each message also has exactly one of:
|
||||
|
||||
- `content`, containing an inline Go template; or
|
||||
- `content_file`, naming a file whose contents are the Go template.
|
||||
@@ -277,7 +283,8 @@ Profile sources resolve matching IDs in this order:
|
||||
2. the ordinary configured source selected by a profile file, `fs.FS`, or
|
||||
configured profile directory;
|
||||
3. application fallback profiles supplied with `WithFallbackProfileFS`; and
|
||||
4. embedded built-in profiles.
|
||||
4. Promptkit-owned profiles; and
|
||||
5. maintained external catalog profiles.
|
||||
|
||||
A profile source supplies a complete definition; definitions and their fields
|
||||
are not merged across sources. A higher-precedence source falls back only when
|
||||
@@ -320,33 +327,32 @@ that backend's connection and credential metadata. Profile files do not repeat
|
||||
those values. A configured, application fallback, or in-memory profile with
|
||||
the same profile ID takes precedence.
|
||||
|
||||
| Provider | ID | Backend | Model |
|
||||
| --- | --- | --- | --- |
|
||||
| aion-labs | `aion-2` | `openrouter` | `aion-labs/aion-2.0` |
|
||||
| anthropic | `claude-fable-latest` | `openrouter` | `~anthropic/claude-fable-latest` |
|
||||
| anthropic | `claude-haiku-latest` | `openrouter` | `~anthropic/claude-haiku-latest` |
|
||||
| anthropic | `claude-opus-latest` | `openrouter` | `~anthropic/claude-opus-latest` |
|
||||
| anthropic | `claude-sonnet-latest` | `openrouter` | `~anthropic/claude-sonnet-latest` |
|
||||
| deepseek | `deepseek-3-2` | `openrouter` | `deepseek/deepseek-v3.2` |
|
||||
| deepseek | `deepseek-4-flash` | `openrouter` | `deepseek/deepseek-v4-flash` |
|
||||
| deepseek | `deepseek-4-pro` | `openrouter` | `deepseek/deepseek-v4-pro` |
|
||||
| google | `gemini-2-flash` | `openrouter` | `google/gemini-2.5-flash` |
|
||||
| google | `gemini-2-flash-lite` | `openrouter` | `google/gemini-2.5-flash-lite` |
|
||||
| google | `gemini-2-pro` | `openrouter` | `google/gemini-2.5-pro` |
|
||||
| google | `gemini-3-flash-lite` | `openrouter` | `google/gemini-3.1-flash-lite` |
|
||||
| google | `gemini-flash-latest` | `openrouter` | `~google/gemini-flash-latest` |
|
||||
| google | `gemini-pro-latest` | `openrouter` | `~google/gemini-pro-latest` |
|
||||
| google | `gemma-4-31b` | `openrouter` | `google/gemma-4-31b-it:exacto` |
|
||||
| google | `rakestrawhome-gemma-4-31b` | `rakestrawhome` | `google/gemma-4-31b-it` |
|
||||
| minimax | `minimax-m2` | `openrouter` | `minimax/minimax-m2.5` |
|
||||
| minimax | `minimax-m3` | `openrouter` | `minimax/minimax-m3` |
|
||||
| mistral | `mistral-large-2512` | `openrouter` | `mistralai/mistral-large-2512` |
|
||||
| mistral | `mistral-medium-3-5` | `openrouter` | `mistralai/mistral-medium-3-5` |
|
||||
| mistral | `mistral-small-3` | `openrouter` | `mistralai/mistral-small-3.2-24b-instruct` |
|
||||
| mistral | `mistral-small-4` | `openrouter` | `mistralai/mistral-small-2603` |
|
||||
| nvidia | `nemotron-3-ultra` | `openrouter` | `nvidia/nemotron-3-ultra-550b-a55b` |
|
||||
| openai | `gpt-5-mini` | `openrouter` | `openai/gpt-5.4-mini` |
|
||||
| openai | `gpt-5-nano` | `openrouter` | `openai/gpt-5.4-nano` |
|
||||
Promptkit owns two opt-in profiles. The following definitions are current
|
||||
selections, not automatic defaults:
|
||||
|
||||
| ID | Current definition |
|
||||
| --- | --- |
|
||||
| `promptkit-openrouter-standard` | Inherits `gemini-flash-latest`. |
|
||||
| `promptkit-openrouter-auto` | Selects backend `openrouter` and model `openrouter/auto`. |
|
||||
|
||||
Select either name through an existing request `profile_id` or prompt
|
||||
`default_profile`. Consumers can replace either definition or its inherited
|
||||
base through the ordinary profile-source precedence above. The
|
||||
[release policy](policy/release.md#catalog-profile-compatibility) defines the
|
||||
cross-version guarantee for these names.
|
||||
|
||||
## External Profile Inventories
|
||||
|
||||
The external profile inventory is defined by the selected catalog modules, not
|
||||
duplicated here. Browse the exact data selected by this Promptkit version:
|
||||
|
||||
- [OpenRouter catalog v2.20260926.0 profiles](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/tag/v2.20260926.0/catalog/profiles)
|
||||
- [Rakestrawhome catalog v2.20260926.0 profiles](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/tag/v2.20260926.0/catalog/profiles)
|
||||
|
||||
External profile entries are volatile across catalog releases. The
|
||||
[release policy](policy/release.md#catalog-profile-compatibility) owns the precise
|
||||
compatibility boundary; a selected module version identifies tested local data,
|
||||
not live provider availability.
|
||||
|
||||
## Schemas
|
||||
|
||||
|
||||
@@ -55,6 +55,12 @@ Each ordinary message contains its `role` and string `content`. A
|
||||
cache-controlled message instead uses a text content block containing `type`,
|
||||
`text`, and `cache_control`; an empty cache-control TTL is omitted.
|
||||
|
||||
Promptkit sends only `developer`, `system`, `user`, and `assistant` roles and
|
||||
does so without provider-specific translation. Tool and deprecated function
|
||||
payloads are outside this text-message contract. A backend or model that
|
||||
rejects an otherwise supported role or context returns its ordinary provider
|
||||
error, which follows the normal generation-error path.
|
||||
|
||||
The effective direct or prompt-rendered session ID is trimmed, limited to 256
|
||||
Unicode code points, and sent when nonempty as top-level `session_id`. It is
|
||||
never also sent as a session header.
|
||||
@@ -107,6 +113,14 @@ data, a second JSON value, absent choices, missing content, `null` content,
|
||||
non-string content, and size overflow are malformed responses and return no
|
||||
partial result.
|
||||
|
||||
Each of the five token fields retains whether the provider reported a
|
||||
non-null integer, including zero. Omitted and `null` fields are unknown for
|
||||
observations. A strictly decoded successful envelope can yield observed usage
|
||||
even when a later missing choice or content makes the attempt malformed.
|
||||
Invalid framing and non-2xx error bodies supply no observed usage; provider
|
||||
work may still have occurred. Reported counts are not inferred, reconciled,
|
||||
or proof of complete cost accounting.
|
||||
|
||||
For a non-2xx status, Promptkit recognizes one JSON document with a top-level
|
||||
object-valued `error` member. Its optional `message` and `type` fields must be
|
||||
strings, and `code` may be a string or JSON number. Valid supported fields are
|
||||
@@ -133,6 +147,33 @@ identity and the exact transport error for `errors.Is` and `errors.As` checks.
|
||||
The rendered error does not include the selected endpoint, request headers,
|
||||
request content, credentials, or provider body.
|
||||
|
||||
## Transient Status Retries
|
||||
|
||||
The built-in path retries only HTTP 429, 500, 502, 503, and 504. It makes two
|
||||
additional attempts by default; engine configuration or an individual run can
|
||||
choose a different nonnegative count, including zero for single-attempt
|
||||
behavior. The same budget applies separately to initial generation and each
|
||||
structural repair generation. An injected `LLMClient` is dispatched once by
|
||||
Promptkit regardless of this setting. A custom `http.Client` on the built-in
|
||||
path remains subject to these retries.
|
||||
|
||||
Each retry uses equal jitter between half and all of an exponential local
|
||||
delay, starting at 500 milliseconds and capped at 8 seconds. The first
|
||||
`Retry-After` header value sets a minimum not-before time measured from
|
||||
response receipt if it is valid unsigned decimal seconds or an HTTP date.
|
||||
Invalid values use local delay; past dates impose no extra delay. The provider
|
||||
minimum is not capped at 8 seconds. Retry waiting is cancellable and holds no
|
||||
active backend permit. A caller without a deadline can wait a long time for
|
||||
provider timing.
|
||||
|
||||
The request body, selected URL, and resolved authorization are fixed across
|
||||
attempts of one generation. Each attempt creates a fresh HTTP request and
|
||||
receives its own configured generation and HTTP-client timeout after capacity
|
||||
admission. The caller context covers every attempt and wait without a reset.
|
||||
Transport failures, malformed responses, noneligible statuses, and invalid
|
||||
generated content are not transient retries. A positive budget ending in
|
||||
eligible statuses reports retry exhaustion with the final provider cause.
|
||||
|
||||
## Timeout And Cancellation
|
||||
|
||||
Timeouts are layered:
|
||||
|
||||
@@ -14,7 +14,7 @@ stable JSON values.
|
||||
|
||||
## Construction And Pool Lifecycle
|
||||
|
||||
Each root `NewEngine` call obtains a normalized capacity-policy snapshot from
|
||||
Each internal engine construction obtains a normalized capacity-policy snapshot from
|
||||
its immutable backend registry and constructs a new `Manager`. The manager
|
||||
creates one pool for each limited backend ID. It has no package-global mutable
|
||||
state, background workers, shutdown protocol, or persistence, so engines with
|
||||
@@ -54,38 +54,48 @@ reserve another bounded slot.
|
||||
|
||||
## FIFO Generation Permits
|
||||
|
||||
`NewClient` wraps the engine's selected internal model client after public
|
||||
client adaptation or built-in client construction. Initial generation and the
|
||||
default repairer receive the same wrapper. Their requests retain the same
|
||||
effective backend, credential, numeric-presence metadata, and structured-output
|
||||
settings, so scheduling does not change provider omission semantics between
|
||||
calls.
|
||||
The engine supplies one [generation coordinator](generation.md) to initial
|
||||
generation and the default repairer. Their requests retain the same effective
|
||||
backend, credential, numeric-presence metadata, structured-output settings,
|
||||
and retry budget. Scheduling does not change provider omission semantics
|
||||
between calls.
|
||||
|
||||
For each `Generate` call, the wrapper selects a pool from the request's
|
||||
effective backend ID. An unlimited call passes directly to the next client. A
|
||||
limited call acquires an active permit, invokes the next client, and defers
|
||||
permit release so ordinary returns and panic unwinding both restore capacity.
|
||||
For each model attempt, the coordinator selects a pool from the request's
|
||||
effective backend ID. An unlimited call uses the unrestricted path. A limited
|
||||
call acquires an active permit and defers release around that attempt, so
|
||||
ordinary returns and panic unwinding both restore capacity.
|
||||
Preparation and validation never hold an active permit.
|
||||
|
||||
The manager's `Acquire` operation owns that active-permit acquisition and
|
||||
returns an idempotent release function. The coordinator uses it around
|
||||
individual attempts. The earlier one-shot wrapper still delegates to the same
|
||||
operation for focused internal callers and tests, but engine assembly no
|
||||
longer uses it. There is one queue and one active limit per limited backend pool.
|
||||
|
||||
When all active permits are occupied, calls join a mutex-protected FIFO waiter
|
||||
list. Releasing a permit transfers it directly to the oldest remaining waiter
|
||||
before making it generally available. Pools do not order work relative to
|
||||
other backend IDs.
|
||||
|
||||
The wrapper passes generation requests, responses, and collaborator errors
|
||||
through unchanged. It owns scheduling only; the concrete model client remains
|
||||
responsible for provider transport behavior. The runner, rather than the
|
||||
The coordinator does not retry injected clients; it normalizes a nil successful
|
||||
response as malformed generation. The built-in HTTP client remains responsible
|
||||
for provider transport behavior. The runner, rather than the
|
||||
capacity layer, sums all five usage fields from the initial response and every
|
||||
completed repair response into the successful run result.
|
||||
|
||||
An invocation-local observer receives attempt events after active permit
|
||||
release, including canceled acquisition without dispatch. A retry wait holds
|
||||
the logical admission lease but no active permit. Observer callbacks must not
|
||||
block on work that requires the same run's admission slot.
|
||||
|
||||
## Cancellation And Release
|
||||
|
||||
Admission checks the caller context before reserving a slot. A call canceled
|
||||
while waiting for an active permit removes its waiter under the same pool lock
|
||||
used to grant permits. If cancellation removes the waiter first, the wrapped
|
||||
client is not invoked. If a concurrent grant wins first, the call owns the
|
||||
permit and invokes the client with the original context, allowing the client
|
||||
to observe cancellation normally.
|
||||
used to grant permits. If cancellation removes the waiter first, no model
|
||||
attempt is dispatched. If a concurrent grant wins first, the call owns the
|
||||
permit; the coordinator checks the context before dispatch and releases the
|
||||
permit if cancellation has already occurred.
|
||||
|
||||
This grant-or-cancel decision prevents lost and double-released permits.
|
||||
Admission leases and active permits are released after success, collaborator
|
||||
@@ -111,6 +121,8 @@ release. The
|
||||
[external package capacity tests](../../capacity_contract_test.go) own the
|
||||
assembled public-engine behavior for configured limits, capacity errors,
|
||||
endpoint identity, engine independence, and injected clients. The
|
||||
[reliable execution contract tests](../../reliable_execution_contract_test.go)
|
||||
cover shared and independent backend pools during retry and repair. The
|
||||
[prepared-execution contract tests](../../prepared_execution_contract_test.go)
|
||||
own the public prepared-capacity boundary. The
|
||||
[root error-boundary tests](../../errors_internal_test.go) own preservation of
|
||||
|
||||
50
docs/internal/generation.md
Normal file
50
docs/internal/generation.md
Normal file
@@ -0,0 +1,50 @@
|
||||
# Internal Generation Coordination
|
||||
|
||||
`internal/generation` coordinates one model-generation request through the
|
||||
existing engine-local [capacity manager](capacity.md) and the
|
||||
[built-in HTTP client](llm.md). The engine supplies the same coordinator to
|
||||
initial generation and the default output repairer.
|
||||
|
||||
The coordinator has explicit built-in and injected-client constructors. The
|
||||
built-in path prepares the provider request once, then acquires and releases an
|
||||
active backend permit around each actual HTTP attempt. Only HTTP 429, 500, 502,
|
||||
503, and 504 can lead to another attempt. The injected-client path acquires a
|
||||
permit and invokes its client once, regardless of the internal retry budget.
|
||||
The runner continues to own the logical execution's admission lease. A request
|
||||
can override the engine's retry budget; the runner freezes that choice for
|
||||
prepared execution and passes it to each repair generation.
|
||||
|
||||
Retry waits hold no active permit. The local delay uses equal jitter between
|
||||
half and all of a capped exponential base, beginning at 500 milliseconds and
|
||||
capping at 8 seconds. A valid provider not-before time can extend that delay;
|
||||
the caller context interrupts it. The next attempt reacquires the ordinary
|
||||
FIFO permit. Preparation failures dispatch nothing. When a positive retry
|
||||
budget ends with eligible failures, the internal exhaustion error retains the
|
||||
final cause and actual dispatched-attempt count.
|
||||
|
||||
When an invocation-local [observation recorder](observation.md) is present in
|
||||
the context, each acquisition emits an attempt event, including canceled
|
||||
capacity waits without dispatch. Each actual retry wait emits a delay event,
|
||||
including interrupted waits. Active permits are released before callbacks;
|
||||
generation duration ends before callback delivery. Built-in successful-envelope
|
||||
usage retains reported-field presence, while injected clients use explicit
|
||||
presence when supplied and otherwise treat zero fields as unknown.
|
||||
Malformed successful envelopes can retain strictly decoded usage; error
|
||||
response bodies are never mined for it. Arbitrary injected failures carry
|
||||
unknown usage. Transport, timeout, malformed, and noneligible status failures
|
||||
stop the sequence without retry exhaustion. Context is checked before each
|
||||
wait, acquisition, and dispatch.
|
||||
|
||||
The public client adapter translates explicitly supplied public transport and
|
||||
malformed-response identities into internal categories while retaining the
|
||||
original error chain. This keeps injected attempt classifications consistent
|
||||
with terminal observations without parsing diagnostic text or retrying injected
|
||||
clients.
|
||||
|
||||
Focused coordinator tests use controlled transport and wait functions. They
|
||||
cover eligibility, exhaustion, cancellation, backoff, request freezing, permit
|
||||
release, and the injected-client path. The capacity manager tests own permit
|
||||
acquisition and idempotent release. Public
|
||||
[integration tests](../../reliable_execution_contract_test.go) cover retry,
|
||||
repair, observation, frozen prepared execution, and shared-backend behavior
|
||||
through the assembled facade.
|
||||
@@ -10,7 +10,7 @@ owns the observable outbound HTTP contract. The
|
||||
[framework format reference](../formats.md) owns the profile and prompt
|
||||
settings consumed by the client.
|
||||
|
||||
The concrete client remains under `internal/llm`. The root engine assembles it
|
||||
The concrete client remains under `internal/llm`. The internal engine assembles it
|
||||
as the default implementation behind Promptkit's public client boundary.
|
||||
|
||||
## Components And Flow
|
||||
@@ -21,32 +21,39 @@ uses internal domain values for rendered prompts, execution targets,
|
||||
structured output, responses, and token usage.
|
||||
|
||||
The runner supplies a fully resolved target after applying backend, profile,
|
||||
and request precedence. The client uses its endpoint, credential metadata,
|
||||
generation fields, and extra parameters. `BackendID` remains routing metadata
|
||||
and request precedence, plus canonical provider-bound text messages. The
|
||||
client uses its endpoint, credential metadata, generation fields, and extra
|
||||
parameters. `BackendID` remains routing metadata
|
||||
for the generation boundary and is not mapped into the provider payload.
|
||||
|
||||
Construction trims and validates a nonempty configured base URL and clones any
|
||||
supplied `http.Client` so Promptkit can apply its timeout default without
|
||||
mutating the caller's client. An empty configured base remains valid because a
|
||||
resolved request target may supply the endpoint. Generation then:
|
||||
resolved request target may supply the endpoint. Generation preparation then:
|
||||
|
||||
1. validates shared execution-setting invariants and the final selected base
|
||||
endpoint;
|
||||
2. maps the internal request into the OpenAI-compatible chat payload;
|
||||
3. validates and merges extra parameters;
|
||||
4. composes `/chat/completions` through parsed URL path operations;
|
||||
5. resolves authentication;
|
||||
6. performs the outbound request under the applicable deadlines; and
|
||||
7. decodes one strictly framed, size-bounded successful response object and
|
||||
maps its first choice and token usage, or decodes bounded structured
|
||||
non-success detail.
|
||||
5. resolves authentication and freezes the URL, body, and optional credential.
|
||||
|
||||
Each execution of that prepared request creates a fresh HTTP request, body
|
||||
reader, and attempt timeout context. It closes the response body and returns
|
||||
the HTTP status, decoded response or error, reported token values and presence,
|
||||
and any valid provider `Retry-After` not-before time. A strictly decoded
|
||||
successful envelope can retain usage even if its first choice or content is
|
||||
missing. Invalid framing and non-success error bodies supply no usage. The
|
||||
ordinary `Generate` method prepares and executes once and returns no partial
|
||||
response on failure.
|
||||
|
||||
`internal/llm` owns the set of reserved OpenAI-compatible request fields used
|
||||
when validating extra parameters. Backend registration consumes the same rule
|
||||
without making the model client depend on registry configuration.
|
||||
|
||||
The implementation has no retry loop, tool-call support, provider catalog,
|
||||
inbound HTTP behavior, or durable session store.
|
||||
This client does not schedule retries; the [generation coordinator](generation.md)
|
||||
does so around its prepared attempts. The client has no tool-call support,
|
||||
provider catalog, inbound HTTP behavior, or durable session store.
|
||||
|
||||
## Prepared Generation
|
||||
|
||||
@@ -95,6 +102,12 @@ whitespace-only content. Missing choices, missing or `null` content, and
|
||||
non-string content are malformed responses. Output validation and correction
|
||||
eligibility remain outside this package.
|
||||
|
||||
Successful-envelope usage is decoded with independent presence bits for each
|
||||
token field: omission and `null` are unknown, while a reported integer zero is
|
||||
known. Strictly decoded usage can still be observed if later choice/content
|
||||
checks fail; invalid framing or a non-success error body contributes none.
|
||||
Existing successful-result accounting remains value-based.
|
||||
|
||||
For a non-success response, `ProviderHTTPError` retains the HTTP status and
|
||||
only normalized detail from the bounded recognized envelope. It retains
|
||||
`ErrUnexpectedStatus` through unwrapping. The client owns response closure;
|
||||
|
||||
48
docs/internal/observation.md
Normal file
48
docs/internal/observation.md
Normal file
@@ -0,0 +1,48 @@
|
||||
# Internal Execution Observation
|
||||
|
||||
`internal/observation` owns one invocation's run ID, presence-aware token
|
||||
aggregation, measured phase totals, and synchronous metadata event delivery.
|
||||
Its event values and callback type live in `internal/domain` so internal
|
||||
requests can carry them without depending on the recorder package. The
|
||||
recorder supplies run IDs to the runner and receives attempt and
|
||||
retry-delay measurements from the generation coordinator. Ordinary `Run`
|
||||
lifecycle instrumentation spans the facade and runner boundaries.
|
||||
Prepared execution starts only after its private payload is atomically claimed;
|
||||
the facade receives that invocation's recorder privately and emits the terminal
|
||||
event after public error mapping. Failed claims and preparation emit nothing.
|
||||
|
||||
The facade masks inherited recorder and prepared-claim state when entering
|
||||
preparation or execution. Delegated calls from an injected collaborator retain
|
||||
the caller's cancellation, deadline, and unrelated context values, but do not
|
||||
reuse the enclosing invocation's accounting. Only the new request's observer
|
||||
(or the claimed handle's retained observer) enables its observation lifecycle.
|
||||
|
||||
A recorder keeps only counters, resolved identity fields, and the selected
|
||||
callback. It has no global history, queue, goroutine, or mutex held across a
|
||||
callback. The caller supplies normalized failure metadata; the recorder never
|
||||
parses an error string or receives provider bodies, credentials, headers, or
|
||||
other content. Events are values with no mutable containers. Callback panics
|
||||
propagate to the caller.
|
||||
Per-operation callbacks are ordered and synchronous; different operations may
|
||||
invoke the same callback concurrently. The recorder marks each measured phase's
|
||||
end before delivery, so callback work does not inflate provider duration.
|
||||
Terminal phase durations sum measured phases, while wall duration may also
|
||||
include preparation, validation, and callbacks. Panics do not promise a
|
||||
terminal event.
|
||||
|
||||
Attempt usage is known when its explicit presence bit is true. For legacy
|
||||
injected clients without presence metadata, nonzero counts are known and
|
||||
zeros are unknown. Aggregate values sum known contributions; `Known` means
|
||||
at least one reported value, while `Complete` means every dispatched attempt
|
||||
reported the field. Zero dispatched attempts produce known, complete zero
|
||||
totals. Non-dispatched acquisitions contribute capacity wait but no usage or
|
||||
generation duration. Terminal totals count dispatched attempts, transient
|
||||
attempts above the first in a generation, and repair generations with at least
|
||||
one dispatch. A terminal event is an aggregate and must not be added to its
|
||||
attempt events.
|
||||
|
||||
The [recorder tests](../../internal/observation/recorder_test.go) own usage
|
||||
presence, phase sums, event ordering, empty-run accounting, identity snapshots,
|
||||
and callback panic behavior. The [generation coordinator](generation.md) owns
|
||||
attempt and retry-delay measurements and emits their completion events after
|
||||
releasing active permits.
|
||||
@@ -11,25 +11,29 @@ contributor workflow and validation.
|
||||
|
||||
| Component | Implemented responsibility | References |
|
||||
| --- | --- | --- |
|
||||
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, and profile-inspection values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, typed capacity and generation error mapping, engine-local profile-source assembly including application fallbacks, and bounded output-repair assembly. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
|
||||
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, profile-inspection, and ordinary-run observation values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go) |
|
||||
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||
| `internal/backend` | Constructs each engine's immutable registry from the maintained built-in definitions and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
|
||||
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
||||
| `internal/backend` | Constructs each engine's immutable registry from maintained definitions and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
|
||||
| `internal/catalog` | Strictly validates imported immutable maintained backend and profile catalog assets before engine assembly uses them. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) |
|
||||
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||
| `internal/engine` | Assembles external catalogs, Promptkit-owned profiles, repositories, registry, capacity, collaborators, repairer, and the one use-case runner for each engine instance. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) |
|
||||
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, and attempt observations. | [Internal generation coordination](generation.md) |
|
||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
|
||||
| `internal/profile` | Loads strictly decoded, locally validated execution profiles from filesystem and `fs.FS` sources, overlays raw sources with error-preserving fallback, and resolves inherited profiles. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go), [internal sources](sources.md#profiles-and-built-ins) |
|
||||
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select maintained built-in backends. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
||||
| `internal/profile` | Owns immutable in-memory and filesystem profile repositories, local profile validation, raw-source overlays with error-preserving fallback, and inherited-profile resolution. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go), [internal sources](sources.md#profiles-and-built-ins) |
|
||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts. | [Internal execution observation](observation.md) |
|
||||
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary and single-claim prepared execution across internal sources, rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||
|
||||
The root package assembles these internal components without exposing their
|
||||
The internal engine assembles these internal components without exposing their
|
||||
representations. Consumers depend only on the root facade.
|
||||
|
||||
## Maintenance
|
||||
|
||||
@@ -9,8 +9,8 @@ artifact, rendering, and validation behavior, while the
|
||||
[model-client document](llm.md) owns generation behavior and failure
|
||||
categories.
|
||||
|
||||
The runner remains under `internal/usecase` and is assembled by the root
|
||||
Promptkit engine. Its concrete type is not part of the public API.
|
||||
The runner remains under `internal/usecase` and is assembled by
|
||||
`internal/engine`. Its concrete type is not part of the public API.
|
||||
The [framework format reference](../formats.md) owns prompt, profile, schema,
|
||||
and override semantics consumed by the runner.
|
||||
|
||||
@@ -18,13 +18,13 @@ and override semantics consumed by the runner.
|
||||
|
||||
`Runner` coordinates narrow internal interfaces for prompt definitions,
|
||||
profiles, backend resolution, artifacts, rendering, model generation, and
|
||||
validation. The root engine supplies one immutable registry containing the
|
||||
validation. The internal engine supplies one immutable registry containing the
|
||||
built-in backend and validated consumer additions, one engine-local run
|
||||
admitter, and a model client wrapped by the same capacity manager. Validation
|
||||
admitter, and a generation coordinator using the same capacity manager. Validation
|
||||
plans and provider-facing schema metadata come from the validator's preparation
|
||||
interface.
|
||||
The root engine supplies one default output repairer through the explicit
|
||||
runner constructor, using the same capacity-wrapped client as initial
|
||||
The internal engine supplies one default output repairer through the explicit
|
||||
runner constructor, using the same coordinator as initial
|
||||
generation. The no-repair runner constructor remains available for focused
|
||||
internal callers and tests.
|
||||
|
||||
@@ -84,7 +84,8 @@ profile, or backend:
|
||||
schema metadata from it when required;
|
||||
2. load and hash input artifacts;
|
||||
3. render messages and the prompt-defined session;
|
||||
4. apply any direct session ID;
|
||||
4. apply any direct session ID, then append already-normalized request messages
|
||||
after the rendered definition messages;
|
||||
5. hash the effective rendered prompt; and
|
||||
6. construct the prepared value and preparation timing.
|
||||
|
||||
@@ -113,7 +114,10 @@ the prompt session template, and is applied after ordinary message rendering.
|
||||
A blank direct value retains prompt-template behavior. The runner clears the
|
||||
template only on a value copy of the definition, so the definition hash always
|
||||
describes the original source while the rendered-prompt hash includes the
|
||||
effective direct or rendered session.
|
||||
effective direct or rendered session and the complete effective message
|
||||
sequence. The rendered-prompt hash uses a versioned, length-framed SHA-256
|
||||
encoding of that session, every role and content value, and cache-control
|
||||
presence and values; its hexadecimal value is opaque.
|
||||
|
||||
The registry is read-only after engine construction. Concurrent `Prepare` and
|
||||
`Run` calls resolve independent defensive backend values and keep all
|
||||
@@ -130,6 +134,15 @@ and validates that artifact with the plan compiled during completion. Invalid
|
||||
generated content remains a validation result; an inability to generate or
|
||||
perform validation is an operational error.
|
||||
|
||||
When the public facade installs an invocation-local observation recorder,
|
||||
ordinary `Run` reuses its run ID and carries it to the generation coordinator
|
||||
through context. Prompt and profile identities are enriched only after their
|
||||
respective resolutions succeed, before admission. The facade starts observation
|
||||
before request conversion and emits its terminal event after public error
|
||||
mapping and after the runner's deferred admission release. Preparation and
|
||||
prepared execution follows the same terminal mapping after a successful claim;
|
||||
invalid or discarded handles do not acquire an observer.
|
||||
|
||||
Validation preparation and execution honor cancellation at every
|
||||
Promptkit-controlled boundary and do not publish a partial plan or result.
|
||||
Schema reads are bounded and context-checked between chunks; JSON decoding,
|
||||
@@ -141,21 +154,23 @@ method must return before cancellation can take precedence over its outcome.
|
||||
The admission lease covers completion-phase preparation, initial generation,
|
||||
validation, every repair, and every exit. It bounds accepted work without
|
||||
serializing preparation or validation behind the active-generation limit.
|
||||
The wrapped model client separately acquires a FIFO active permit only around
|
||||
each actual generation call.
|
||||
The coordinator separately acquires a FIFO active permit only around each
|
||||
actual model attempt. It releases the permit before a transient retry wait.
|
||||
|
||||
After a failed `basic`, JSON, or JSON Schema validation with a positive frozen
|
||||
budget, the installed repairer can make a bounded corrective call. Each request
|
||||
starts with a fresh copy of the complete original rendered messages, includes
|
||||
only the latest nonempty candidate as an assistant message, and appends one
|
||||
corrective user message. Empty candidates omit that assistant message. The
|
||||
starts with a fresh copy of the complete effective message sequence (the
|
||||
configured prefix followed by the request suffix), includes only the latest
|
||||
nonempty candidate as an assistant message, and appends one corrective user
|
||||
message. Empty candidates omit that assistant message. The
|
||||
correction carries validation diagnostics as JSON data bounded to 64 KiB; the
|
||||
full diagnostics remain in the validation result.
|
||||
|
||||
Repair receives the effective execution target, explicit numeric-presence bits,
|
||||
credential, backend identity, session ID, and structured-output specification.
|
||||
credential, backend identity, session ID, structured-output specification, and
|
||||
frozen transient retry budget.
|
||||
The same request constructor supplies those common fields to initial and repair
|
||||
generation. The default repairer uses the same wrapped client as initial
|
||||
generation. The default repairer uses the same coordinator as initial
|
||||
generation, so each repair reacquires the selected backend's active permit
|
||||
while remaining inside its original admission lease. Repair never performs a
|
||||
second bounded admission, and repaired outputs use the operation's existing
|
||||
@@ -174,7 +189,8 @@ credential during execution, reaches generation and every repair attempt.
|
||||
Result usage is the field-wise sum of all five usage values from the initial
|
||||
response and every completed repair response. Final raw output, artifact, and
|
||||
validation state still come from the last candidate. A repair error returns no
|
||||
partial run result or partial usage.
|
||||
partial run result or partial result usage, although the invocation-local
|
||||
observer retains completed-attempt usage and reports it in the terminal event.
|
||||
|
||||
## Failure Categories
|
||||
|
||||
@@ -213,8 +229,9 @@ behavior, hashing, generation and validation outcomes, backend propagation,
|
||||
bounded repair progression, initial/repair request parity, cumulative usage,
|
||||
shared initial/repair capacity, credentials and redaction, error categories,
|
||||
artifact metadata, and timing. The
|
||||
[capacity subsystem document](capacity.md) identifies the focused pool,
|
||||
waiter, and wrapped-client tests.
|
||||
[capacity subsystem document](capacity.md) identifies the focused pool and
|
||||
waiter tests. The [generation document](generation.md) owns attempt and retry
|
||||
coordination tests.
|
||||
|
||||
Changes to orchestration should continue to use the existing package
|
||||
interfaces, keep request state local to an invocation, and preserve the shared
|
||||
|
||||
@@ -6,7 +6,7 @@ This document describes Promptkit's implemented internal source, artifact,
|
||||
rendering, and output-validation behavior. The
|
||||
[architecture policy](../policy/architecture.md) owns the library boundary and
|
||||
dependency rules. None of these internal packages is a supported consumer API,
|
||||
and the root engine assembles them behind its public source options and values.
|
||||
and the internal engine assembles them behind the public facade's source options and values.
|
||||
The [framework format reference](../formats.md) owns the exact file fields,
|
||||
validation modes, built-in catalog, and source precedence.
|
||||
|
||||
@@ -21,6 +21,12 @@ paths, content opening, and root containment. Each lookup remains a
|
||||
point-in-time scan: definitions and catalogs are not cached, and file-backed
|
||||
message content is opened only for the exact selected candidate.
|
||||
|
||||
Message roles are normalized through the shared domain owner by trimming
|
||||
Unicode whitespace and lowercasing. Only `developer`, `system`, `user`, and
|
||||
`assistant` are published; invalid roles remain selected prompt-definition
|
||||
failures rather than becoming request errors. Cache-control metadata uses the
|
||||
same shared domain normalization and defensive-copy rule.
|
||||
|
||||
Operating-system sources enforce containment against canonical roots and
|
||||
targets so symlinks cannot escape. Injected `fs.FS` sources enforce containment
|
||||
in their clean relative path namespace. A single-file source uses the selected
|
||||
@@ -39,8 +45,10 @@ duplicate detection, and source containment:
|
||||
|
||||
## Profiles And Built-Ins
|
||||
|
||||
`internal/profile` loads, locally validates, overlays, and resolves execution
|
||||
profiles from an operating-system filesystem or an `fs.FS`. A file contains
|
||||
`internal/profile` owns immutable in-memory and filesystem repositories, local
|
||||
validation, overlays, and resolution for execution profiles. In-memory
|
||||
definitions are normalized and validated once at that repository boundary and
|
||||
are defensively copied on storage and lookup. A file contains
|
||||
exactly one YAML document and its trimmed YAML `id` is its only selection
|
||||
identity; filenames do not confer authority. Each point lookup reads discovered
|
||||
files once for their metadata and reuses the selected file's bytes for strict
|
||||
@@ -51,6 +59,23 @@ and permits inherited target fields only when a base is named. File-backed
|
||||
bounded JSON-value owner before a profile is published. OpenAI-compatible
|
||||
reserved-field policy remains with the model-client and backend-registry owners.
|
||||
|
||||
`LoadFSRepository` is the eager immutable loading boundary for internal
|
||||
catalog consumers. It discovers and strictly validates every raw profile once,
|
||||
preserves safe source metadata including explicitly present YAML fields, and
|
||||
publishes independently copied values from memory. It does not resolve profile
|
||||
inheritance. Configured consumer sources continue to use the lazy point lookup
|
||||
repositories described above.
|
||||
|
||||
`internal/catalog` validates the imported immutable OpenRouter and
|
||||
Rakestrawhome asset modules as one private adapter boundary. It enforces their
|
||||
manifest, layout, profile ownership, inheritance, and secret-safety rules
|
||||
before returning raw catalog sources. Internal engine assembly uses those validated
|
||||
catalogs as the lowest-precedence profile source.
|
||||
|
||||
Normalized external catalog profile IDs with the exact `promptkit-` prefix are
|
||||
rejected at this adapter boundary. That reservation applies only to imported
|
||||
catalogs; configured consumer sources retain the ordinary profile-ID rules.
|
||||
|
||||
The overlay repository consults the next repository only when the
|
||||
higher-precedence repository reports that a profile is absent. A reliably
|
||||
selected malformed profile stops fallback, while an unrelated malformed file
|
||||
@@ -59,12 +84,15 @@ backend registry membership because the available registry belongs to the
|
||||
assembled engine; the runner checks membership during preparation and exact
|
||||
profile inspection.
|
||||
|
||||
The root engine assembles one raw composite catalog in precedence order:
|
||||
The internal engine assembles one raw composite catalog in precedence order:
|
||||
in-memory profiles, one ordinary configured source, an application fallback
|
||||
source, then the embedded built-in catalog. An explicit file or `fs.FS` profile
|
||||
source replaces `Config.ProfileDir` within the ordinary configured-source
|
||||
category. One outer resolving repository wraps that complete raw catalog, so
|
||||
each base lookup observes the same precedence and shadowing rules.
|
||||
source, Promptkit-owned profiles, then the maintained external catalog. It
|
||||
constructs the owned definitions through the in-memory repository and validates
|
||||
their maintained-only resolution and backend membership before consumer sources
|
||||
are overlaid. An explicit file or `fs.FS` profile source replaces
|
||||
`Config.ProfileDir` within the ordinary configured-source category. One outer
|
||||
resolving repository wraps that complete raw catalog, so each base lookup
|
||||
observes the same precedence and shadowing rules.
|
||||
|
||||
The resolving repository traverses every selected chain afresh, retains no
|
||||
cache, detects cycles, limits a chain to 32 profiles, merges root-to-leaf into a
|
||||
@@ -79,15 +107,12 @@ or schema sources. It does not retain that lookup for a later execution.
|
||||
Prepared execution instead freezes the fully resolved target; a later ordinary
|
||||
operation performs a fresh traversal.
|
||||
|
||||
`internal/profile/builtin` embeds the maintained built-in profile catalog.
|
||||
Every embedded profile selects a maintained built-in backend and inherits that
|
||||
backend's endpoint and credential environment-variable name from the built-in
|
||||
backend registry rather than repeating those values. Profile loading and
|
||||
overlay behavior are owned by the
|
||||
[profile repository tests](../../internal/profile/repository_test.go), while
|
||||
catalog completeness, the backend-selection invariant, and duplicate IDs are
|
||||
owned by the
|
||||
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
|
||||
The imported external catalog provides external profiles and matching backend
|
||||
definitions. The internal engine owns the Promptkit profiles documented in the
|
||||
[framework format reference](../formats.md#built-in-profile-catalog). Profile
|
||||
loading and overlay behavior are owned by the [profile repository tests](../../internal/profile/repository_test.go), while
|
||||
catalog completeness, backend selection, and duplicate IDs are owned by the
|
||||
[catalog adapter tests](../../internal/catalog/catalog_test.go).
|
||||
|
||||
## Ordinary Artifacts
|
||||
|
||||
@@ -122,8 +147,9 @@ Session and message parsing and execution remain synchronous. The renderer
|
||||
checks cancellation before and after each parse and execution boundary,
|
||||
between artifact conversion chunks, around each message, and before publishing
|
||||
the complete prompt. It cannot interrupt template work already in progress and
|
||||
never publishes a partial prompt after observing cancellation. It carries
|
||||
message roles, session IDs, and cache control into the rendered prompt. The
|
||||
never publishes a partial prompt after observing cancellation. It validates and
|
||||
canonicalizes message roles before carrying roles, session IDs, and cache
|
||||
control into the rendered prompt. The
|
||||
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
|
||||
|
||||
## Schemas And Output Validation
|
||||
|
||||
118
docs/oss/attribution.md
Normal file
118
docs/oss/attribution.md
Normal file
@@ -0,0 +1,118 @@
|
||||
# Open Source Attribution Notices
|
||||
|
||||
These entries identify upstream material used by Promptkit. The
|
||||
[OSS policy](../policy/oss.md) owns the format and maintenance rules; the
|
||||
[project grant](../../COPYRIGHT) does not replace third-party terms.
|
||||
|
||||
The module entries cover the five external modules used by `go list -deps -test
|
||||
./...` at adoption, including the transitive `golang.org/x/text` dependency.
|
||||
Other modules present only in dependency module graphs are not thereby
|
||||
incorporated into Promptkit. No separately copied third-party code or asset
|
||||
notices were identified in Promptkit's source; adapted policy documents are
|
||||
attributed separately below. Standard license texts in `licenses/` are shared
|
||||
reference texts, not additional project grants.
|
||||
|
||||
## Promptkit OpenRouter Catalog
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2`
|
||||
at `v2.20260926.0`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: MIT
|
||||
|
||||
License and notices: [MIT](licenses/MIT.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/COPYRIGHT) |
|
||||
[upstream LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/LICENSE).
|
||||
|
||||
Source: [OpenRouter catalog at the imported commit](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3).
|
||||
|
||||
## Promptkit Rakestrawhome Catalog
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2`
|
||||
at `v2.20260926.0`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: MIT
|
||||
|
||||
License and notices: [MIT](licenses/MIT.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/COPYRIGHT) |
|
||||
[upstream LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/LICENSE).
|
||||
|
||||
Source: [Rakestrawhome catalog at the imported commit](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596).
|
||||
|
||||
## jsonschema
|
||||
|
||||
Selected module: `github.com/santhosh-tekuri/jsonschema/v6` at `v6.0.2`.
|
||||
|
||||
No project-specific copyright statement was supplied in the selected source's
|
||||
LICENSE, README, or Go source files; none is invented here.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
License and notices: [Apache 2.0](licenses/Apache-2.0.txt) |
|
||||
[upstream LICENSE](https://github.com/santhosh-tekuri/jsonschema/blob/29cbed948d24a04700eb94436416b18a07953b71/LICENSE).
|
||||
|
||||
Source: [jsonschema at the imported commit](https://github.com/santhosh-tekuri/jsonschema/tree/29cbed948d24a04700eb94436416b18a07953b71).
|
||||
|
||||
## Go Text
|
||||
|
||||
Selected module: `golang.org/x/text` at `v0.14.0` (transitive dependency).
|
||||
|
||||
Copyright (c) 2009 The Go Authors. All rights reserved.
|
||||
|
||||
Individual source files retain their additional year-specific Go Authors
|
||||
notices in the linked source tree.
|
||||
|
||||
SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause.txt) |
|
||||
[upstream LICENSE](https://github.com/golang/text/blob/6c97a165dd661335ff7bce6104a008558123c353/LICENSE) |
|
||||
[additional patent grant](https://github.com/golang/text/blob/6c97a165dd661335ff7bce6104a008558123c353/PATENTS).
|
||||
|
||||
Source: [Go Text at the imported commit](https://github.com/golang/text/tree/6c97a165dd661335ff7bce6104a008558123c353).
|
||||
|
||||
## Go YAML v3
|
||||
|
||||
Selected module: `gopkg.in/yaml.v3` at `v3.0.1`.
|
||||
|
||||
Copyright (c) 2006-2010 Kirill Simonov
|
||||
|
||||
Copyright (c) 2006-2011 Kirill Simonov
|
||||
|
||||
Copyright (c) 2011-2019 Canonical Ltd
|
||||
|
||||
Copyright 2011-2016 Canonical Ltd.
|
||||
|
||||
SPDX-License-Identifier: MIT AND Apache-2.0
|
||||
|
||||
The licenses apply to different files, not an either/or choice. The upstream
|
||||
LICENSE identifies the libyaml-derived MIT files and the Apache-licensed
|
||||
remainder; NOTICE supplies the final copyright statement above.
|
||||
|
||||
License and notices: [MIT](licenses/MIT.txt) |
|
||||
[Apache 2.0](licenses/Apache-2.0.txt) |
|
||||
[upstream LICENSE and file scope](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/LICENSE) |
|
||||
[NOTICE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/NOTICE).
|
||||
|
||||
Source: [Go YAML at the imported commit](https://github.com/go-yaml/yaml/tree/f6f7691b1fdeb513f56608cd2c32c51f8194bf51).
|
||||
|
||||
## Standard Repository Policies
|
||||
|
||||
Material: adapted standard policy documents under `docs/policy/`, with
|
||||
Promptkit-specific ownership, library boundaries, contributor guidance, and
|
||||
release/OSS conventions. These documents have been modified from the templates.
|
||||
|
||||
No project-specific copyright statement is supplied by the standard policy
|
||||
repository's README or LICENSE; none is invented here.
|
||||
|
||||
SPDX-License-Identifier: CC-BY-SA-4.0
|
||||
|
||||
The adapted policy documentation retains these upstream terms; this does not
|
||||
change Promptkit's library grant.
|
||||
|
||||
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt).
|
||||
|
||||
Source: maintainer-provided internal standard policy templates. The source
|
||||
repository is non-public; no public source or notice URL is available.
|
||||
202
docs/oss/licenses/Apache-2.0.txt
Normal file
202
docs/oss/licenses/Apache-2.0.txt
Normal file
@@ -0,0 +1,202 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
11
docs/oss/licenses/BSD-3-Clause.txt
Normal file
11
docs/oss/licenses/BSD-3-Clause.txt
Normal file
@@ -0,0 +1,11 @@
|
||||
Copyright (c) <year> <owner>.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
170
docs/oss/licenses/CC-BY-SA-4.0.txt
Normal file
170
docs/oss/licenses/CC-BY-SA-4.0.txt
Normal file
@@ -0,0 +1,170 @@
|
||||
Creative Commons Attribution-ShareAlike 4.0 International
|
||||
|
||||
Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible.
|
||||
|
||||
Using Creative Commons Public Licenses
|
||||
|
||||
Creative Commons public licenses provide a standard set of terms and conditions that creators and other rights holders may use to share original works of authorship and other material subject to copyright and certain other rights specified in the public license below. The following considerations are for informational purposes only, are not exhaustive, and do not form part of our licenses.
|
||||
|
||||
Considerations for licensors: Our public licenses are intended for use by those authorized to give the public permission to use material in ways otherwise restricted by copyright and certain other rights. Our licenses are irrevocable. Licensors should read and understand the terms and conditions of the license they choose before applying it. Licensors should also secure all rights necessary before applying our licenses so that the public can reuse the material as expected. Licensors should clearly mark any material not subject to the license. This includes other CC-licensed material, or material used under an exception or limitation to copyright. More considerations for licensors.
|
||||
|
||||
Considerations for the public: By using one of our public licenses, a licensor grants the public permission to use the licensed material under specified terms and conditions. If the licensor’s permission is not necessary for any reason–for example, because of any applicable exception or limitation to copyright–then that use is not regulated by the license. Our licenses grant only permissions under copyright and certain other rights that a licensor has authority to grant. Use of the licensed material may still be restricted for other reasons, including because others have copyright or other rights in the material. A licensor may make special requests, such as asking that all changes be marked or described.
|
||||
|
||||
Although not required by our licenses, you are encouraged to respect those requests where reasonable. More considerations for the public.
|
||||
|
||||
Creative Commons Attribution-ShareAlike 4.0 International Public License
|
||||
|
||||
By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-ShareAlike 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions.
|
||||
|
||||
Section 1 – Definitions.
|
||||
|
||||
a. Adapted Material means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image.
|
||||
|
||||
b. Adapter's License means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License.
|
||||
|
||||
c. BY-SA Compatible License means a license listed at creativecommons.org/compatiblelicenses, approved by Creative Commons as essentially the equivalent of this Public License.
|
||||
|
||||
d. Copyright and Similar Rights means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights.
|
||||
|
||||
e. Effective Technological Measures means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements.
|
||||
|
||||
f. Exceptions and Limitations means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material.
|
||||
|
||||
g. License Elements means the license attributes listed in the name of a Creative Commons Public License. The License Elements of this Public License are Attribution and ShareAlike.
|
||||
|
||||
h. Licensed Material means the artistic or literary work, database, or other material to which the Licensor applied this Public License.
|
||||
|
||||
i. Licensed Rights means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license.
|
||||
|
||||
j. Licensor means the individual(s) or entity(ies) granting rights under this Public License.
|
||||
|
||||
k. Share means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them.
|
||||
|
||||
l. Sui Generis Database Rights means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world.
|
||||
|
||||
m. You means the individual or entity exercising the Licensed Rights under this Public License. Your has a corresponding meaning.
|
||||
|
||||
Section 2 – Scope.
|
||||
|
||||
a. License grant.
|
||||
|
||||
1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to:
|
||||
|
||||
A. reproduce and Share the Licensed Material, in whole or in part; and
|
||||
|
||||
B. produce, reproduce, and Share Adapted Material.
|
||||
|
||||
2. Exceptions and Limitations. For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions.
|
||||
|
||||
3. Term. The term of this Public License is specified in Section 6(a).
|
||||
|
||||
4. Media and formats; technical modifications allowed. The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so. The Licensor waives and/or agrees not to assert any right or authority to forbid You from making technical modifications necessary to exercise the Licensed Rights, including technical modifications necessary to circumvent Effective Technological Measures. For purposes of this Public License, simply making modifications authorized by this Section 2(a)(4) never produces Adapted Material.
|
||||
|
||||
5. Downstream recipients.
|
||||
|
||||
A. Offer from the Licensor – Licensed Material. Every recipient of the Licensed Material automatically receives an offer from the Licensor to exercise the Licensed Rights under the terms and conditions of this Public License.
|
||||
|
||||
B. Additional offer from the Licensor – Adapted Material. Every recipient of Adapted Material from You automatically receives an offer from the Licensor to exercise the Licensed Rights in the Adapted Material under the conditions of the Adapter’s License You apply.
|
||||
|
||||
C. No downstream restrictions. You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material.
|
||||
|
||||
6. No endorsement. Nothing in this Public License constitutes or may be construed as permission to assert or imply that You are, or that Your use of the Licensed Material is, connected with, or sponsored, endorsed, or granted official status by, the Licensor or others designated to receive attribution as provided in Section 3(a)(1)(A)(i).
|
||||
|
||||
b. Other rights.
|
||||
|
||||
1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights; however, to the extent possible, the Licensor waives and/or agrees not to assert any such rights held by the Licensor to the limited extent necessary to allow You to exercise the Licensed Rights, but not otherwise.
|
||||
|
||||
2. Patent and trademark rights are not licensed under this Public License.
|
||||
|
||||
3. To the extent possible, the Licensor waives any right to collect royalties from You for the exercise of the Licensed Rights, whether directly or through a collecting society under any voluntary or waivable statutory or compulsory licensing scheme. In all other cases the Licensor expressly reserves any right to collect such royalties.
|
||||
|
||||
Section 3 – License Conditions.
|
||||
|
||||
Your exercise of the Licensed Rights is expressly made subject to the following conditions.
|
||||
|
||||
a. Attribution.
|
||||
|
||||
1. If You Share the Licensed Material (including in modified form), You must:
|
||||
|
||||
A. retain the following if it is supplied by the Licensor with the Licensed Material:
|
||||
|
||||
i. identification of the creator(s) of the Licensed Material and any others designated to receive attribution, in any reasonable manner requested by the Licensor (including by pseudonym if designated);
|
||||
|
||||
ii. a copyright notice;
|
||||
|
||||
iii. a notice that refers to this Public License;
|
||||
|
||||
iv. a notice that refers to the disclaimer of warranties;
|
||||
|
||||
v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable;
|
||||
|
||||
B. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and
|
||||
|
||||
C. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License.
|
||||
|
||||
2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material. For example, it may be reasonable to satisfy the conditions by providing a URI or hyperlink to a resource that includes the required information.
|
||||
|
||||
3. If requested by the Licensor, You must remove any of the information required by Section 3(a)(1)(A) to the extent reasonably practicable.
|
||||
|
||||
b. ShareAlike.In addition to the conditions in Section 3(a), if You Share Adapted Material You produce, the following conditions also apply.
|
||||
|
||||
1. The Adapter’s License You apply must be a Creative Commons license with the same License Elements, this version or later, or a BY-SA Compatible License.
|
||||
|
||||
2. You must include the text of, or the URI or hyperlink to, the Adapter's License You apply. You may satisfy this condition in any reasonable manner based on the medium, means, and context in which You Share Adapted Material.
|
||||
|
||||
3. You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, Adapted Material that restrict exercise of the rights granted under the Adapter's License You apply.
|
||||
|
||||
Section 4 – Sui Generis Database Rights.
|
||||
|
||||
Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material:
|
||||
|
||||
a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database;
|
||||
|
||||
b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material, including for purposes of Section 3(b); and
|
||||
|
||||
c. You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database.
|
||||
For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights.
|
||||
|
||||
Section 5 – Disclaimer of Warranties and Limitation of Liability.
|
||||
|
||||
a. Unless otherwise separately undertaken by the Licensor, to the extent possible, the Licensor offers the Licensed Material as-is and as-available, and makes no representations or warranties of any kind concerning the Licensed Material, whether express, implied, statutory, or other. This includes, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not known or discoverable. Where disclaimers of warranties are not allowed in full or in part, this disclaimer may not apply to You.
|
||||
|
||||
b. To the extent possible, in no event will the Licensor be liable to You on any legal theory (including, without limitation, negligence) or otherwise for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising out of this Public License or use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. Where a limitation of liability is not allowed in full or in part, this limitation may not apply to You.
|
||||
|
||||
c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability.
|
||||
|
||||
Section 6 – Term and Termination.
|
||||
|
||||
a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically.
|
||||
|
||||
b. Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates:
|
||||
|
||||
1. automatically as of the date the violation is cured, provided it is cured within 30 days of Your discovery of the violation; or
|
||||
|
||||
2. upon express reinstatement by the Licensor.
|
||||
|
||||
c. For the avoidance of doubt, this Section 6(b) does not affect any right the Licensor may have to seek remedies for Your violations of this Public License.
|
||||
|
||||
d. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License.
|
||||
|
||||
e. Sections 1, 5, 6, 7, and 8 survive termination of this Public License.
|
||||
|
||||
Section 7 – Other Terms and Conditions.
|
||||
|
||||
a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed.
|
||||
|
||||
b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License.
|
||||
|
||||
Section 8 – Interpretation.
|
||||
|
||||
a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License.
|
||||
|
||||
b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions.
|
||||
|
||||
c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor.
|
||||
|
||||
d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority.
|
||||
|
||||
Creative Commons is not a party to its public licenses. Notwithstanding, Creative Commons may elect to apply one of its public licenses to material it publishes and in those instances will be considered the “Licensor.” Except for the limited purpose of indicating that material is shared under a Creative Commons public license or as otherwise permitted by the Creative Commons policies published at creativecommons.org/policies, Creative Commons does not authorize the use of the trademark “Creative Commons” or any other trademark or logo of Creative Commons without its prior written consent including, without limitation, in connection with any unauthorized modifications to any of its public licenses or any other arrangements, understandings, or agreements concerning use of licensed material. For the avoidance of doubt, this paragraph does not form part of the public licenses.
|
||||
|
||||
Creative Commons may be contacted at creativecommons.org.
|
||||
18
docs/oss/licenses/MIT.txt
Normal file
18
docs/oss/licenses/MIT.txt
Normal file
@@ -0,0 +1,18 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) <year> <copyright holders>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
|
||||
following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial
|
||||
portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
|
||||
LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
|
||||
EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
||||
USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
88
docs/policy/agents.md
Normal file
88
docs/policy/agents.md
Normal file
@@ -0,0 +1,88 @@
|
||||
# Agent Policy
|
||||
|
||||
**Base Policy Version:** 1.1
|
||||
|
||||
**Last Updated:** 2026-09-23
|
||||
|
||||
## Purpose
|
||||
|
||||
Agents must follow these house rules before recommending, planning, or
|
||||
implementing repository changes.
|
||||
|
||||
## Implementation Discipline
|
||||
|
||||
Write the simplest idiomatic code that satisfies the user's request.
|
||||
Simplify the mechanism, not the user's choices. Do not implicitly strengthen
|
||||
requirements, restrictions, or guarantees.
|
||||
|
||||
**Keep implementation machinery proportionate.** Use straightforward,
|
||||
idiomatic code and introduce only the machinery needed to support the requested
|
||||
behavior and flexibility. Optimize for conceptual simplicity and
|
||||
maintainability, not for the fewest lines, fewest options, or narrowest product
|
||||
surface.
|
||||
|
||||
**Do not invent restrictions.** Do not reject, prohibit, constrain, cap,
|
||||
harden, or otherwise narrow behavior unless required by the user, an explicit
|
||||
repository contract, or an unavoidable correctness constraint. Preserve
|
||||
operator and caller choice where policy has not been established. If an
|
||||
additional restriction appears advisable, present it as a decision rather than
|
||||
silently implementing it.
|
||||
|
||||
A correctness constraint must follow from the requested behavior or an
|
||||
established invariant, not from the convenience of the chosen implementation.
|
||||
Explain the specific failure it prevents and why a less restrictive
|
||||
implementation cannot satisfy the contract.
|
||||
|
||||
**An implementation detail is not repository policy.** Tests and documentation
|
||||
must not elevate incidental behavior into a durable guarantee or restriction
|
||||
unless that behavior was expressly selected as part of the contract. Existing
|
||||
explicit contracts remain authoritative; changing or weakening one requires a
|
||||
deliberate decision rather than being inferred from this principle.
|
||||
|
||||
Agent-authored plans, tests, documentation, and prior implementations are not
|
||||
evidence that an additional restriction was deliberately approved. Identify the
|
||||
authorizing requirement or decision; when that authority is unclear, surface
|
||||
the ambiguity rather than treating repetition as approval.
|
||||
|
||||
**Do not invent guarantees.** Do not strengthen public contracts, guarantees,
|
||||
or invariants beyond those expressly requested by the user.
|
||||
|
||||
**Use the standard library.** Prefer simple, idiomatic standard-library
|
||||
implementations when they satisfy the requested behavior. Do not add custom
|
||||
power-loss protection or defenses against privileged concurrent filesystem
|
||||
actors unless expressly required. Preserve existing repository contracts.
|
||||
|
||||
## Git Commit Policy
|
||||
|
||||
By default, create a commit after completing each coherent unit of work, using a
|
||||
concise, plain-English message describing what changed. This policy is standing
|
||||
authorization to create those local commits; a separate user request is not
|
||||
required. An explicit user instruction to leave work uncommitted takes precedence.
|
||||
|
||||
A unit of work is a focused, reviewable outcome, including its relevant tests and
|
||||
documentation. Do not commit after every intermediate edit or create empty commits.
|
||||
Complete the appropriate validation before committing. If work is incomplete or
|
||||
validation is blocked, report that state rather than presenting it as completed.
|
||||
|
||||
Before committing:
|
||||
|
||||
1. Run `git status --short` and review the relevant diff, including any staged
|
||||
changes. Identify changes that predate the current unit of work.
|
||||
2. Exclude secrets, local environment files, generated junk, and unrelated changes.
|
||||
3. Stage only the files or hunks belonging to the completed unit. Use `git add .`
|
||||
only when every included change belongs to that unit. Do not disturb unrelated
|
||||
staged or unstaged work.
|
||||
4. Review `git diff --staged` and run appropriate checks for the change. Do not
|
||||
repeat checks already passed for unchanged content merely to create a commit.
|
||||
5. Review `git log --oneline -n 10` and use
|
||||
`git commit -m "<concise plain-English commit message>"`.
|
||||
6. Report the commit identifier and any remaining uncommitted work.
|
||||
|
||||
If the unit cannot be safely separated from pre-existing work, leave it uncommitted
|
||||
and explain the overlap rather than silently including that work. If a commit
|
||||
fails, preserve the changes and report the failure; do not bypass hooks.
|
||||
|
||||
Default commit authorization does not authorize pushing, publishing, amending,
|
||||
resetting, reverting, or rewriting history. Those actions require a separate user
|
||||
instruction. Do not use `git commit -a` or `git commit --no-verify` to bypass the
|
||||
staging and validation discipline above.
|
||||
@@ -1,5 +1,9 @@
|
||||
# Architecture Policy
|
||||
|
||||
**Base Policy Version:** 1.0
|
||||
|
||||
**Last Updated:** 2026-09-23
|
||||
|
||||
## Purpose
|
||||
|
||||
This document defines Promptkit's current high-level architecture and the
|
||||
@@ -17,53 +21,25 @@ The module root contains package `promptkit`, which is the public facade. It
|
||||
provides the supported engine, configuration and source options, requests,
|
||||
results, public values, extension interfaces, profiles, and error sentinels.
|
||||
|
||||
The implemented internal components consist of:
|
||||
The [internal component overview](../internal/overview.md) owns the inventory
|
||||
of implemented packages and their concrete responsibilities. This policy owns
|
||||
the boundaries and dependency rules those components must preserve.
|
||||
|
||||
- `internal/domain`, which owns framework data values and source-neutral
|
||||
invariants shared by later internal components;
|
||||
- `internal/backend`, which owns validated immutable OpenAI-compatible backend
|
||||
definitions and the maintained built-in definitions;
|
||||
- `internal/capacity`, which owns engine-local bounded run admission and
|
||||
model-generation scheduling for limited backends;
|
||||
- `internal/defaults`, which owns application-neutral framework defaults and
|
||||
constructs the default execution target;
|
||||
- `internal/filecatalog`, which discovers YAML files and provides source-path
|
||||
helpers for filesystem and `fs.FS` consumers;
|
||||
- `internal/jsonvalue`, which validates and defensively copies JSON-compatible
|
||||
extra-parameter trees;
|
||||
- `internal/promptdef`, which loads and validates prompt definitions from
|
||||
filesystem and `fs.FS` sources;
|
||||
- `internal/profile`, which loads, validates, and overlays execution profiles
|
||||
from filesystem and `fs.FS` sources;
|
||||
- `internal/profile/builtin`, which embeds the built-in execution profile
|
||||
catalog;
|
||||
- `internal/prompt`, which renders prompt messages from Go templates;
|
||||
- `internal/artifact`, which resolves ordinary inline and unrestricted
|
||||
caller-selected file references;
|
||||
- `internal/validate`, which validates basic, JSON, and JSON Schema output
|
||||
using filesystem and `fs.FS` schema sources;
|
||||
- `internal/llm`, which defines the provider-neutral generation boundary and
|
||||
implements outbound OpenAI-compatible chat requests; and
|
||||
- `internal/usecase`, which coordinates preparation and execution across the
|
||||
internal framework components.
|
||||
|
||||
The `examples/go-library/prepare` and `examples/go-library/run` packages are
|
||||
maintained downstream consumers of the root facade. They do not expose library
|
||||
packages or participate in internal assembly.
|
||||
|
||||
The root facade assembles one immutable backend registry, one capacity manager,
|
||||
The internal engine assembles one immutable backend registry, one capacity manager,
|
||||
the internal repositories, renderer, validator, outbound client, and use-case
|
||||
runner while translating public values and errors at the library boundary. The
|
||||
registry contains built-ins plus validated engine-scoped consumer additions.
|
||||
The facade constructs the capacity manager from the registry's immutable
|
||||
runner. The registry contains validated maintained definitions plus engine-scoped
|
||||
consumer additions. The root facade translates public values and errors at the
|
||||
library boundary.
|
||||
The internal engine constructs the capacity manager from the registry's immutable
|
||||
policy snapshot, wraps the selected built-in or injected model client, and
|
||||
supplies bounded admission to the runner. The defaults and renderer depend on
|
||||
the domain model. Prompt-definition and profile repositories use the domain
|
||||
model, file catalog, and YAML decoder. The built-in profile repository supplies
|
||||
an embedded `fs.FS` to the profile package. Artifact reading uses the domain
|
||||
model and application-neutral defaults. Validation uses the domain model, file
|
||||
catalog, and JSON Schema implementation. The model client uses the domain
|
||||
model, application-neutral defaults, and an injected or standard-library HTTP
|
||||
model, file catalog, and YAML decoder. The catalog adapter validates imported
|
||||
immutable maintained data, including its reserved namespace, before internal
|
||||
engine assembly supplies it to the profile package. Artifact reading uses the
|
||||
domain model and application-neutral defaults. Validation uses the domain model,
|
||||
file catalog, and JSON Schema implementation. The model client uses the domain
|
||||
model and application-neutral defaults, and an injected or standard-library HTTP
|
||||
client. The use-case runner depends on the narrow interfaces owned by each
|
||||
internal component, including backend lookup and run admission.
|
||||
|
||||
@@ -125,8 +101,10 @@ consumer contract requires a public package.
|
||||
Each package must have one cohesive responsibility and a clear dependency
|
||||
direction. Internal packages must not expose their types merely to simplify
|
||||
wiring, and the public facade must not leak internal representations through
|
||||
exported signatures. New public packages require a durable consumer need that
|
||||
cannot be served cleanly by the root facade.
|
||||
exported signatures. Production packages under `internal/` must not import the
|
||||
module root; the root facade may depend on internal implementation packages.
|
||||
New public packages require a durable consumer need that cannot be served
|
||||
cleanly by the root facade.
|
||||
|
||||
The [internal component overview](../internal/overview.md) must be updated as
|
||||
packages are implemented or their responsibilities change.
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
# Documentation Policy
|
||||
|
||||
**Base Policy Version:** 1.5
|
||||
|
||||
**Last Updated:** 2026-09-26
|
||||
|
||||
## Purpose
|
||||
|
||||
This policy assigns each Promptkit documentation topic to one canonical owner.
|
||||
@@ -68,17 +72,24 @@ mechanisms, not secret values.
|
||||
|
||||
| Topic | Canonical owner | Owned content | Content owned elsewhere |
|
||||
| --- | --- | --- | --- |
|
||||
| Project orientation | `README.md` | What Promptkit is, its current usability, module identity, license summary, and links onward. | Exact API contracts, contributor procedures, architecture detail, and release steps. |
|
||||
| Project orientation | `README.md` | What Promptkit is, its current usability, module identity, license summary, a minimal successful quickstart, and links onward. | Exact API contracts, contributor procedures, architecture detail, and release steps. |
|
||||
| Contributor workflow | `docs/development.md` | Task-oriented reading guide, local workflow, validation commands, and repository hygiene. | Architecture rules, API semantics, subsystem behavior, and release procedure. |
|
||||
| Current architecture | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, package boundaries, invariants, and non-goals. | Concrete component inventory, implementation mechanics, contributor procedures, decision history, and future work. |
|
||||
| Agent policy | `docs/policy/agents.md` | Agent implementation discipline, proportionality, idiomatic simplicity, local commit authorization and discipline, and rules against inventing restrictions, guarantees, or repository policy. | Public contracts, architecture, testing, and contributor procedures. |
|
||||
| Documentation organization | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and document lifecycle. | Library architecture or runtime behavior. |
|
||||
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression policy, and test maintenance. | Subsystem behavior, exact public contracts, subsystem-specific test inventories, and implementation plans. |
|
||||
| Release procedure | `docs/release.md`, when present | Required release validation, version and tag procedure, release ordering, and post-publication checks. | General contributor workflow, public API semantics, and decision history. |
|
||||
| Release policy | `docs/policy/release.md` | Versioning, compatibility, release-note requirements, validation obligations, immutability, recovery, and adoption scope. | Executable commands, publication mechanics, and historical release summaries. |
|
||||
| Licensing and attribution policy | `docs/policy/oss.md` | Project licensing conventions, attribution format, text and notice-link rules, discoverability, maintenance, and concrete exceptions. | Component entries, license texts, release commands, and dependency security review. |
|
||||
| Project grant | `COPYRIGHT` | Project identity, holders and years, SPDX expression, grant, warranty disclaimer, and third-party statement. | Complete license texts and component attribution. |
|
||||
| Project license texts | `LICENSE` | Complete project license and incorporated companion texts. | General licensing policy and dependency attribution. |
|
||||
| Upstream attribution | `docs/oss/attribution.md` | Component names, copyright statements, license identification, and license, notice, and source links. | General policy and complete standard texts. |
|
||||
| Standard dependency license texts | `docs/oss/licenses/` | Complete, verbatim texts shared by attribution entries. | Project-specific notices, attribution prose, and policy interpretation. |
|
||||
| Release procedure | `docs/release.md` | Maintainer prerequisites, candidate preparation, concrete publication guards and commands, verification, and recovery steps. | Release-policy requirements, public API semantics, and historical summaries. |
|
||||
| Exact exported Go API | Go declarations and GoDoc, as APIs are implemented | Exported names, signatures, types, values, errors, and exact behavioral contracts. | Task-oriented consumer walkthroughs, implementation details, and future API proposals. |
|
||||
| Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, built-in profiles, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. |
|
||||
| Consumer guidance | `docs/consumers/`, when consumer workflows require dedicated guidance | Task-oriented use of implemented public APIs, minimal examples, and consumer responsibilities. | Exact exported declarations and internal mechanics. |
|
||||
| Durable integration contracts | `docs/integrations/`, when integrations exist | External formats and protocols, compatibility behavior, and upstream or downstream responsibilities. | Internal transformations and public Go declarations. |
|
||||
| Supplemental release guidance | None. `docs/releases/` may be used when a release benefits from a changelog or migration guide. | No canonical content. These files may briefly summarize release-specific changes, compatibility, and consumer migration paths, and may be corrected, consolidated, archived, or removed when no longer useful. | Public API and behavior contracts, formats, integrations, architecture, release procedure, and the authoritative annotated-tag release record. |
|
||||
| Release-note history | `docs/releases/` | Mandatory version-matched historical summaries for releases made under the adopted release policy; the note at the immutable tag records that release. | Current APIs, behavior, formats, compatibility contracts, and release procedures. |
|
||||
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal documents. | Normative architecture, contributor workflow, external contracts, and proposed components. |
|
||||
| Internal subsystem behavior | Other files under `docs/internal/`, when a subsystem needs durable detail | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, public API definitions, and future package plans. |
|
||||
| Architectural decision history | `docs/adr/`, when repository-local decisions require records | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, and task sequencing. |
|
||||
@@ -110,22 +121,30 @@ but must link to its canonical definition rather than restate it.
|
||||
The [framework format reference](../formats.md) owns exact prompt, profile, and
|
||||
schema-file contracts. Integration documents own external wire formats.
|
||||
|
||||
### Supplemental Release Guidance
|
||||
### Release Documentation
|
||||
|
||||
Files under `docs/releases/` may provide changelog-style summaries and
|
||||
migration guidance for a particular release. They are navigation and
|
||||
orientation aids, not canonical owners of public APIs, behavior, formats,
|
||||
integrations, architecture, release procedure, or other durable facts. When a
|
||||
reader needs detail beyond a short release-specific note, the release document
|
||||
must link to the applicable canonical documentation rather than reproduce its
|
||||
contract.
|
||||
The [release policy](release.md) owns release requirements; the
|
||||
[repository release procedure](../release.md) owns their concrete execution.
|
||||
Version-matched checked-in notes are required for future releases under that
|
||||
policy. The source tag is the release identity, and the note in its immutable
|
||||
tree is the historical summary; annotated tag messages complement that record.
|
||||
|
||||
The annotated tag message required by the
|
||||
[release procedure](../release.md#write-the-release-note) remains the
|
||||
authoritative release record. Supplemental release documents may be corrected,
|
||||
consolidated, archived, or removed at any time when they are no longer useful,
|
||||
provided maintained documentation does not depend on them and the annotated
|
||||
tag record remains intact.
|
||||
Release notes do not own current API or behavior contracts. Link to the
|
||||
canonical owners for details instead of reproducing those contracts. Published
|
||||
notes are not disposable guidance and must not be replaced under the same
|
||||
version. Corrections belong in a subsequent release.
|
||||
|
||||
Adoption is prospective. Previously published tags and historical notes retain
|
||||
their original records and are not reformatted or rewritten to satisfy the new
|
||||
requirements.
|
||||
|
||||
### Project Licensing And Attribution
|
||||
|
||||
The [OSS policy](oss.md) owns licensing conventions, the attribution House
|
||||
Format, and README/release navigation. COPYRIGHT owns the project grant;
|
||||
LICENSE supplies complete texts. Component-specific information belongs in
|
||||
attribution entries, not parallel README or release-note notice lists. The
|
||||
release procedure owns concrete footer URLs and publication mechanics.
|
||||
|
||||
### Security Topics
|
||||
|
||||
@@ -178,9 +197,8 @@ durable owners, update incoming links, and archive or remove the roadmap
|
||||
according to repository practice. Do not preserve completed roadmaps as a
|
||||
second current-state reference.
|
||||
|
||||
Supplemental release documents may likewise be removed without preserving a
|
||||
replacement. Before removal, update maintained incoming links so current
|
||||
documentation does not depend on an optional historical guide.
|
||||
Retain release notes as historical records under the release policy. Do not
|
||||
apply temporary-roadmap retirement rules to published release notes.
|
||||
|
||||
Before completing documentation work:
|
||||
|
||||
|
||||
135
docs/policy/oss.md
Normal file
135
docs/policy/oss.md
Normal file
@@ -0,0 +1,135 @@
|
||||
# Open Source Licensing And Attribution Policy
|
||||
|
||||
**Base Policy Version:** 1.2
|
||||
|
||||
**Last Updated:** 2026-09-26
|
||||
|
||||
Adapted from the standard policy; see [provenance and licensing](../oss/attribution.md#standard-repository-policies).
|
||||
|
||||
## Purpose And Scope
|
||||
|
||||
This policy defines Promptkit's project licensing conventions and House Format
|
||||
for upstream attribution. It covers relevant incorporated or distributed
|
||||
components, including transitive dependencies and copied code, documentation,
|
||||
or assets. Treatment follows the component's terms and actual use, not merely
|
||||
its direct-dependency status or common ownership. Development tools do not
|
||||
automatically require attribution.
|
||||
|
||||
Promptkit is a Go library with an author-selected LGPL-3.0-or-later grant in
|
||||
[COPYRIGHT](../../COPYRIGHT). Its public source repository is
|
||||
`https://gitea.maximumdirect.net/eric/promptkit`; it publishes source-only Go
|
||||
module releases through annotated tags, not binaries or separate hosted release
|
||||
pages. The [release procedure](../release.md) owns publication mechanics.
|
||||
|
||||
The House Format does not establish license compatibility or resolve every
|
||||
obligation arising from use or distribution. Adoption does not relicense
|
||||
existing material or override third-party terms.
|
||||
|
||||
## Canonical Locations
|
||||
|
||||
The [documentation policy](documentation.md) assigns ownership:
|
||||
|
||||
- `COPYRIGHT`: project name, holders and years, SPDX expression, grant,
|
||||
warranty disclaimer, and statement about third-party material.
|
||||
- `LICENSE`: complete, unmodified project license texts, including incorporated
|
||||
companion text. Promptkit includes LGPLv3 followed by GPLv3.
|
||||
- `docs/oss/attribution.md`: canonical component entries.
|
||||
- `docs/oss/licenses/`: complete, verbatim standard dependency texts, including
|
||||
companions where applicable. Shared texts can serve multiple entries.
|
||||
|
||||
Keep explanations and project-specific notices outside standard texts. Preserve
|
||||
distinct versions and exceptions, using recognizable filenames.
|
||||
|
||||
## Project Licensing
|
||||
|
||||
The house defaults for new projects are GPL-3.0-or-later for applications and
|
||||
LGPL-3.0-or-later for libraries. An author's explicit choice takes precedence
|
||||
without an exception rationale, extra approval, or compensating controls.
|
||||
Promptkit retains its selected library grant; do not infer different licenses
|
||||
for individual packages or relicense existing code by adopting this policy.
|
||||
|
||||
Keep COPYRIGHT, LICENSE, SPDX expressions, and README descriptions consistent.
|
||||
Use accurate holders and years; do not assume the maintainer owns every
|
||||
contribution or advance years automatically. Update grant, disclaimer, and text
|
||||
references together when the author deliberately changes licensing. Preserve
|
||||
third-party terms and required existing notices. No blanket per-file header
|
||||
requirement is introduced.
|
||||
|
||||
## Upstream Attribution House Format
|
||||
|
||||
Each component entry contains five fields:
|
||||
|
||||
1. Project name, normally as the heading.
|
||||
2. Upstream copyright statements, preserving their wording.
|
||||
3. Applicable SPDX license identifier or expression.
|
||||
4. License and notice links.
|
||||
5. Source repository link at the exact imported commit.
|
||||
|
||||
These are fields, not a line limit. Include multiple statements or links where
|
||||
needed. Do not invent absent copyright statements or SPDX identifiers. Describe
|
||||
nonstandard terms through their linked text, preserve alternatives and
|
||||
exceptions, and record a selected option when relevant. Do not silently resolve
|
||||
an unsettled licensing choice.
|
||||
|
||||
## License Texts And Upstream Links
|
||||
|
||||
Link standard texts to local copies. Link project-specific notices to the
|
||||
upstream copies at the exact commit corresponding to the imported version or
|
||||
copied material. Moving branches and version tags are not substitutes for
|
||||
commit-specific component links.
|
||||
|
||||
Classify material by content, not filename: an upstream LICENSE or COPYING can
|
||||
mix standard text with project-specific notices or exceptions. Link both when
|
||||
needed. Make incorporated companion texts directly reachable from each entry.
|
||||
|
||||
For project-specific license and attribution material, direct upstream links
|
||||
are accepted as satisfying reproduction, inclusion, or copy requirements under
|
||||
the adopted house interpretation. Do not add local-copy requirements or request
|
||||
repeated approval solely because a license uses those terms. This concerns
|
||||
notice reproduction, not other concrete obligations such as marking modified
|
||||
files or supplying corresponding source. Central entries do not authorize
|
||||
stripping required notices from incorporated source.
|
||||
|
||||
## Discoverability
|
||||
|
||||
End README with friendly statements linking to the actual grant in COPYRIGHT
|
||||
and to the attribution document. Each new checked-in release note ends with
|
||||
links whose visible text is `COPYRIGHT | LICENSE | ATTRIBUTION`, using absolute
|
||||
URLs at the release's immutable tag or corresponding immutable source revision.
|
||||
These release links differ from imported-component links, which use commits.
|
||||
|
||||
The release procedure owns concrete URL construction. If a hosted download page
|
||||
is deliberately introduced, reuse the checked-in note and preserve its footer
|
||||
there. This convention does not require a new hosted page, downloadable license
|
||||
assets, source bundles, or another packaging format. Do not maintain duplicate
|
||||
notice lists.
|
||||
|
||||
## Maintenance And Exceptions
|
||||
|
||||
Update attribution in the change that adds, removes, upgrades, copies, or changes
|
||||
distribution of a relevant component. Inspect the selected source's actual
|
||||
licenses/notices rather than assuming they are unchanged. Update entries,
|
||||
commit links, and texts together; retain history through Git.
|
||||
|
||||
Review affected statements, standard-text fidelity, and link destinations during
|
||||
ordinary change review. Future-release URLs can be checked against candidate
|
||||
paths before the tag exists remotely. Do not add publication-time network checks,
|
||||
CI polling, recurring legal sign-offs, mandatory scanners, or an SBOM just to
|
||||
implement this policy. Attribution maintenance is preparation, not a release gate.
|
||||
|
||||
Raise exceptions only for concrete obligations or unresolved rights questions
|
||||
the House Format does not address, including distribution-specific issues.
|
||||
Record their selected treatment here or in the affected entry and resolve them
|
||||
before the affected use/distribution, without reopening settled choices.
|
||||
Common ownership is not a blanket exception. Adapted standard policy documents
|
||||
retain their upstream documentation license and identify modifications in their
|
||||
provenance entry; this does not change Promptkit's library grant.
|
||||
|
||||
The maintainer-provided standard policy templates are internal and non-public.
|
||||
Their attribution entry identifies that source in prose and retains the
|
||||
applicable license locally; do not link to or require publication of the
|
||||
internal repository.
|
||||
|
||||
Apply conventions prospectively. Do not rewrite published tags, notes, binaries,
|
||||
or checksums to retrofit formatting. Handle actual omissions deliberately under
|
||||
applicable terms and the [release correction policy](release.md#immutability-and-failure-recovery).
|
||||
151
docs/policy/release.md
Normal file
151
docs/policy/release.md
Normal file
@@ -0,0 +1,151 @@
|
||||
# Release Policy
|
||||
|
||||
**Base Policy Version:** 1.2
|
||||
|
||||
**Last Updated:** 2026-09-26
|
||||
|
||||
## Purpose And Adoption Scope
|
||||
|
||||
This policy owns Promptkit's release requirements and compatibility rules.
|
||||
The [release procedure](../release.md) owns the commands used to satisfy them.
|
||||
This scoped adoption applies to releases prepared after its adoption on
|
||||
2026-09-23. Published tags and historical notes are not rewritten to retrofit
|
||||
the new note format.
|
||||
|
||||
## Release Model And Compatibility
|
||||
|
||||
Promptkit publishes source-only Go module releases from committed, pushed
|
||||
`main` candidates through annotated `vMAJOR.MINOR.PATCH` tags. It publishes
|
||||
no binaries, hosted releases, or other artifacts and uses maintainer-run
|
||||
validation rather than hosted CI. Versions must be unused locally and on
|
||||
`origin`; the existing procedure accepts stable version tags only.
|
||||
|
||||
Before v1, minor releases may change the public API and patches preserve
|
||||
compatibility within their minor line, subject to the catalog-data provision
|
||||
below. Release notes must identify compatibility impact, public API changes,
|
||||
and required consumer action. The source tag identifies the exact release;
|
||||
the checked-in note at that tag records its historical summary.
|
||||
|
||||
## Catalog Profile Compatibility
|
||||
|
||||
External catalog profile data may be added, removed, renamed, or modified
|
||||
without deprecation or advance notice. This exception is limited to profile
|
||||
data imported from the maintained catalog modules. It does not weaken ordinary
|
||||
compatibility for Promptkit's public Go API, configuration and format semantics,
|
||||
runtime behavior, catalog-module APIs, manifest schema, backend definitions, or
|
||||
consumer-owned definitions.
|
||||
|
||||
Within the catalog versions selected and tested by a Promptkit release, and
|
||||
unless a consumer overrides the name or its base, Promptkit keeps the literal
|
||||
profile IDs `promptkit-openrouter-standard` and
|
||||
`promptkit-openrouter-auto` available. The names do not promise a permanent
|
||||
model, backend, routing behavior, settings, cost, output, or upstream
|
||||
availability. The [framework format reference](../formats.md#built-in-profile-catalog)
|
||||
owns their current definitions.
|
||||
|
||||
`v0.10.0` is the initial minor release for this compatibility transition.
|
||||
Afterward, catalog-only updates and adjustments to the current selections may
|
||||
ship in a patch release when all other compatibility obligations remain intact.
|
||||
External profile removals can require consumer action; they are not automatic
|
||||
migrations.
|
||||
|
||||
Catalog adoption and release validation remain local and offline. They do not
|
||||
probe provider endpoints, request model lists, or make inference calls.
|
||||
|
||||
## Release Documentation
|
||||
|
||||
Every new release must include `docs/releases/<version>.md` in the tagged
|
||||
source, using the complete tag as the filename. Its title contains Promptkit
|
||||
and that exact version. Use this structure:
|
||||
|
||||
```markdown
|
||||
# Promptkit vMAJOR.MINOR.PATCH
|
||||
|
||||
This release ...
|
||||
|
||||
## Summary
|
||||
|
||||
## Compatibility
|
||||
|
||||
## Upgrade
|
||||
|
||||
## Changes
|
||||
```
|
||||
|
||||
State “No action required” or “None” where appropriate rather than omitting a
|
||||
section. Keep exact current contracts in their canonical documents and link
|
||||
to them. Update affected canonical documentation in the same candidate.
|
||||
|
||||
These notes are historical records, not optional disposable guidance. Published
|
||||
release-note contents must not be replaced under the same version. Corrections
|
||||
belong in a subsequent release. Annotated-tag messages complement the checked-in
|
||||
note: generate the annotation from that note and the validated commit rather
|
||||
than maintaining a second summary.
|
||||
|
||||
## Validation And Publication
|
||||
|
||||
New release notes prepared after the 2026-09-26 OSS-policy adoption end with
|
||||
the [licensing footer](oss.md#discoverability). The release procedure owns
|
||||
absolute, release-pinned URLs. This is prospective preparation work, not a new
|
||||
approval or network-validation gate; historical notes and tags are not reformatted.
|
||||
|
||||
The [publication script](../../scripts/release.sh) runs the
|
||||
[local checks](../../scripts/release-check.sh) once for the final candidate.
|
||||
Retain clean-candidate, synchronized-main, unused-version, module-hygiene,
|
||||
release-note structure, and tag-target checks. Publish only the selected tag
|
||||
ref. Changing the candidate requires fresh validation. No provider endpoint
|
||||
probes or inference calls are release checks.
|
||||
|
||||
Release-note content and retirement of completed roadmaps are preparation
|
||||
responsibilities, not separate human sign-off gates. Preserve planned and
|
||||
deferred work and repair links. Ordinary change review covers unintended files,
|
||||
secrets, and private data; do not repeat it as a release-specific approval.
|
||||
|
||||
A successful tag push to origin completes the release. Never wait for CI to
|
||||
finish or succeed. CI results, remote read-back, and Go module availability
|
||||
are informational follow-ups, not completion gates. An ambiguous push result
|
||||
requires confirming publication on origin, not waiting for CI.
|
||||
|
||||
## Immutability And Failure Recovery
|
||||
|
||||
Never move, replace, recreate, or delete a published tag or replace published
|
||||
release contents. If validation fails before publication, correct the candidate,
|
||||
rerun the complete checks, and repeat the publication guards. Remove an
|
||||
unpublished local tag only after confirming it was not published.
|
||||
|
||||
If publication or subsequent investigation reveals a defect, leave the
|
||||
published release intact. Correct the defect on main, select a new patch or
|
||||
other semantically appropriate version, write its matching release note, and
|
||||
repeat the complete procedure. If publication status is uncertain, inspect
|
||||
the authoritative remote before deciding whether a tag is unpublished; do not
|
||||
overwrite it to retry.
|
||||
|
||||
Transient network, authentication, CI, or module-availability failures do not
|
||||
by themselves warrant a new version. Investigate separately and publish a
|
||||
corrective release only when published contents actually need changing.
|
||||
|
||||
## Policy Exceptions And Deliberate Scope
|
||||
|
||||
This adoption follows the maintainer-approved lean workflow rather than every
|
||||
base-policy guard:
|
||||
|
||||
- Collision checks concern origin, the authoritative publication destination,
|
||||
rather than every configured remote. Local and origin tag checks, synchronized
|
||||
main, and an explicit selected-ref push protect the actual release channel.
|
||||
- Automated local validation and publication-state checks replace manual
|
||||
mechanical inspections and separate human sign-offs. Ordinary change review,
|
||||
editorial responsibility for release notes, and roadmap housekeeping remain;
|
||||
repeating them as approvals would add friction without distinct protection.
|
||||
- Successful tag publication completes this source-only release. CI can be slow
|
||||
or unreliable, and module availability depends on external services. Neither
|
||||
is a completion gate. Local candidate validation and explicit tag-target
|
||||
checks protect publication; ambiguous push results are resolved against origin.
|
||||
Actual defects are corrected in a new release, never by replacing a tag.
|
||||
|
||||
Ignored local files are not prohibited merely because they exist: they are not
|
||||
included in the tagged commit. The clean-candidate guard and ordinary review of
|
||||
tracked changes protect the published contents.
|
||||
|
||||
The catalog profile provision preserves an already approved compatibility
|
||||
boundary; it does not extend volatility to backend definitions or runtime
|
||||
contracts. Further release-governance changes require deliberate approval.
|
||||
@@ -1,5 +1,9 @@
|
||||
# Testing Policy
|
||||
|
||||
**Base Policy Version:** 1.0
|
||||
|
||||
**Last Updated:** 2026-09-23
|
||||
|
||||
## Purpose
|
||||
|
||||
Our tests exist to make **incorrect changes expensive and correct changes cheap**.
|
||||
@@ -50,18 +54,25 @@ Examples of appropriate seams include clocks, randomness, subprocesses, remote A
|
||||
## Test execution requirements
|
||||
|
||||
Promptkit currently uses maintainer-run validation rather than hosted CI.
|
||||
Maintainers run the complete local workflow in the
|
||||
[development guide](../development.md#maintainer-validation) before accepting
|
||||
changes. That guide is the canonical owner of exact commands, formatting,
|
||||
documentation-link validation, and repository-hygiene checks.
|
||||
Ordinary work requires validation proportionate to its scope and risk. Do not
|
||||
repeat successful checks for unchanged content merely to create a commit.
|
||||
Release candidates require the complete local workflow in the
|
||||
[development guide](../development.md#maintainer-validation). That guide routes
|
||||
contributors to the executable checks for formatting, documentation links,
|
||||
and repository hygiene.
|
||||
Introducing hosted CI later would supplement, not silently redefine, this
|
||||
documented validation model.
|
||||
|
||||
Maintainer validation must include ordinary and race-enabled package tests,
|
||||
static analysis, a complete build, and execution of both maintained offline
|
||||
consumer examples. The preparation example protects assembled preparation and
|
||||
Full release-candidate validation includes race-enabled package tests, explicit
|
||||
static analysis, and execution with automated semantic checks of the three maintained
|
||||
offline consumer examples. The test run builds the packages; the examples
|
||||
also exercise ordinary compilation. Separate ordinary tests and a standalone
|
||||
build are not additional release gates. Ordinary tests remain useful during
|
||||
development. The preparation example protects assembled preparation and
|
||||
inspection behavior. The execution example separately protects assembled
|
||||
`Run`, injected-client, validation, usage, and result behavior.
|
||||
`Run`, injected-client, validation, usage, and result behavior. The observation
|
||||
example protects assembled built-in retries, structured proposal and assessment
|
||||
runs, and application-owned interpretation of metadata.
|
||||
|
||||
Tests in the default suite must be deterministic, offline, and independent of
|
||||
real credentials. They must not invoke paid APIs, use live network
|
||||
@@ -87,7 +98,7 @@ Use each test type where it protects a distinct risk:
|
||||
interaction, while replacing live or nondeterministic external boundaries.
|
||||
- External-package root tests exercise the public facade as a Go consumer,
|
||||
while internal package tests own focused implementation behavior.
|
||||
- The maintained offline preparation and execution examples protect distinct
|
||||
- The maintained offline preparation, execution, and observation examples protect distinct
|
||||
representative assembled consumer workflows without contacting a model
|
||||
provider.
|
||||
- Fixtures should be minimal, synthetic, versioned with the behavior they
|
||||
|
||||
295
docs/release.md
295
docs/release.md
@@ -1,242 +1,97 @@
|
||||
# Release Procedure
|
||||
|
||||
## Release Model
|
||||
This runbook implements the [release policy](policy/release.md). A successful
|
||||
push of the selected annotated tag to origin completes the release. Never wait
|
||||
for CI to finish or succeed.
|
||||
|
||||
Promptkit publishes a Go library through source commits and semantic Go module
|
||||
tags. It does not publish runnable binaries or binary packages and does not
|
||||
currently use hosted CI. The release maintainer performs and records the
|
||||
required validation.
|
||||
## Prepare The Candidate
|
||||
|
||||
`v0.1.0` is the initial published release. Later releases use semantic
|
||||
`vMAJOR.MINOR.PATCH` tags. Before `v1`, minor releases may change the public
|
||||
API and patch releases preserve compatibility within their minor line. Every
|
||||
pre-`v1` release note must summarize compatibility, identify public API
|
||||
changes, and state any action required of consumers.
|
||||
Required tools are Git, Go with race-detector support, POSIX shell, and Python
|
||||
3. Publication requires push access to origin. Work from the repository root.
|
||||
|
||||
Promptkit releases are source-only. The annotated tag message is the release
|
||||
note; there is no separate hosted release or binary packaging step.
|
||||
Select an unused version and create `docs/releases/<version>.md` using the
|
||||
[required structure](policy/release.md#release-documentation). Describe
|
||||
compatibility impact and consumer actions and update affected canonical docs.
|
||||
For catalog updates, record selected catalog versions and any consumer action,
|
||||
linking to the catalog policy rather than repeating its contracts.
|
||||
|
||||
## Establish The Candidate
|
||||
Follow the [OSS policy](policy/oss.md) when preparing licensing and attribution
|
||||
changes. Each new note must end with the following navigation, replacing
|
||||
`vMAJOR.MINOR.PATCH` with the selected release tag:
|
||||
|
||||
Choose a version that has not been published and export it as
|
||||
`RELEASE_VERSION`. Run every command in this procedure from the Promptkit
|
||||
repository root in the same POSIX shell. Do not reuse `v0.1.0` or another
|
||||
existing version.
|
||||
```markdown
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/docs/oss/attribution.md)
|
||||
```
|
||||
|
||||
The following guard derives the release commit from `HEAD` and stops on a
|
||||
missing or malformed version, a checkout other than synchronized `main`,
|
||||
uncommitted changes, an active Go workspace, a module replacement, a vendor
|
||||
tree, or an existing local or remote tag:
|
||||
Check these file paths against the candidate; a future tag need not resolve
|
||||
remotely during preparation. The annotated tag derives from this same note,
|
||||
including its footer. Promptkit publishes no separate hosted release body or
|
||||
download page. If such a channel is deliberately introduced, reuse the note
|
||||
and preserve its footer. No extra network checks, artifacts, or approval gates
|
||||
are introduced, and published notes are not retrofitted.
|
||||
|
||||
Retire completed roadmaps, preserving planned or deferred work and repairing
|
||||
links. This is housekeeping, not a separate approval gate. Review changes
|
||||
through the ordinary development workflow, commit them, and push main to
|
||||
origin. No repeated release-specific diff review is required.
|
||||
|
||||
## Validate And Publish
|
||||
|
||||
Run the following with the chosen unused version (replace the placeholder):
|
||||
|
||||
```sh
|
||||
set -eu
|
||||
|
||||
: "${RELEASE_VERSION:?export an unpublished vMAJOR.MINOR.PATCH version}"
|
||||
if ! printf '%s\n' "$RELEASE_VERSION" |
|
||||
grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$'
|
||||
then
|
||||
printf '%s\n' "invalid release version: $RELEASE_VERSION" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
RELEASE_COMMIT=$(git rev-parse --verify 'HEAD^{commit}')
|
||||
export RELEASE_COMMIT
|
||||
|
||||
check_release_candidate() {
|
||||
test "$(git branch --show-current)" = main
|
||||
test -z "$(git status --porcelain)"
|
||||
|
||||
gowork_value=$(go env GOWORK)
|
||||
case "$gowork_value" in
|
||||
''|off) ;;
|
||||
*)
|
||||
printf '%s\n' "active Go workspace: $gowork_value" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
test -z "$(git ls-files go.work go.work.sum)"
|
||||
test ! -e vendor
|
||||
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||
then
|
||||
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
git fetch origin main --tags
|
||||
test "$RELEASE_COMMIT" = \
|
||||
"$(git rev-parse --verify 'refs/remotes/origin/main^{commit}')"
|
||||
|
||||
if git show-ref --verify --quiet "refs/tags/$RELEASE_VERSION"
|
||||
then
|
||||
printf '%s\n' "local tag already exists: $RELEASE_VERSION" >&2
|
||||
return 1
|
||||
fi
|
||||
if test -n "$(
|
||||
git ls-remote --tags origin \
|
||||
"refs/tags/$RELEASE_VERSION" \
|
||||
"refs/tags/$RELEASE_VERSION^{}"
|
||||
)"
|
||||
then
|
||||
printf '%s\n' "remote tag already exists: $RELEASE_VERSION" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
check_release_candidate
|
||||
./scripts/release.sh vMAJOR.MINOR.PATCH
|
||||
```
|
||||
|
||||
Do not continue unless the guard completes successfully. In particular, push
|
||||
the intended commit through the normal `main` branch workflow before release;
|
||||
the tag procedure is not a substitute for publishing the source commit.
|
||||
The [publication script](../scripts/release.sh) checks a clean, synchronized
|
||||
main candidate, the version and unused tag, and the release note's title and
|
||||
required headings. It runs the [local checks](../scripts/release-check.sh)
|
||||
once, rechecks candidate identity and publication state, generates the
|
||||
annotation from the committed note plus the validated commit, and pushes only
|
||||
the selected annotated tag.
|
||||
|
||||
## Validate The Candidate
|
||||
There is no separate tag-message authoring, example-output inspection,
|
||||
metadata inspection, or human sign-off. Compatibility and release-note content
|
||||
remain ordinary editorial responsibilities. Development validation is useful
|
||||
but does not require a second full run before invoking the publication script.
|
||||
|
||||
Confirm the module and root package metadata:
|
||||
The script stops successfully after the tag push. CI completion, CI success,
|
||||
remote read-back, and Go module availability are not release completion gates.
|
||||
Do not wait or poll for them. Provider probes and inference calls are never
|
||||
part of release validation.
|
||||
|
||||
## Catalog Profile Compatibility
|
||||
|
||||
The [release policy](policy/release.md#catalog-profile-compatibility) owns the
|
||||
catalog compatibility boundary. This heading preserves older incoming links.
|
||||
|
||||
## Failure Recovery
|
||||
|
||||
If local checks fail, fix the candidate, commit and push changes, then rerun
|
||||
the publication command. It never automatically removes a tag.
|
||||
|
||||
If a push fails or its result is ambiguous, set `RELEASE_VERSION` to the
|
||||
attempted version and inspect origin before retrying:
|
||||
|
||||
```sh
|
||||
go list -m -f '{{.Path}} {{.GoVersion}}'
|
||||
go list -f '{{.Name}} {{.ImportPath}}' .
|
||||
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}"
|
||||
git rev-parse "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}"
|
||||
```
|
||||
|
||||
The output must be:
|
||||
Matching remote tag and peeled commit values confirm publication, even if the
|
||||
push command reported an error. If the remote query fails, publication status
|
||||
is still unknown. Do not delete or recreate the local tag until absence on
|
||||
origin is confirmed. Once confirmed unpublished, an unsuccessful attempt's
|
||||
local tag can be removed and the publication command rerun. Never overwrite
|
||||
a published tag.
|
||||
|
||||
```text
|
||||
gitea.maximumdirect.net/eric/promptkit 1.25.5
|
||||
promptkit gitea.maximumdirect.net/eric/promptkit
|
||||
```
|
||||
|
||||
As a release prerequisite, run the complete
|
||||
[maintainer validation workflow](development.md#maintainer-validation) against
|
||||
the clean candidate. Do not substitute a partial command list: the development
|
||||
guide owns the tests, race checks, analysis, build, both offline examples,
|
||||
formatting, Markdown links, generated-output and credential review, and
|
||||
repository hygiene. Record the successful workflow result with the candidate.
|
||||
|
||||
## Write The Release Note
|
||||
|
||||
Prepare a plain-text annotated-tag message outside the repository and export
|
||||
its path as `RELEASE_NOTES_FILE`. Use this form, replacing each summary with
|
||||
release-specific text; write `None.` when there are no public API changes or
|
||||
consumer actions:
|
||||
|
||||
```text
|
||||
Promptkit vMAJOR.MINOR.PATCH
|
||||
|
||||
Validated commit: full commit ID
|
||||
Compatibility: compatibility summary
|
||||
Public API changes: changes or None.
|
||||
Consumer action: required action or None.
|
||||
```
|
||||
|
||||
After writing it, require all release-note fields, the selected version, and
|
||||
the validated commit to be present:
|
||||
|
||||
```sh
|
||||
: "${RELEASE_NOTES_FILE:?export the path to the release-note file}"
|
||||
test -f "$RELEASE_NOTES_FILE"
|
||||
test -s "$RELEASE_NOTES_FILE"
|
||||
grep -F "Promptkit $RELEASE_VERSION" "$RELEASE_NOTES_FILE"
|
||||
grep -F "Validated commit: $RELEASE_COMMIT" "$RELEASE_NOTES_FILE"
|
||||
grep -F 'Compatibility:' "$RELEASE_NOTES_FILE"
|
||||
grep -F 'Public API changes:' "$RELEASE_NOTES_FILE"
|
||||
grep -F 'Consumer action:' "$RELEASE_NOTES_FILE"
|
||||
```
|
||||
|
||||
Inspect the complete message and confirm that it accurately records the
|
||||
compatibility impact, public API changes, and required consumer action.
|
||||
|
||||
## Create And Inspect The Tag
|
||||
|
||||
Run the candidate guard again immediately before tag creation. This ensures
|
||||
that validation or release-note preparation did not change the checkout and
|
||||
that the commit is still published and untagged:
|
||||
|
||||
```sh
|
||||
check_release_candidate
|
||||
```
|
||||
|
||||
Create the annotated tag from the prepared release note and bind it explicitly
|
||||
to the validated commit:
|
||||
|
||||
```sh
|
||||
git tag --annotate "$RELEASE_VERSION" \
|
||||
--file "$RELEASE_NOTES_FILE" \
|
||||
"$RELEASE_COMMIT"
|
||||
```
|
||||
|
||||
Inspect both the tag message and its source commit before publication:
|
||||
|
||||
```sh
|
||||
test "$(git cat-file -t "refs/tags/$RELEASE_VERSION")" = tag
|
||||
git show --no-patch --decorate "refs/tags/$RELEASE_VERSION"
|
||||
test "$(
|
||||
git rev-parse --verify "refs/tags/$RELEASE_VERSION^{commit}"
|
||||
)" = "$RELEASE_COMMIT"
|
||||
```
|
||||
|
||||
If inspection finds an error, delete the unpublished local tag, correct the
|
||||
release note or candidate, and repeat the guards. Never move or recreate a tag
|
||||
that has been published.
|
||||
|
||||
## Publish The Selected Tag
|
||||
|
||||
Push only the selected tag ref. Do not use `git push --tags`:
|
||||
|
||||
```sh
|
||||
git push origin \
|
||||
"refs/tags/$RELEASE_VERSION:refs/tags/$RELEASE_VERSION"
|
||||
```
|
||||
|
||||
## Verify Publication
|
||||
|
||||
Compare the remote annotated-tag object with the local object, then compare the
|
||||
remote peeled source commit with the validated commit:
|
||||
|
||||
```sh
|
||||
remote_tag=$(
|
||||
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" |
|
||||
awk 'NR == 1 { print $1 }'
|
||||
)
|
||||
remote_commit=$(
|
||||
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION^{}" |
|
||||
awk 'NR == 1 { print $1 }'
|
||||
)
|
||||
test -n "$remote_tag"
|
||||
test "$remote_tag" = \
|
||||
"$(git rev-parse --verify "refs/tags/$RELEASE_VERSION")"
|
||||
test "$remote_commit" = "$RELEASE_COMMIT"
|
||||
```
|
||||
|
||||
Finally, resolve the version as an ordinary Go module in a temporary module
|
||||
outside this repository and without a workspace or replacement:
|
||||
|
||||
```sh
|
||||
resolution_dir=$(mktemp -d)
|
||||
(
|
||||
trap 'rm -rf "$resolution_dir"' 0 1 2 15
|
||||
cd "$resolution_dir"
|
||||
GOWORK=off go mod init example.com/promptkit-release-check
|
||||
GOWORK=off go mod download \
|
||||
"gitea.maximumdirect.net/eric/promptkit@$RELEASE_VERSION"
|
||||
resolved_version=$(
|
||||
GOWORK=off go list -m -f '{{.Version}}' \
|
||||
"gitea.maximumdirect.net/eric/promptkit@$RELEASE_VERSION"
|
||||
)
|
||||
test "$resolved_version" = "$RELEASE_VERSION"
|
||||
)
|
||||
```
|
||||
|
||||
Promptkit must publish and verify the required version before Scriptorium or
|
||||
another consumer publishes a release that depends on it. This ordering does
|
||||
not replace the consumer project's own release procedure. Released consumers
|
||||
must select the published Promptkit tag through ordinary module resolution,
|
||||
without a workspace, replacement, vendored Promptkit source, or unpublished
|
||||
revision.
|
||||
CI or module-resolution problems may be investigated separately; temporary
|
||||
service, network, or authentication failures do not require another release.
|
||||
Only an actual defect requiring changes to published contents warrants a new
|
||||
appropriate version. See the [recovery policy](policy/release.md#immutability-and-failure-recovery).
|
||||
|
||||
## Policy Changes
|
||||
|
||||
Document and approve a durable policy change before introducing hosted
|
||||
automation, binary artifacts, or different release governance. Update this
|
||||
procedure in the same change so maintainers do not rely on hidden release
|
||||
requirements.
|
||||
Update the policy and this procedure together when changing release governance.
|
||||
Consumers own their dependency-availability and release checks; their readiness
|
||||
does not delay completion of a Promptkit release.
|
||||
|
||||
44
docs/releases/v0.10.0.md
Normal file
44
docs/releases/v0.10.0.md
Normal file
@@ -0,0 +1,44 @@
|
||||
# Promptkit v0.10.0
|
||||
|
||||
This supplemental guide summarizes the `v0.10.0` changes and migration needs.
|
||||
The annotated tag remains the authoritative release record.
|
||||
|
||||
## Transition Summary
|
||||
|
||||
`v0.10.0` adopts independently versioned OpenRouter and Rakestrawhome catalog
|
||||
modules whose external profile data can change between catalog releases. It
|
||||
also introduces the maintained literal names
|
||||
`promptkit-openrouter-standard` and `promptkit-openrouter-auto`.
|
||||
|
||||
The public package now delegates assembly to an internal engine while retaining
|
||||
the existing Go API. This refactor requires no consumer code changes.
|
||||
|
||||
This release selects OpenRouter `v2.20260922.1` and Rakestrawhome
|
||||
`v2.20260922.0`; their exact inventories are linked from the
|
||||
[format reference](../formats.md#external-profile-inventories).
|
||||
|
||||
The maintained names provide continuity only within the catalog versions
|
||||
selected and tested by a Promptkit release and when consumers do not override
|
||||
the name or its base. Their current selections may change. External catalog
|
||||
profile IDs can be removed, renamed, or modified, so an application that
|
||||
selects one directly may need to update its configuration when upgrading.
|
||||
|
||||
## Consumer Migration
|
||||
|
||||
Before upgrading, review every profile ID used by prompts, requests, and
|
||||
application-owned inheritance. Choose a maintained name, accept an external
|
||||
catalog profile's volatility, or define a complete application-owned profile
|
||||
when the application must control its target and settings. An alias that
|
||||
inherits from an external base is not a frozen copy.
|
||||
|
||||
In particular, the selected OpenRouter catalog removes `deepseek-3-2`,
|
||||
`deepseek-4-flash`, and `deepseek-4-pro`. Consumers using those IDs must choose
|
||||
an available profile or supply their own complete definition; the new
|
||||
`deepseek-flash-latest` and `deepseek-pro-latest` profiles are not automatic
|
||||
aliases and need not preserve the previous settings.
|
||||
|
||||
See the [release policy](../release.md#catalog-profile-compatibility) for the
|
||||
compatibility boundary, the [format reference](../formats.md#built-in-profile-catalog)
|
||||
for current Promptkit-owned definitions and selected external inventories, and
|
||||
the [consumer guide](../consumers/pkg-promptkit.md#choose-a-profile-ownership-model)
|
||||
for configuration choices.
|
||||
76
docs/releases/v0.11.0.md
Normal file
76
docs/releases/v0.11.0.md
Normal file
@@ -0,0 +1,76 @@
|
||||
# Promptkit v0.11.0
|
||||
|
||||
This release adds reliable, observable model execution and changes Promptkit's
|
||||
license to LGPL-3.0-or-later.
|
||||
|
||||
## Summary
|
||||
|
||||
- Built-in model calls can recover from selected transient HTTP failures.
|
||||
- Optional execution observations report attempts, timing, outcomes, and known
|
||||
token usage, including work completed before a run fails.
|
||||
- Public errors distinguish transport failures, malformed responses, and retry
|
||||
exhaustion without parsing diagnostic text.
|
||||
|
||||
## Compatibility
|
||||
|
||||
The built-in client now defaults to two additional transient attempts. This can
|
||||
increase latency, request counts, and cost; retries do not guarantee exactly-once
|
||||
generation. Injected `LLMClient` implementations retain their own retry policy
|
||||
and are not automatically replayed. Existing generation timeouts apply per
|
||||
attempt, while the caller context governs the complete operation. See the
|
||||
[provider integration contract](../integrations/openai-compatible-chat.md).
|
||||
|
||||
Public retry and observation controls are additive. Existing successful-result
|
||||
usage semantics remain unchanged, and operational failures still return no
|
||||
partial `RunResult`. Observations distinguish unknown usage from reported zero;
|
||||
they are not a complete billing ledger. See the
|
||||
[consumer guide](../consumers/pkg-promptkit.md) and
|
||||
[observation GoDoc](../../observations.go).
|
||||
|
||||
Promptkit is now licensed under LGPL-3.0-or-later. Review the
|
||||
[license summary](../../README.md#license), [LGPL text](../../LICENSE), and
|
||||
[accompanying GPL text](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/COPYING) before adoption or distribution.
|
||||
Previously published releases and dependency licenses are unchanged.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Update the dependency:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.11.0
|
||||
```
|
||||
|
||||
To retain single-attempt behavior for built-in generation, set
|
||||
`Config.TransientRetries` to a pointer to zero. `RunRequest.TransientRetries`
|
||||
can override the engine choice per request. Review any existing application or
|
||||
HTTP-transport retry loops to avoid unintentionally multiplying attempts.
|
||||
Transient retries and structural repairs have separate budgets; provide a
|
||||
caller deadline when a total execution time limit is needed.
|
||||
|
||||
Observation is opt-in through `RunRequest.Observer`, including for prepared
|
||||
execution. Callbacks are synchronous; shared observers must support concurrent
|
||||
runs. Injected clients can supply `GenerateResponse.UsagePresence` to explicitly
|
||||
report known zero counts. No observer or usage-presence changes are required
|
||||
for existing consumers. See the [consumer guide](../consumers/pkg-promptkit.md)
|
||||
for adoption and the [offline example](../../examples/go-library/observe/main.go)
|
||||
for a structured proposal-and-assessment workflow.
|
||||
|
||||
## Changes
|
||||
|
||||
- Centralized built-in retry coordination with cancellable backoff and provider
|
||||
retry timing. Retries and structural repair share existing backend capacity;
|
||||
backoff releases active permits without releasing logical run admission.
|
||||
- Added `ErrTransport`, `ErrMalformedResponse`, `ErrRetriesExhausted`, and
|
||||
`RetryExhaustedError`, retaining applicable underlying error identities.
|
||||
- Added request-scoped lifecycle observations and presence-aware token accounting
|
||||
for ordinary and prepared execution, without recording prompt/response bodies.
|
||||
- Isolated delegated operations' observation state and made injected-client
|
||||
attempt classifications consistent with terminal classifications.
|
||||
- Added an offline observation example and regression coverage for retry,
|
||||
repair, capacity, error, and observation interactions.
|
||||
- Adopted LGPL-3.0-or-later and streamlined the documented local release process.
|
||||
|
||||
Backend catalog selections are unchanged from v0.10.0: OpenRouter
|
||||
`v2.20260922.1` and Rakestrawhome `v2.20260922.0`. See the
|
||||
[catalog compatibility policy](../policy/release.md#catalog-profile-compatibility)
|
||||
and [profile inventories](../formats.md#external-profile-inventories).
|
||||
43
docs/releases/v0.11.1.md
Normal file
43
docs/releases/v0.11.1.md
Normal file
@@ -0,0 +1,43 @@
|
||||
# Promptkit v0.11.1
|
||||
|
||||
This patch updates the maintained catalogs to MIT-licensed releases and
|
||||
improves licensing documentation.
|
||||
|
||||
## Summary
|
||||
|
||||
Both OpenRouter and Rakestrawhome catalogs now use `v2.20260926.0`. These
|
||||
catalog releases change licensing and documentation only; their profiles,
|
||||
backend definitions, and Go code are unchanged.
|
||||
|
||||
## Compatibility
|
||||
|
||||
No public API, runtime behavior, or profile changes are introduced.
|
||||
Promptkit remains LGPL-3.0-or-later; the catalogs are now MIT-licensed.
|
||||
See the [project grant](../../COPYRIGHT),
|
||||
[component attribution](../oss/attribution.md), and
|
||||
[catalog compatibility policy](../policy/release.md#catalog-profile-compatibility).
|
||||
|
||||
## Upgrade
|
||||
|
||||
Update the module dependency:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.11.1
|
||||
```
|
||||
|
||||
No code or configuration migration is required from v0.11.0.
|
||||
Consumers upgrading from earlier versions should also read the
|
||||
[v0.11.0 migration guide](v0.11.0.md).
|
||||
|
||||
## Changes
|
||||
|
||||
- Adopt the MIT-licensed catalog releases and update inventory links.
|
||||
- Add centralized attribution and dependency license texts.
|
||||
- Separate the project grant into COPYRIGHT and consolidate the LGPLv3 and
|
||||
incorporated GPLv3 texts in LICENSE, replacing the separate COPYING file.
|
||||
- Align documentation ownership and future release navigation with the
|
||||
[OSS policy](../policy/oss.md), without adding release gates.
|
||||
- Repair the historical v0.11.0 GPL-text link and retire the completed
|
||||
licensing roadmap.
|
||||
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.1/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.1/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.1/docs/oss/attribution.md)
|
||||
125
docs/releases/v0.9.0.md
Normal file
125
docs/releases/v0.9.0.md
Normal file
@@ -0,0 +1,125 @@
|
||||
# Promptkit v0.9.0
|
||||
|
||||
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||
changes from `v0.8.0` to `v0.9.0`. The annotated `v0.9.0` tag is the
|
||||
authoritative release record. Exact current contracts belong to the linked
|
||||
GoDoc and durable documentation.
|
||||
|
||||
## Summary
|
||||
|
||||
`v0.9.0` adds stateless request-message composition and separates maintained
|
||||
provider data from Promptkit's core implementation:
|
||||
|
||||
- callers can append already-rendered messages to a configured prompt for
|
||||
application-owned conversations and semantic correction workflows;
|
||||
- the public package now publishes constants for the four supported text-chat
|
||||
roles, and prompt definitions use the same normalized role vocabulary; and
|
||||
- the OpenRouter and Rakestrawhome backend/profile catalogs now come from two
|
||||
independently versioned Go module dependencies.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This release adds one field and four constants to the public API and removes no
|
||||
public declarations. Existing keyed `RunRequest` literals that omit
|
||||
`AppendedMessages` retain their behavior. Adding the field changes the struct
|
||||
shape, so consumers using positional `RunRequest` literals must convert them
|
||||
to keyed literals.
|
||||
|
||||
Message roles in prompt definitions are now trimmed, lowercased, and required
|
||||
to be `developer`, `system`, `user`, or `assistant`. Definitions using another
|
||||
role that earlier releases accepted as an arbitrary nonblank string now fail
|
||||
prompt loading. In particular, the text-only message contract does not support
|
||||
`tool` or the deprecated `function` role. Otherwise valid roles with different
|
||||
case or surrounding whitespace are normalized rather than rejected.
|
||||
|
||||
The catalog extraction preserves Promptkit's public API, built-in backend and
|
||||
profile IDs, configuration, precedence, credential handling, and capacity
|
||||
behavior. Consumers do not import or register either catalog themselves, and
|
||||
no configuration migration is required. Promptkit now selects two
|
||||
independently versioned data dependencies and guarantees only the catalog
|
||||
versions selected and tested by this Promptkit release.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Update the module dependency with:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.9.0
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
Convert any positional `RunRequest` literals to keyed literals. Review prompt
|
||||
definitions for unsupported roles, then run the consuming project's ordinary
|
||||
and race-enabled tests.
|
||||
|
||||
## Appended Request Messages
|
||||
|
||||
`RunRequest.AppendedMessages` accepts caller-owned `RenderedMessage` values
|
||||
that Promptkit validates, copies, and appends after every rendered
|
||||
prompt-definition message in caller order. Promptkit does not template this
|
||||
content, retain conversation state between calls, impose a retry policy, or
|
||||
apply a message-count, byte-size, token, or context-window limit. Upstream
|
||||
rejections continue through the ordinary generation-error boundary.
|
||||
|
||||
This primitive supports application-owned conversation continuations and
|
||||
domain-aware correction loops while preserving Promptkit's existing
|
||||
preparation, hashing, prepared-execution, structural repair, backend-capacity,
|
||||
credential, and cancellation behavior. Appended content can include sensitive
|
||||
model output or application feedback; prepared values expose the complete
|
||||
effective messages by design, while default request formatting reports only
|
||||
the appended-message count.
|
||||
|
||||
See the
|
||||
[consumer appended-message example](../consumers/pkg-promptkit.md#append-already-rendered-messages),
|
||||
the [`RunRequest`, `RenderedMessage`, and `CacheControl` GoDoc](../../types.go),
|
||||
and the [OpenAI-compatible request contract](../integrations/openai-compatible-chat.md#request-body)
|
||||
for current behavior.
|
||||
|
||||
## Supported Message Roles
|
||||
|
||||
The new `RoleDeveloper`, `RoleSystem`, `RoleUser`, and `RoleAssistant`
|
||||
constants identify the complete role vocabulary accepted by Promptkit's
|
||||
text-chat message model. The same validation and normalization now apply to
|
||||
prompt-definition messages and request-supplied appended messages. Promptkit
|
||||
does not translate between roles; provider- or model-specific rejection of an
|
||||
otherwise supported role remains an upstream generation error.
|
||||
|
||||
See the [message format reference](../formats.md#messages-and-templates) for
|
||||
the canonical prompt-definition contract.
|
||||
|
||||
## Independently Versioned Backend Catalogs
|
||||
|
||||
Promptkit imports immutable catalog data for its maintained OpenRouter and
|
||||
Rakestrawhome backends and profiles. Engine construction validates and
|
||||
assembles both catalogs behind the existing built-in registry and profile
|
||||
precedence rules. Promptkit no longer keeps duplicate embedded profile assets
|
||||
or hard-coded definitions for those maintained backends.
|
||||
|
||||
The module versions in Promptkit's `go.mod` identify the catalog releases
|
||||
tested with this release. The [built-in backend and profile format
|
||||
reference](../formats.md#built-in-backends) remains the canonical consumer
|
||||
contract, while the [internal source documentation](../internal/sources.md#profiles-and-built-ins)
|
||||
describes the dependency boundary.
|
||||
|
||||
## Public API Changes
|
||||
|
||||
The release adds:
|
||||
|
||||
- `RunRequest.AppendedMessages`;
|
||||
- `RoleDeveloper`;
|
||||
- `RoleSystem`;
|
||||
- `RoleUser`; and
|
||||
- `RoleAssistant`.
|
||||
|
||||
No public declaration was removed.
|
||||
|
||||
## Consumer Action
|
||||
|
||||
- Convert positional `RunRequest` literals to keyed literals.
|
||||
- Replace unsupported prompt-definition roles with an appropriate supported
|
||||
role, or keep richer tool-call protocols in an application-owned client.
|
||||
- Treat appended messages and prepared effective messages according to the
|
||||
application's sensitive-data policy.
|
||||
- Do not add direct catalog imports or registration calls; existing Promptkit
|
||||
construction and configuration remain correct.
|
||||
- Run consumer ordinary and race-enabled tests after updating the module.
|
||||
@@ -38,8 +38,13 @@ consumers.
|
||||
|
||||
## Ideas
|
||||
|
||||
No ideas are currently awaiting selection. Active feature work belongs in its
|
||||
focused roadmap rather than this catalog.
|
||||
### Remaining Standard Policy Alignment
|
||||
|
||||
Review the internal standard's architecture and testing clarifications about
|
||||
adopted constraints, threat scope, and tests that must not invent contracts.
|
||||
Adapt relevant guidance to Promptkit's library boundaries without introducing
|
||||
new runtime restrictions or release gates. This is separate from the completed
|
||||
licensing-documentation alignment.
|
||||
|
||||
## Entry Format
|
||||
|
||||
|
||||
354
engine.go
354
engine.go
@@ -6,23 +6,12 @@ import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile/builtin"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||
internalengine "gitea.maximumdirect.net/eric/promptkit/internal/engine"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/observation"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||
)
|
||||
|
||||
// ErrInvalidConfig identifies invalid engine construction, including missing
|
||||
@@ -73,6 +62,19 @@ var (
|
||||
// [GenerationError]. Errors returned by an injected LLMClient remain
|
||||
// available through errors.Is.
|
||||
ErrLLMGenerate = errors.New("failed to generate output")
|
||||
// ErrTransport identifies a failed HTTP request by the built-in model
|
||||
// client. Generation errors in this category also match ErrLLMGenerate and
|
||||
// retain the transport cause for errors.Is and errors.As. It does not imply
|
||||
// that the provider received no work.
|
||||
ErrTransport = errors.New("model transport failed")
|
||||
// ErrMalformedResponse identifies an invalid successful model response or
|
||||
// a nil successful response from an injected client. Generation errors in
|
||||
// this category also match ErrLLMGenerate.
|
||||
ErrMalformedResponse = errors.New("malformed model response")
|
||||
// ErrRetriesExhausted identifies a built-in generation sequence whose
|
||||
// positive transient retry budget ended with another eligible HTTP status.
|
||||
// RetryExhaustedError retains the final cause and actual attempt count.
|
||||
ErrRetriesExhausted = errors.New("transient model retries exhausted")
|
||||
// ErrValidation identifies an operational failure to load or compile a
|
||||
// schema or validate output. A completed validation whose Status is
|
||||
// ValidationFailed is returned in RunResult without this error.
|
||||
@@ -89,7 +91,7 @@ var (
|
||||
// model generation. Injected collaborators may still be invoked concurrently
|
||||
// across different backend pools or for unlimited backends.
|
||||
type Engine struct {
|
||||
runner *usecase.Runner
|
||||
internal *internalengine.Engine
|
||||
}
|
||||
|
||||
// Config selects the directory-backed sources and built-in model-client
|
||||
@@ -100,7 +102,7 @@ type Config struct {
|
||||
// prompt source.
|
||||
PromptDir string
|
||||
// ProfileDir is an optional ordinary configured source whose profiles take
|
||||
// precedence over application fallback and embedded built-in profiles. An
|
||||
// precedence over application fallback and maintained catalog profiles. An
|
||||
// empty value selects the lower-precedence sources unless a profile-source
|
||||
// option supplies the ordinary source.
|
||||
ProfileDir string
|
||||
@@ -116,6 +118,11 @@ type Config struct {
|
||||
// Timeout or the 10-minute default. The supplied client is not mutated. This
|
||||
// field is ignored when WithLLMClient is used.
|
||||
HTTPClient *http.Client
|
||||
// TransientRetries selects additional built-in HTTP attempts after eligible
|
||||
// provider statuses. Nil defaults to two; zero disables retries. Negative
|
||||
// values are invalid. Injected LLMClient implementations are never retried
|
||||
// automatically.
|
||||
TransientRetries *int
|
||||
}
|
||||
|
||||
// Option customizes engine construction.
|
||||
@@ -138,20 +145,7 @@ func (f optionFunc) apply(options *engineOptions) error {
|
||||
}
|
||||
|
||||
type engineOptions struct {
|
||||
llmClient llm.Client
|
||||
artifactReader artifactadapter.Reader
|
||||
promptDefs promptdef.Repository
|
||||
profiles profile.Repository
|
||||
fallbackProfiles profile.Repository
|
||||
memoryProfiles profile.Repository
|
||||
backends []domain.Backend
|
||||
validator validate.Validator
|
||||
promptSource bool
|
||||
profileSource bool
|
||||
fallbackProfileSource bool
|
||||
memorySource bool
|
||||
validatorSource bool
|
||||
artifactSource bool
|
||||
internal internalengine.Options
|
||||
}
|
||||
|
||||
// WithLLMClient replaces the built-in model client used by [Engine.Run] and
|
||||
@@ -167,7 +161,7 @@ func WithLLMClient(client LLMClient) Option {
|
||||
if client == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.llmClient = publicLLMClientAdapter{client: client}
|
||||
options.internal.SetLLMClient(publicLLMClientAdapter{client: client})
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -182,8 +176,7 @@ func WithArtifactReader(reader ArtifactReader) Option {
|
||||
if reader == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.artifactReader = publicArtifactReaderAdapter{reader: reader}
|
||||
options.artifactSource = true
|
||||
options.internal.SetArtifactReader(publicArtifactReaderAdapter{reader: reader})
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -197,15 +190,7 @@ func WithArtifactReader(reader ArtifactReader) Option {
|
||||
// prompt-source options.
|
||||
func WithPromptFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.promptDefs = promptdef.NewFSRepository(fsys, root)
|
||||
options.promptSource = true
|
||||
return nil
|
||||
return options.internal.SetPromptFS(fsys, root)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -216,13 +201,7 @@ func WithPromptFS(fsys fs.FS, root string) Option {
|
||||
// This option replaces Config.PromptDir and earlier prompt-source options.
|
||||
func WithPromptFile(path string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
fsys, root, err := fileSource(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
options.promptDefs = promptdef.NewFileRepository(fsys, root, filepath.Dir(path))
|
||||
options.promptSource = true
|
||||
return nil
|
||||
return options.internal.SetPromptFile(path)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -236,15 +215,7 @@ func WithPromptFile(path string) Option {
|
||||
// FS profile-source options, but remains below WithProfiles in precedence.
|
||||
func WithProfileFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.profiles = profile.NewFSRepository(fsys, root)
|
||||
options.profileSource = true
|
||||
return nil
|
||||
return options.internal.SetProfileFS(fsys, root)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -258,13 +229,7 @@ func WithProfileFS(fsys fs.FS, root string) Option {
|
||||
// precedence.
|
||||
func WithProfileFile(path string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
fsys, root, err := fileSource(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
options.profiles = profile.NewFSRepository(fsys, root)
|
||||
options.profileSource = true
|
||||
return nil
|
||||
return options.internal.SetProfileFile(path)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -273,7 +238,7 @@ func WithProfileFile(path string) Option {
|
||||
//
|
||||
// Profile lookup checks, in order, profiles supplied by WithProfiles; the
|
||||
// ordinary configured source selected by WithProfileFile, WithProfileFS, or
|
||||
// Config.ProfileDir; this fallback source; and Promptkit's embedded built-in
|
||||
// Config.ProfileDir; this fallback source; and Promptkit's maintained catalog
|
||||
// profiles. Each source supplies a complete profile definition; profile fields
|
||||
// are not merged between sources. Only an absent profile ID proceeds to the
|
||||
// next source. A matching read, parse, duplicate, validation, or credential
|
||||
@@ -289,15 +254,7 @@ func WithProfileFile(path string) Option {
|
||||
// failover.
|
||||
func WithFallbackProfileFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.fallbackProfiles = profile.NewFSRepository(fsys, root)
|
||||
options.fallbackProfileSource = true
|
||||
return nil
|
||||
return options.internal.SetFallbackProfileFS(fsys, root)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -312,13 +269,11 @@ func WithFallbackProfileFS(fsys fs.FS, root string) Option {
|
||||
// earlier in-memory set rather than merging it.
|
||||
func WithProfiles(profiles ...Profile) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
repo, err := newMemoryProfileRepository(profiles)
|
||||
if err != nil {
|
||||
return err
|
||||
definitions := make([]domain.ExecutionProfile, len(profiles))
|
||||
for index, publicProfile := range profiles {
|
||||
definitions[index] = toDomainProfile(publicProfile)
|
||||
}
|
||||
options.memoryProfiles = repo
|
||||
options.memorySource = true
|
||||
return nil
|
||||
return options.internal.SetMemoryProfiles(definitions)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -330,15 +285,7 @@ func WithProfiles(profiles ...Profile) Option {
|
||||
// replaces Config.SchemaDir and earlier schema-source options.
|
||||
func WithSchemaFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.validator = validate.NewFSValidator(fsys, root)
|
||||
options.validatorSource = true
|
||||
return nil
|
||||
return options.internal.SetSchemaFS(fsys, root)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -349,13 +296,7 @@ func WithSchemaFS(fsys fs.FS, root string) Option {
|
||||
// replaces Config.SchemaDir and earlier schema-source options.
|
||||
func WithSchemaFile(path string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
fsys, root, err := fileSource(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
options.validator = validate.NewFSValidator(fsys, root)
|
||||
options.validatorSource = true
|
||||
return nil
|
||||
return options.internal.SetSchemaFile(path)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -370,7 +311,8 @@ func WithSchemaFile(path string) Option {
|
||||
// NewEngine returns an error matching ErrInvalidConfig for invalid
|
||||
// configuration, options, or backend-capacity policies. Each constructed
|
||||
// Engine has independent backend-capacity pools. Construction does not perform
|
||||
// model requests or require credentials.
|
||||
// model requests or require credentials. A negative Config.TransientRetries
|
||||
// is invalid.
|
||||
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
var options engineOptions
|
||||
for _, opt := range opts {
|
||||
@@ -382,105 +324,14 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
}
|
||||
}
|
||||
|
||||
promptDefs := options.promptDefs
|
||||
if !options.promptSource {
|
||||
if strings.TrimSpace(cfg.PromptDir) == "" {
|
||||
return nil, fmt.Errorf("%w: prompt directory is required", ErrInvalidConfig)
|
||||
}
|
||||
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
|
||||
}
|
||||
|
||||
profiles := newProfileRepository(cfg.ProfileDir, options)
|
||||
|
||||
backendRegistry, err := backend.NewRegistry(options.backends)
|
||||
assembled, err := internalengine.New(
|
||||
internalengine.Config{PromptDir: cfg.PromptDir, ProfileDir: cfg.ProfileDir, SchemaDir: cfg.SchemaDir, Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: copyIntPtr(cfg.TransientRetries)},
|
||||
options.internal,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err)
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
|
||||
capacityManager, err := capacity.NewManager(backendRegistry.CapacityPolicies())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to construct backend capacity manager: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
|
||||
validator := options.validator
|
||||
if !options.validatorSource {
|
||||
schemaDir := cfg.SchemaDir
|
||||
if strings.TrimSpace(schemaDir) == "" {
|
||||
schemaDir = defaults.SchemaDirDefault
|
||||
}
|
||||
validator = validate.NewStandardValidator(schemaDir)
|
||||
}
|
||||
|
||||
llmClient := options.llmClient
|
||||
if llmClient == nil {
|
||||
var err error
|
||||
llmClient, err = llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
|
||||
Timeout: cfg.Timeout,
|
||||
HTTPClient: cfg.HTTPClient,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
}
|
||||
llmClient = capacity.NewClient(capacityManager, llmClient)
|
||||
|
||||
artifacts := options.artifactReader
|
||||
if !options.artifactSource {
|
||||
artifacts = artifactadapter.NewCompositeReader()
|
||||
}
|
||||
|
||||
return &Engine{
|
||||
runner: usecase.NewRunnerWithRepairer(
|
||||
promptDefs,
|
||||
profiles,
|
||||
backendRegistry,
|
||||
artifacts,
|
||||
prompt.NewGoRenderer(),
|
||||
llmClient,
|
||||
validator,
|
||||
usecase.NewDefaultOutputRepairer(llmClient),
|
||||
capacityManager,
|
||||
),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newProfileRepository(profileDir string, options engineOptions) profile.Repository {
|
||||
repository := builtin.NewRepository()
|
||||
|
||||
if options.fallbackProfileSource {
|
||||
repository = profile.NewOverlayRepository(options.fallbackProfiles, repository)
|
||||
}
|
||||
|
||||
if options.profileSource {
|
||||
repository = profile.NewOverlayRepository(options.profiles, repository)
|
||||
} else if strings.TrimSpace(profileDir) != "" {
|
||||
repository = profile.NewOverlayRepository(profile.NewFilesystemRepository(profileDir), repository)
|
||||
}
|
||||
|
||||
if options.memorySource {
|
||||
repository = profile.NewOverlayRepository(options.memoryProfiles, repository)
|
||||
}
|
||||
|
||||
return profile.NewResolvingRepository(repository)
|
||||
}
|
||||
|
||||
func fileSource(name string) (fs.FS, string, error) {
|
||||
if strings.TrimSpace(name) == "" {
|
||||
return nil, "", ErrInvalidConfig
|
||||
}
|
||||
dir := filepath.Dir(name)
|
||||
base := filepath.Base(name)
|
||||
if base == "." || base == string(filepath.Separator) {
|
||||
return nil, "", ErrInvalidConfig
|
||||
}
|
||||
info, err := os.Stat(name)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, name, err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, name)
|
||||
}
|
||||
return os.DirFS(dir), filepath.ToSlash(base), nil
|
||||
return &Engine{internal: assembled}, nil
|
||||
}
|
||||
|
||||
// InspectPrompt resolves one explicit prompt definition without selecting a
|
||||
@@ -519,11 +370,11 @@ func (e *Engine) InspectPrompt(
|
||||
promptID string,
|
||||
promptVersion string,
|
||||
) (*PromptInspection, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
inspection, err := e.runner.InspectPrompt(ctx, promptID, promptVersion)
|
||||
inspection, err := e.internal.InspectPrompt(ctx, promptID, promptVersion)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
@@ -559,11 +410,11 @@ func (e *Engine) InspectPrompt(
|
||||
// profile loading matches ErrProfileLoad while preserving the context error.
|
||||
// InspectProfile returns no partial result on error.
|
||||
func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*ProfileInspection, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
inspection, err := e.runner.InspectProfile(ctx, profileID)
|
||||
inspection, err := e.internal.InspectProfile(ctx, profileID)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
@@ -571,6 +422,8 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
|
||||
}
|
||||
|
||||
// Prepare resolves and renders a prompt request without calling an LLM.
|
||||
// It appends any validated RunRequest.AppendedMessages after rendered
|
||||
// definition messages in the returned caller-owned snapshot.
|
||||
//
|
||||
// Prepare selects the prompt and profile, resolves any selected backend and
|
||||
// effective execution settings, resolves the output contract, loads and hashes
|
||||
@@ -586,9 +439,10 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
|
||||
// applicable. Cancellation is passed to the active collaborator and is
|
||||
// reported in the applicable operation category; no general errors.Is
|
||||
// relationship to ctx.Err is promised. Prepare returns no partial result on
|
||||
// error.
|
||||
// error. It validates RunRequest.TransientRetries but makes no model attempts.
|
||||
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
ctx = observation.WithoutExecutionState(ctx)
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
@@ -597,7 +451,7 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
prepared, err := e.runner.Prepare(ctx, domainReq)
|
||||
prepared, err := e.internal.Prepare(ctx, domainReq)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
@@ -605,7 +459,8 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
||||
}
|
||||
|
||||
// PrepareExecution completely prepares a prompt request without calling the
|
||||
// configured LLMClient or reserving backend admission capacity.
|
||||
// configured LLMClient or reserving backend admission capacity. Validated
|
||||
// RunRequest.AppendedMessages are included in the frozen effective messages.
|
||||
//
|
||||
// The returned opaque handle is bound to this Engine and permits one
|
||||
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
|
||||
@@ -618,9 +473,12 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
||||
// returns does not invalidate the handle or propagate to RunPrepared.
|
||||
// PrepareExecution returns the same error categories as [Engine.Prepare] and
|
||||
// returns no handle on error. A nil Engine returns an error matching
|
||||
// ErrInvalidConfig.
|
||||
// ErrInvalidConfig. The effective transient retry budget is frozen privately
|
||||
// for RunPrepared and is absent from Details. A supplied Observer is retained
|
||||
// privately for RunPrepared; preparation itself emits no events.
|
||||
func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
ctx = observation.WithoutExecutionState(ctx)
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
@@ -629,7 +487,7 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
prepared, err := e.runner.PrepareExecution(ctx, domainReq)
|
||||
prepared, err := e.internal.PrepareExecution(ctx, domainReq)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
@@ -637,7 +495,8 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
|
||||
}
|
||||
|
||||
// Run prepares a request, invokes the configured LLMClient, and validates the
|
||||
// generated output.
|
||||
// generated output. Each call resolves current sources and composes a fresh,
|
||||
// stateless effective prompt with any validated RunRequest.AppendedMessages.
|
||||
//
|
||||
// A content-validation failure is a successful run whose
|
||||
// RunResult.Validation has Status ValidationFailed. When its output contract
|
||||
@@ -647,20 +506,56 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
|
||||
// repair attempts. An inability to generate or validate returns an error and
|
||||
// no partial result.
|
||||
//
|
||||
// The built-in client retries eligible provider statuses using the effective
|
||||
// Config or request budget. With R additional transient attempts and P repair
|
||||
// generations, one run can dispatch at most (R+1)×(P+1) Promptkit-controlled
|
||||
// model attempts. Each dispatched attempt receives its own generation and
|
||||
// HTTP-client timeout; the caller context governs the full operation.
|
||||
// Injected LLMClient implementations are not retried automatically.
|
||||
// With Observer, Run emits started before facade request conversion or a nil
|
||||
// engine check, and finished on every normal return after public error mapping
|
||||
// and resource release. The event RunID matches a successful result. Without
|
||||
// Observer, result timing retains its existing convention; observed wall
|
||||
// duration can be slightly longer because it spans the facade boundary.
|
||||
//
|
||||
// Run can return every error category documented by [Engine.Prepare], plus
|
||||
// ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is
|
||||
// ErrCapacityExceeded and ErrLLMGenerate. Built-in HTTP execution failures
|
||||
// additionally match ErrTransport; malformed successful responses and nil
|
||||
// successful injected responses additionally match ErrMalformedResponse.
|
||||
// An engine admission rejection is
|
||||
// discoverable as [CapacityError] and still matches ErrCapacityExceeded. It
|
||||
// occurs before artifacts, schemas, rendering, or model generation because the
|
||||
// selected backend's admission capacity is full; it does not match
|
||||
// ErrInvalidRequest or ErrLLMGenerate. A built-in OpenAI-compatible non-2xx
|
||||
// response is discoverable as [GenerationError]. Errors from injected clients
|
||||
// remain available through errors.Is. Cancellation while waiting for
|
||||
// response is discoverable as [GenerationError]. Eligible-status exhaustion
|
||||
// with a positive budget is discoverable as [RetryExhaustedError]. Injected
|
||||
// errors remain available through errors.Is. Cancellation while waiting for
|
||||
// model-generation capacity matches both ErrLLMGenerate and the context error.
|
||||
// Cancellation otherwise follows the active collaborator's documented
|
||||
// behavior. A nil Engine returns ErrInvalidConfig. Run returns no partial
|
||||
// result on error.
|
||||
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
ctx = observation.WithoutExecutionState(ctx)
|
||||
if req.Observer == nil {
|
||||
return e.run(ctx, req)
|
||||
}
|
||||
recorder, idErr := observation.New(func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) })
|
||||
recorder.Start()
|
||||
if idErr != nil {
|
||||
err := fmt.Errorf("failed to create run id: %w", idErr)
|
||||
terminal := observedFinish(nil, err)
|
||||
recorder.Finish(observation.FinishData{Outcome: terminal.Outcome, Failure: terminal.Failure})
|
||||
return nil, err
|
||||
}
|
||||
result, err := e.run(observation.WithRecorder(ctx, recorder), req)
|
||||
terminal := observedFinish(result, err)
|
||||
recorder.Finish(observation.FinishData{Outcome: terminal.Outcome, Failure: terminal.Failure,
|
||||
StatusCode: terminal.StatusCode, RetriesExhausted: terminal.RetriesExhausted})
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (e *Engine) run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
@@ -669,7 +564,7 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
result, err := e.runner.Run(ctx, domainReq)
|
||||
result, err := e.internal.Run(ctx, domainReq)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
@@ -697,11 +592,22 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
// preserving documented collaborator and context identities. An engine
|
||||
// admission rejection is discoverable as [CapacityError] and still matches
|
||||
// ErrCapacityExceeded. A built-in OpenAI-compatible non-2xx response is
|
||||
// discoverable as [GenerationError]. A completed content-validation rejection,
|
||||
// including repair exhaustion, is returned in RunResult, not as an operational
|
||||
// error. An operational error returns no partial RunResult.
|
||||
// discoverable as [GenerationError]. Built-in HTTP execution failures match
|
||||
// ErrTransport; malformed successful responses and nil successful injected
|
||||
// responses match ErrMalformedResponse. Positive-budget eligible-status
|
||||
// exhaustion is discoverable as [RetryExhaustedError]. A completed
|
||||
// content-validation rejection, including repair exhaustion, is returned in
|
||||
// RunResult, not as an operational
|
||||
// error. RunPrepared uses the retry choice frozen during PrepareExecution. An
|
||||
// operational error returns no partial RunResult. A retained Observer begins
|
||||
// only after a successful atomic claim, with identities from the frozen
|
||||
// snapshot, and receives a terminal event on normal return after public error
|
||||
// mapping. Invalid, foreign, discarded, or reused handles emit no events.
|
||||
// Callbacks run outside the handle and capacity locks; panics propagate and
|
||||
// do not promise a terminal event.
|
||||
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
ctx = observation.WithoutExecutionState(ctx)
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
@@ -709,9 +615,21 @@ func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
|
||||
if prepared != nil {
|
||||
internal = prepared.internal
|
||||
}
|
||||
result, err := e.runner.RunPrepared(ctx, internal)
|
||||
var recorder *observation.Recorder
|
||||
result, err := e.internal.RunPrepared(observation.WithClaimSink(ctx, &recorder), internal)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
mapped := mapPublicError(err)
|
||||
if recorder != nil {
|
||||
terminal := observedFinish(nil, mapped)
|
||||
recorder.Finish(observation.FinishData{Outcome: terminal.Outcome, Failure: terminal.Failure,
|
||||
StatusCode: terminal.StatusCode, RetriesExhausted: terminal.RetriesExhausted})
|
||||
}
|
||||
return nil, mapped
|
||||
}
|
||||
return fromDomainRunResult(result), nil
|
||||
publicResult := fromDomainRunResult(result)
|
||||
if recorder != nil {
|
||||
terminal := observedFinish(publicResult, nil)
|
||||
recorder.Finish(observation.FinishData{Outcome: terminal.Outcome})
|
||||
}
|
||||
return publicResult, nil
|
||||
}
|
||||
|
||||
@@ -220,6 +220,14 @@ func TestRunRequestFormattingRedactsDirectAPIKey(t *testing.T) {
|
||||
"transcript": promptkit.InlineWithURI(inputURI, inputBody),
|
||||
},
|
||||
Vars: map[string]string{"audience": variableValue},
|
||||
AppendedMessages: []promptkit.RenderedMessage{{
|
||||
Role: "private-role-sentinel",
|
||||
Content: "private-appended-content-sentinel",
|
||||
CacheControl: &promptkit.CacheControl{
|
||||
Type: "private-cache-sentinel",
|
||||
TTL: "private-ttl-sentinel",
|
||||
},
|
||||
}},
|
||||
}
|
||||
|
||||
for _, formatted := range []string{
|
||||
@@ -229,7 +237,7 @@ func TestRunRequestFormattingRedactsDirectAPIKey(t *testing.T) {
|
||||
fmt.Sprintf("%+v", req),
|
||||
fmt.Sprintf("%#v", req),
|
||||
} {
|
||||
for _, privateValue := range []string{secret, inputURI, inputBody, variableValue} {
|
||||
for _, privateValue := range []string{secret, inputURI, inputBody, variableValue, "private-role-sentinel", "private-appended-content-sentinel", "private-cache-sentinel", "private-ttl-sentinel"} {
|
||||
if strings.Contains(formatted, privateValue) {
|
||||
t.Fatalf("formatted RunRequest leaked private value %q: %s", privateValue, formatted)
|
||||
}
|
||||
@@ -240,6 +248,7 @@ func TestRunRequestFormattingRedactsDirectAPIKey(t *testing.T) {
|
||||
"APIKeySet:true",
|
||||
"Inputs:1",
|
||||
"Vars:1",
|
||||
"AppendedMessages:1",
|
||||
} {
|
||||
if !strings.Contains(formatted, summary) {
|
||||
t.Fatalf("formatted RunRequest omitted structural summary %q: %s", summary, formatted)
|
||||
@@ -1536,31 +1545,21 @@ func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
|
||||
name string
|
||||
profileID string
|
||||
backendID string
|
||||
endpoint string
|
||||
apiKeyEnv string
|
||||
model string
|
||||
}{
|
||||
{
|
||||
name: "OpenRouter",
|
||||
profileID: "mistral-small-3",
|
||||
backendID: promptkit.BackendOpenRouter,
|
||||
endpoint: "https://openrouter.ai/api/v1",
|
||||
apiKeyEnv: "OPENROUTER_API_KEY",
|
||||
model: "mistralai/mistral-small-3.2-24b-instruct",
|
||||
},
|
||||
{
|
||||
name: "Rakestrawhome",
|
||||
profileID: "rakestrawhome-gemma-4-31b",
|
||||
backendID: promptkit.BackendRakestrawHome,
|
||||
endpoint: "https://inference.ai.rakestrawhome.com/v1",
|
||||
apiKeyEnv: "RAKESTRAWHOME_INFERENCE_API_KEY",
|
||||
model: "google/gemma-4-31b-it",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Setenv(tc.apiKeyEnv, "test-key")
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: frameworkPromptDir,
|
||||
SchemaDir: frameworkSchemaDir,
|
||||
@@ -1572,6 +1571,7 @@ func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: frameworkMarkdownSummaryPromptID,
|
||||
ProfileID: tc.profileID,
|
||||
APIKey: "test-key",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||
@@ -1582,10 +1582,7 @@ func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
|
||||
}
|
||||
if prepared.SelectedProfileID != tc.profileID ||
|
||||
prepared.SelectedBackendID != tc.backendID ||
|
||||
prepared.EffectiveModelParams.BackendID != tc.backendID ||
|
||||
prepared.EffectiveModelParams.Endpoint != tc.endpoint ||
|
||||
prepared.EffectiveModelParams.APIKeyEnv != tc.apiKeyEnv ||
|
||||
prepared.EffectiveModelParams.Model != tc.model {
|
||||
prepared.EffectiveModelParams.BackendID != tc.backendID {
|
||||
t.Fatalf("unexpected built-in preparation: %#v", prepared)
|
||||
}
|
||||
})
|
||||
@@ -2549,6 +2546,38 @@ func TestInMemoryProfileExtraParamsAreCopiedAcrossPublicBoundary(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithProfilesCopiesDefinitionsWhenApplied(t *testing.T) {
|
||||
nested := map[string]any{"value": "when option is created"}
|
||||
option := promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "profile",
|
||||
Endpoint: "http://copy.example/v1",
|
||||
Model: "copy-model",
|
||||
ExtraParams: map[string]any{"nested": nested},
|
||||
})
|
||||
nested["value"] = "when option is applied"
|
||||
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir}, option)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
nested["value"] = "after construction"
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: frameworkMarkdownSummaryPromptID,
|
||||
ProfileID: "profile",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with copied profile: %v", err)
|
||||
}
|
||||
if got := prepared.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"]; got != "when option is applied" {
|
||||
t.Fatalf("profile copy value = %q, want value at option application", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithProfilesRejectsInvalidExtraParams(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
21
errors.go
21
errors.go
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/generation"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||
@@ -21,6 +22,17 @@ func mapPublicError(err error) error {
|
||||
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
||||
return &CapacityError{BackendID: internalCapacityError.BackendID}
|
||||
}
|
||||
var internalExhausted *generation.ExhaustedError
|
||||
if errors.As(err, &internalExhausted) && internalExhausted != nil {
|
||||
mapped := newRetryExhaustedError(internalExhausted.Attempts(), mapPublicError(internalExhausted.Unwrap()))
|
||||
if errors.Is(err, usecase.ErrValidation) && !errors.Is(mapped, ErrValidation) {
|
||||
return fmt.Errorf("%w: %w", ErrValidation, mapped)
|
||||
}
|
||||
if publicErr := publicErrorFor(err); publicErr != nil && !errors.Is(mapped, publicErr) {
|
||||
return fmt.Errorf("%w: %w", publicErr, mapped)
|
||||
}
|
||||
return mapped
|
||||
}
|
||||
publicErr := publicErrorFor(err)
|
||||
var providerHTTPError *llm.ProviderHTTPError
|
||||
if errors.As(err, &providerHTTPError) && providerHTTPError != nil {
|
||||
@@ -38,7 +50,14 @@ func mapPublicError(err error) error {
|
||||
if publicErr == nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("%w: %w", publicErr, err)
|
||||
switch {
|
||||
case errors.Is(err, llm.ErrTransport):
|
||||
return fmt.Errorf("%w: %w: %w", publicErr, ErrTransport, err)
|
||||
case errors.Is(err, llm.ErrMalformedResponse):
|
||||
return fmt.Errorf("%w: %w: %w", publicErr, ErrMalformedResponse, err)
|
||||
default:
|
||||
return fmt.Errorf("%w: %w", publicErr, err)
|
||||
}
|
||||
}
|
||||
|
||||
func publicErrorFor(err error) error {
|
||||
|
||||
@@ -73,3 +73,10 @@ func TestMapPublicErrorPreservesValidationAroundGenerationError(t *testing.T) {
|
||||
t.Fatalf("mapped error exposes internal ProviderHTTPError: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapPublicErrorDoesNotTreatRequestEncodingAsTransport(t *testing.T) {
|
||||
err := mapPublicError(fmt.Errorf("%w: %w", usecase.ErrLLMGenerate, llm.ErrRequestFailed))
|
||||
if !errors.Is(err, ErrLLMGenerate) || errors.Is(err, ErrTransport) {
|
||||
t.Fatalf("request encoding category = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
157
examples/go-library/observe/main.go
Normal file
157
examples/go-library/observe/main.go
Normal file
@@ -0,0 +1,157 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
//go:embed prompts/*.yaml schemas/*.json
|
||||
var assets embed.FS
|
||||
|
||||
type scriptedTransport struct {
|
||||
proposalCalls int
|
||||
}
|
||||
|
||||
func (t *scriptedTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
if err := req.Context().Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var request struct {
|
||||
Model string `json:"model"`
|
||||
}
|
||||
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch request.Model {
|
||||
case "proposal-model":
|
||||
t.proposalCalls++
|
||||
if t.proposalCalls == 1 {
|
||||
return httpResponse(http.StatusServiceUnavailable, `{"error":{}}`), nil
|
||||
}
|
||||
return modelResponse(`{"title":"Queue cleanup","risk":"low"}`, 10, 8), nil
|
||||
case "assessment-model":
|
||||
return modelResponse(`{"recommendation":"accept","confidence":0.9}`, 7, 5), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected model in offline transport")
|
||||
}
|
||||
}
|
||||
|
||||
func modelResponse(content string, promptTokens, completionTokens int) *http.Response {
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"choices": []any{map[string]any{"message": map[string]string{"content": content}}},
|
||||
"usage": map[string]int{"prompt_tokens": promptTokens, "completion_tokens": completionTokens},
|
||||
})
|
||||
return httpResponse(http.StatusOK, string(body))
|
||||
}
|
||||
|
||||
func httpResponse(status int, body string) *http.Response {
|
||||
return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))}
|
||||
}
|
||||
|
||||
type eventLog struct{ events []promptkit.ExecutionEvent }
|
||||
|
||||
func (l *eventLog) record(event promptkit.ExecutionEvent) { l.events = append(l.events, event) }
|
||||
|
||||
func (l *eventLog) terminal() (promptkit.ExecutionEvent, error) {
|
||||
if len(l.events) == 0 || l.events[len(l.events)-1].Kind != promptkit.ExecutionFinished {
|
||||
return promptkit.ExecutionEvent{}, fmt.Errorf("execution did not finish")
|
||||
}
|
||||
return l.events[len(l.events)-1], nil
|
||||
}
|
||||
|
||||
type summary struct {
|
||||
ProposalAttempts uint64 `json:"proposal_attempts"`
|
||||
ProposalTransientRetries uint64 `json:"proposal_transient_retries"`
|
||||
ProposalUsageComplete bool `json:"proposal_usage_complete"`
|
||||
AssessmentAttempts uint64 `json:"assessment_attempts"`
|
||||
AssessmentTransientRetries uint64 `json:"assessment_transient_retries"`
|
||||
AssessmentOutcome string `json:"assessment_outcome"`
|
||||
}
|
||||
|
||||
func run() (summary, error) {
|
||||
transport := &scriptedTransport{}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}},
|
||||
promptkit.WithPromptFS(assets, "prompts"),
|
||||
promptkit.WithSchemaFS(assets, "schemas"),
|
||||
promptkit.WithBackend(promptkit.Backend{ID: "offline-backend", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1}),
|
||||
promptkit.WithProfiles(
|
||||
promptkit.Profile{ID: "proposal-profile", BackendID: "offline-backend", Model: "proposal-model"},
|
||||
promptkit.Profile{ID: "assessment-profile", BackendID: "offline-backend", Model: "assessment-model"},
|
||||
))
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
|
||||
var proposalLog eventLog
|
||||
proposal, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, Observer: proposalLog.record,
|
||||
})
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
if !proposal.Validation.IsValid {
|
||||
return summary{}, fmt.Errorf("proposal failed structural validation")
|
||||
}
|
||||
proposalTerminal, err := proposalLog.terminal()
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
var assessmentLog eventLog
|
||||
assessment, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(proposal.RawOutput)}, Observer: assessmentLog.record,
|
||||
})
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
if !assessment.Validation.IsValid {
|
||||
return summary{}, fmt.Errorf("assessment failed structural validation")
|
||||
}
|
||||
assessmentTerminal, err := assessmentLog.terminal()
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
var proposalValue struct {
|
||||
Risk string `json:"risk"`
|
||||
}
|
||||
var assessmentValue struct {
|
||||
Recommendation string `json:"recommendation"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(proposal.RawOutput), &proposalValue); err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
if err := json.Unmarshal([]byte(assessment.RawOutput), &assessmentValue); err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
decision := "rejected"
|
||||
if proposalValue.Risk == "low" && assessmentValue.Recommendation == "accept" {
|
||||
decision = "accepted"
|
||||
}
|
||||
return summary{
|
||||
ProposalAttempts: proposalTerminal.Attempts,
|
||||
ProposalTransientRetries: proposalTerminal.TransientRetries,
|
||||
ProposalUsageComplete: proposalTerminal.Usage.PromptTokens.Complete,
|
||||
AssessmentAttempts: assessmentTerminal.Attempts,
|
||||
AssessmentTransientRetries: assessmentTerminal.TransientRetries,
|
||||
AssessmentOutcome: decision,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
value, err := run()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := json.NewEncoder(os.Stdout).Encode(value); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
17
examples/go-library/observe/prompts/assessment.yaml
Normal file
17
examples/go-library/observe/prompts/assessment.yaml
Normal file
@@ -0,0 +1,17 @@
|
||||
id: example.assessment
|
||||
version: "1.0.0"
|
||||
default_profile: assessment-profile
|
||||
description: Assess a structured proposal independently of schema validity.
|
||||
inputs:
|
||||
- name: proposal
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content: Return a concise assessment as JSON.
|
||||
- role: user
|
||||
content: '{{input "proposal"}}'
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: assessment.schema.json
|
||||
17
examples/go-library/observe/prompts/proposal.yaml
Normal file
17
examples/go-library/observe/prompts/proposal.yaml
Normal file
@@ -0,0 +1,17 @@
|
||||
id: example.proposal
|
||||
version: "1.0.0"
|
||||
default_profile: proposal-profile
|
||||
description: Propose a bounded improvement from a synthetic idea.
|
||||
inputs:
|
||||
- name: idea
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content: Return a concise proposal as JSON.
|
||||
- role: user
|
||||
content: '{{input "idea"}}'
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: proposal.schema.json
|
||||
10
examples/go-library/observe/schemas/assessment.schema.json
Normal file
10
examples/go-library/observe/schemas/assessment.schema.json
Normal file
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"required": ["recommendation", "confidence"],
|
||||
"properties": {
|
||||
"recommendation": {"type": "string"},
|
||||
"confidence": {"type": "number"}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
10
examples/go-library/observe/schemas/proposal.schema.json
Normal file
10
examples/go-library/observe/schemas/proposal.schema.json
Normal file
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"required": ["title", "risk"],
|
||||
"properties": {
|
||||
"title": {"type": "string"},
|
||||
"risk": {"type": "string"}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
@@ -18,7 +18,7 @@ func (r RunRequest) GoString() string {
|
||||
|
||||
func (r RunRequest) redactedString() string {
|
||||
return fmt.Sprintf(
|
||||
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t}",
|
||||
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t AppendedMessages:%d}",
|
||||
r.PromptID,
|
||||
r.PromptVersion,
|
||||
r.ProfileID,
|
||||
@@ -27,6 +27,7 @@ func (r RunRequest) redactedString() string {
|
||||
len(r.Vars),
|
||||
r.Execution != nil,
|
||||
r.Validation != nil,
|
||||
len(r.AppendedMessages),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -39,6 +39,23 @@ func TestBuiltInGenerationError(t *testing.T) {
|
||||
assertGenerationError(t, err, http.StatusServiceUnavailable, "", "", "")
|
||||
}
|
||||
|
||||
func TestBuiltInGenerationErrorWithAppendedMessages(t *testing.T) {
|
||||
const messageMarker = "combined-request-provider-marker"
|
||||
engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity,
|
||||
`{"error":{"message":"`+messageMarker+`"}}`)
|
||||
request := generationErrorRunRequest()
|
||||
request.AppendedMessages = []promptkit.RenderedMessage{
|
||||
{Role: promptkit.RoleAssistant, Content: "previous response"},
|
||||
{Role: promptkit.RoleUser, Content: "consumer correction"},
|
||||
}
|
||||
|
||||
result, err := engine.Run(context.Background(), request)
|
||||
if result != nil {
|
||||
t.Fatalf("Run result = %#v, want nil", result)
|
||||
}
|
||||
assertGenerationError(t, err, http.StatusUnprocessableEntity, "", "", messageMarker)
|
||||
}
|
||||
|
||||
func TestBuiltInRepairGenerationError(t *testing.T) {
|
||||
const (
|
||||
codeMarker = "repair-code-marker"
|
||||
@@ -77,6 +94,89 @@ func TestBuiltInRepairGenerationError(t *testing.T) {
|
||||
assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker)
|
||||
}
|
||||
|
||||
func TestBuiltInGenerationFailureCategories(t *testing.T) {
|
||||
transportCause := errors.New("private transport diagnostic")
|
||||
config := contractConfig(frameworkSchemaDir)
|
||||
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return nil, transportCause
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := engine.Run(context.Background(), generationErrorRunRequest())
|
||||
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrTransport) || !errors.Is(err, transportCause) {
|
||||
t.Fatalf("transport result=%#v, error=%v", result, err)
|
||||
}
|
||||
if errors.Is(err, promptkit.ErrMalformedResponse) || strings.Contains(fmt.Sprintf("%#v", err), transportCause.Error()) {
|
||||
t.Fatalf("transport category or formatting = %v", err)
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
body string
|
||||
}{
|
||||
{name: "invalid framing", body: `{"choices":[]}{}`},
|
||||
{name: "missing choices", body: `{"choices":[]}`},
|
||||
{name: "missing content", body: `{"choices":[{"message":{}}]}`},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
engine := newBuiltInGenerationErrorEngine(t, http.StatusOK, tc.body)
|
||||
result, err := engine.Run(context.Background(), generationErrorRunRequest())
|
||||
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrMalformedResponse) || errors.Is(err, promptkit.ErrTransport) {
|
||||
t.Fatalf("malformed result=%#v, error=%v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltInGenerationPreservesContextIdentity(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
cause error
|
||||
}{
|
||||
{name: "canceled", cause: context.Canceled},
|
||||
{name: "deadline", cause: context.DeadlineExceeded},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
config := contractConfig(frameworkSchemaDir)
|
||||
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return nil, tc.cause
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := engine.Run(context.Background(), generationErrorRunRequest())
|
||||
if result != nil || !errors.Is(err, tc.cause) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrTransport) {
|
||||
t.Fatalf("context result=%#v, error=%v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInjectedGenerationFailureCategories(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
client *fakeLLMClient
|
||||
malformed bool
|
||||
}{
|
||||
{name: "nil response", client: &fakeLLMClient{}, malformed: true},
|
||||
{name: "arbitrary error", client: &fakeLLMClient{err: errors.New("injected failure")}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(tc.client))
|
||||
result, err := engine.Run(context.Background(), generationErrorRunRequest())
|
||||
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || errors.Is(err, promptkit.ErrTransport) || errors.Is(err, promptkit.ErrMalformedResponse) != tc.malformed {
|
||||
t.Fatalf("injected result=%#v, error=%v", result, err)
|
||||
}
|
||||
if tc.client.err != nil && !errors.Is(err, tc.client.err) {
|
||||
t.Fatalf("injected cause lost: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertGenerationError(t *testing.T, err error, statusCode int, code, providerType, message string) {
|
||||
t.Helper()
|
||||
|
||||
@@ -108,6 +208,8 @@ func newBuiltInGenerationErrorEngine(t *testing.T, statusCode int, body string)
|
||||
t.Helper()
|
||||
|
||||
config := contractConfig(frameworkSchemaDir)
|
||||
noRetries := 0
|
||||
config.TransientRetries = &noRetries
|
||||
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
StatusCode: statusCode,
|
||||
|
||||
2
go.mod
2
go.mod
@@ -3,6 +3,8 @@ module gitea.maximumdirect.net/eric/promptkit
|
||||
go 1.25.5
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
4
go.sum
4
go.sum
@@ -1,3 +1,7 @@
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0 h1:wdab+R+lgMZxzYwhbAuCrW7zdSQ1cW9ovBR6UzLmZR4=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0/go.mod h1:a3VBbh9oaGXD6ruxhtwHj+AWW6DY/SSyby1QtMmYSUo=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0 h1:baCCIB6Ee5t0S4Zc++NTBIPcJEv5UPS6DlSrpZfykwg=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0/go.mod h1:mtgD1E0SGm0e/I8Ffhz+AYX7GN7RKUWJXehNFIKHs4Y=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
|
||||
@@ -22,16 +22,7 @@ const (
|
||||
// backend.
|
||||
RakestrawHomeID = "rakestrawhome"
|
||||
|
||||
openRouterEndpoint = "https://openrouter.ai/api/v1"
|
||||
openRouterAPIKeyEnv = "OPENROUTER_API_KEY"
|
||||
|
||||
openRouterConcurrencyLimit = 16
|
||||
|
||||
rakestrawHomeEndpoint = "https://inference.ai.rakestrawhome.com/v1"
|
||||
rakestrawHomeAPIKeyEnv = "RAKESTRAWHOME_INFERENCE_API_KEY"
|
||||
|
||||
rakestrawHomeConcurrencyLimit = 4
|
||||
defaultQueueCapacity = 1024
|
||||
defaultQueueCapacity = 1024
|
||||
)
|
||||
|
||||
// ErrBackendNotFound identifies a registry lookup for an unknown backend ID.
|
||||
@@ -44,16 +35,15 @@ type Registry struct {
|
||||
backends map[string]domain.Backend
|
||||
}
|
||||
|
||||
// NewRegistry constructs a registry containing the built-in definitions
|
||||
// followed by the supplied additions. Every ID must be unique.
|
||||
func NewRegistry(additions []domain.Backend) (*Registry, error) {
|
||||
builtIns := builtInBackends()
|
||||
// NewRegistry constructs a registry containing maintained definitions followed
|
||||
// by consumer additions. Every ID must be unique across both groups.
|
||||
func NewRegistry(maintained, additions []domain.Backend) (*Registry, error) {
|
||||
registry := &Registry{
|
||||
backends: make(map[string]domain.Backend, len(builtIns)+len(additions)),
|
||||
backends: make(map[string]domain.Backend, len(maintained)+len(additions)),
|
||||
}
|
||||
|
||||
definitions := make([]domain.Backend, 0, len(builtIns)+len(additions))
|
||||
definitions = append(definitions, builtIns...)
|
||||
definitions := make([]domain.Backend, 0, len(maintained)+len(additions))
|
||||
definitions = append(definitions, maintained...)
|
||||
definitions = append(definitions, additions...)
|
||||
|
||||
for _, definition := range definitions {
|
||||
@@ -65,7 +55,7 @@ func NewRegistry(additions []domain.Backend) (*Registry, error) {
|
||||
return nil, fmt.Errorf("backend ID %q is already registered", definition.ID)
|
||||
}
|
||||
|
||||
normalized, err := normalizeBackend(definition)
|
||||
normalized, err := NormalizeDefinition(definition)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -75,23 +65,6 @@ func NewRegistry(additions []domain.Backend) (*Registry, error) {
|
||||
return registry, nil
|
||||
}
|
||||
|
||||
func builtInBackends() []domain.Backend {
|
||||
return []domain.Backend{
|
||||
{
|
||||
ID: OpenRouterID,
|
||||
Endpoint: openRouterEndpoint,
|
||||
APIKeyEnv: openRouterAPIKeyEnv,
|
||||
ConcurrencyLimit: openRouterConcurrencyLimit,
|
||||
},
|
||||
{
|
||||
ID: RakestrawHomeID,
|
||||
Endpoint: rakestrawHomeEndpoint,
|
||||
APIKeyEnv: rakestrawHomeAPIKeyEnv,
|
||||
ConcurrencyLimit: rakestrawHomeConcurrencyLimit,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// GetBackend returns a defensive copy of the backend registered with id.
|
||||
func (r *Registry) GetBackend(id string) (domain.Backend, error) {
|
||||
if r == nil {
|
||||
@@ -128,7 +101,8 @@ func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy {
|
||||
return policies
|
||||
}
|
||||
|
||||
func normalizeBackend(definition domain.Backend) (domain.Backend, error) {
|
||||
// NormalizeDefinition validates and defensively copies one backend definition.
|
||||
func NormalizeDefinition(definition domain.Backend) (domain.Backend, error) {
|
||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(definition.Endpoint)
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err)
|
||||
|
||||
@@ -2,6 +2,7 @@ package backend_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -11,48 +12,24 @@ import (
|
||||
|
||||
const validEndpoint = "https://backend.example/v1"
|
||||
|
||||
func TestRegistryIncludesExactBuiltInDefinitions(t *testing.T) {
|
||||
registry, err := backend.NewRegistry(nil)
|
||||
func TestRegistryIncludesMaintainedDefinitions(t *testing.T) {
|
||||
maintained := []domain.Backend{
|
||||
{ID: backend.OpenRouterID, Endpoint: validEndpoint, ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true},
|
||||
{ID: backend.RakestrawHomeID, Endpoint: "https://second.example/v1", ConcurrencyLimit: 4, QueueCapacity: 5, QueueCapacitySet: true},
|
||||
}
|
||||
registry, err := backend.NewRegistry(maintained, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
id string
|
||||
endpoint string
|
||||
apiKeyEnv string
|
||||
concurrent int
|
||||
}{
|
||||
{
|
||||
name: "OpenRouter",
|
||||
id: backend.OpenRouterID,
|
||||
endpoint: "https://openrouter.ai/api/v1",
|
||||
apiKeyEnv: "OPENROUTER_API_KEY",
|
||||
concurrent: 16,
|
||||
},
|
||||
{
|
||||
name: "Rakestrawhome",
|
||||
id: backend.RakestrawHomeID,
|
||||
endpoint: "https://inference.ai.rakestrawhome.com/v1",
|
||||
apiKeyEnv: "RAKESTRAWHOME_INFERENCE_API_KEY",
|
||||
concurrent: 4,
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
definition, err := registry.GetBackend(tc.id)
|
||||
for _, expected := range maintained {
|
||||
t.Run(expected.ID, func(t *testing.T) {
|
||||
definition, err := registry.GetBackend(expected.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("look up built-in: %v", err)
|
||||
t.Fatalf("look up maintained definition: %v", err)
|
||||
}
|
||||
if definition.ID != tc.id ||
|
||||
definition.Endpoint != tc.endpoint ||
|
||||
definition.APIKeyEnv != tc.apiKeyEnv ||
|
||||
definition.ConcurrencyLimit != tc.concurrent ||
|
||||
definition.QueueCapacity != 1024 ||
|
||||
!definition.QueueCapacitySet ||
|
||||
definition.ExtraParams != nil {
|
||||
t.Fatalf("unexpected built-in definition: %#v", definition)
|
||||
if !reflect.DeepEqual(definition, expected) {
|
||||
t.Fatalf("unexpected maintained definition: %#v", definition)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -60,12 +37,12 @@ func TestRegistryIncludesExactBuiltInDefinitions(t *testing.T) {
|
||||
policies := registry.CapacityPolicies()
|
||||
if len(policies) != 2 ||
|
||||
policies[backend.OpenRouterID] != (domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 16,
|
||||
QueueCapacity: 1024,
|
||||
ConcurrencyLimit: 2,
|
||||
QueueCapacity: 3,
|
||||
}) ||
|
||||
policies[backend.RakestrawHomeID] != (domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 4,
|
||||
QueueCapacity: 1024,
|
||||
QueueCapacity: 5,
|
||||
}) {
|
||||
t.Fatalf("unexpected built-in capacity policies: %#v", policies)
|
||||
}
|
||||
@@ -77,7 +54,7 @@ func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
|
||||
"count": int64(7),
|
||||
"nested": nested,
|
||||
}
|
||||
registry, err := backend.NewRegistry([]domain.Backend{
|
||||
registry, err := backend.NewRegistry(nil, []domain.Backend{
|
||||
{
|
||||
ID: " custom ",
|
||||
Endpoint: " https://custom.example/openai/v1 ",
|
||||
@@ -140,12 +117,11 @@ func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
|
||||
}
|
||||
|
||||
policies := registry.CapacityPolicies()
|
||||
if len(policies) != 3 {
|
||||
if len(policies) != 1 {
|
||||
t.Fatalf("unexpected capacity policy count: %#v", policies)
|
||||
}
|
||||
policies["custom"] = domain.BackendCapacityPolicy{}
|
||||
delete(policies, backend.OpenRouterID)
|
||||
delete(policies, backend.RakestrawHomeID)
|
||||
delete(policies, "custom")
|
||||
againPolicies := registry.CapacityPolicies()
|
||||
if againPolicies["custom"] != (domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 3,
|
||||
@@ -153,12 +129,6 @@ func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
|
||||
}) {
|
||||
t.Fatalf("capacity policy map mutated registry state: %#v", againPolicies)
|
||||
}
|
||||
if _, ok := againPolicies[backend.OpenRouterID]; !ok {
|
||||
t.Fatalf("OpenRouter capacity policy deletion mutated registry state: %#v", againPolicies)
|
||||
}
|
||||
if _, ok := againPolicies[backend.RakestrawHomeID]; !ok {
|
||||
t.Fatalf("Rakestrawhome capacity policy deletion mutated registry state: %#v", againPolicies)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) {
|
||||
@@ -234,7 +204,7 @@ func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
tc.definition.ID = "custom"
|
||||
tc.definition.Endpoint = validEndpoint
|
||||
registry, err := backend.NewRegistry([]domain.Backend{tc.definition})
|
||||
registry, err := backend.NewRegistry(nil, []domain.Backend{tc.definition})
|
||||
if tc.wantError {
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid capacity policy error")
|
||||
@@ -298,7 +268,7 @@ func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry(tc.additions)
|
||||
_, err := backend.NewRegistry(nil, tc.additions)
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate ID error")
|
||||
}
|
||||
@@ -312,7 +282,7 @@ func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
|
||||
func TestNewRegistryValidatesIDs(t *testing.T) {
|
||||
for _, id := range []string{"", " \t\n "} {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
_, err := backend.NewRegistry(nil, []domain.Backend{{
|
||||
ID: id,
|
||||
Endpoint: validEndpoint,
|
||||
}})
|
||||
@@ -341,7 +311,7 @@ func TestNewRegistryValidatesEndpoints(t *testing.T) {
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
_, err := backend.NewRegistry(nil, []domain.Backend{{
|
||||
ID: "custom",
|
||||
Endpoint: tc.endpoint,
|
||||
}})
|
||||
@@ -355,7 +325,7 @@ func TestNewRegistryValidatesEndpoints(t *testing.T) {
|
||||
func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) {
|
||||
for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
_, err := backend.NewRegistry(nil, []domain.Backend{{
|
||||
ID: "custom",
|
||||
Endpoint: validEndpoint,
|
||||
APIKeyEnv: name,
|
||||
@@ -378,7 +348,7 @@ func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
_, err := backend.NewRegistry(nil, []domain.Backend{{
|
||||
ID: "custom",
|
||||
Endpoint: validEndpoint,
|
||||
ExtraParams: tc.extraParams,
|
||||
@@ -391,7 +361,7 @@ func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRegistryLookupReportsNotFound(t *testing.T) {
|
||||
registry, err := backend.NewRegistry(nil)
|
||||
registry, err := backend.NewRegistry(nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
}
|
||||
|
||||
@@ -28,13 +28,13 @@ func (c *client) Generate(
|
||||
ctx context.Context,
|
||||
req domain.GenerateRequest,
|
||||
) (*domain.GenerateResponse, error) {
|
||||
pool := c.manager.getPool(req.Target.BackendID)
|
||||
if pool == nil {
|
||||
if c.manager.getPool(req.Target.BackendID) == nil {
|
||||
return c.next.Generate(ctx, req)
|
||||
}
|
||||
if err := pool.acquire(ctx); err != nil {
|
||||
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer pool.releaseActive()
|
||||
defer release()
|
||||
return c.next.Generate(ctx, req)
|
||||
}
|
||||
|
||||
@@ -107,6 +107,26 @@ func (m *Manager) getPool(backendID string) *pool {
|
||||
return m.pools[backendID]
|
||||
}
|
||||
|
||||
// Limited reports whether a backend uses an active-generation pool.
|
||||
func (m *Manager) Limited(backendID string) bool { return m.getPool(backendID) != nil }
|
||||
|
||||
// Acquire obtains one active-generation permit for a backend. Its release
|
||||
// function is idempotent. Backends without a configured pool are unlimited.
|
||||
func (m *Manager) Acquire(ctx context.Context, backendID string) (func(), error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pool := m.getPool(backendID)
|
||||
if pool == nil {
|
||||
return releaseNothing, nil
|
||||
}
|
||||
if err := pool.acquire(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var once sync.Once
|
||||
return func() { once.Do(pool.releaseActive) }, nil
|
||||
}
|
||||
|
||||
func (p *pool) acquire(ctx context.Context) error {
|
||||
p.mu.Lock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
|
||||
@@ -60,6 +60,26 @@ func TestNewManagerRejectsInvalidPolicies(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerAcquireReleaseIsIdempotent(t *testing.T) {
|
||||
manager, err := NewManager(map[string]domain.BackendCapacityPolicy{
|
||||
"limited": {ConcurrencyLimit: 1},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release, err := manager.Acquire(context.Background(), "limited")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release()
|
||||
release()
|
||||
other, err := manager.Acquire(context.Background(), "limited")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other()
|
||||
}
|
||||
|
||||
func TestManagerAdmissionIsBoundedAndReleaseIsIdempotent(t *testing.T) {
|
||||
policies := map[string]domain.BackendCapacityPolicy{
|
||||
"limited": {
|
||||
|
||||
249
internal/catalog/catalog.go
Normal file
249
internal/catalog/catalog.go
Normal file
@@ -0,0 +1,249 @@
|
||||
// Package catalog validates immutable maintained backend catalog assets.
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
|
||||
// Source identifies one immutable backend catalog asset tree.
|
||||
type Source struct {
|
||||
Name string
|
||||
ExpectedBackendID string
|
||||
FS fs.FS
|
||||
Root string
|
||||
}
|
||||
|
||||
// Set is the validated maintained backend and raw profile catalog.
|
||||
type Set struct {
|
||||
Backends []domain.Backend
|
||||
Profiles profile.Repository
|
||||
profileIDs []string
|
||||
}
|
||||
|
||||
// Load validates and combines immutable catalog sources in source order.
|
||||
func Load(sources ...Source) (Set, error) {
|
||||
if len(sources) == 0 {
|
||||
return Set{}, errors.New("at least one catalog source is required")
|
||||
}
|
||||
loaded := Set{Backends: make([]domain.Backend, 0, len(sources))}
|
||||
names := map[string]bool{}
|
||||
backendIDs := map[string]bool{}
|
||||
profileIDs := map[string]bool{}
|
||||
for _, source := range sources {
|
||||
if err := validateSource(source, names); err != nil {
|
||||
return Set{}, err
|
||||
}
|
||||
names[source.Name] = true
|
||||
definition, err := loadBackend(source)
|
||||
if err != nil {
|
||||
return Set{}, err
|
||||
}
|
||||
if backendIDs[definition.ID] {
|
||||
return Set{}, fmt.Errorf("catalog %s: backend ID duplicates an earlier catalog", source.Name)
|
||||
}
|
||||
backendIDs[definition.ID] = true
|
||||
repository, metadata, err := profile.LoadFSRepository(context.Background(), source.FS, path.Join(source.Root, "profiles"))
|
||||
if err != nil {
|
||||
return Set{}, fmt.Errorf("catalog %s: %w", source.Name, err)
|
||||
}
|
||||
if len(metadata) == 0 {
|
||||
return Set{}, fmt.Errorf("catalog %s: profiles must not be empty", source.Name)
|
||||
}
|
||||
resolvingRepository := profile.NewResolvingRepository(repository)
|
||||
for _, entry := range metadata {
|
||||
if strings.HasPrefix(entry.ID, "promptkit-") {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s uses the reserved promptkit- profile ID prefix", source.Name, entry.Path)
|
||||
}
|
||||
if profileIDs[entry.ID] {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s duplicates an earlier profile ID", source.Name, entry.Path)
|
||||
}
|
||||
if containsField(entry.ExplicitFields, "endpoint") || containsField(entry.ExplicitFields, "api_key_env") {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s contains connection metadata", source.Name, entry.Path)
|
||||
}
|
||||
value, err := repository.GetProfile(context.Background(), entry.ID)
|
||||
if err != nil {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s: %w", source.Name, entry.Path, err)
|
||||
}
|
||||
if err := rejectSecretKeys(value.ExtraParams); err != nil {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s: prohibited extra parameter key", source.Name, entry.Path)
|
||||
}
|
||||
resolved, err := resolvingRepository.GetProfile(context.Background(), entry.ID)
|
||||
if err != nil {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s has invalid profile inheritance", source.Name, entry.Path)
|
||||
}
|
||||
if resolved.BackendID != definition.ID {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s selects a different backend", source.Name, entry.Path)
|
||||
}
|
||||
profileIDs[entry.ID] = true
|
||||
loaded.profileIDs = append(loaded.profileIDs, entry.ID)
|
||||
}
|
||||
if loaded.Profiles == nil {
|
||||
loaded.Profiles = repository
|
||||
} else {
|
||||
loaded.Profiles = profile.NewOverlayRepository(loaded.Profiles, repository)
|
||||
}
|
||||
loaded.Backends = append(loaded.Backends, definition)
|
||||
}
|
||||
sort.Strings(loaded.profileIDs)
|
||||
return loaded, nil
|
||||
}
|
||||
|
||||
func validateSource(source Source, names map[string]bool) error {
|
||||
if strings.TrimSpace(source.Name) == "" || source.Name != strings.TrimSpace(source.Name) {
|
||||
return errors.New("catalog source name must not be blank")
|
||||
}
|
||||
if names[source.Name] {
|
||||
return fmt.Errorf("duplicate catalog source name %q", source.Name)
|
||||
}
|
||||
if source.FS == nil {
|
||||
return fmt.Errorf("catalog %s: filesystem is nil", source.Name)
|
||||
}
|
||||
if source.Root == "." || source.Root == "" || !fs.ValidPath(source.Root) {
|
||||
return fmt.Errorf("catalog %s: asset root is invalid", source.Name)
|
||||
}
|
||||
if strings.TrimSpace(source.ExpectedBackendID) == "" {
|
||||
return fmt.Errorf("catalog %s: expected backend ID is blank", source.Name)
|
||||
}
|
||||
return validateLayout(source)
|
||||
}
|
||||
|
||||
func validateLayout(source Source) error {
|
||||
manifestPath := path.Join(source.Root, "backend.json")
|
||||
profilesRoot := path.Join(source.Root, "profiles")
|
||||
return fs.WalkDir(source.FS, source.Root, func(assetPath string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if assetPath == source.Root {
|
||||
if !entry.IsDir() {
|
||||
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if entry.IsDir() {
|
||||
if assetPath == profilesRoot || strings.HasPrefix(assetPath, profilesRoot+"/") {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
|
||||
}
|
||||
if assetPath == manifestPath && entry.Type().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
if strings.HasPrefix(assetPath, profilesRoot+"/") && entry.Type().IsRegular() && strings.HasSuffix(assetPath, ".yml") {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
|
||||
})
|
||||
}
|
||||
|
||||
func loadBackend(source Source) (domain.Backend, error) {
|
||||
data, err := fs.ReadFile(source.FS, path.Join(source.Root, "backend.json"))
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: %w", source.Name, err)
|
||||
}
|
||||
type manifest struct {
|
||||
SchemaVersion *int `json:"schema_version"`
|
||||
ID *string `json:"id"`
|
||||
Endpoint *string `json:"endpoint"`
|
||||
APIKeyEnv *string `json:"api_key_env"`
|
||||
ConcurrencyLimit *int `json:"concurrency_limit"`
|
||||
QueueCapacity *int `json:"queue_capacity"`
|
||||
ExtraParams json.RawMessage `json:"extra_params"`
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.DisallowUnknownFields()
|
||||
var value manifest
|
||||
if err := decoder.Decode(&value); err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
|
||||
}
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
|
||||
}
|
||||
if value.SchemaVersion == nil || *value.SchemaVersion != 1 || value.ID == nil || value.Endpoint == nil || value.APIKeyEnv == nil || value.ConcurrencyLimit == nil || value.QueueCapacity == nil || value.ExtraParams == nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: required field is missing or unsupported", source.Name)
|
||||
}
|
||||
if *value.ID != source.ExpectedBackendID {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend ID does not match expected ID", source.Name)
|
||||
}
|
||||
if strings.TrimSpace(*value.APIKeyEnv) == "" {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: api key environment variable must not be blank", source.Name)
|
||||
}
|
||||
extraParams, err := decodeExtraParams(value.ExtraParams)
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid extra parameters", source.Name)
|
||||
}
|
||||
if err := rejectSecretKeys(extraParams); err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: prohibited extra parameter key", source.Name)
|
||||
}
|
||||
normalized, err := backend.NormalizeDefinition(domain.Backend{ID: *value.ID, Endpoint: *value.Endpoint, APIKeyEnv: *value.APIKeyEnv, ExtraParams: extraParams, ConcurrencyLimit: *value.ConcurrencyLimit, QueueCapacity: *value.QueueCapacity, QueueCapacitySet: true})
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid backend definition", source.Name)
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
func decodeExtraParams(data []byte) (map[string]any, error) {
|
||||
if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
|
||||
return nil, nil
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.UseNumber()
|
||||
var value map[string]any
|
||||
if err := decoder.Decode(&value); err != nil || value == nil {
|
||||
return nil, errors.New("extra parameters must be an object or null")
|
||||
}
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return nil, errors.New("extra parameters must contain exactly one JSON value")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func containsField(fields []string, target string) bool {
|
||||
for _, field := range fields {
|
||||
if field == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func rejectSecretKeys(value map[string]any) error {
|
||||
return rejectSecretValue(value)
|
||||
}
|
||||
|
||||
func rejectSecretValue(value any) error {
|
||||
switch value := value.(type) {
|
||||
case map[string]any:
|
||||
for key, child := range value {
|
||||
switch strings.ToLower(key) {
|
||||
case "api_key", "apikey", "authorization", "credential", "credentials", "password", "secret", "token", "access_token":
|
||||
return errors.New("prohibited key")
|
||||
}
|
||||
if err := rejectSecretValue(child); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, child := range value {
|
||||
if err := rejectSecretValue(child); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
356
internal/catalog/catalog_test.go
Normal file
356
internal/catalog/catalog_test.go
Normal file
@@ -0,0 +1,356 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2"
|
||||
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2"
|
||||
)
|
||||
|
||||
func TestLoadPublishedCatalogs(t *testing.T) {
|
||||
loaded, err := Load(
|
||||
Source{Name: "OpenRouter", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
|
||||
Source{Name: "Rakestrawhome", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("load published catalogs: %v", err)
|
||||
}
|
||||
if loaded.Profiles == nil || len(loaded.profileIDs) == 0 {
|
||||
t.Fatalf("published catalogs have no usable profiles: %#v", loaded)
|
||||
}
|
||||
if _, err := loaded.Profiles.GetProfile(context.Background(), "gemini-flash-latest"); err != nil {
|
||||
t.Fatalf("load maintained standard-profile base: %v", err)
|
||||
}
|
||||
|
||||
expectedBackends := map[string]struct {
|
||||
endpoint string
|
||||
apiKeyEnv string
|
||||
concurrencyLimit int
|
||||
queueCapacity int
|
||||
}{
|
||||
"openrouter": {
|
||||
endpoint: "https://openrouter.ai/api/v1",
|
||||
apiKeyEnv: "OPENROUTER_API_KEY",
|
||||
concurrencyLimit: 16,
|
||||
queueCapacity: 1024,
|
||||
},
|
||||
"rakestrawhome": {
|
||||
endpoint: "https://inference.ai.rakestrawhome.com/v1",
|
||||
apiKeyEnv: "RAKESTRAWHOME_INFERENCE_API_KEY",
|
||||
concurrencyLimit: 4,
|
||||
queueCapacity: 1024,
|
||||
},
|
||||
}
|
||||
if len(loaded.Backends) != len(expectedBackends) {
|
||||
t.Fatalf("published backend count = %d, want %d", len(loaded.Backends), len(expectedBackends))
|
||||
}
|
||||
for _, definition := range loaded.Backends {
|
||||
expected, found := expectedBackends[definition.ID]
|
||||
if !found {
|
||||
t.Fatalf("unexpected published backend %q", definition.ID)
|
||||
}
|
||||
if definition.Endpoint != expected.endpoint ||
|
||||
definition.APIKeyEnv != expected.apiKeyEnv ||
|
||||
definition.ConcurrencyLimit != expected.concurrencyLimit ||
|
||||
definition.QueueCapacity != expected.queueCapacity ||
|
||||
!definition.QueueCapacitySet ||
|
||||
definition.ExtraParams != nil {
|
||||
t.Fatalf("unexpected published backend definition: %#v", definition)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidSources(t *testing.T) {
|
||||
for name, sources := range map[string][]Source{
|
||||
"none": nil,
|
||||
"blank name": {{Name: " ", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root}},
|
||||
"duplicate name": {
|
||||
{Name: "same", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
|
||||
{Name: "same", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
|
||||
},
|
||||
"nil filesystem": {{Name: "missing", ExpectedBackendID: "openrouter", Root: openrouter.Root}},
|
||||
"invalid root": {{Name: "invalid-root", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: "."}},
|
||||
"blank expected backend": {{Name: "blank-backend", ExpectedBackendID: " ", FS: openrouter.FS(), Root: openrouter.Root}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := Load(sources...); err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidLayouts(t *testing.T) {
|
||||
tests := map[string]func(fstest.MapFS){
|
||||
"unexpected file": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/notes.txt"] = &fstest.MapFile{Data: []byte("unexpected")}
|
||||
},
|
||||
"unexpected directory": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/unexpected"] = &fstest.MapFile{Mode: fs.ModeDir}
|
||||
},
|
||||
"nonregular manifest": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/backend.json"].Mode = fs.ModeSymlink
|
||||
},
|
||||
"wrong profile extension": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/extra.yaml"] = &fstest.MapFile{Data: []byte(validProfile("extra", "one"))}
|
||||
},
|
||||
"nonregular profile": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Mode = fs.ModeSymlink
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fsys := validCatalogFS("one")
|
||||
mutate(fsys)
|
||||
if _, err := Load(testSource("one", fsys)); err == nil {
|
||||
t.Fatal("expected invalid layout error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidManifests(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"malformed": `{`,
|
||||
"trailing value": validManifest("one", "TEST_API_KEY", "null") + `{}`,
|
||||
"missing fields": `{"schema_version":1,"id":"one"}`,
|
||||
"unsupported version": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"schema_version":1`, `"schema_version":2`, 1),
|
||||
"unknown field": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"extra_params":null`, `"extra_params":null,"unknown":true`, 1),
|
||||
"blank API key env": validManifest("one", " ", "null"),
|
||||
"invalid API key env": validManifest("one", "LEAK-MARKER", "null"),
|
||||
"invalid endpoint": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `https://one.example/v1`, `ftp://leak-marker.invalid/v1`, 1),
|
||||
"zero concurrency": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"concurrency_limit":2`, `"concurrency_limit":0`, 1),
|
||||
"non-object parameters": validManifest("one", "TEST_API_KEY", `[]`),
|
||||
"secret parameter": validManifest("one", "TEST_API_KEY", `{"nested":{"token":"leak-marker"}}`),
|
||||
}
|
||||
for name, manifest := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fsys := validCatalogFS("one")
|
||||
fsys["catalog/backend.json"].Data = []byte(manifest)
|
||||
_, err := Load(testSource("one", fsys))
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid manifest error")
|
||||
}
|
||||
if strings.Contains(strings.ToLower(err.Error()), "leak-marker") {
|
||||
t.Fatalf("catalog error exposed manifest content: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPreservesManifestJSONNumbers(t *testing.T) {
|
||||
fsys := validCatalogFS("one")
|
||||
fsys["catalog/backend.json"].Data = []byte(validManifest(
|
||||
"one",
|
||||
"TEST_API_KEY",
|
||||
`{"large":9007199254740993,"nested":[1.25]}`,
|
||||
))
|
||||
loaded, err := Load(testSource("one", fsys))
|
||||
if err != nil {
|
||||
t.Fatalf("load catalog: %v", err)
|
||||
}
|
||||
if got := loaded.Backends[0].ExtraParams["large"]; got != json.Number("9007199254740993") {
|
||||
t.Fatalf("large JSON integer = %#v, want preserved json.Number", got)
|
||||
}
|
||||
nested := loaded.Backends[0].ExtraParams["nested"].([]any)
|
||||
if nested[0] != json.Number("1.25") {
|
||||
t.Fatalf("nested JSON number = %#v, want preserved json.Number", nested[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidCatalogProfiles(t *testing.T) {
|
||||
tests := map[string]func(fstest.MapFS){
|
||||
"empty": func(fsys fstest.MapFS) {
|
||||
delete(fsys, "catalog/profiles/one-profile.yml")
|
||||
fsys["catalog/profiles"] = &fstest.MapFile{Mode: fs.ModeDir}
|
||||
},
|
||||
"malformed": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: [")
|
||||
},
|
||||
"raw API key": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key: leak-marker\n")
|
||||
},
|
||||
"endpoint field": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "endpoint: ''\n")
|
||||
},
|
||||
"API key environment field": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key_env: ''\n")
|
||||
},
|
||||
"owner mismatch": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "other"))
|
||||
},
|
||||
"missing base": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: leak-marker\n")
|
||||
},
|
||||
"cyclic base": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: second\n")
|
||||
fsys["catalog/profiles/second.yml"] = &fstest.MapFile{Data: []byte("id: second\nbase_profile: one-profile\n")}
|
||||
},
|
||||
"secret profile parameter": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "extra_params:\n nested:\n password: leak-marker\n")
|
||||
},
|
||||
"unknown field is redacted": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "leak_marker: leak-marker\n")
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fsys := validCatalogFS("one")
|
||||
mutate(fsys)
|
||||
_, err := Load(testSource("one", fsys))
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid profile error")
|
||||
}
|
||||
if strings.Contains(strings.ToLower(err.Error()), "leak-marker") {
|
||||
t.Fatalf("catalog error exposed profile content: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadReservesPromptkitProfilePrefixForCatalogs(t *testing.T) {
|
||||
tests := map[string]struct {
|
||||
profileID string
|
||||
wantError bool
|
||||
}{
|
||||
"maintained name": {
|
||||
profileID: "promptkit-openrouter-standard",
|
||||
wantError: true,
|
||||
},
|
||||
"other reserved name": {
|
||||
profileID: "promptkit-example",
|
||||
wantError: true,
|
||||
},
|
||||
"ordinary name": {
|
||||
profileID: "ordinary-profile",
|
||||
},
|
||||
}
|
||||
for name, test := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fsys := validCatalogFS("one")
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile(test.profileID, "one"))
|
||||
|
||||
_, err := Load(testSource("one", fsys))
|
||||
if (err != nil) != test.wantError {
|
||||
t.Fatalf("Load() error = %v, want error %t", err, test.wantError)
|
||||
}
|
||||
if err != nil && !strings.Contains(err.Error(), "catalog one: one-profile.yml") {
|
||||
t.Fatalf("Load() error = %v, want catalog source and path context", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsCrossCatalogConflicts(t *testing.T) {
|
||||
t.Run("duplicate backend", func(t *testing.T) {
|
||||
second := catalogFS("one", map[string]string{
|
||||
"catalog/profiles/second.yml": validProfile("second", "one"),
|
||||
}, "null")
|
||||
_, err := Load(
|
||||
Source{Name: "first", ExpectedBackendID: "one", FS: validCatalogFS("one"), Root: "catalog"},
|
||||
Source{Name: "second", ExpectedBackendID: "one", FS: second, Root: "catalog"},
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate backend error")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("duplicate profile", func(t *testing.T) {
|
||||
first := catalogFS("one", map[string]string{
|
||||
"catalog/profiles/shared.yml": validProfile("shared", "one"),
|
||||
}, "null")
|
||||
second := catalogFS("two", map[string]string{
|
||||
"catalog/profiles/shared.yml": validProfile("shared", "two"),
|
||||
}, "null")
|
||||
_, err := Load(testSource("one", first), testSource("two", second))
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate profile error")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("cross-catalog base", func(t *testing.T) {
|
||||
first := catalogFS("one", map[string]string{
|
||||
"catalog/profiles/base.yml": validProfile("base", "one"),
|
||||
}, "null")
|
||||
second := catalogFS("two", map[string]string{
|
||||
"catalog/profiles/child.yml": "id: child\nbase_profile: base\n",
|
||||
}, "null")
|
||||
_, err := Load(testSource("one", first), testSource("two", second))
|
||||
if err == nil {
|
||||
t.Fatal("expected cross-catalog base error")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestLoadReturnsDefensiveCatalogValues(t *testing.T) {
|
||||
fsys := catalogFS("one", map[string]string{
|
||||
"catalog/profiles/one-profile.yml": validProfile("one-profile", "one") + "extra_params:\n nested:\n value: profile\n",
|
||||
}, `{"nested":{"value":"backend"}}`)
|
||||
loaded, err := Load(testSource("one", fsys))
|
||||
if err != nil {
|
||||
t.Fatalf("load catalog: %v", err)
|
||||
}
|
||||
loaded.Backends[0].ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||
profileValue, err := loaded.Profiles.GetProfile(context.Background(), "one-profile")
|
||||
if err != nil {
|
||||
t.Fatalf("load profile: %v", err)
|
||||
}
|
||||
profileValue.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||
|
||||
again, err := Load(testSource("one", fsys))
|
||||
if err != nil {
|
||||
t.Fatalf("reload catalog: %v", err)
|
||||
}
|
||||
if got := again.Backends[0].ExtraParams["nested"].(map[string]any)["value"]; got != "backend" {
|
||||
t.Fatalf("backend mutation escaped returned set: %#v", got)
|
||||
}
|
||||
againProfile, err := loaded.Profiles.GetProfile(context.Background(), "one-profile")
|
||||
if err != nil {
|
||||
t.Fatalf("reload profile: %v", err)
|
||||
}
|
||||
if got := againProfile.ExtraParams["nested"].(map[string]any)["value"]; got != "profile" {
|
||||
t.Fatalf("profile mutation escaped returned value: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func testSource(id string, fsys fs.FS) Source {
|
||||
return Source{Name: id, ExpectedBackendID: id, FS: fsys, Root: "catalog"}
|
||||
}
|
||||
|
||||
func validCatalogFS(id string) fstest.MapFS {
|
||||
return catalogFS(id, map[string]string{
|
||||
"catalog/profiles/" + id + "-profile.yml": validProfile(id+"-profile", id),
|
||||
}, "null")
|
||||
}
|
||||
|
||||
func catalogFS(id string, profiles map[string]string, extraParams string) fstest.MapFS {
|
||||
fsys := fstest.MapFS{
|
||||
"catalog/backend.json": &fstest.MapFile{Data: []byte(validManifest(id, "TEST_API_KEY", extraParams))},
|
||||
}
|
||||
for name, content := range profiles {
|
||||
fsys[name] = &fstest.MapFile{Data: []byte(content)}
|
||||
}
|
||||
return fsys
|
||||
}
|
||||
|
||||
func validManifest(id, apiKeyEnv, extraParams string) string {
|
||||
return fmt.Sprintf(
|
||||
`{"schema_version":1,"id":%q,"endpoint":%q,"api_key_env":%q,"concurrency_limit":2,"queue_capacity":3,"extra_params":%s}`,
|
||||
id,
|
||||
"https://"+id+".example/v1",
|
||||
apiKeyEnv,
|
||||
extraParams,
|
||||
)
|
||||
}
|
||||
|
||||
func validProfile(id, backendID string) string {
|
||||
return fmt.Sprintf("id: %s\nbackend: %s\nmodel: test-model\n", id, backendID)
|
||||
}
|
||||
|
||||
var _ fs.FS = openrouter.FS()
|
||||
@@ -60,15 +60,18 @@ type CacheControl struct {
|
||||
|
||||
// RunRequest represents a request to generate a single artifact.
|
||||
type RunRequest struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
SessionID string
|
||||
APIKey string `json:"-" yaml:"-"`
|
||||
Inputs map[string]ArtifactRef
|
||||
Vars map[string]string
|
||||
Execution *ExecutionTargetOverride
|
||||
Validation *OutputContract
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
SessionID string
|
||||
APIKey string `json:"-" yaml:"-"`
|
||||
Inputs map[string]ArtifactRef
|
||||
Vars map[string]string
|
||||
Execution *ExecutionTargetOverride
|
||||
Validation *OutputContract
|
||||
AppendedMessages []RenderedMessage
|
||||
TransientRetries *int
|
||||
Observer ExecutionObserver `json:"-"`
|
||||
}
|
||||
|
||||
// RunResult represents the complete result of a prompt execution run.
|
||||
@@ -281,6 +284,8 @@ type GenerateRequest struct {
|
||||
Target ExecutionTarget
|
||||
TargetPresence ExecutionTargetPresence
|
||||
StructuredOutput *StructuredOutputSpec
|
||||
TransientRetries uint64
|
||||
Generation uint64
|
||||
}
|
||||
|
||||
// StructuredOutputType indicates which provider-level output mode is requested.
|
||||
@@ -305,8 +310,18 @@ type StructuredOutputJSONSpec struct {
|
||||
|
||||
// GenerateResponse is the response received from the LLM client.
|
||||
type GenerateResponse struct {
|
||||
Content string
|
||||
Usage TokenUsage
|
||||
Content string
|
||||
Usage TokenUsage
|
||||
UsagePresence *TokenUsagePresence
|
||||
}
|
||||
|
||||
// TokenUsagePresence records which token fields a provider reported.
|
||||
type TokenUsagePresence struct {
|
||||
PromptTokens bool
|
||||
CompletionTokens bool
|
||||
TotalTokens bool
|
||||
CachedTokens bool
|
||||
CacheWriteTokens bool
|
||||
}
|
||||
|
||||
// TokenUsage tracks token consumption.
|
||||
|
||||
82
internal/domain/message.go
Normal file
82
internal/domain/message.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
RoleDeveloper = "developer"
|
||||
RoleSystem = "system"
|
||||
RoleUser = "user"
|
||||
RoleAssistant = "assistant"
|
||||
)
|
||||
|
||||
// NormalizeMessageRole validates and canonicalizes a provider-bound chat role.
|
||||
func NormalizeMessageRole(role string) (string, error) {
|
||||
if !utf8.ValidString(role) {
|
||||
return "", errors.New("message role must be valid UTF-8")
|
||||
}
|
||||
|
||||
normalized := strings.ToLower(strings.TrimSpace(role))
|
||||
switch normalized {
|
||||
case RoleDeveloper, RoleSystem, RoleUser, RoleAssistant:
|
||||
return normalized, nil
|
||||
default:
|
||||
return "", errors.New("message role must be developer, system, user, or assistant")
|
||||
}
|
||||
}
|
||||
|
||||
// NormalizeCacheControl validates, canonicalizes, and copies cache metadata.
|
||||
func NormalizeCacheControl(control *CacheControl) (*CacheControl, error) {
|
||||
if control == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if !utf8.ValidString(string(control.Type)) {
|
||||
return nil, errors.New("cache control type must be valid UTF-8")
|
||||
}
|
||||
if !utf8.ValidString(control.TTL) {
|
||||
return nil, errors.New("cache control ttl must be valid UTF-8")
|
||||
}
|
||||
|
||||
cacheType := strings.TrimSpace(string(control.Type))
|
||||
if cacheType == "" {
|
||||
return nil, errors.New("cache control type is required")
|
||||
}
|
||||
if CacheControlType(cacheType) != CacheControlEphemeral {
|
||||
return nil, errors.New("unsupported type")
|
||||
}
|
||||
|
||||
ttl := strings.TrimSpace(control.TTL)
|
||||
if ttl != "" && ttl != "1h" {
|
||||
return nil, errors.New("unsupported ttl")
|
||||
}
|
||||
|
||||
return &CacheControl{Type: CacheControlType(cacheType), TTL: ttl}, nil
|
||||
}
|
||||
|
||||
// CloneRenderedMessages returns a deep copy of rendered messages.
|
||||
func CloneRenderedMessages(messages []RenderedMessage) []RenderedMessage {
|
||||
cloned := make([]RenderedMessage, len(messages))
|
||||
copyRenderedMessages(cloned, messages)
|
||||
return cloned
|
||||
}
|
||||
|
||||
// ConcatRenderedMessages returns an independently owned concatenation of messages.
|
||||
func ConcatRenderedMessages(prefix, suffix []RenderedMessage) []RenderedMessage {
|
||||
messages := make([]RenderedMessage, len(prefix)+len(suffix))
|
||||
copyRenderedMessages(messages, prefix)
|
||||
copyRenderedMessages(messages[len(prefix):], suffix)
|
||||
return messages
|
||||
}
|
||||
|
||||
func copyRenderedMessages(destination, source []RenderedMessage) {
|
||||
for index, message := range source {
|
||||
destination[index] = message
|
||||
if message.CacheControl != nil {
|
||||
cacheControl := *message.CacheControl
|
||||
destination[index].CacheControl = &cacheControl
|
||||
}
|
||||
}
|
||||
}
|
||||
121
internal/domain/message_test.go
Normal file
121
internal/domain/message_test.go
Normal file
@@ -0,0 +1,121 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNormalizeMessageRole(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "developer", input: RoleDeveloper, want: RoleDeveloper},
|
||||
{name: "system", input: RoleSystem, want: RoleSystem},
|
||||
{name: "user", input: RoleUser, want: RoleUser},
|
||||
{name: "assistant", input: RoleAssistant, want: RoleAssistant},
|
||||
{name: "surrounding whitespace and mixed case", input: " \u2003UsEr\u2003 ", want: RoleUser},
|
||||
{name: "blank", input: " \t\n ", wantErr: true},
|
||||
{name: "tool", input: "tool", wantErr: true},
|
||||
{name: "function", input: "function", wantErr: true},
|
||||
{name: "custom", input: "custom-role", wantErr: true},
|
||||
{name: "invalid UTF-8", input: string([]byte{0xff}), wantErr: true},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, err := NormalizeMessageRole(test.input)
|
||||
if test.wantErr {
|
||||
if err == nil {
|
||||
t.Fatal("expected an error")
|
||||
}
|
||||
if strings.Contains(err.Error(), test.input) {
|
||||
t.Fatalf("error exposed the input: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("NormalizeMessageRole() error = %v", err)
|
||||
}
|
||||
if got != test.want {
|
||||
t.Fatalf("NormalizeMessageRole() = %q, want %q", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeCacheControl(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input *CacheControl
|
||||
want *CacheControl
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "nil", input: nil, want: nil},
|
||||
{
|
||||
name: "canonical values",
|
||||
input: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"},
|
||||
want: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"},
|
||||
},
|
||||
{
|
||||
name: "trims values",
|
||||
input: &CacheControl{Type: " ephemeral ", TTL: " 1h\t"},
|
||||
want: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"},
|
||||
},
|
||||
{name: "empty type", input: &CacheControl{}, wantErr: true},
|
||||
{name: "unsupported type", input: &CacheControl{Type: "persistent"}, wantErr: true},
|
||||
{name: "unsupported ttl", input: &CacheControl{Type: CacheControlEphemeral, TTL: "5m"}, wantErr: true},
|
||||
{name: "invalid type UTF-8", input: &CacheControl{Type: CacheControlType(string([]byte{0xff}))}, wantErr: true},
|
||||
{name: "invalid ttl UTF-8", input: &CacheControl{Type: CacheControlEphemeral, TTL: string([]byte{0xff})}, wantErr: true},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, err := NormalizeCacheControl(test.input)
|
||||
if test.wantErr {
|
||||
if err == nil {
|
||||
t.Fatal("expected an error")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("NormalizeCacheControl() error = %v", err)
|
||||
}
|
||||
if got == nil || test.want == nil {
|
||||
if got != test.want {
|
||||
t.Fatalf("NormalizeCacheControl() = %#v, want %#v", got, test.want)
|
||||
}
|
||||
return
|
||||
}
|
||||
if *got != *test.want {
|
||||
t.Fatalf("NormalizeCacheControl() = %#v, want %#v", got, test.want)
|
||||
}
|
||||
if got == test.input {
|
||||
t.Fatal("normalized cache control aliases its input")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderedMessageCloning(t *testing.T) {
|
||||
prefix := []RenderedMessage{{Role: RoleSystem, Content: "prefix", CacheControl: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"}}}
|
||||
suffix := []RenderedMessage{{Role: RoleUser, Content: " suffix "}, {Role: RoleAssistant, Content: "", CacheControl: &CacheControl{Type: CacheControlEphemeral}}}
|
||||
|
||||
cloned := CloneRenderedMessages(prefix)
|
||||
combined := ConcatRenderedMessages(prefix, suffix)
|
||||
if len(combined) != 3 || combined[0].Content != "prefix" || combined[1].Content != " suffix " || combined[2].Content != "" {
|
||||
t.Fatalf("unexpected combined messages: %#v", combined)
|
||||
}
|
||||
if cloned[0].CacheControl == prefix[0].CacheControl || combined[0].CacheControl == prefix[0].CacheControl || combined[2].CacheControl == suffix[1].CacheControl {
|
||||
t.Fatal("cloned cache controls alias their inputs")
|
||||
}
|
||||
|
||||
prefix[0].Content = "changed"
|
||||
prefix[0].CacheControl.TTL = ""
|
||||
suffix[1].CacheControl.Type = "changed"
|
||||
if cloned[0].Content != "prefix" || cloned[0].CacheControl.TTL != "1h" || combined[0].Content != "prefix" || combined[0].CacheControl.TTL != "1h" || combined[2].CacheControl.Type != CacheControlEphemeral {
|
||||
t.Fatalf("cloned messages changed with their inputs: cloned=%#v combined=%#v", cloned, combined)
|
||||
}
|
||||
}
|
||||
90
internal/domain/observation.go
Normal file
90
internal/domain/observation.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package domain
|
||||
|
||||
import "time"
|
||||
|
||||
// ExecutionObserver receives one immutable metadata event synchronously.
|
||||
type ExecutionObserver func(ExecutionEvent)
|
||||
|
||||
type ExecutionEventKind string
|
||||
|
||||
const (
|
||||
ExecutionStarted ExecutionEventKind = "started"
|
||||
ExecutionAttempt ExecutionEventKind = "attempt"
|
||||
ExecutionRetryDelay ExecutionEventKind = "retry_delay"
|
||||
ExecutionFinished ExecutionEventKind = "finished"
|
||||
)
|
||||
|
||||
type ExecutionOutcome string
|
||||
|
||||
const (
|
||||
ExecutionOutcomeSuccess ExecutionOutcome = "success"
|
||||
ExecutionOutcomeInvalidOutput ExecutionOutcome = "invalid_output"
|
||||
ExecutionOutcomeError ExecutionOutcome = "error"
|
||||
)
|
||||
|
||||
type ExecutionFailureKind string
|
||||
|
||||
const (
|
||||
ExecutionFailureNone ExecutionFailureKind = ""
|
||||
ExecutionFailureCanceled ExecutionFailureKind = "canceled"
|
||||
ExecutionFailureDeadline ExecutionFailureKind = "deadline"
|
||||
ExecutionFailureTransport ExecutionFailureKind = "transport"
|
||||
ExecutionFailureProviderStatus ExecutionFailureKind = "provider_status"
|
||||
ExecutionFailureMalformedResponse ExecutionFailureKind = "malformed_response"
|
||||
ExecutionFailureCapacity ExecutionFailureKind = "capacity"
|
||||
ExecutionFailureInvalidConfig ExecutionFailureKind = "invalid_config"
|
||||
ExecutionFailureInvalidRequest ExecutionFailureKind = "invalid_request"
|
||||
ExecutionFailurePromptNotFound ExecutionFailureKind = "prompt_not_found"
|
||||
ExecutionFailureProfileNotFound ExecutionFailureKind = "profile_not_found"
|
||||
ExecutionFailurePromptLoad ExecutionFailureKind = "prompt_load"
|
||||
ExecutionFailureProfileLoad ExecutionFailureKind = "profile_load"
|
||||
ExecutionFailureArtifactLoad ExecutionFailureKind = "artifact_load"
|
||||
ExecutionFailurePromptRender ExecutionFailureKind = "prompt_render"
|
||||
ExecutionFailureValidation ExecutionFailureKind = "validation"
|
||||
ExecutionFailureGeneration ExecutionFailureKind = "generation"
|
||||
ExecutionFailureOther ExecutionFailureKind = "other"
|
||||
)
|
||||
|
||||
// ObservedTokenCount distinguishes omitted usage from reported zero.
|
||||
type ObservedTokenCount struct {
|
||||
Value int
|
||||
Known bool
|
||||
Complete bool
|
||||
}
|
||||
|
||||
// ObservedTokenUsage reports presence-aware token counts.
|
||||
type ObservedTokenUsage struct {
|
||||
PromptTokens ObservedTokenCount
|
||||
CompletionTokens ObservedTokenCount
|
||||
TotalTokens ObservedTokenCount
|
||||
CachedTokens ObservedTokenCount
|
||||
CacheWriteTokens ObservedTokenCount
|
||||
}
|
||||
|
||||
// ExecutionEvent contains metadata only; it carries no mutable containers or
|
||||
// error values. Nonterminal events leave aggregate counters at zero.
|
||||
type ExecutionEvent struct {
|
||||
Kind ExecutionEventKind
|
||||
RunID string
|
||||
Time time.Time
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
BackendID string
|
||||
Model string
|
||||
Generation uint64
|
||||
Attempt uint64
|
||||
Dispatched bool
|
||||
CapacityWait time.Duration
|
||||
GenerationDuration time.Duration
|
||||
RetryDelay time.Duration
|
||||
Duration time.Duration
|
||||
Outcome ExecutionOutcome
|
||||
Failure ExecutionFailureKind
|
||||
StatusCode int
|
||||
RetriesExhausted bool
|
||||
Usage ObservedTokenUsage
|
||||
Attempts uint64
|
||||
TransientRetries uint64
|
||||
RepairCalls uint64
|
||||
}
|
||||
15
internal/domain/retries.go
Normal file
15
internal/domain/retries.go
Normal file
@@ -0,0 +1,15 @@
|
||||
package domain
|
||||
|
||||
import "fmt"
|
||||
|
||||
// ResolveTransientRetries selects a nonnegative retry budget. A nil selection
|
||||
// inherits the supplied default.
|
||||
func ResolveTransientRetries(defaultBudget uint64, selection *int) (uint64, error) {
|
||||
if selection == nil {
|
||||
return defaultBudget, nil
|
||||
}
|
||||
if *selection < 0 {
|
||||
return 0, fmt.Errorf("transient retries must not be negative")
|
||||
}
|
||||
return uint64(*selection), nil
|
||||
}
|
||||
285
internal/engine/engine.go
Normal file
285
internal/engine/engine.go
Normal file
@@ -0,0 +1,285 @@
|
||||
// Package engine assembles Promptkit's internal execution components.
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2"
|
||||
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/catalog"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/generation"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||
)
|
||||
|
||||
// Config carries construction settings selected by the public facade.
|
||||
type Config struct {
|
||||
PromptDir string
|
||||
ProfileDir string
|
||||
SchemaDir string
|
||||
Timeout time.Duration
|
||||
HTTPClient *http.Client
|
||||
TransientRetries *int
|
||||
}
|
||||
|
||||
// Options collects construction choices from the public facade.
|
||||
type Options struct {
|
||||
llmClient llm.Client
|
||||
artifactReader artifact.Reader
|
||||
promptDefinitions promptdef.Repository
|
||||
profiles profile.Repository
|
||||
fallbackProfiles profile.Repository
|
||||
memoryProfiles profile.Repository
|
||||
backends []domain.Backend
|
||||
validator validate.Validator
|
||||
promptSource bool
|
||||
profileSource bool
|
||||
fallbackProfileSource bool
|
||||
memorySource bool
|
||||
validatorSource bool
|
||||
artifactSource bool
|
||||
}
|
||||
|
||||
// SetLLMClient selects the client collaborator for one engine.
|
||||
func (o *Options) SetLLMClient(client llm.Client) { o.llmClient = client }
|
||||
|
||||
// SetArtifactReader selects the artifact collaborator for one engine.
|
||||
func (o *Options) SetArtifactReader(reader artifact.Reader) {
|
||||
o.artifactReader = reader
|
||||
o.artifactSource = true
|
||||
}
|
||||
|
||||
// SetPromptFS selects a prompt source rooted in fsys.
|
||||
func (o *Options) SetPromptFS(fsys fs.FS, root string) error {
|
||||
if err := validateSourceFS(fsys, root); err != nil {
|
||||
return err
|
||||
}
|
||||
o.promptDefinitions = promptdef.NewFSRepository(fsys, root)
|
||||
o.promptSource = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetPromptFile selects a single prompt file.
|
||||
func (o *Options) SetPromptFile(path string) error {
|
||||
fsys, root, directory, err := fileSource(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
o.promptDefinitions = promptdef.NewFileRepository(fsys, root, directory)
|
||||
o.promptSource = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetProfileFS selects an ordinary profile source rooted in fsys.
|
||||
func (o *Options) SetProfileFS(fsys fs.FS, root string) error {
|
||||
if err := validateSourceFS(fsys, root); err != nil {
|
||||
return err
|
||||
}
|
||||
o.profiles = profile.NewFSRepository(fsys, root)
|
||||
o.profileSource = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetProfileFile selects a single ordinary profile file.
|
||||
func (o *Options) SetProfileFile(path string) error {
|
||||
fsys, root, _, err := fileSource(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
o.profiles = profile.NewFSRepository(fsys, root)
|
||||
o.profileSource = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetFallbackProfileFS selects a fallback profile source rooted in fsys.
|
||||
func (o *Options) SetFallbackProfileFS(fsys fs.FS, root string) error {
|
||||
if err := validateSourceFS(fsys, root); err != nil {
|
||||
return err
|
||||
}
|
||||
o.fallbackProfiles = profile.NewFSRepository(fsys, root)
|
||||
o.fallbackProfileSource = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetMemoryProfiles validates and stores application-owned profile definitions.
|
||||
func (o *Options) SetMemoryProfiles(definitions []domain.ExecutionProfile) error {
|
||||
repository, err := profile.NewMemoryRepository(definitions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
o.memoryProfiles = repository
|
||||
o.memorySource = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddBackend appends one backend registration.
|
||||
func (o *Options) AddBackend(backend domain.Backend) { o.backends = append(o.backends, backend) }
|
||||
|
||||
// SetSchemaFS selects a schema source rooted in fsys.
|
||||
func (o *Options) SetSchemaFS(fsys fs.FS, root string) error {
|
||||
if err := validateSourceFS(fsys, root); err != nil {
|
||||
return err
|
||||
}
|
||||
o.validator = validate.NewFSValidator(fsys, root)
|
||||
o.validatorSource = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetSchemaFile selects a single schema file.
|
||||
func (o *Options) SetSchemaFile(path string) error {
|
||||
fsys, root, _, err := fileSource(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
o.validator = validate.NewFSValidator(fsys, root)
|
||||
o.validatorSource = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// Engine forwards domain operations to the single assembled use-case runner.
|
||||
type Engine struct {
|
||||
runner *usecase.Runner
|
||||
}
|
||||
|
||||
// New assembles an engine from facade-selected configuration and collaborators.
|
||||
func New(cfg Config, options Options) (*Engine, error) {
|
||||
defaultRetries, err := domain.ResolveTransientRetries(2, cfg.TransientRetries)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
promptDefinitions := options.promptDefinitions
|
||||
if !options.promptSource {
|
||||
if strings.TrimSpace(cfg.PromptDir) == "" {
|
||||
return nil, fmt.Errorf("prompt directory is required")
|
||||
}
|
||||
promptDefinitions = promptdef.NewFilesystemRepository(cfg.PromptDir)
|
||||
}
|
||||
|
||||
maintained, err := catalog.Load(
|
||||
catalog.Source{Name: "OpenRouter", ExpectedBackendID: backend.OpenRouterID, FS: openrouter.FS(), Root: openrouter.Root},
|
||||
catalog.Source{Name: "Rakestrawhome", ExpectedBackendID: backend.RakestrawHomeID, FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load maintained catalogs: %w", err)
|
||||
}
|
||||
|
||||
maintainedProfiles, err := newMaintainedProfileRepository(maintained.Profiles, maintained.Backends)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to construct Promptkit-maintained profiles: %w", err)
|
||||
}
|
||||
|
||||
profiles := newProfileRepository(cfg.ProfileDir, options, maintainedProfiles)
|
||||
registry, err := backend.NewRegistry(maintained.Backends, options.backends)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to construct backend registry: %w", err)
|
||||
}
|
||||
manager, err := capacity.NewManager(registry.CapacityPolicies())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to construct backend capacity manager: %w", err)
|
||||
}
|
||||
|
||||
validator := options.validator
|
||||
if !options.validatorSource {
|
||||
schemaDir := cfg.SchemaDir
|
||||
if strings.TrimSpace(schemaDir) == "" {
|
||||
schemaDir = defaults.SchemaDirDefault
|
||||
}
|
||||
validator = validate.NewStandardValidator(schemaDir)
|
||||
}
|
||||
var client llm.Client
|
||||
if options.llmClient == nil {
|
||||
builtIn, newErr := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient})
|
||||
if newErr != nil {
|
||||
return nil, newErr
|
||||
}
|
||||
client = generation.NewBuiltIn(manager, builtIn)
|
||||
} else {
|
||||
client = generation.NewInjected(manager, options.llmClient)
|
||||
}
|
||||
reader := options.artifactReader
|
||||
if !options.artifactSource {
|
||||
reader = artifact.NewCompositeReader()
|
||||
}
|
||||
|
||||
return &Engine{runner: usecase.NewRunnerWithRepairer(promptDefinitions, profiles, registry, reader, prompt.NewGoRenderer(), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries)}, nil
|
||||
}
|
||||
|
||||
func newProfileRepository(profileDir string, options Options, maintained profile.Repository) profile.Repository {
|
||||
repository := maintained
|
||||
if options.fallbackProfileSource {
|
||||
repository = profile.NewOverlayRepository(options.fallbackProfiles, repository)
|
||||
}
|
||||
if options.profileSource {
|
||||
repository = profile.NewOverlayRepository(options.profiles, repository)
|
||||
} else if strings.TrimSpace(profileDir) != "" {
|
||||
repository = profile.NewOverlayRepository(profile.NewFilesystemRepository(profileDir), repository)
|
||||
}
|
||||
if options.memorySource {
|
||||
repository = profile.NewOverlayRepository(options.memoryProfiles, repository)
|
||||
}
|
||||
return profile.NewResolvingRepository(repository)
|
||||
}
|
||||
|
||||
func validateSourceFS(fsys fs.FS, root string) error {
|
||||
if fsys == nil {
|
||||
return fmt.Errorf("source filesystem is nil")
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return fmt.Errorf("source root is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fileSource(name string) (fs.FS, string, string, error) {
|
||||
if strings.TrimSpace(name) == "" {
|
||||
return nil, "", "", fmt.Errorf("source path is required")
|
||||
}
|
||||
directory := filepath.Dir(name)
|
||||
base := filepath.Base(name)
|
||||
if base == "." || base == string(filepath.Separator) {
|
||||
return nil, "", "", fmt.Errorf("source path must name a file")
|
||||
}
|
||||
info, err := os.Stat(name)
|
||||
if err != nil {
|
||||
return nil, "", "", fmt.Errorf("failed to access source file %q: %w", name, err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil, "", "", fmt.Errorf("source path %q must be a file", name)
|
||||
}
|
||||
return os.DirFS(directory), filepath.ToSlash(base), directory, nil
|
||||
}
|
||||
|
||||
func (e *Engine) InspectPrompt(ctx context.Context, id, version string) (*domain.PromptInspection, error) {
|
||||
return e.runner.InspectPrompt(ctx, id, version)
|
||||
}
|
||||
func (e *Engine) InspectProfile(ctx context.Context, id string) (*domain.ProfileInspection, error) {
|
||||
return e.runner.InspectProfile(ctx, id)
|
||||
}
|
||||
func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) {
|
||||
return e.runner.Prepare(ctx, request)
|
||||
}
|
||||
func (e *Engine) PrepareExecution(ctx context.Context, request domain.RunRequest) (*usecase.PreparedExecution, error) {
|
||||
return e.runner.PrepareExecution(ctx, request)
|
||||
}
|
||||
func (e *Engine) Run(ctx context.Context, request domain.RunRequest) (*domain.RunResult, error) {
|
||||
return e.runner.Run(ctx, request)
|
||||
}
|
||||
func (e *Engine) RunPrepared(ctx context.Context, prepared *usecase.PreparedExecution) (*domain.RunResult, error) {
|
||||
return e.runner.RunPrepared(ctx, prepared)
|
||||
}
|
||||
50
internal/engine/maintained_profiles.go
Normal file
50
internal/engine/maintained_profiles.go
Normal file
@@ -0,0 +1,50 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
|
||||
func maintainedProfileDefinitions() []domain.ExecutionProfile {
|
||||
return []domain.ExecutionProfile{
|
||||
{
|
||||
ID: "promptkit-openrouter-standard",
|
||||
BaseProfileID: "gemini-flash-latest",
|
||||
},
|
||||
{
|
||||
ID: "promptkit-openrouter-auto",
|
||||
BackendID: backend.OpenRouterID,
|
||||
Model: "openrouter/auto",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newMaintainedProfileRepository(catalogProfiles profile.Repository, maintainedBackends []domain.Backend) (profile.Repository, error) {
|
||||
definitions := maintainedProfileDefinitions()
|
||||
owned, err := profile.NewMemoryRepository(definitions)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create Promptkit-maintained profiles: %w", err)
|
||||
}
|
||||
|
||||
combined := profile.NewOverlayRepository(owned, catalogProfiles)
|
||||
resolved := profile.NewResolvingRepository(combined)
|
||||
registry, err := backend.NewRegistry(maintainedBackends, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("construct maintained backend registry: %w", err)
|
||||
}
|
||||
for _, definition := range definitions {
|
||||
effective, err := resolved.GetProfile(context.Background(), definition.ID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve Promptkit-maintained profile %q: %w", definition.ID, err)
|
||||
}
|
||||
if _, err := registry.GetBackend(effective.BackendID); err != nil {
|
||||
return nil, fmt.Errorf("Promptkit-maintained profile %q selects unavailable backend %q: %w", definition.ID, effective.BackendID, err)
|
||||
}
|
||||
}
|
||||
|
||||
return combined, nil
|
||||
}
|
||||
78
internal/engine/maintained_profiles_test.go
Normal file
78
internal/engine/maintained_profiles_test.go
Normal file
@@ -0,0 +1,78 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
|
||||
func TestNewMaintainedProfileRepository(t *testing.T) {
|
||||
newCatalogProfiles := func(t *testing.T, definitions []domain.ExecutionProfile) profile.Repository {
|
||||
t.Helper()
|
||||
repository, err := profile.NewMemoryRepository(definitions)
|
||||
if err != nil {
|
||||
t.Fatalf("create catalog profiles: %v", err)
|
||||
}
|
||||
return repository
|
||||
}
|
||||
newBackend := func(id string) domain.Backend {
|
||||
return domain.Backend{ID: id, Endpoint: "https://" + id + ".example/v1", APIKeyEnv: "TEST_API_KEY"}
|
||||
}
|
||||
|
||||
t.Run("resolves inherited and direct definitions", func(t *testing.T) {
|
||||
repository, err := newMaintainedProfileRepository(
|
||||
newCatalogProfiles(t, []domain.ExecutionProfile{{
|
||||
ID: "gemini-flash-latest",
|
||||
BackendID: backend.OpenRouterID,
|
||||
Model: "catalog-selected-model",
|
||||
}}),
|
||||
[]domain.Backend{newBackend(backend.OpenRouterID)},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("create maintained profiles: %v", err)
|
||||
}
|
||||
|
||||
resolved := profile.NewResolvingRepository(repository)
|
||||
standard, err := resolved.GetProfile(context.Background(), "promptkit-openrouter-standard")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve inherited profile: %v", err)
|
||||
}
|
||||
if standard.BackendID != backend.OpenRouterID || standard.Model != "catalog-selected-model" {
|
||||
t.Fatalf("unexpected inherited profile: %#v", standard)
|
||||
}
|
||||
|
||||
auto, err := resolved.GetProfile(context.Background(), "promptkit-openrouter-auto")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve direct profile: %v", err)
|
||||
}
|
||||
definitions := maintainedProfileDefinitions()
|
||||
if auto.BackendID != definitions[1].BackendID || auto.Model != definitions[1].Model {
|
||||
t.Fatalf("unexpected direct profile: %#v", auto)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects a missing inherited base", func(t *testing.T) {
|
||||
if _, err := newMaintainedProfileRepository(
|
||||
newCatalogProfiles(t, nil),
|
||||
[]domain.Backend{newBackend(backend.OpenRouterID)},
|
||||
); err == nil {
|
||||
t.Fatal("expected missing base error")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects an unavailable effective backend", func(t *testing.T) {
|
||||
if _, err := newMaintainedProfileRepository(
|
||||
newCatalogProfiles(t, []domain.ExecutionProfile{{
|
||||
ID: "gemini-flash-latest",
|
||||
BackendID: "other",
|
||||
Model: "catalog-selected-model",
|
||||
}}),
|
||||
[]domain.Backend{newBackend("other")},
|
||||
); err == nil {
|
||||
t.Fatal("expected unavailable backend error")
|
||||
}
|
||||
})
|
||||
}
|
||||
294
internal/generation/coordinator.go
Normal file
294
internal/generation/coordinator.go
Normal file
@@ -0,0 +1,294 @@
|
||||
// Package generation coordinates model attempts, backend permits, and bounded
|
||||
// built-in HTTP retries for one generation request.
|
||||
package generation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/observation"
|
||||
)
|
||||
|
||||
const (
|
||||
initialBackoff = 500 * time.Millisecond
|
||||
maximumBackoff = 8 * time.Second
|
||||
)
|
||||
|
||||
// Coordinator implements the internal model client using the existing backend
|
||||
// capacity manager. Exactly one of builtIn and injected is selected.
|
||||
type Coordinator struct {
|
||||
manager *capacity.Manager
|
||||
builtIn *llm.OpenAICompatibleClient
|
||||
injected llm.Client
|
||||
now func() time.Time
|
||||
jitter func(time.Duration) time.Duration
|
||||
wait func(context.Context, time.Duration) error
|
||||
}
|
||||
|
||||
// NewBuiltIn selects the built-in HTTP retry path.
|
||||
func NewBuiltIn(manager *capacity.Manager, client *llm.OpenAICompatibleClient) *Coordinator {
|
||||
return &Coordinator{manager: manager, builtIn: client, now: time.Now, jitter: randomJitter, wait: waitContext}
|
||||
}
|
||||
|
||||
// NewInjected selects one opaque client dispatch with no automatic retry.
|
||||
func NewInjected(manager *capacity.Manager, client llm.Client) *Coordinator {
|
||||
return &Coordinator{manager: manager, injected: client, now: time.Now, jitter: randomJitter, wait: waitContext}
|
||||
}
|
||||
|
||||
// ExhaustedError retains the final eligible status failure and the number of
|
||||
// attempts actually dispatched.
|
||||
type ExhaustedError struct {
|
||||
attempts uint64
|
||||
cause error
|
||||
}
|
||||
|
||||
func (e *ExhaustedError) Error() string {
|
||||
if e == nil {
|
||||
return "transient model retries exhausted"
|
||||
}
|
||||
return fmt.Sprintf("transient model retries exhausted after %d attempts", e.attempts)
|
||||
}
|
||||
|
||||
func (e *ExhaustedError) Unwrap() error {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return e.cause
|
||||
}
|
||||
|
||||
func (e *ExhaustedError) Attempts() uint64 {
|
||||
if e == nil {
|
||||
return 0
|
||||
}
|
||||
return e.attempts
|
||||
}
|
||||
|
||||
// Generate executes either one opaque client call or a frozen built-in HTTP
|
||||
// sequence with one permit per actual dispatch.
|
||||
func (c *Coordinator) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
if c.injected != nil {
|
||||
return c.generateInjected(ctx, req)
|
||||
}
|
||||
prepared, err := c.builtIn.PrepareGeneration(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var attempts, retriesUsed uint64
|
||||
recorder := observation.FromContext(ctx)
|
||||
generation := req.Generation
|
||||
if generation == 0 {
|
||||
generation = 1
|
||||
}
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ordinal := attempts + 1
|
||||
outcome, dispatched, capacityWait, generationDuration := c.execute(ctx, req.Target.BackendID, prepared)
|
||||
if dispatched {
|
||||
attempts++
|
||||
}
|
||||
if recorder != nil {
|
||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
|
||||
Dispatched: dispatched, CapacityWait: capacityWait, GenerationDuration: generationDuration,
|
||||
Outcome: attemptOutcome(outcome.Err), Failure: attemptFailure(outcome.Err),
|
||||
StatusCode: outcome.StatusCode, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
|
||||
}
|
||||
if outcome.Err == nil {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return outcome.Response, nil
|
||||
}
|
||||
if !dispatched || !eligibleStatus(outcome) {
|
||||
return nil, outcome.Err
|
||||
}
|
||||
if retriesUsed == req.TransientRetries {
|
||||
if retriesUsed == 0 {
|
||||
return nil, outcome.Err
|
||||
}
|
||||
return nil, &ExhaustedError{attempts: attempts, cause: outcome.Err}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
retryOrdinal := retriesUsed + 1
|
||||
delay := retryDelay(c.backoff(retryOrdinal), outcome.RetryNotBefore, c.now())
|
||||
waitStart := c.now()
|
||||
waitErr := c.wait(ctx, delay)
|
||||
if recorder != nil {
|
||||
recorder.Delay(observation.DelayData{Generation: generation, Attempt: attempts,
|
||||
Elapsed: elapsed(waitStart, c.now()), Failure: attemptFailure(waitErr)})
|
||||
}
|
||||
if waitErr != nil {
|
||||
return nil, waitErr
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
retriesUsed++
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
recorder := observation.FromContext(ctx)
|
||||
generation := req.Generation
|
||||
if generation == 0 {
|
||||
generation = 1
|
||||
}
|
||||
start := c.now()
|
||||
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
|
||||
capacityWait := c.capacityWait(req.Target.BackendID, start)
|
||||
if err != nil {
|
||||
if recorder != nil {
|
||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: attemptFailure(err)})
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer release()
|
||||
if err := ctx.Err(); err != nil {
|
||||
release()
|
||||
if recorder != nil {
|
||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: attemptFailure(err)})
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
dispatchStart := c.now()
|
||||
response, err := c.injected.Generate(ctx, req)
|
||||
duration := elapsed(dispatchStart, c.now())
|
||||
if err == nil && response == nil {
|
||||
err = llm.ErrMalformedResponse
|
||||
}
|
||||
release()
|
||||
if recorder != nil {
|
||||
data := observation.AttemptData{Generation: generation, Attempt: 1, Dispatched: true, CapacityWait: capacityWait, GenerationDuration: duration,
|
||||
Outcome: attemptOutcome(err), Failure: attemptFailure(err)}
|
||||
if err == nil {
|
||||
data.Usage = response.Usage
|
||||
data.UsagePresence = response.UsagePresence
|
||||
}
|
||||
recorder.Attempt(data)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration) {
|
||||
start := c.now()
|
||||
release, err := c.manager.Acquire(ctx, backendID)
|
||||
capacityWait = c.capacityWait(backendID, start)
|
||||
if err != nil {
|
||||
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0
|
||||
}
|
||||
defer release()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0
|
||||
}
|
||||
dispatchStart := c.now()
|
||||
outcome = prepared.Execute(ctx)
|
||||
return outcome, true, capacityWait, elapsed(dispatchStart, c.now())
|
||||
}
|
||||
|
||||
func (c *Coordinator) capacityWait(backendID string, start time.Time) time.Duration {
|
||||
if !c.manager.Limited(backendID) {
|
||||
return 0
|
||||
}
|
||||
return elapsed(start, c.now())
|
||||
}
|
||||
|
||||
func elapsed(start, end time.Time) time.Duration {
|
||||
if d := end.Sub(start); d > 0 {
|
||||
return d
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func attemptOutcome(err error) domain.ExecutionOutcome {
|
||||
if err == nil {
|
||||
return domain.ExecutionOutcomeSuccess
|
||||
}
|
||||
return domain.ExecutionOutcomeError
|
||||
}
|
||||
|
||||
func attemptFailure(err error) domain.ExecutionFailureKind {
|
||||
if err == nil {
|
||||
return domain.ExecutionFailureNone
|
||||
}
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return domain.ExecutionFailureCanceled
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return domain.ExecutionFailureDeadline
|
||||
}
|
||||
if errors.Is(err, llm.ErrUnexpectedStatus) {
|
||||
return domain.ExecutionFailureProviderStatus
|
||||
}
|
||||
if errors.Is(err, llm.ErrTransport) {
|
||||
return domain.ExecutionFailureTransport
|
||||
}
|
||||
if errors.Is(err, llm.ErrMalformedResponse) {
|
||||
return domain.ExecutionFailureMalformedResponse
|
||||
}
|
||||
if errors.Is(err, llm.ErrInvalidRequest) {
|
||||
return domain.ExecutionFailureInvalidRequest
|
||||
}
|
||||
return domain.ExecutionFailureGeneration
|
||||
}
|
||||
|
||||
func eligibleStatus(outcome llm.AttemptOutcome) bool {
|
||||
if !errors.Is(outcome.Err, llm.ErrUnexpectedStatus) {
|
||||
return false
|
||||
}
|
||||
switch outcome.StatusCode {
|
||||
case 429, 500, 502, 503, 504:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Coordinator) backoff(ordinal uint64) time.Duration {
|
||||
base := initialBackoff
|
||||
for step := uint64(1); step < ordinal && base < maximumBackoff; step++ {
|
||||
base *= 2
|
||||
}
|
||||
half := base / 2
|
||||
return half + c.jitter(half)
|
||||
}
|
||||
|
||||
func retryDelay(local time.Duration, notBefore, now time.Time) time.Duration {
|
||||
if providerDelay := notBefore.Sub(now); providerDelay > local {
|
||||
return providerDelay
|
||||
}
|
||||
return local
|
||||
}
|
||||
|
||||
func randomJitter(max time.Duration) time.Duration {
|
||||
return time.Duration(rand.Int64N(int64(max) + 1))
|
||||
}
|
||||
|
||||
func waitContext(ctx context.Context, delay time.Duration) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
278
internal/generation/coordinator_test.go
Normal file
278
internal/generation/coordinator_test.go
Normal file
@@ -0,0 +1,278 @@
|
||||
package generation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
)
|
||||
|
||||
type transportFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f transportFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
|
||||
|
||||
func response(status int, body string) *http.Response {
|
||||
return &http.Response{StatusCode: status, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}
|
||||
}
|
||||
|
||||
func testRequest(retries uint64) domain.GenerateRequest {
|
||||
return domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}},
|
||||
Target: domain.ExecutionTarget{BackendID: "limited", Endpoint: "https://example.invalid/v1", Model: "model"},
|
||||
TransientRetries: retries,
|
||||
}
|
||||
}
|
||||
|
||||
func testCoordinator(t *testing.T, transport transportFunc) *Coordinator {
|
||||
t.Helper()
|
||||
manager, err := capacity.NewManager(map[string]domain.BackendCapacityPolicy{
|
||||
"limited": {ConcurrencyLimit: 1, QueueCapacity: 1},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: transport}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
coordinator := NewBuiltIn(manager, client)
|
||||
coordinator.jitter = func(time.Duration) time.Duration { return 0 }
|
||||
coordinator.wait = func(ctx context.Context, _ time.Duration) error { return ctx.Err() }
|
||||
return coordinator
|
||||
}
|
||||
|
||||
func TestBuiltInRetryEligibilityAndExhaustion(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
status int
|
||||
eligible bool
|
||||
}{
|
||||
{status: 429, eligible: true}, {status: 500, eligible: true},
|
||||
{status: 502, eligible: true}, {status: 503, eligible: true},
|
||||
{status: 504, eligible: true}, {status: 400}, {status: 501},
|
||||
} {
|
||||
t.Run(http.StatusText(tc.status), func(t *testing.T) {
|
||||
calls := 0
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return response(tc.status, `{"error":{}}`), nil
|
||||
}
|
||||
return response(200, `{"choices":[{"message":{"content":"ok"}}]}`), nil
|
||||
})
|
||||
result, err := coordinator.Generate(context.Background(), testRequest(1))
|
||||
if tc.eligible {
|
||||
if err != nil || result == nil || result.Content != "ok" || calls != 2 {
|
||||
t.Fatalf("result=%+v, error=%v, calls=%d", result, err, calls)
|
||||
}
|
||||
} else if result != nil || err == nil || calls != 1 {
|
||||
t.Fatalf("result=%+v, error=%v, calls=%d", result, err, calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
for _, budget := range []uint64{0, 2} {
|
||||
calls := 0
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
return response(503, `{"error":{}}`), nil
|
||||
})
|
||||
_, err := coordinator.Generate(context.Background(), testRequest(budget))
|
||||
var exhausted *ExhaustedError
|
||||
if !errors.Is(err, llm.ErrUnexpectedStatus) || calls != int(budget+1) || errors.As(err, &exhausted) != (budget > 0) {
|
||||
t.Fatalf("budget=%d calls=%d error=%v", budget, calls, err)
|
||||
}
|
||||
if budget > 0 && exhausted.Attempts() != budget+1 {
|
||||
t.Fatalf("attempts=%d", exhausted.Attempts())
|
||||
}
|
||||
}
|
||||
|
||||
calls := 0
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return response(503, `{"error":{}}`), nil
|
||||
}
|
||||
return response(400, `{"error":{}}`), nil
|
||||
})
|
||||
_, err := coordinator.Generate(context.Background(), testRequest(3))
|
||||
var exhausted *ExhaustedError
|
||||
if calls != 2 || !errors.Is(err, llm.ErrUnexpectedStatus) || errors.As(err, &exhausted) {
|
||||
t.Fatalf("later permanent failure: calls=%d, error=%v", calls, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltInRetryWaitHonorsContextAndProviderMinimum(t *testing.T) {
|
||||
when := time.Date(2026, 9, 23, 12, 0, 0, 0, time.UTC)
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) { return response(503, `{"error":{}}`), nil })
|
||||
if got := coordinator.backoff(1); got != 250*time.Millisecond {
|
||||
t.Fatalf("first backoff=%v", got)
|
||||
}
|
||||
if got := coordinator.backoff(math.MaxUint64); got != 4*time.Second {
|
||||
t.Fatalf("capped backoff=%v", got)
|
||||
}
|
||||
coordinator.jitter = func(max time.Duration) time.Duration { return max }
|
||||
if got := coordinator.backoff(math.MaxUint64); got != 8*time.Second {
|
||||
t.Fatalf("maximum equal jitter=%v", got)
|
||||
}
|
||||
if got := retryDelay(500*time.Millisecond, when.Add(3*time.Second), when); got != 3*time.Second {
|
||||
t.Fatalf("provider minimum=%v", got)
|
||||
}
|
||||
if got := retryDelay(500*time.Millisecond, when.Add(-time.Second), when); got != 500*time.Millisecond {
|
||||
t.Fatalf("past provider minimum=%v", got)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
calls := 0
|
||||
coordinator = testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
return response(503, `{"error":{}}`), nil
|
||||
})
|
||||
coordinator.wait = func(ctx context.Context, delay time.Duration) error {
|
||||
if delay < 250*time.Millisecond {
|
||||
t.Fatalf("delay=%v", delay)
|
||||
}
|
||||
cancel()
|
||||
return ctx.Err()
|
||||
}
|
||||
_, err := coordinator.Generate(ctx, testRequest(2))
|
||||
if !errors.Is(err, context.Canceled) || calls != 1 {
|
||||
t.Fatalf("canceled wait: error=%v calls=%d", err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltInRetryUsesSeparateAttemptDeadlines(t *testing.T) {
|
||||
var contexts []context.Context
|
||||
coordinator := testCoordinator(t, func(req *http.Request) (*http.Response, error) {
|
||||
contexts = append(contexts, req.Context())
|
||||
if len(contexts) == 1 {
|
||||
return response(503, `{"error":{}}`), nil
|
||||
}
|
||||
return response(200, `{"choices":[{"message":{"content":"ok"}}]}`), nil
|
||||
})
|
||||
req := testRequest(1)
|
||||
req.Target.TimeoutSeconds = 1
|
||||
if _, err := coordinator.Generate(context.Background(), req); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(contexts) != 2 || contexts[0] == contexts[1] {
|
||||
t.Fatalf("attempt contexts=%d", len(contexts))
|
||||
}
|
||||
for _, ctx := range contexts {
|
||||
if _, ok := ctx.Deadline(); !ok || !errors.Is(ctx.Err(), context.Canceled) {
|
||||
t.Fatalf("attempt deadline/error=%t/%v", ok, ctx.Err())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanceledCapacityWaitDispatchesNothing(t *testing.T) {
|
||||
calls := 0
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
return response(200, `{"choices":[{"message":{"content":"ok"}}]}`), nil
|
||||
})
|
||||
release, err := coordinator.manager.Acquire(context.Background(), "limited")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer release()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
result := make(chan error, 1)
|
||||
go func() { _, err := coordinator.Generate(ctx, testRequest(2)); result <- err }()
|
||||
cancel()
|
||||
if err := <-result; !errors.Is(err, context.Canceled) || calls != 0 {
|
||||
t.Fatalf("capacity cancellation: error=%v calls=%d", err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltInRetryFreesPermitAndFreezesCredential(t *testing.T) {
|
||||
const envName = "PROMPTKIT_GENERATION_RETRY_TEST_KEY"
|
||||
t.Setenv(envName, "first-key")
|
||||
var calls atomic.Int64
|
||||
var firstAuth, retryAuth string
|
||||
coordinator := testCoordinator(t, func(req *http.Request) (*http.Response, error) {
|
||||
switch calls.Add(1) {
|
||||
case 1:
|
||||
firstAuth = req.Header.Get("Authorization")
|
||||
return response(503, `{"error":{}}`), nil
|
||||
case 2:
|
||||
return response(200, `{"choices":[{"message":{"content":"other"}}]}`), nil
|
||||
default:
|
||||
retryAuth = req.Header.Get("Authorization")
|
||||
return response(200, `{"choices":[{"message":{"content":"retry"}}]}`), nil
|
||||
}
|
||||
})
|
||||
entered := make(chan struct{})
|
||||
resume := make(chan struct{})
|
||||
coordinator.wait = func(ctx context.Context, _ time.Duration) error {
|
||||
close(entered)
|
||||
select {
|
||||
case <-resume:
|
||||
return ctx.Err()
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
req := testRequest(1)
|
||||
req.Target.APIKeyEnv = envName
|
||||
first := make(chan error, 1)
|
||||
go func() { _, err := coordinator.Generate(context.Background(), req); first <- err }()
|
||||
<-entered
|
||||
t.Setenv(envName, "later-key")
|
||||
other, err := coordinator.Generate(context.Background(), testRequest(0))
|
||||
if err != nil || other == nil || other.Content != "other" {
|
||||
t.Fatalf("other call while backoff blocked = %+v, %v", other, err)
|
||||
}
|
||||
close(resume)
|
||||
if err := <-first; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls.Load() != 3 || firstAuth != "Bearer first-key" || retryAuth != firstAuth {
|
||||
t.Fatalf("calls=%d, first auth=%q, retry auth=%q", calls.Load(), firstAuth, retryAuth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInjectedClientDispatchesOnceAndReleasesAfterPanic(t *testing.T) {
|
||||
manager, err := capacity.NewManager(map[string]domain.BackendCapacityPolicy{"limited": {ConcurrencyLimit: 1}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &injectedClient{panicFirst: true}
|
||||
coordinator := NewInjected(manager, client)
|
||||
func() {
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Fatal("expected injected panic")
|
||||
}
|
||||
}()
|
||||
_, _ = coordinator.Generate(context.Background(), testRequest(5))
|
||||
}()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
result, err := coordinator.Generate(ctx, testRequest(5))
|
||||
if err != nil || result == nil || client.calls != 2 {
|
||||
t.Fatalf("after panic result=%+v error=%v calls=%d", result, err, client.calls)
|
||||
}
|
||||
}
|
||||
|
||||
type injectedClient struct {
|
||||
calls int
|
||||
panicFirst bool
|
||||
}
|
||||
|
||||
func (c *injectedClient) Generate(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
c.calls++
|
||||
if c.panicFirst && c.calls == 1 {
|
||||
panic("injected panic")
|
||||
}
|
||||
return &domain.GenerateResponse{Content: "ok"}, nil
|
||||
}
|
||||
201
internal/generation/observation_test.go
Normal file
201
internal/generation/observation_test.go
Normal file
@@ -0,0 +1,201 @@
|
||||
package generation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/observation"
|
||||
)
|
||||
|
||||
func recordEvents(t *testing.T, ctx context.Context, events *[]domain.ExecutionEvent) context.Context {
|
||||
t.Helper()
|
||||
recorder, err := observation.New(func(event domain.ExecutionEvent) { *events = append(*events, event) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return observation.WithRecorder(ctx, recorder)
|
||||
}
|
||||
|
||||
func TestObservedRetryAndExhaustion(t *testing.T) {
|
||||
for _, status := range []int{200, 503} {
|
||||
calls := 0
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if calls == 1 || status == 503 {
|
||||
return response(503, `{"error":{}}`), nil
|
||||
}
|
||||
return response(200, `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0,"completion_tokens":2}}`), nil
|
||||
})
|
||||
var events []domain.ExecutionEvent
|
||||
ctx := recordEvents(t, context.Background(), &events)
|
||||
_, err := coordinator.Generate(ctx, testRequest(1))
|
||||
if status == 200 && err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status == 503 {
|
||||
var exhausted *ExhaustedError
|
||||
if !errors.As(err, &exhausted) {
|
||||
t.Fatalf("error=%v", err)
|
||||
}
|
||||
}
|
||||
if len(events) != 3 || events[0].Kind != domain.ExecutionAttempt || events[1].Kind != domain.ExecutionRetryDelay || events[2].Kind != domain.ExecutionAttempt {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
if events[0].Attempt != 1 || events[1].Attempt != 1 || events[2].Attempt != 2 || !events[0].Dispatched || !events[2].Dispatched {
|
||||
t.Fatalf("indexes=%+v", events)
|
||||
}
|
||||
if events[0].Failure != domain.ExecutionFailureProviderStatus || events[1].RetryDelay < 0 {
|
||||
t.Fatalf("first=%+v delay=%+v", events[0], events[1])
|
||||
}
|
||||
if status == 200 && (!events[2].Usage.PromptTokens.Known || events[2].Usage.PromptTokens.Value != 0 || events[2].Usage.TotalTokens.Known) {
|
||||
t.Fatalf("usage=%+v", events[2].Usage)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestObservedInterruptedWaitAndCanceledCapacity(t *testing.T) {
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) { return response(503, `{"error":{}}`), nil })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
var events []domain.ExecutionEvent
|
||||
ctx = recordEvents(t, ctx, &events)
|
||||
coordinator.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
|
||||
_, err := coordinator.Generate(ctx, testRequest(1))
|
||||
if !errors.Is(err, context.Canceled) || len(events) != 2 || events[1].Kind != domain.ExecutionRetryDelay || events[1].Failure != domain.ExecutionFailureCanceled {
|
||||
t.Fatalf("error=%v events=%+v", err, events)
|
||||
}
|
||||
|
||||
coordinator = testCoordinator(t, func(*http.Request) (*http.Response, error) { t.Fatal("unexpected dispatch"); return nil, nil })
|
||||
release, err := coordinator.manager.Acquire(context.Background(), "limited")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer release()
|
||||
ctx, cancel = context.WithCancel(context.Background())
|
||||
events = nil
|
||||
ctx = recordEvents(t, ctx, &events)
|
||||
entered := make(chan struct{})
|
||||
ctx = ¬ifyingContext{Context: ctx, entered: entered}
|
||||
done := make(chan error, 1)
|
||||
go func() { _, err := coordinator.Generate(ctx, testRequest(0)); done <- err }()
|
||||
<-entered
|
||||
cancel()
|
||||
if err := <-done; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("error=%v", err)
|
||||
}
|
||||
if len(events) != 1 || events[0].Dispatched || events[0].Failure != domain.ExecutionFailureCanceled {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
}
|
||||
|
||||
type notifyingContext struct {
|
||||
context.Context
|
||||
entered chan struct{}
|
||||
}
|
||||
|
||||
func (c *notifyingContext) Done() <-chan struct{} {
|
||||
select {
|
||||
case <-c.entered:
|
||||
default:
|
||||
close(c.entered)
|
||||
}
|
||||
return c.Context.Done()
|
||||
}
|
||||
|
||||
func TestObservedCallbackReleasesPermitAndCanCancelRetry(t *testing.T) {
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) { return response(503, `{"error":{}}`), nil })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
var events []domain.ExecutionEvent
|
||||
recorder, err := observation.New(func(event domain.ExecutionEvent) {
|
||||
events = append(events, event)
|
||||
if event.Kind != domain.ExecutionAttempt {
|
||||
return
|
||||
}
|
||||
permitCtx, stop := context.WithTimeout(context.Background(), time.Second)
|
||||
defer stop()
|
||||
release, err := coordinator.manager.Acquire(permitCtx, "limited")
|
||||
if err != nil {
|
||||
t.Errorf("callback could not acquire permit: %v", err)
|
||||
} else {
|
||||
release()
|
||||
}
|
||||
cancel()
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = coordinator.Generate(observation.WithRecorder(ctx, recorder), testRequest(2))
|
||||
if !errors.Is(err, context.Canceled) || len(events) != 1 {
|
||||
t.Fatalf("error=%v events=%+v", err, events)
|
||||
}
|
||||
}
|
||||
|
||||
type fixedClient struct {
|
||||
response *domain.GenerateResponse
|
||||
err error
|
||||
}
|
||||
|
||||
func (c fixedClient) Generate(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
return c.response, c.err
|
||||
}
|
||||
|
||||
func TestObservedInjectedUsageAndRepairGeneration(t *testing.T) {
|
||||
manager, err := capacity.NewManager(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var events []domain.ExecutionEvent
|
||||
ctx := recordEvents(t, context.Background(), &events)
|
||||
legacy := NewInjected(manager, fixedClient{response: &domain.GenerateResponse{Usage: domain.TokenUsage{PromptTokens: 3}}})
|
||||
if _, err := legacy.Generate(ctx, testRequest(4)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
presence := &domain.TokenUsagePresence{CompletionTokens: true}
|
||||
repair := NewInjected(manager, fixedClient{response: &domain.GenerateResponse{UsagePresence: presence}})
|
||||
req := testRequest(4)
|
||||
req.Generation = 2
|
||||
if _, err := repair.Generate(ctx, req); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(events) != 2 || !events[0].Usage.PromptTokens.Known || events[0].Usage.CompletionTokens.Known || !events[1].Usage.CompletionTokens.Known || events[1].Usage.CompletionTokens.Value != 0 || events[1].Usage.PromptTokens.Known {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
if events[1].Generation != 2 || events[1].Attempt != 1 {
|
||||
t.Fatalf("repair indexes=%+v", events[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestObservedSuccessfulGenerationBeforeRepairFailure(t *testing.T) {
|
||||
manager, err := capacity.NewManager(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var events []domain.ExecutionEvent
|
||||
recorder, err := observation.New(func(event domain.ExecutionEvent) { events = append(events, event) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := observation.WithRecorder(context.Background(), recorder)
|
||||
initial := NewInjected(manager, fixedClient{response: &domain.GenerateResponse{Usage: domain.TokenUsage{PromptTokens: 5}}})
|
||||
if _, err := initial.Generate(ctx, testRequest(0)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repair := NewInjected(manager, fixedClient{err: errors.New("repair unavailable")})
|
||||
req := testRequest(0)
|
||||
req.Generation = 2
|
||||
if _, err := repair.Generate(ctx, req); err == nil {
|
||||
t.Fatal("expected repair failure")
|
||||
}
|
||||
recorder.Finish(observation.FinishData{Outcome: domain.ExecutionOutcomeError, Failure: domain.ExecutionFailureGeneration})
|
||||
if len(events) != 3 || events[0].Outcome != domain.ExecutionOutcomeSuccess || events[1].Outcome != domain.ExecutionOutcomeError || events[2].Attempts != 2 || events[2].RepairCalls != 1 {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
if got := events[2].Usage.PromptTokens; got.Value != 5 || !got.Known || got.Complete {
|
||||
t.Fatalf("prompt usage=%+v", got)
|
||||
}
|
||||
}
|
||||
@@ -7,9 +7,11 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -21,6 +23,7 @@ var (
|
||||
ErrInvalidConfig = errors.New("invalid llm client configuration")
|
||||
ErrInvalidRequest = errors.New("invalid generate request")
|
||||
ErrRequestFailed = errors.New("llm request failed")
|
||||
ErrTransport = errors.New("llm transport failed")
|
||||
ErrUnexpectedStatus = errors.New("llm returned non-success status")
|
||||
ErrMalformedResponse = errors.New("malformed llm response")
|
||||
)
|
||||
@@ -36,7 +39,7 @@ func (e *requestFailedError) Error() string {
|
||||
}
|
||||
|
||||
func (e *requestFailedError) Unwrap() []error {
|
||||
return []error{ErrRequestFailed, e.cause}
|
||||
return []error{ErrRequestFailed, ErrTransport, e.cause}
|
||||
}
|
||||
|
||||
type OpenAICompatibleConfig struct {
|
||||
@@ -86,6 +89,40 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
|
||||
}
|
||||
|
||||
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
prepared, err := c.PrepareGeneration(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outcome := prepared.Execute(ctx)
|
||||
if outcome.Err != nil {
|
||||
return nil, outcome.Err
|
||||
}
|
||||
return outcome.Response, nil
|
||||
}
|
||||
|
||||
// PreparedGeneration holds one encoded request and resolved authentication for
|
||||
// repeated attempts. Each Execute call creates a fresh HTTP request and context.
|
||||
type PreparedGeneration struct {
|
||||
endpoint string
|
||||
payload []byte
|
||||
apiKey string
|
||||
timeoutSeconds int
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// AttemptOutcome describes one HTTP execution without exposing response bodies
|
||||
// or headers to the orchestration layer.
|
||||
type AttemptOutcome struct {
|
||||
Response *domain.GenerateResponse
|
||||
Err error
|
||||
StatusCode int
|
||||
Usage domain.TokenUsage
|
||||
UsagePresence *domain.TokenUsagePresence
|
||||
RetryNotBefore time.Time
|
||||
}
|
||||
|
||||
// PrepareGeneration validates and freezes one built-in request before dispatch.
|
||||
func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (*PreparedGeneration, error) {
|
||||
if err := domain.ValidateExecutionTargetSettings(req.Target); err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
@@ -117,22 +154,6 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
|
||||
}
|
||||
|
||||
requestContext := ctx
|
||||
if req.Target.TimeoutSeconds > 0 {
|
||||
var cancel context.CancelFunc
|
||||
requestContext, cancel = context.WithTimeout(
|
||||
ctx,
|
||||
time.Duration(req.Target.TimeoutSeconds)*time.Second,
|
||||
)
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(requestContext, http.MethodPost, endpoint, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
apiKey := strings.TrimSpace(req.Target.APIKey)
|
||||
envName := strings.TrimSpace(req.Target.APIKeyEnv)
|
||||
if apiKey == "" && envName != "" {
|
||||
@@ -144,55 +165,140 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
||||
}
|
||||
return nil, fmt.Errorf("%w: api key is required", ErrInvalidRequest)
|
||||
}
|
||||
if apiKey != "" {
|
||||
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
}
|
||||
|
||||
httpClient := c.httpClient
|
||||
if httpClient == nil {
|
||||
httpClient = &http.Client{Timeout: defaults.LLMRequestTimeoutDefault}
|
||||
}
|
||||
return &PreparedGeneration{
|
||||
endpoint: endpoint, payload: payload, apiKey: apiKey,
|
||||
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
|
||||
}, nil
|
||||
}
|
||||
|
||||
httpResp, err := httpClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, &requestFailedError{cause: err}
|
||||
// Execute dispatches one attempt. The returned response is non-nil only when
|
||||
// the successful envelope contains a usable first choice.
|
||||
func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
|
||||
requestContext := ctx
|
||||
if p.timeoutSeconds > 0 {
|
||||
var cancel context.CancelFunc
|
||||
requestContext, cancel = context.WithTimeout(ctx, time.Duration(p.timeoutSeconds)*time.Second)
|
||||
defer cancel()
|
||||
}
|
||||
httpReq, err := http.NewRequestWithContext(requestContext, http.MethodPost, p.endpoint, bytes.NewReader(p.payload))
|
||||
if err != nil {
|
||||
return AttemptOutcome{Err: fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)}
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
if p.apiKey != "" {
|
||||
httpReq.Header.Set("Authorization", "Bearer "+p.apiKey)
|
||||
}
|
||||
|
||||
httpResp, err := p.httpClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return AttemptOutcome{Err: &requestFailedError{cause: err}}
|
||||
}
|
||||
receivedAt := time.Now()
|
||||
defer httpResp.Body.Close()
|
||||
outcome := AttemptOutcome{StatusCode: httpResp.StatusCode}
|
||||
|
||||
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
|
||||
return nil, providerHTTPErrorFromBody(
|
||||
outcome.RetryNotBefore = retryNotBefore(httpResp.Header.Values("Retry-After"), receivedAt)
|
||||
outcome.Err = providerHTTPErrorFromBody(
|
||||
httpResp.StatusCode,
|
||||
httpResp.ContentLength,
|
||||
httpResp.Body,
|
||||
)
|
||||
if err := observedContextError(requestContext, ctx, outcome.Err); err != nil {
|
||||
outcome.Err = err
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
if httpResp.ContentLength > maxOpenAIChatResponseBytes {
|
||||
return nil, openAIChatResponseTooLargeError()
|
||||
outcome.Err = openAIChatResponseTooLargeError()
|
||||
return outcome
|
||||
}
|
||||
|
||||
wireResp, err := decodeOpenAIChatResponse(httpResp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
outcome.Err = err
|
||||
if contextErr := observedContextError(requestContext, ctx, err); contextErr != nil {
|
||||
outcome.Err = contextErr
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
outcome.Usage = domain.TokenUsage{
|
||||
PromptTokens: tokenCount(wireResp.Usage.PromptTokens), CompletionTokens: tokenCount(wireResp.Usage.CompletionTokens),
|
||||
TotalTokens: tokenCount(wireResp.Usage.TotalTokens), CachedTokens: tokenCount(wireResp.Usage.PromptTokensDetails.CachedTokens),
|
||||
CacheWriteTokens: tokenCount(wireResp.Usage.CacheWriteTokens),
|
||||
}
|
||||
outcome.UsagePresence = &domain.TokenUsagePresence{
|
||||
PromptTokens: wireResp.Usage.PromptTokens != nil, CompletionTokens: wireResp.Usage.CompletionTokens != nil,
|
||||
TotalTokens: wireResp.Usage.TotalTokens != nil, CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens != nil,
|
||||
CacheWriteTokens: wireResp.Usage.CacheWriteTokens != nil,
|
||||
}
|
||||
|
||||
if len(wireResp.Choices) == 0 {
|
||||
return nil, fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
|
||||
outcome.Err = fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
|
||||
return outcome
|
||||
}
|
||||
content := wireResp.Choices[0].Message.Content
|
||||
if content == nil {
|
||||
return nil, fmt.Errorf("%w: first choice has missing message content", ErrMalformedResponse)
|
||||
outcome.Err = fmt.Errorf("%w: first choice has missing message content", ErrMalformedResponse)
|
||||
return outcome
|
||||
}
|
||||
outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence}
|
||||
return outcome
|
||||
}
|
||||
|
||||
return &domain.GenerateResponse{
|
||||
Content: *content,
|
||||
Usage: domain.TokenUsage{
|
||||
PromptTokens: wireResp.Usage.PromptTokens,
|
||||
CompletionTokens: wireResp.Usage.CompletionTokens,
|
||||
TotalTokens: wireResp.Usage.TotalTokens,
|
||||
CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens,
|
||||
CacheWriteTokens: wireResp.Usage.CacheWriteTokens,
|
||||
},
|
||||
}, nil
|
||||
func observedContextError(attempt, caller context.Context, cause error) error {
|
||||
if err := caller.Err(); err != nil {
|
||||
return &requestFailedError{cause: err}
|
||||
}
|
||||
if err := attempt.Err(); err != nil {
|
||||
return &requestFailedError{cause: err}
|
||||
}
|
||||
if errors.Is(cause, context.Canceled) || errors.Is(cause, context.DeadlineExceeded) {
|
||||
return &requestFailedError{cause: cause}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func retryNotBefore(values []string, receivedAt time.Time) time.Time {
|
||||
if len(values) == 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
value := strings.TrimSpace(values[0])
|
||||
if value == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
digits := true
|
||||
for _, r := range value {
|
||||
if r < '0' || r > '9' {
|
||||
digits = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if digits {
|
||||
seconds, err := strconv.ParseUint(value, 10, 64)
|
||||
if err != nil || seconds > uint64(math.MaxInt64/int64(time.Second)) {
|
||||
return receivedAt.Add(time.Duration(math.MaxInt64))
|
||||
}
|
||||
return receivedAt.Add(time.Duration(seconds) * time.Second)
|
||||
}
|
||||
if when, err := http.ParseTime(value); err == nil {
|
||||
if when.Before(receivedAt) {
|
||||
return receivedAt
|
||||
}
|
||||
return when
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
func tokenCount(value *int) int {
|
||||
if value == nil {
|
||||
return 0
|
||||
}
|
||||
return *value
|
||||
}
|
||||
|
||||
func decodeOpenAIChatResponse(body io.Reader) (openAIChatResponse, error) {
|
||||
@@ -204,6 +310,9 @@ func decodeOpenAIChatResponse(body io.Reader) (openAIChatResponse, error) {
|
||||
|
||||
var response openAIChatResponse
|
||||
if err := decoder.Decode(&response); err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return openAIChatResponse{}, &requestFailedError{cause: err}
|
||||
}
|
||||
if limited.N == 0 {
|
||||
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||
}
|
||||
@@ -215,6 +324,9 @@ func decodeOpenAIChatResponse(body io.Reader) (openAIChatResponse, error) {
|
||||
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return openAIChatResponse{}, &requestFailedError{cause: err}
|
||||
}
|
||||
if limited.N == 0 {
|
||||
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||
}
|
||||
@@ -388,13 +500,13 @@ type openAIChatResponse struct {
|
||||
Message openAIChatResponseMessage `json:"message"`
|
||||
} `json:"choices"`
|
||||
Usage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
PromptTokens *int `json:"prompt_tokens"`
|
||||
CompletionTokens *int `json:"completion_tokens"`
|
||||
TotalTokens *int `json:"total_tokens"`
|
||||
PromptTokensDetails struct {
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
CachedTokens *int `json:"cached_tokens"`
|
||||
} `json:"prompt_tokens_details"`
|
||||
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||
CacheWriteTokens *int `json:"cache_write_tokens"`
|
||||
} `json:"usage"`
|
||||
}
|
||||
|
||||
|
||||
@@ -363,6 +363,7 @@ func TestOpenAICompatibleClientRequestMapping(t *testing.T) {
|
||||
run func(*testing.T)
|
||||
}{
|
||||
{name: "complete request and response mapping", run: checkCompleteRequestAndResponseMapping},
|
||||
{name: "canonical role and exact content", run: checkCanonicalRoleAndExactContentMapping},
|
||||
{name: "cache-controlled message", run: checkCacheControlledMessageMapping},
|
||||
{name: "empty cache-control TTL", run: checkEmptyCacheControlTTLOmission},
|
||||
{name: "session ID", run: checkSessionIDMapping},
|
||||
@@ -377,6 +378,34 @@ func TestOpenAICompatibleClientRequestMapping(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func checkCanonicalRoleAndExactContentMapping(t *testing.T) {
|
||||
provider := newRecordingProvider(t)
|
||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
||||
const content = " exact content\nwith whitespace "
|
||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{
|
||||
Role: domain.RoleDeveloper,
|
||||
Content: content,
|
||||
}}},
|
||||
Target: domain.ExecutionTarget{Model: "model"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
|
||||
var messages []struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
request := provider.lastRequest(t)
|
||||
if !request.decodeField(t, "messages", &messages) || len(messages) != 1 {
|
||||
t.Fatalf("messages = %#v, want one message", messages)
|
||||
}
|
||||
if messages[0].Role != domain.RoleDeveloper || messages[0].Content != content {
|
||||
t.Fatalf("message = %#v, want role %q and exact content %q", messages[0], domain.RoleDeveloper, content)
|
||||
}
|
||||
}
|
||||
|
||||
func checkCompleteRequestAndResponseMapping(t *testing.T) {
|
||||
provider := newRecordingProvider(t)
|
||||
provider.respond(http.StatusOK, `{
|
||||
@@ -852,6 +881,44 @@ func checkCacheUsageMapping(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleUsagePresence(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
usage string
|
||||
want domain.TokenUsage
|
||||
presence domain.TokenUsagePresence
|
||||
}{
|
||||
{name: "absent"},
|
||||
{name: "null", usage: `"usage":null,`},
|
||||
{name: "partial nested", usage: `"usage":{"prompt_tokens":7,"prompt_tokens_details":{"cached_tokens":0}},`,
|
||||
want: domain.TokenUsage{PromptTokens: 7}, presence: domain.TokenUsagePresence{PromptTokens: true, CachedTokens: true}},
|
||||
{name: "explicit null fields", usage: `"usage":{"prompt_tokens":null,"completion_tokens":0,"prompt_tokens_details":{"cached_tokens":null}},`,
|
||||
presence: domain.TokenUsagePresence{CompletionTokens: true}},
|
||||
{name: "all reported", usage: `"usage":{"prompt_tokens":1,"completion_tokens":2,"total_tokens":3,"prompt_tokens_details":{"cached_tokens":4},"cache_write_tokens":5},`,
|
||||
want: domain.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3, CachedTokens: 4, CacheWriteTokens: 5},
|
||||
presence: domain.TokenUsagePresence{PromptTokens: true, CompletionTokens: true, TotalTokens: true, CachedTokens: true, CacheWriteTokens: true}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
provider := newRecordingProvider(t)
|
||||
provider.respond(http.StatusOK, `{`+tc.usage+`"choices":[{"message":{"content":"ok"}}]}`)
|
||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
|
||||
resp, err := client.Generate(context.Background(), ordinaryGenerateRequest())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp.Usage != tc.want || resp.UsagePresence == nil || *resp.UsagePresence != tc.presence {
|
||||
t.Fatalf("usage=%+v, presence=%+v", resp.Usage, resp.UsagePresence)
|
||||
}
|
||||
})
|
||||
}
|
||||
provider := newRecordingProvider(t)
|
||||
provider.respond(http.StatusOK, `{"usage":{"prompt_tokens":"seven"},"choices":[{"message":{"content":"ok"}}]}`)
|
||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
|
||||
if resp, err := client.Generate(context.Background(), ordinaryGenerateRequest()); resp != nil || !errors.Is(err, ErrMalformedResponse) {
|
||||
t.Fatalf("invalid numeric usage response=%+v, error=%v", resp, err)
|
||||
}
|
||||
}
|
||||
|
||||
func checkOptionalResponseFieldOmission(t *testing.T) {
|
||||
provider := newRecordingProvider(t)
|
||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
||||
|
||||
225
internal/llm/prepared_generation_test.go
Normal file
225
internal/llm/prepared_generation_test.go
Normal file
@@ -0,0 +1,225 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestPreparedGenerationFreezesPayloadAndAuthentication(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
initialKey string
|
||||
wantAuth string
|
||||
}{
|
||||
{name: "absent optional key"},
|
||||
{name: "present key", initialKey: "original-key", wantAuth: "Bearer original-key"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
const envName = "PROMPTKIT_PREPARED_GENERATION_TEST_KEY"
|
||||
t.Setenv(envName, tc.initialKey)
|
||||
var bodies, auth, urls []string
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||
BaseURL: "https://example.invalid/v1", Model: "default-model",
|
||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
body, err := io.ReadAll(req.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bodies = append(bodies, string(body))
|
||||
auth = append(auth, req.Header.Get("Authorization"))
|
||||
urls = append(urls, req.URL.String())
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"ok"}}]}`))}, nil
|
||||
})},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "original"}}},
|
||||
Target: domain.ExecutionTarget{APIKeyEnv: envName},
|
||||
}
|
||||
prepared, err := client.PrepareGeneration(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Prompt.Messages[0].Content = "changed"
|
||||
t.Setenv(envName, "later-key")
|
||||
for range 2 {
|
||||
outcome := prepared.Execute(context.Background())
|
||||
if outcome.Err != nil || outcome.Response == nil || outcome.Response.Content != "ok" {
|
||||
t.Fatalf("Execute = %+v", outcome)
|
||||
}
|
||||
}
|
||||
if len(bodies) != 2 || bodies[0] != bodies[1] || !strings.Contains(bodies[0], "original") || strings.Contains(bodies[0], "changed") || !reflect.DeepEqual(auth, []string{tc.wantAuth, tc.wantAuth}) || !reflect.DeepEqual(urls, []string{"https://example.invalid/v1/chat/completions", "https://example.invalid/v1/chat/completions"}) {
|
||||
t.Fatalf("bodies=%v auth=%v urls=%v", bodies, auth, urls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedGenerationAttemptsHaveIndependentContextsAndCloseBodies(t *testing.T) {
|
||||
var contexts []context.Context
|
||||
var closed int
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||
BaseURL: "https://example.invalid/v1", Model: "model",
|
||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
if err := req.Context().Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
contexts = append(contexts, req.Context())
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: &countingBody{
|
||||
Reader: strings.NewReader(`{"choices":[{"message":{"content":"ok"}}]}`),
|
||||
closed: &closed,
|
||||
}}, nil
|
||||
})},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||
Target: domain.ExecutionTarget{TimeoutSeconds: 1},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range 2 {
|
||||
if outcome := prepared.Execute(context.Background()); outcome.Err != nil {
|
||||
t.Fatal(outcome.Err)
|
||||
}
|
||||
}
|
||||
if len(contexts) != 2 || contexts[0] == contexts[1] || closed != 2 {
|
||||
t.Fatalf("contexts=%d, closed=%d", len(contexts), closed)
|
||||
}
|
||||
for _, ctx := range contexts {
|
||||
if _, ok := ctx.Deadline(); !ok || !errors.Is(ctx.Err(), context.Canceled) {
|
||||
t.Fatalf("attempt context deadline/error = %v/%v", ok, ctx.Err())
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if outcome := prepared.Execute(ctx); !errors.Is(outcome.Err, context.Canceled) {
|
||||
t.Fatalf("canceled attempt = %+v", outcome)
|
||||
}
|
||||
if outcome := prepared.Execute(context.Background()); outcome.Err != nil || outcome.Response == nil {
|
||||
t.Fatalf("later attempt = %+v", outcome)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedGenerationRetainsUsageFromMalformedChoice(t *testing.T) {
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||
BaseURL: "https://example.invalid/v1", Model: "model",
|
||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"usage":{"prompt_tokens":0},"choices":[]}`))}, nil
|
||||
})},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outcome := prepared.Execute(context.Background())
|
||||
if !errors.Is(outcome.Err, ErrMalformedResponse) || outcome.Response != nil || outcome.UsagePresence == nil || !outcome.UsagePresence.PromptTokens || outcome.Usage.PromptTokens != 0 {
|
||||
t.Fatalf("malformed choice = %+v", outcome)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedGenerationDoesNotUseInvalidlyFramedUsage(t *testing.T) {
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||
BaseURL: "https://example.invalid/v1", Model: "model",
|
||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"usage":{"prompt_tokens":9},"choices":[]}{}`))}, nil
|
||||
})},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outcome := prepared.Execute(context.Background())
|
||||
if !errors.Is(outcome.Err, ErrMalformedResponse) || outcome.UsagePresence != nil || outcome.Usage != (domain.TokenUsage{}) {
|
||||
t.Fatalf("invalid framing = %+v", outcome)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedGenerationRetainsReadCancellation(t *testing.T) {
|
||||
closed := false
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||
BaseURL: "https://example.invalid/v1", Model: "model",
|
||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: &readErrorBody{err: context.Canceled, closed: &closed}}, nil
|
||||
})},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outcome := prepared.Execute(context.Background())
|
||||
if !errors.Is(outcome.Err, context.Canceled) || !errors.Is(outcome.Err, ErrTransport) || errors.Is(outcome.Err, ErrMalformedResponse) || !closed {
|
||||
t.Fatalf("read cancellation = %+v, closed=%t", outcome, closed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryNotBefore(t *testing.T) {
|
||||
received := time.Date(2026, 9, 23, 12, 0, 0, 0, time.UTC)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
value string
|
||||
want time.Time
|
||||
}{
|
||||
{name: "seconds", value: " 12 ", want: received.Add(12 * time.Second)},
|
||||
{name: "date", value: "Wed, 23 Sep 2026 12:00:15 GMT", want: received.Add(15 * time.Second)},
|
||||
{name: "past date", value: "Wed, 23 Sep 2026 11:59:59 GMT", want: received},
|
||||
{name: "negative", value: "-2"},
|
||||
{name: "fractional", value: "1.5"},
|
||||
{name: "malformed", value: "tomorrow"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := retryNotBefore([]string{tc.value, "999"}, received); !got.Equal(tc.want) {
|
||||
t.Fatalf("not before = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
if got := retryNotBefore([]string{"999999999999999999999999999999999999"}, received); !got.After(received) {
|
||||
t.Fatalf("large valid delay wrapped: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
type countingBody struct {
|
||||
io.Reader
|
||||
closed *int
|
||||
}
|
||||
|
||||
func (b *countingBody) Close() error {
|
||||
*b.closed++
|
||||
return nil
|
||||
}
|
||||
|
||||
type readErrorBody struct {
|
||||
err error
|
||||
closed *bool
|
||||
}
|
||||
|
||||
func (b *readErrorBody) Read([]byte) (int, error) { return 0, b.err }
|
||||
func (b *readErrorBody) Close() error { *b.closed = true; return nil }
|
||||
318
internal/observation/recorder.go
Normal file
318
internal/observation/recorder.go
Normal file
@@ -0,0 +1,318 @@
|
||||
// Package observation owns invocation-local execution identities, metadata
|
||||
// accounting, and synchronous event delivery.
|
||||
package observation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
// Recorder accumulates one execution's measured attempts and usage. Its owner
|
||||
// calls methods in execution order; it has no shared history or background work.
|
||||
type Recorder struct {
|
||||
runID string
|
||||
startedAt time.Time
|
||||
now func() time.Time
|
||||
observer domain.ExecutionObserver
|
||||
selection Selection
|
||||
attempts uint64
|
||||
retries uint64
|
||||
repairCalls uint64
|
||||
lastRepairGeneration uint64
|
||||
finished bool
|
||||
capacityWait time.Duration
|
||||
generationDuration time.Duration
|
||||
retryDelay time.Duration
|
||||
usage usageTotals
|
||||
}
|
||||
|
||||
// Selection contains resolved identities only. Unresolved fields stay empty.
|
||||
type Selection struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
BackendID string
|
||||
Model string
|
||||
}
|
||||
|
||||
// AttemptData describes one completed active-permit acquisition and optional
|
||||
// generation dispatch. Non-dispatched attempts contribute no token usage.
|
||||
type AttemptData struct {
|
||||
Generation uint64
|
||||
Attempt uint64
|
||||
Dispatched bool
|
||||
CapacityWait time.Duration
|
||||
GenerationDuration time.Duration
|
||||
Outcome domain.ExecutionOutcome
|
||||
Failure domain.ExecutionFailureKind
|
||||
StatusCode int
|
||||
Usage domain.TokenUsage
|
||||
UsagePresence *domain.TokenUsagePresence
|
||||
}
|
||||
|
||||
// DelayData describes one completed or interrupted retry wait.
|
||||
type DelayData struct {
|
||||
Generation uint64
|
||||
Attempt uint64
|
||||
Elapsed time.Duration
|
||||
Failure domain.ExecutionFailureKind
|
||||
}
|
||||
|
||||
// FinishData describes the terminal outcome after resource release.
|
||||
type FinishData struct {
|
||||
Outcome domain.ExecutionOutcome
|
||||
Failure domain.ExecutionFailureKind
|
||||
StatusCode int
|
||||
RetriesExhausted bool
|
||||
}
|
||||
|
||||
// New creates an invocation-local recorder and run ID. It returns the recorder
|
||||
// even if ID creation fails, allowing an error terminal event with an empty ID.
|
||||
func New(observer domain.ExecutionObserver) (*Recorder, error) {
|
||||
runID, err := NewRunID()
|
||||
return newRecorder(runID, time.Now, observer), err
|
||||
}
|
||||
|
||||
func newRecorder(runID string, now func() time.Time, observer domain.ExecutionObserver) *Recorder {
|
||||
return &Recorder{runID: runID, startedAt: now(), now: now, observer: observer,
|
||||
usage: newUsageTotals()}
|
||||
}
|
||||
|
||||
// NewRunID creates the identity shared by observations and successful results.
|
||||
func NewRunID() (string, error) {
|
||||
var b [16]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
b[6] = (b[6] & 0x0f) | 0x40
|
||||
b[8] = (b[8] & 0x3f) | 0x80
|
||||
return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]), nil
|
||||
}
|
||||
|
||||
// RunID returns this invocation's identity.
|
||||
func (r *Recorder) RunID() string { return r.runID }
|
||||
|
||||
// Start delivers the opening event with only identities already resolved.
|
||||
func (r *Recorder) Start() {
|
||||
event := r.event(domain.ExecutionStarted)
|
||||
r.deliver(event)
|
||||
}
|
||||
|
||||
// UpdateSelection replaces the known resolved identities for later events.
|
||||
func (r *Recorder) UpdateSelection(selection Selection) { r.selection = selection }
|
||||
|
||||
// Attempt records one completed acquisition and optional dispatch.
|
||||
func (r *Recorder) Attempt(data AttemptData) {
|
||||
r.capacityWait = addDuration(r.capacityWait, data.CapacityWait)
|
||||
if data.Dispatched {
|
||||
r.attempts++
|
||||
if data.Attempt > 1 {
|
||||
r.retries++
|
||||
}
|
||||
if data.Generation > 1 && data.Generation != r.lastRepairGeneration {
|
||||
r.repairCalls++
|
||||
r.lastRepairGeneration = data.Generation
|
||||
}
|
||||
r.generationDuration = addDuration(r.generationDuration, data.GenerationDuration)
|
||||
r.usage.add(data.Usage, data.UsagePresence)
|
||||
}
|
||||
event := r.event(domain.ExecutionAttempt)
|
||||
event.Generation = data.Generation
|
||||
event.Attempt = data.Attempt
|
||||
event.Dispatched = data.Dispatched
|
||||
event.CapacityWait = data.CapacityWait
|
||||
if data.Dispatched {
|
||||
event.GenerationDuration = data.GenerationDuration
|
||||
}
|
||||
event.Outcome = data.Outcome
|
||||
event.Failure = data.Failure
|
||||
event.StatusCode = data.StatusCode
|
||||
if data.Dispatched {
|
||||
event.Usage = observedUsage(data.Usage, data.UsagePresence)
|
||||
}
|
||||
r.deliver(event)
|
||||
}
|
||||
|
||||
// Delay records the measured time spent waiting after one failed attempt.
|
||||
func (r *Recorder) Delay(data DelayData) {
|
||||
r.retryDelay = addDuration(r.retryDelay, data.Elapsed)
|
||||
event := r.event(domain.ExecutionRetryDelay)
|
||||
event.Generation = data.Generation
|
||||
event.Attempt = data.Attempt
|
||||
event.RetryDelay = data.Elapsed
|
||||
event.Failure = data.Failure
|
||||
r.deliver(event)
|
||||
}
|
||||
|
||||
// Finish delivers terminal totals once on a normal return. Its caller owns the
|
||||
// operation's resource cleanup and failure classification.
|
||||
func (r *Recorder) Finish(data FinishData) {
|
||||
if r.finished {
|
||||
return
|
||||
}
|
||||
r.finished = true
|
||||
event := r.event(domain.ExecutionFinished)
|
||||
event.Outcome = data.Outcome
|
||||
event.Failure = data.Failure
|
||||
event.StatusCode = data.StatusCode
|
||||
event.RetriesExhausted = data.RetriesExhausted
|
||||
event.Attempts = r.attempts
|
||||
event.TransientRetries = r.retries
|
||||
event.RepairCalls = r.repairCalls
|
||||
event.CapacityWait = r.capacityWait
|
||||
event.GenerationDuration = r.generationDuration
|
||||
event.RetryDelay = r.retryDelay
|
||||
event.Duration = event.Time.Sub(r.startedAt)
|
||||
if event.Duration < 0 {
|
||||
event.Duration = 0
|
||||
}
|
||||
event.Usage = r.usage.snapshot(r.attempts)
|
||||
r.deliver(event)
|
||||
}
|
||||
|
||||
func (r *Recorder) event(kind domain.ExecutionEventKind) domain.ExecutionEvent {
|
||||
return domain.ExecutionEvent{
|
||||
Kind: kind, RunID: r.runID, Time: r.now(), PromptID: r.selection.PromptID,
|
||||
PromptVersion: r.selection.PromptVersion, ProfileID: r.selection.ProfileID,
|
||||
BackendID: r.selection.BackendID, Model: r.selection.Model,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Recorder) deliver(event domain.ExecutionEvent) {
|
||||
if r.observer != nil {
|
||||
r.observer(event)
|
||||
}
|
||||
}
|
||||
|
||||
type contextKey struct{}
|
||||
|
||||
// WithoutExecutionState starts a public operation without inheriting another
|
||||
// invocation's recorder or prepared-claim sink. Other context values and all
|
||||
// cancellation/deadline behavior remain with the caller's context.
|
||||
func WithoutExecutionState(ctx context.Context) context.Context {
|
||||
return executionBoundary{Context: ctx}
|
||||
}
|
||||
|
||||
type executionBoundary struct{ context.Context }
|
||||
|
||||
func (c executionBoundary) Value(key any) any {
|
||||
switch key.(type) {
|
||||
case contextKey, claimSinkKey:
|
||||
return nil
|
||||
default:
|
||||
return c.Context.Value(key)
|
||||
}
|
||||
}
|
||||
|
||||
// WithRecorder carries one invocation's recorder through internal client calls.
|
||||
func WithRecorder(ctx context.Context, recorder *Recorder) context.Context {
|
||||
return context.WithValue(ctx, contextKey{}, recorder)
|
||||
}
|
||||
|
||||
// FromContext returns the invocation-local recorder, if one was installed.
|
||||
func FromContext(ctx context.Context) *Recorder {
|
||||
recorder, _ := ctx.Value(contextKey{}).(*Recorder)
|
||||
return recorder
|
||||
}
|
||||
|
||||
type claimSinkKey struct{}
|
||||
|
||||
// WithClaimSink lets the facade learn which invocation actually claimed a
|
||||
// prepared handle, so it can emit the terminal event after public mapping.
|
||||
func WithClaimSink(ctx context.Context, sink **Recorder) context.Context {
|
||||
return context.WithValue(ctx, claimSinkKey{}, sink)
|
||||
}
|
||||
|
||||
// RegisterClaim records an invocation after its prepared handle is claimed.
|
||||
func RegisterClaim(ctx context.Context, recorder *Recorder) {
|
||||
if sink, ok := ctx.Value(claimSinkKey{}).(**Recorder); ok && sink != nil {
|
||||
*sink = recorder
|
||||
}
|
||||
}
|
||||
|
||||
type usageField struct {
|
||||
value int
|
||||
known, complete bool
|
||||
}
|
||||
type usageTotals struct {
|
||||
prompt, completion, total, cached, cacheWrite usageField
|
||||
}
|
||||
|
||||
func newUsageTotals() usageTotals {
|
||||
return usageTotals{prompt: usageField{complete: true}, completion: usageField{complete: true},
|
||||
total: usageField{complete: true}, cached: usageField{complete: true}, cacheWrite: usageField{complete: true}}
|
||||
}
|
||||
|
||||
func (u *usageTotals) add(usage domain.TokenUsage, presence *domain.TokenUsagePresence) {
|
||||
known := usagePresence(usage, presence)
|
||||
u.prompt.add(usage.PromptTokens, known.PromptTokens)
|
||||
u.completion.add(usage.CompletionTokens, known.CompletionTokens)
|
||||
u.total.add(usage.TotalTokens, known.TotalTokens)
|
||||
u.cached.add(usage.CachedTokens, known.CachedTokens)
|
||||
u.cacheWrite.add(usage.CacheWriteTokens, known.CacheWriteTokens)
|
||||
}
|
||||
|
||||
func (u usageTotals) snapshot(attempts uint64) domain.ObservedTokenUsage {
|
||||
return domain.ObservedTokenUsage{
|
||||
PromptTokens: u.prompt.snapshot(attempts), CompletionTokens: u.completion.snapshot(attempts),
|
||||
TotalTokens: u.total.snapshot(attempts), CachedTokens: u.cached.snapshot(attempts),
|
||||
CacheWriteTokens: u.cacheWrite.snapshot(attempts),
|
||||
}
|
||||
}
|
||||
|
||||
func (f *usageField) add(value int, known bool) {
|
||||
if known {
|
||||
f.value += value
|
||||
f.known = true
|
||||
} else {
|
||||
f.complete = false
|
||||
}
|
||||
}
|
||||
|
||||
func (f usageField) snapshot(attempts uint64) domain.ObservedTokenCount {
|
||||
if attempts == 0 {
|
||||
return domain.ObservedTokenCount{Known: true, Complete: true}
|
||||
}
|
||||
return domain.ObservedTokenCount{Value: f.value, Known: f.known, Complete: f.complete}
|
||||
}
|
||||
|
||||
func observedUsage(usage domain.TokenUsage, presence *domain.TokenUsagePresence) domain.ObservedTokenUsage {
|
||||
known := usagePresence(usage, presence)
|
||||
return domain.ObservedTokenUsage{
|
||||
PromptTokens: observedCount(usage.PromptTokens, known.PromptTokens),
|
||||
CompletionTokens: observedCount(usage.CompletionTokens, known.CompletionTokens),
|
||||
TotalTokens: observedCount(usage.TotalTokens, known.TotalTokens),
|
||||
CachedTokens: observedCount(usage.CachedTokens, known.CachedTokens),
|
||||
CacheWriteTokens: observedCount(usage.CacheWriteTokens, known.CacheWriteTokens),
|
||||
}
|
||||
}
|
||||
|
||||
func usagePresence(usage domain.TokenUsage, presence *domain.TokenUsagePresence) domain.TokenUsagePresence {
|
||||
if presence != nil {
|
||||
return *presence
|
||||
}
|
||||
return domain.TokenUsagePresence{
|
||||
PromptTokens: usage.PromptTokens != 0, CompletionTokens: usage.CompletionTokens != 0,
|
||||
TotalTokens: usage.TotalTokens != 0, CachedTokens: usage.CachedTokens != 0,
|
||||
CacheWriteTokens: usage.CacheWriteTokens != 0,
|
||||
}
|
||||
}
|
||||
|
||||
func observedCount(value int, known bool) domain.ObservedTokenCount {
|
||||
if !known {
|
||||
return domain.ObservedTokenCount{}
|
||||
}
|
||||
return domain.ObservedTokenCount{Value: value, Known: true, Complete: true}
|
||||
}
|
||||
|
||||
func addDuration(total, next time.Duration) time.Duration {
|
||||
if next > 0 && total > time.Duration(math.MaxInt64)-next {
|
||||
return time.Duration(math.MaxInt64)
|
||||
}
|
||||
return total + next
|
||||
}
|
||||
159
internal/observation/recorder_test.go
Normal file
159
internal/observation/recorder_test.go
Normal file
@@ -0,0 +1,159 @@
|
||||
package observation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestExecutionBoundaryPreservesCallerContext(t *testing.T) {
|
||||
type callerKey struct{}
|
||||
parent, stop := context.WithTimeout(context.WithValue(context.Background(), callerKey{}, "caller"), time.Minute)
|
||||
defer stop()
|
||||
parent, cancel := context.WithCancelCause(parent)
|
||||
defer cancel(nil)
|
||||
recorder := newRecorder("outer", time.Now, nil)
|
||||
sink := recorder
|
||||
parent = WithClaimSink(WithRecorder(parent, recorder), &sink)
|
||||
child := WithoutExecutionState(parent)
|
||||
if FromContext(child) != nil || FromContext(parent) != recorder {
|
||||
t.Fatal("execution boundary did not isolate recorder state")
|
||||
}
|
||||
RegisterClaim(child, nil)
|
||||
if sink != recorder {
|
||||
t.Fatal("execution boundary retained the parent claim sink")
|
||||
}
|
||||
wantDeadline, _ := parent.Deadline()
|
||||
gotDeadline, ok := child.Deadline()
|
||||
if !ok || gotDeadline != wantDeadline || child.Value(callerKey{}) != "caller" || child.Done() != parent.Done() {
|
||||
t.Fatal("execution boundary changed caller context")
|
||||
}
|
||||
cause := errors.New("caller canceled")
|
||||
cancel(cause)
|
||||
if !errors.Is(child.Err(), context.Canceled) || context.Cause(child) != cause {
|
||||
t.Fatal("execution boundary lost cancellation or its cause")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecorderAccountsDispatchedAttemptsAndReportedUsage(t *testing.T) {
|
||||
start := time.Date(2026, 9, 23, 12, 0, 0, 0, time.UTC)
|
||||
tick := 0
|
||||
now := func() time.Time {
|
||||
value := start.Add(time.Duration(tick) * time.Second)
|
||||
tick++
|
||||
return value
|
||||
}
|
||||
var events []domain.ExecutionEvent
|
||||
recorder := newRecorder("run-1", now, func(event domain.ExecutionEvent) { events = append(events, event) })
|
||||
recorder.Start()
|
||||
recorder.UpdateSelection(Selection{PromptID: "prompt", PromptVersion: "v1", ProfileID: "profile", BackendID: "backend", Model: "model"})
|
||||
recorder.Attempt(AttemptData{
|
||||
Generation: 1, Attempt: 1, Dispatched: true,
|
||||
CapacityWait: 2 * time.Millisecond, GenerationDuration: 10 * time.Millisecond,
|
||||
Outcome: domain.ExecutionOutcomeSuccess,
|
||||
UsagePresence: &domain.TokenUsagePresence{PromptTokens: true},
|
||||
})
|
||||
recorder.Delay(DelayData{Generation: 1, Attempt: 1, Elapsed: 7 * time.Millisecond})
|
||||
recorder.Attempt(AttemptData{
|
||||
Generation: 1, Attempt: 2, Dispatched: true,
|
||||
CapacityWait: 3 * time.Millisecond, GenerationDuration: 20 * time.Millisecond,
|
||||
Outcome: domain.ExecutionOutcomeSuccess,
|
||||
Usage: domain.TokenUsage{PromptTokens: 2, TotalTokens: 5},
|
||||
UsagePresence: &domain.TokenUsagePresence{PromptTokens: true, TotalTokens: true},
|
||||
})
|
||||
recorder.Attempt(AttemptData{
|
||||
Generation: 2, Attempt: 1, Dispatched: false,
|
||||
CapacityWait: 4 * time.Millisecond, GenerationDuration: 99 * time.Millisecond,
|
||||
Outcome: domain.ExecutionOutcomeError, Failure: domain.ExecutionFailureCanceled,
|
||||
Usage: domain.TokenUsage{PromptTokens: 99},
|
||||
})
|
||||
recorder.Attempt(AttemptData{
|
||||
Generation: 2, Attempt: 1, Dispatched: true,
|
||||
CapacityWait: 5 * time.Millisecond, GenerationDuration: 30 * time.Millisecond,
|
||||
Outcome: domain.ExecutionOutcomeError, Failure: domain.ExecutionFailureProviderStatus,
|
||||
StatusCode: 503, Usage: domain.TokenUsage{PromptTokens: 3},
|
||||
})
|
||||
recorder.Finish(FinishData{Outcome: domain.ExecutionOutcomeError, Failure: domain.ExecutionFailureProviderStatus, StatusCode: 503, RetriesExhausted: true})
|
||||
recorder.Finish(FinishData{})
|
||||
|
||||
if len(events) != 7 {
|
||||
t.Fatalf("event count=%d", len(events))
|
||||
}
|
||||
if events[0].PromptID != "" || events[1].PromptID != "prompt" || events[0].RunID != "run-1" || events[1].RunID != "run-1" {
|
||||
t.Fatalf("identity snapshots=%+v, %+v", events[0], events[1])
|
||||
}
|
||||
if events[1].Usage.PromptTokens != (domain.ObservedTokenCount{Known: true, Complete: true}) ||
|
||||
events[4].Dispatched || events[4].GenerationDuration != 0 || events[4].Usage.PromptTokens.Known {
|
||||
t.Fatalf("attempt usage or undispatched event=%+v, %+v", events[1], events[4])
|
||||
}
|
||||
last := events[len(events)-1]
|
||||
if last.Attempts != 3 || last.TransientRetries != 1 || last.RepairCalls != 1 ||
|
||||
last.CapacityWait != 14*time.Millisecond || last.GenerationDuration != 60*time.Millisecond ||
|
||||
last.RetryDelay != 7*time.Millisecond || last.Duration != 7*time.Second {
|
||||
t.Fatalf("terminal counters=%+v", last)
|
||||
}
|
||||
if last.Usage.PromptTokens != (domain.ObservedTokenCount{Value: 5, Known: true, Complete: true}) ||
|
||||
last.Usage.TotalTokens != (domain.ObservedTokenCount{Value: 5, Known: true, Complete: false}) ||
|
||||
last.Usage.CachedTokens != (domain.ObservedTokenCount{}) {
|
||||
t.Fatalf("aggregate usage=%+v", last.Usage)
|
||||
}
|
||||
if last.Failure != domain.ExecutionFailureProviderStatus || !last.RetriesExhausted || last.StatusCode != 503 {
|
||||
t.Fatalf("terminal failure=%+v", last)
|
||||
}
|
||||
if events[1].Attempts != 0 || events[1].TransientRetries != 0 || events[1].RepairCalls != 0 {
|
||||
t.Fatalf("attempt event carried terminal totals=%+v", events[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecorderZeroAttemptsAndExplicitUnknownUsage(t *testing.T) {
|
||||
var events []domain.ExecutionEvent
|
||||
recorder := newRecorder("", time.Now, func(event domain.ExecutionEvent) { events = append(events, event) })
|
||||
recorder.Finish(FinishData{Outcome: domain.ExecutionOutcomeError, Failure: domain.ExecutionFailureOther})
|
||||
if len(events) != 1 || events[0].RunID != "" || events[0].Attempts != 0 ||
|
||||
events[0].Usage.PromptTokens != (domain.ObservedTokenCount{Known: true, Complete: true}) ||
|
||||
events[0].Usage.CacheWriteTokens != (domain.ObservedTokenCount{Known: true, Complete: true}) {
|
||||
t.Fatalf("zero-attempt terminal=%+v", events)
|
||||
}
|
||||
|
||||
recorder = newRecorder("run-2", time.Now, func(event domain.ExecutionEvent) { events = append(events, event) })
|
||||
recorder.Attempt(AttemptData{Dispatched: true, Generation: 1, Attempt: 1,
|
||||
Usage: domain.TokenUsage{PromptTokens: 7}, UsagePresence: &domain.TokenUsagePresence{}})
|
||||
recorder.Finish(FinishData{Outcome: domain.ExecutionOutcomeSuccess})
|
||||
last := events[len(events)-1]
|
||||
if last.Usage.PromptTokens != (domain.ObservedTokenCount{}) {
|
||||
t.Fatalf("explicitly unknown nonzero field=%+v", last.Usage.PromptTokens)
|
||||
}
|
||||
_ = newRecorder("run-3", time.Now, nil) // A nil callback still permits recording.
|
||||
}
|
||||
|
||||
func TestRecorderOrderContextAndCallbackPanic(t *testing.T) {
|
||||
var kinds []domain.ExecutionEventKind
|
||||
recorder := newRecorder("run", time.Now, func(event domain.ExecutionEvent) { kinds = append(kinds, event.Kind) })
|
||||
ctx := WithRecorder(context.Background(), recorder)
|
||||
if FromContext(ctx) != recorder || FromContext(context.Background()) != nil {
|
||||
t.Fatal("recorder context propagation failed")
|
||||
}
|
||||
recorder.Start()
|
||||
recorder.Attempt(AttemptData{Generation: 1, Attempt: 1})
|
||||
recorder.Delay(DelayData{Generation: 1, Attempt: 1})
|
||||
recorder.Finish(FinishData{Outcome: domain.ExecutionOutcomeSuccess})
|
||||
if !reflect.DeepEqual(kinds, []domain.ExecutionEventKind{
|
||||
domain.ExecutionStarted, domain.ExecutionAttempt, domain.ExecutionRetryDelay, domain.ExecutionFinished,
|
||||
}) {
|
||||
t.Fatalf("callback order=%v", kinds)
|
||||
}
|
||||
|
||||
recorder = newRecorder("panic", time.Now, func(domain.ExecutionEvent) { panic("observer panic") })
|
||||
func() {
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Fatal("observer panic was swallowed")
|
||||
}
|
||||
}()
|
||||
recorder.Start()
|
||||
}()
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
id: aion-2
|
||||
backend: openrouter
|
||||
model: aion-labs/aion-2.0
|
||||
temperature: 0.72
|
||||
reasoning_effort: high
|
||||
top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: claude-fable-latest
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-fable-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 600
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: claude-haiku-latest
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-haiku-latest"
|
||||
reasoning_effort: medium
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: claude-opus-latest
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-opus-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: claude-sonnet-latest
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-sonnet-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: deepseek-3-2
|
||||
backend: openrouter
|
||||
model: deepseek/deepseek-v3.2
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: deepseek-4-flash
|
||||
backend: openrouter
|
||||
model: deepseek/deepseek-v4-flash
|
||||
#reasoning_effort: medium
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: deepseek-4-pro
|
||||
backend: openrouter
|
||||
model: deepseek/deepseek-v4-pro
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemini-2-flash-lite
|
||||
backend: openrouter
|
||||
model: "google/gemini-2.5-flash-lite"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemini-2-flash
|
||||
backend: openrouter
|
||||
model: "google/gemini-2.5-flash"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemini-2-pro
|
||||
backend: openrouter
|
||||
model: "google/gemini-2.5-pro"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemini-3-flash-lite
|
||||
backend: openrouter
|
||||
model: "google/gemini-3.1-flash-lite"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemini-flash-latest
|
||||
backend: openrouter
|
||||
model: "~google/gemini-flash-latest"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemini-pro-latest
|
||||
backend: openrouter
|
||||
model: "~google/gemini-pro-latest"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemma-4-31b
|
||||
backend: openrouter
|
||||
model: google/gemma-4-31b-it:exacto
|
||||
temperature: 0.15
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,3 +0,0 @@
|
||||
id: rakestrawhome-gemma-4-31b
|
||||
backend: rakestrawhome
|
||||
model: google/gemma-4-31b-it
|
||||
@@ -1,8 +0,0 @@
|
||||
id: minimax-m2
|
||||
backend: openrouter
|
||||
model: minimax/minimax-m2.5
|
||||
temperature: 0.5
|
||||
reasoning_effort: high
|
||||
top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: minimax-m3
|
||||
backend: openrouter
|
||||
model: minimax/minimax-m3
|
||||
#temperature: 0.5
|
||||
reasoning_effort: high
|
||||
#top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: mistral-large-2512
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-large-2512
|
||||
temperature: 0.15
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
@@ -1,7 +0,0 @@
|
||||
id: mistral-medium-3-5
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-medium-3-5
|
||||
temperature: 0.15
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
@@ -1,6 +0,0 @@
|
||||
id: mistral-small-3
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-small-3.2-24b-instruct
|
||||
temperature: 0.05
|
||||
top_p: 1.0
|
||||
timeout_seconds: 180
|
||||
@@ -1,7 +0,0 @@
|
||||
id: mistral-small-4
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-small-2603
|
||||
temperature: 0.1
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
@@ -1,6 +0,0 @@
|
||||
id: nemotron-3-ultra
|
||||
backend: openrouter
|
||||
model: nvidia/nemotron-3-ultra-550b-a55b
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: gpt-5-mini
|
||||
backend: openrouter
|
||||
model: "openai/gpt-5.4-mini"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: gpt-5-nano
|
||||
backend: openrouter
|
||||
model: "openai/gpt-5.4-nano"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,16 +0,0 @@
|
||||
package builtin
|
||||
|
||||
import (
|
||||
"embed"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
|
||||
const assetRoot = "assets"
|
||||
|
||||
//go:embed assets/**/*.yml
|
||||
var assets embed.FS
|
||||
|
||||
func NewRepository() profile.Repository {
|
||||
return profile.NewFSRepository(assets, assetRoot)
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
package builtin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
|
||||
repo := NewRepository()
|
||||
ids := loadBuiltInProfileIDs(t)
|
||||
if len(ids) == 0 {
|
||||
t.Fatal("expected built-in profiles")
|
||||
}
|
||||
|
||||
for id := range ids {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
p, err := repo.GetProfile(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("expected built-in profile %q to load, got %v", id, err)
|
||||
}
|
||||
if p.ID != id {
|
||||
t.Fatalf("expected profile id %q, got %q", id, p.ID)
|
||||
}
|
||||
if !builtInBackendIDs[p.BackendID] {
|
||||
t.Fatalf("expected profile %q to select a maintained built-in, got %q", id, p.BackendID)
|
||||
}
|
||||
if p.Endpoint != "" || p.APIKeyEnv != "" {
|
||||
t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltInProfilesDoNotContainDuplicateIDsOrRawAPIKeys(t *testing.T) {
|
||||
loadBuiltInProfileIDs(t)
|
||||
}
|
||||
|
||||
func TestRakestrawhomeGemmaProfileUsesNativeDefaults(t *testing.T) {
|
||||
p, err := NewRepository().GetProfile(context.Background(), "rakestrawhome-gemma-4-31b")
|
||||
if err != nil {
|
||||
t.Fatalf("load Rakestrawhome Gemma profile: %v", err)
|
||||
}
|
||||
if p.ID != "rakestrawhome-gemma-4-31b" ||
|
||||
p.BackendID != backend.RakestrawHomeID ||
|
||||
p.Model != "google/gemma-4-31b-it" ||
|
||||
p.Endpoint != "" ||
|
||||
p.Temperature != 0 ||
|
||||
p.MaxTokens != 0 ||
|
||||
p.TopP != 0 ||
|
||||
p.TimeoutSeconds != 0 ||
|
||||
p.ServiceTier != "" ||
|
||||
p.ReasoningEffort != "" ||
|
||||
p.APIKeyEnv != "" ||
|
||||
p.APIKeyRequired ||
|
||||
p.ExtraParams != nil {
|
||||
t.Fatalf("unexpected Rakestrawhome Gemma profile: %#v", p)
|
||||
}
|
||||
}
|
||||
|
||||
var builtInBackendIDs = map[string]bool{
|
||||
backend.OpenRouterID: true,
|
||||
backend.RakestrawHomeID: true,
|
||||
}
|
||||
|
||||
func loadBuiltInProfileIDs(t *testing.T) map[string]string {
|
||||
t.Helper()
|
||||
|
||||
ids := map[string]string{}
|
||||
err := fs.WalkDir(assets, assetRoot, func(name string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() || !strings.HasSuffix(name, ".yml") {
|
||||
return nil
|
||||
}
|
||||
|
||||
data, err := assets.ReadFile(name)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read built-in profile %s: %v", name, err)
|
||||
}
|
||||
|
||||
var raw map[string]any
|
||||
if err := yaml.Unmarshal(data, &raw); err != nil {
|
||||
t.Fatalf("failed to decode built-in profile %s: %v", name, err)
|
||||
}
|
||||
if _, ok := raw["api_key"]; ok {
|
||||
t.Fatalf("built-in profile %s contains raw api_key", name)
|
||||
}
|
||||
backendID, ok := raw["backend"].(string)
|
||||
if !ok || !builtInBackendIDs[backendID] {
|
||||
t.Fatalf("built-in profile %s does not select a maintained built-in: %#v", name, raw["backend"])
|
||||
}
|
||||
if _, ok := raw["endpoint"]; ok {
|
||||
t.Fatalf("built-in profile %s repeats endpoint", name)
|
||||
}
|
||||
if _, ok := raw["api_key_env"]; ok {
|
||||
t.Fatalf("built-in profile %s repeats api_key_env", name)
|
||||
}
|
||||
id, ok := raw["id"].(string)
|
||||
if !ok || strings.TrimSpace(id) == "" {
|
||||
t.Fatalf("built-in profile %s has missing id", name)
|
||||
}
|
||||
if previous, ok := ids[id]; ok {
|
||||
t.Fatalf("duplicate built-in profile id %q in %s and %s", id, previous, name)
|
||||
}
|
||||
ids[id] = name
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to walk built-in profiles: %v", err)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user