30 Commits

Author SHA1 Message Date
e361c97bb5 Document the v0.2.0 upgrade path 2026-07-29 23:22:38 +00:00
be67707582 Keep capacity cancellation tests independent of implementation details 2026-07-29 22:41:12 +00:00
e61ab700c7 Document backend capacity management 2026-07-29 21:16:49 +00:00
d2c4051dd0 Wire backend capacity into engine runtime 2026-07-29 21:11:08 +00:00
861da355d8 Add early backend run admission 2026-07-29 21:01:37 +00:00
a752f88166 Add backend capacity manager 2026-07-29 20:55:46 +00:00
238fa90bfa Add backend capacity policy configuration 2026-07-29 20:48:10 +00:00
ffe6d261a9 Add roadmap and implementation plan for backend concurrency scaling 2026-07-29 20:42:45 +00:00
dc39562ff7 Keep session IDs on repair requests and retire completed plans 2026-07-29 20:25:00 +00:00
0a839aa16d Document per-run session and reasoning overrides 2026-07-29 19:47:39 +00:00
f6ee18f6b3 Add direct per-run session overrides 2026-07-29 19:43:26 +00:00
eb8ab215e8 Add tri-state reasoning overrides 2026-07-29 19:36:48 +00:00
f89cb94ed2 Address backend implementation review findings 2026-07-29 19:08:05 +00:00
359b7313f4 Harden backend contracts and documentation 2026-07-29 17:22:55 +00:00
ae210b3c26 Add engine-scoped backend registration 2026-07-29 17:15:51 +00:00
810f80e7c9 Resolve profiles through built-in backends 2026-07-29 17:10:34 +00:00
8d00354c59 Add immutable backend registry foundation 2026-07-29 16:59:43 +00:00
d0010689f3 Prepare the backend registry implementation roadmap 2026-07-29 16:50:36 +00:00
b462153483 Document consumer safety and offline execution 2026-07-29 14:13:52 +00:00
086cf0fc86 Make GoDoc the public API contract 2026-07-29 14:06:54 +00:00
c1cecb1ee8 Resolve public documentation contract questions 2026-07-29 13:53:33 +00:00
bcb327f643 Document the published Promptkit release process 2026-07-28 19:31:32 +00:00
9e68a2bbf7 Add framework documentation and offline example 2026-07-28 04:53:51 +00:00
e4899fb54d Publish the Promptkit engine facade 2026-07-28 04:43:19 +00:00
18b12a25c1 Add internal framework orchestration 2026-07-28 04:33:30 +00:00
7e94ab133b Add OpenAI-compatible model client 2026-07-28 04:29:17 +00:00
62b26fb29e Add artifact reading and output validation 2026-07-28 04:23:50 +00:00
ebc1f3e919 Extract prompt sources and rendering internals 2026-07-28 04:18:09 +00:00
ad1f2674ab Extract Promptkit framework foundation 2026-07-28 04:14:28 +00:00
0d56d986af Bugfix in the release documentation 2026-07-27 22:51:37 -05:00
142 changed files with 21238 additions and 102 deletions

View File

@@ -1,18 +1,44 @@
# Promptkit
Promptkit is the reusable Go prompt-execution framework being separated from
Scriptorium. Its module path is:
Promptkit is a reusable Go library for preparing and executing prompt-defined
LLM workflows. Its module path is:
```text
gitea.maximumdirect.net/eric/promptkit
```
The repository currently provides the independent Go module and its root public
package boundary. Framework behavior and consumer APIs have not yet been
extracted, so there is no installation or usage example at this time.
The root `promptkit` package provides the supported public engine. Consumers
can configure filesystem or in-memory prompt, profile, and schema sources,
prepare requests without generation, run requests with the built-in
OpenAI-compatible client, or inject their own model client and artifact reader.
## Quickstart
Run the maintained offline preparation example from the repository root:
```sh
go run ./examples/go-library/prepare
```
It loads a repository-local prompt, supplies an in-memory profile and inline
input, and prints deterministic preparation metadata without contacting a
provider or requiring credentials. Read the
[example source](examples/go-library/prepare/main.go), the
[Go package consumer guide](docs/consumers/pkg-promptkit.md), and the
[framework format reference](docs/formats.md) to build a consumer workflow.
Contributors should start with the [development guide](docs/development.md).
The [architecture policy](docs/policy/architecture.md) defines the library
boundary and constraints that future framework work must preserve.
boundary and constraints that framework work must preserve.
## Release Guidance
Consumers moving from `v0.1.0` to `v0.2.0` should read the
[v0.2.0 changelog and migration guide](docs/releases/v0.2.0.md).
## Related Project
[Scriptorium](https://gitea.maximumdirect.net/eric/scriptorium) is the CLI and
HTTP application built on Promptkit.
Promptkit is licensed under the [GNU General Public License version 3](LICENSE).

89
architecture_test.go Normal file
View File

@@ -0,0 +1,89 @@
package promptkit_test
import (
"go/ast"
"go/parser"
"go/token"
"io/fs"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
)
const formerModulePath = "gitea.maximumdirect.net/eric/" + "scrip" + "torium"
func TestRepositoryDoesNotImportFormerModule(t *testing.T) {
violations, err := findFormerModuleImports(".")
if err != nil {
t.Fatalf("inspect repository imports: %v", err)
}
if len(violations) > 0 {
t.Fatalf("repository imports the former module:\n%s", strings.Join(violations, "\n"))
}
}
func TestFormerModuleGuardFindsNestedImport(t *testing.T) {
root := t.TempDir()
nested := filepath.Join(root, "nested", "package")
if err := os.MkdirAll(nested, 0o755); err != nil {
t.Fatalf("create nested package: %v", err)
}
sourcePath := filepath.Join(nested, "violation.go")
source := "package nested\n\nimport _ " + strconv.Quote(formerModulePath+"/internal/domain") + "\n"
if err := os.WriteFile(sourcePath, []byte(source), 0o600); err != nil {
t.Fatalf("write nested source: %v", err)
}
violations, err := findFormerModuleImports(root)
if err != nil {
t.Fatalf("inspect nested imports: %v", err)
}
if len(violations) != 1 {
t.Fatalf("violations = %v, want one nested import", violations)
}
if !strings.Contains(violations[0], "violation.go") ||
!strings.Contains(violations[0], formerModulePath+"/internal/domain") {
t.Fatalf("violation = %q, want file and import path", violations[0])
}
}
func findFormerModuleImports(root string) ([]string, error) {
var violations []string
err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if entry.IsDir() {
switch entry.Name() {
case ".git", "generated", "vendor":
return filepath.SkipDir
}
return nil
}
if filepath.Ext(path) != ".go" {
return nil
}
file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.ImportsOnly|parser.ParseComments)
if err != nil {
return err
}
if ast.IsGenerated(file) {
return nil
}
for _, spec := range file.Imports {
importPath, err := strconv.Unquote(spec.Path.Value)
if err != nil {
return err
}
if importPath == formerModulePath || strings.HasPrefix(importPath, formerModulePath+"/") {
violations = append(violations, path+": "+importPath)
}
}
return nil
})
return violations, err
}

39
artifact_reader.go Normal file
View File

@@ -0,0 +1,39 @@
package promptkit
import (
"context"
"errors"
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
var errNilArtifactReaderResponse = errors.New("artifact reader returned nil artifact without error")
type publicArtifactReaderAdapter struct {
reader ArtifactReader
}
var _ artifactadapter.Reader = publicArtifactReaderAdapter{}
func (a publicArtifactReaderAdapter) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
artifact, err := a.reader.Read(ctx, ArtifactRef{
Type: ArtifactRefType(ref.Type),
URI: ref.URI,
Body: ref.Body,
})
if err != nil {
return nil, err
}
if artifact == nil {
return nil, errNilArtifactReaderResponse
}
return &domain.Artifact{
Name: artifact.Name,
ContentType: artifact.ContentType,
Body: copyBytes(artifact.Body),
URI: artifact.URI,
Size: artifact.Size,
Hash: artifact.Hash,
}, nil
}

View File

@@ -0,0 +1,36 @@
package promptkit
import (
"context"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestPublicArtifactReaderAdapterCopiesBody(t *testing.T) {
reader := internalArtifactReaderFake{
artifact: &Artifact{Body: []byte("original")},
}
adapter := publicArtifactReaderAdapter{reader: &reader}
artifact, err := adapter.Read(context.Background(), domain.ArtifactRef{
Type: domain.ArtifactRefInline,
URI: "memory://input",
Body: "input",
})
if err != nil {
t.Fatalf("read artifact: %v", err)
}
artifact.Body[0] = 'X'
if got := string(reader.artifact.Body); got != "original" {
t.Fatalf("reader artifact body was mutated: %q", got)
}
}
type internalArtifactReaderFake struct {
artifact *Artifact
}
func (r *internalArtifactReaderFake) Read(context.Context, ArtifactRef) (*Artifact, error) {
return r.artifact, nil
}

74
backends.go Normal file
View File

@@ -0,0 +1,74 @@
package promptkit
import (
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
// BackendOpenRouter is the reserved ID of Promptkit's built-in OpenRouter
// backend.
const BackendOpenRouter = backend.OpenRouterID
// Backend configures one engine-scoped OpenAI-compatible backend.
//
// Backend has no stable JSON representation. Use keyed literals so additions
// to this configuration value do not break source compatibility.
type Backend struct {
// ID is the stable, case-sensitive registry key. NewEngine trims it and
// requires a non-blank value. BackendOpenRouter is reserved.
ID string
// Endpoint is the OpenAI-compatible base endpoint. NewEngine trims it and
// requires an absolute HTTP or HTTPS URL with a host and without user
// information, a query string, or a fragment. Paths are allowed.
Endpoint string
// APIKeyEnv optionally names the environment variable containing the API
// key. NewEngine trims it and requires the portable form
// [A-Za-z_][A-Za-z0-9_]*. Store only the name, never a credential value.
APIKeyEnv string
// ExtraParams contains backend-wide request defaults. Values must be
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
// must not be model, session_id, messages, temperature, max_tokens, top_p,
// service_tier, reasoning_effort, or response_format. An empty map supplies
// no defaults. NewEngine deeply copies the map.
ExtraParams map[string]any
// ConcurrencyLimit is the maximum number of simultaneous model-generation
// calls allowed for this backend within one Engine. Zero leaves the backend
// unlimited. A negative value makes NewEngine fail with ErrInvalidConfig.
ConcurrencyLimit int
// QueueCapacity controls how many additional Run calls may be admitted
// beyond ConcurrencyLimit. Nil uses 1024 when ConcurrencyLimit is positive;
// a pointer uses its exact value, including zero. The pointed-to value must
// be non-negative, and QueueCapacity must be nil when ConcurrencyLimit is
// zero. Their sum must fit in an int. WithBackend copies the value and does
// not retain the pointer.
QueueCapacity *int
}
// WithBackend adds one Backend registration to the constructed Engine.
//
// Registrations accumulate in option order. Every normalized ID must be unique
// across consumer registrations and built-ins; a duplicate or invalid
// definition makes NewEngine fail with ErrInvalidConfig. In particular,
// BackendOpenRouter cannot be replaced. The immutable registration is scoped
// to the resulting Engine and cannot be enumerated, replaced, removed, or
// mutated after construction. WithBackend does not install package-global
// state.
func WithBackend(backend Backend) Option {
queueCapacity := 0
queueCapacitySet := backend.QueueCapacity != nil
if queueCapacitySet {
queueCapacity = *backend.QueueCapacity
}
return optionFunc(func(options *engineOptions) error {
options.backends = append(options.backends, domain.Backend{
ID: backend.ID,
Endpoint: backend.Endpoint,
APIKeyEnv: backend.APIKeyEnv,
ExtraParams: backend.ExtraParams,
ConcurrencyLimit: backend.ConcurrencyLimit,
QueueCapacity: queueCapacity,
QueueCapacitySet: queueCapacitySet,
})
return nil
})
}

369
capacity_contract_test.go Normal file
View File

@@ -0,0 +1,369 @@
package promptkit_test
import (
"context"
"errors"
"sync"
"testing"
"time"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestEngineLimitsInjectedClientConcurrency(t *testing.T) {
release := make(chan struct{})
client := newCapacityGateClient(release, 8)
engine := newBackendCapacityEngine(t, client, 2, capacityInt(4), nil)
results := make(chan capacityRunResult, 6)
for i := 0; i < 6; i++ {
go runCapacityRequest(engine, context.Background(), promptkit.RunRequest{
PromptID: "prompt",
Execution: &promptkit.ExecutionTargetOverride{
Endpoint: "http://request.example/v1",
},
}, results)
}
first := awaitCapacityRequest(t, client.started)
second := awaitCapacityRequest(t, client.started)
if first.Target.BackendID != "limited" || second.Target.BackendID != "limited" {
t.Fatalf("endpoint override changed backend pool: first=%q second=%q",
first.Target.BackendID, second.Target.BackendID)
}
if active, peak, _ := client.snapshot(); active != 2 || peak != 2 {
t.Fatalf("client concurrency before release=(active=%d peak=%d), want 2", active, peak)
}
close(release)
for i := 0; i < 6; i++ {
outcome := awaitCapacityRun(t, results)
if outcome.err != nil || outcome.result == nil {
t.Fatalf("run outcome=(%+v, %v), want success", outcome.result, outcome.err)
}
}
if _, peak, calls := client.snapshot(); peak > 2 || calls != 6 {
t.Fatalf("client observations=(peak=%d calls=%d), want peak <= 2 and 6 calls", peak, calls)
}
}
func TestEngineRejectsRunBeforeCompletionWhenAdmissionIsFull(t *testing.T) {
artifactRelease := make(chan struct{})
reader := &capacityArtifactReader{
entered: make(chan struct{}, 2),
release: artifactRelease,
}
client := newCapacityGateClient(closedCapacityChannel(), 2)
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), reader)
firstResult := make(chan capacityRunResult, 1)
go runCapacityRequest(engine, context.Background(), capacityInputRequest("http://first.example/v1"), firstResult)
awaitCapacitySignal(t, reader.entered, "first artifact read")
result, err := engine.Run(context.Background(), capacityInputRequest("http://second.example/v1"))
if result != nil {
t.Fatalf("capacity rejection returned partial result: %+v", result)
}
if !errors.Is(err, promptkit.ErrCapacityExceeded) {
t.Fatalf("capacity rejection=%v, want ErrCapacityExceeded", err)
}
if errors.Is(err, promptkit.ErrInvalidRequest) || errors.Is(err, promptkit.ErrLLMGenerate) {
t.Fatalf("capacity rejection had an unrelated category: %v", err)
}
if calls := reader.callCount(); calls != 1 {
t.Fatalf("artifact calls=%d, want only the admitted run", calls)
}
if _, _, calls := client.snapshot(); calls != 0 {
t.Fatalf("client calls=%d before admitted run was released, want 0", calls)
}
close(artifactRelease)
outcome := awaitCapacityRun(t, firstResult)
if outcome.err != nil || outcome.result == nil {
t.Fatalf("first run outcome=(%+v, %v), want success", outcome.result, outcome.err)
}
}
func TestBackendCapacityIsIndependentBetweenEngines(t *testing.T) {
firstRelease := make(chan struct{})
firstClient := newCapacityGateClient(firstRelease, 1)
firstEngine := newBackendCapacityEngine(t, firstClient, 1, capacityInt(0), nil)
secondClient := newCapacityGateClient(closedCapacityChannel(), 1)
secondEngine := newBackendCapacityEngine(t, secondClient, 1, capacityInt(0), nil)
firstResult := make(chan capacityRunResult, 1)
go runCapacityRequest(firstEngine, context.Background(), promptkit.RunRequest{PromptID: "prompt"}, firstResult)
awaitCapacityRequest(t, firstClient.started)
result, err := secondEngine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil || result == nil {
t.Fatalf("second engine run=(%+v, %v), want independent success", result, err)
}
if _, _, calls := secondClient.snapshot(); calls != 1 {
t.Fatalf("second engine client calls=%d, want 1", calls)
}
close(firstRelease)
outcome := awaitCapacityRun(t, firstResult)
if outcome.err != nil || outcome.result == nil {
t.Fatalf("first engine run=(%+v, %v), want success", outcome.result, outcome.err)
}
}
func TestUnlimitedBackendsRetainInjectedClientConcurrency(t *testing.T) {
tests := []struct {
name string
configure func(*testing.T, promptkit.LLMClient) *promptkit.Engine
}{
{
name: "custom backend",
configure: func(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
return newBackendCapacityEngine(t, client, 0, nil, nil)
},
},
{
name: "endpoint-only profile",
configure: func(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithProfiles(promptkit.Profile{
ID: "profile", Endpoint: "http://endpoint.example/v1", Model: "model",
}),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct endpoint-only engine: %v", err)
}
return engine
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
release := make(chan struct{})
client := newCapacityGateClient(release, 2)
engine := tc.configure(t, client)
results := make(chan capacityRunResult, 2)
for i := 0; i < 2; i++ {
go runCapacityRequest(
engine,
context.Background(),
promptkit.RunRequest{PromptID: "prompt"},
results,
)
}
awaitCapacityRequest(t, client.started)
awaitCapacityRequest(t, client.started)
if active, peak, _ := client.snapshot(); active != 2 || peak != 2 {
t.Fatalf("unlimited concurrency=(active=%d peak=%d), want 2", active, peak)
}
close(release)
for i := 0; i < 2; i++ {
outcome := awaitCapacityRun(t, results)
if outcome.err != nil || outcome.result == nil {
t.Fatalf("run outcome=(%+v, %v), want success", outcome.result, outcome.err)
}
}
})
}
}
func TestCapacityExceededSentinelContract(t *testing.T) {
if promptkit.ErrCapacityExceeded == nil {
t.Fatal("ErrCapacityExceeded is nil")
}
for _, unrelated := range []error{
promptkit.ErrInvalidConfig,
promptkit.ErrInvalidRequest,
promptkit.ErrLLMGenerate,
promptkit.ErrValidation,
} {
if errors.Is(promptkit.ErrCapacityExceeded, unrelated) ||
errors.Is(unrelated, promptkit.ErrCapacityExceeded) {
t.Fatalf("ErrCapacityExceeded aliases unrelated sentinel %v", unrelated)
}
}
}
type capacityRunResult struct {
result *promptkit.RunResult
err error
}
func runCapacityRequest(
engine *promptkit.Engine,
ctx context.Context,
request promptkit.RunRequest,
results chan<- capacityRunResult,
) {
result, err := engine.Run(ctx, request)
results <- capacityRunResult{result: result, err: err}
}
func newBackendCapacityEngine(
t *testing.T,
client promptkit.LLMClient,
limit int,
queueCapacity *int,
reader promptkit.ArtifactReader,
) *promptkit.Engine {
t.Helper()
promptFS := contractPromptFS("prompt", "profile", "message")
if reader != nil {
promptFS = contractInputPromptFS()
}
options := []promptkit.Option{
promptkit.WithPromptFS(promptFS, "."),
promptkit.WithBackend(promptkit.Backend{
ID: "limited",
Endpoint: "http://backend.example/v1",
ConcurrencyLimit: limit,
QueueCapacity: queueCapacity,
}),
promptkit.WithProfiles(promptkit.Profile{
ID: "profile", BackendID: "limited", Model: "model",
}),
promptkit.WithLLMClient(client),
}
if reader != nil {
options = append(options, promptkit.WithArtifactReader(reader))
}
engine, err := promptkit.NewEngine(promptkit.Config{}, options...)
if err != nil {
t.Fatalf("construct capacity engine: %v", err)
}
return engine
}
func capacityInputRequest(endpoint string) promptkit.RunRequest {
return promptkit.RunRequest{
PromptID: "input-prompt",
Inputs: map[string]promptkit.ArtifactRef{
"input": promptkit.Inline("input"),
},
Execution: &promptkit.ExecutionTargetOverride{Endpoint: endpoint},
}
}
type capacityGateClient struct {
mu sync.Mutex
active int
peak int
calls int
started chan promptkit.GenerateRequest
release <-chan struct{}
}
func newCapacityGateClient(release <-chan struct{}, buffer int) *capacityGateClient {
return &capacityGateClient{
started: make(chan promptkit.GenerateRequest, buffer),
release: release,
}
}
func (c *capacityGateClient) Generate(
ctx context.Context,
request promptkit.GenerateRequest,
) (*promptkit.GenerateResponse, error) {
c.mu.Lock()
c.calls++
c.active++
if c.active > c.peak {
c.peak = c.active
}
c.mu.Unlock()
defer func() {
c.mu.Lock()
c.active--
c.mu.Unlock()
}()
c.started <- request
select {
case <-c.release:
return &promptkit.GenerateResponse{Content: "ok"}, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}
func (c *capacityGateClient) snapshot() (active, peak, calls int) {
c.mu.Lock()
defer c.mu.Unlock()
return c.active, c.peak, c.calls
}
type capacityArtifactReader struct {
mu sync.Mutex
calls int
entered chan struct{}
release <-chan struct{}
}
func (r *capacityArtifactReader) Read(
ctx context.Context,
_ promptkit.ArtifactRef,
) (*promptkit.Artifact, error) {
r.mu.Lock()
r.calls++
r.mu.Unlock()
r.entered <- struct{}{}
select {
case <-r.release:
return &promptkit.Artifact{Body: []byte("input")}, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}
func (r *capacityArtifactReader) callCount() int {
r.mu.Lock()
defer r.mu.Unlock()
return r.calls
}
func awaitCapacityRequest(
t *testing.T,
requests <-chan promptkit.GenerateRequest,
) promptkit.GenerateRequest {
t.Helper()
select {
case request := <-requests:
return request
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for client invocation")
return promptkit.GenerateRequest{}
}
}
func awaitCapacityRun(t *testing.T, results <-chan capacityRunResult) capacityRunResult {
t.Helper()
select {
case result := <-results:
return result
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for Run")
return capacityRunResult{}
}
}
func awaitCapacitySignal(t *testing.T, signal <-chan struct{}, name string) {
t.Helper()
select {
case <-signal:
case <-time.After(5 * time.Second):
t.Fatalf("timed out waiting for %s", name)
}
}
func capacityInt(value int) *int {
return &value
}
func closedCapacityChannel() <-chan struct{} {
channel := make(chan struct{})
close(channel)
return channel
}

419
convert.go Normal file
View File

@@ -0,0 +1,419 @@
package promptkit
import (
"reflect"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
)
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
execution, err := toDomainExecutionTargetOverride(req.Execution)
if err != nil {
return domain.RunRequest{}, err
}
return domain.RunRequest{
PromptID: req.PromptID,
PromptVersion: req.PromptVersion,
ProfileID: req.ProfileID,
SessionID: req.SessionID,
APIKey: req.APIKey,
Inputs: toDomainArtifactRefMap(req.Inputs),
Vars: copyStringMap(req.Vars),
Execution: execution,
Validation: toDomainOutputContractPtr(req.Validation),
}, nil
}
func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
if prepared == nil {
return nil
}
return &PreparedRun{
PromptID: prepared.PromptID,
PromptVersion: prepared.PromptVersion,
PromptHash: prepared.PromptHash,
SelectedProfileID: prepared.SelectedProfileID,
SelectedBackendID: prepared.SelectedBackendID,
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
OutputContract: fromDomainOutputContract(prepared.OutputContract),
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
InputHashes: copyStringMap(prepared.InputHashes),
SessionID: prepared.SessionID,
RenderedPromptHash: prepared.RenderedPromptHash,
Messages: fromDomainRenderedMessages(prepared.Messages),
StartTime: prepared.StartTime,
EndTime: prepared.EndTime,
DurationMS: prepared.DurationMS,
}
}
func fromDomainRunResult(result *domain.RunResult) *RunResult {
if result == nil {
return nil
}
return &RunResult{
RunID: result.RunID,
Artifact: fromDomainArtifact(result.Artifact),
RawOutput: result.RawOutput,
Validation: fromDomainValidationResult(result.Validation),
PromptID: result.PromptID,
PromptVersion: result.PromptVersion,
PromptHash: result.PromptHash,
SessionID: result.SessionID,
RenderedPromptHash: result.RenderedPromptHash,
SelectedProfileID: result.SelectedProfileID,
SelectedBackendID: result.SelectedBackendID,
ModelName: result.ModelName,
Endpoint: result.Endpoint,
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
InputHashes: copyStringMap(result.InputHashes),
Usage: fromDomainTokenUsage(result.Usage),
StartTime: result.StartTime,
EndTime: result.EndTime,
Duration: result.Duration,
}
}
func fromDomainGenerateRequest(req domain.GenerateRequest) GenerateRequest {
return GenerateRequest{
Prompt: fromDomainRenderedPrompt(req.Prompt),
Target: fromDomainExecutionTarget(req.Target),
TargetPresence: fromDomainExecutionTargetPresence(req.TargetPresence),
StructuredOutput: fromDomainStructuredOutputSpec(req.StructuredOutput),
APIKey: req.Target.APIKey,
}
}
func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse {
if resp == nil {
return nil
}
return &domain.GenerateResponse{
Content: resp.Content,
Usage: toDomainTokenUsage(resp.Usage),
}
}
func fromDomainRenderedPrompt(prompt domain.RenderedPrompt) RenderedPrompt {
return RenderedPrompt{
SessionID: prompt.SessionID,
Messages: fromDomainRenderedMessages(prompt.Messages),
}
}
func toDomainArtifactRefMap(src map[string]ArtifactRef) map[string]domain.ArtifactRef {
if src == nil {
return nil
}
out := make(map[string]domain.ArtifactRef, len(src))
for k, v := range src {
out[k] = toDomainArtifactRef(v)
}
return out
}
func toDomainArtifactRef(ref ArtifactRef) domain.ArtifactRef {
return domain.ArtifactRef{
Type: domain.ArtifactRefType(ref.Type),
URI: ref.URI,
Body: ref.Body,
}
}
func fromDomainArtifact(artifact domain.Artifact) Artifact {
return Artifact{
Name: artifact.Name,
ContentType: artifact.ContentType,
Body: copyBytes(artifact.Body),
URI: artifact.URI,
Size: artifact.Size,
Hash: artifact.Hash,
}
}
func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain.ExecutionTargetOverride, error) {
if override == nil {
return nil, nil
}
extraParams, err := jsonvalue.CopyMap(override.ExtraParams)
if err != nil {
return nil, err
}
return &domain.ExecutionTargetOverride{
Endpoint: override.Endpoint,
Model: override.Model,
Temperature: copyFloat64Ptr(override.Temperature),
MaxTokens: copyIntPtr(override.MaxTokens),
TopP: copyFloat64Ptr(override.TopP),
TimeoutSeconds: copyIntPtr(override.TimeoutSeconds),
ServiceTier: override.ServiceTier,
ReasoningEffort: copyStringPtr(override.ReasoningEffort),
APIKeyEnv: override.APIKeyEnv,
ExtraParams: extraParams,
}, nil
}
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
return ExecutionTarget{
BackendID: target.BackendID,
Endpoint: target.Endpoint,
Model: target.Model,
Temperature: target.Temperature,
MaxTokens: target.MaxTokens,
TopP: target.TopP,
TimeoutSeconds: target.TimeoutSeconds,
ServiceTier: target.ServiceTier,
ReasoningEffort: target.ReasoningEffort,
APIKeyEnv: target.APIKeyEnv,
ExtraParams: copyAnyMap(target.ExtraParams),
}
}
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
return ExecutionTargetPresence{
Temperature: presence.Temperature,
MaxTokens: presence.MaxTokens,
TopP: presence.TopP,
TimeoutSeconds: presence.TimeoutSeconds,
}
}
func toDomainOutputContractPtr(contract *OutputContract) *domain.OutputContract {
if contract == nil {
return nil
}
out := toDomainOutputContract(*contract)
return &out
}
func toDomainOutputContract(contract OutputContract) domain.OutputContract {
return domain.OutputContract{
Format: domain.OutputFormat(contract.Format),
ValidationMode: domain.ValidationMode(contract.ValidationMode),
SchemaPath: contract.SchemaPath,
RepairAttempts: contract.RepairAttempts,
}
}
func fromDomainOutputContract(contract domain.OutputContract) OutputContract {
return OutputContract{
Format: OutputFormat(contract.Format),
ValidationMode: ValidationMode(contract.ValidationMode),
SchemaPath: contract.SchemaPath,
RepairAttempts: contract.RepairAttempts,
}
}
func fromDomainValidationResult(result domain.ValidationResult) ValidationResult {
return ValidationResult{
Status: ValidationStatus(result.Status),
Mode: ValidationMode(result.Mode),
Errors: copyStringSlice(result.Errors),
SchemaPath: result.SchemaPath,
RepairAttempts: result.RepairAttempts,
IsValid: result.IsValid,
}
}
func fromDomainTokenUsage(usage domain.TokenUsage) TokenUsage {
return TokenUsage{
PromptTokens: usage.PromptTokens,
CompletionTokens: usage.CompletionTokens,
TotalTokens: usage.TotalTokens,
CachedTokens: usage.CachedTokens,
CacheWriteTokens: usage.CacheWriteTokens,
}
}
func toDomainTokenUsage(usage TokenUsage) domain.TokenUsage {
return domain.TokenUsage{
PromptTokens: usage.PromptTokens,
CompletionTokens: usage.CompletionTokens,
TotalTokens: usage.TotalTokens,
CachedTokens: usage.CachedTokens,
CacheWriteTokens: usage.CacheWriteTokens,
}
}
func fromDomainRenderedMessages(messages []domain.RenderedMessage) []RenderedMessage {
if messages == nil {
return nil
}
out := make([]RenderedMessage, len(messages))
for i, msg := range messages {
out[i] = RenderedMessage{
Role: msg.Role,
Content: msg.Content,
CacheControl: fromDomainCacheControl(msg.CacheControl),
}
}
return out
}
func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl {
if cacheControl == nil {
return nil
}
return &CacheControl{
Type: CacheControlType(cacheControl.Type),
TTL: cacheControl.TTL,
}
}
func fromDomainStructuredOutputSpec(spec *domain.StructuredOutputSpec) *StructuredOutputSpec {
if spec == nil {
return nil
}
out := &StructuredOutputSpec{
Type: StructuredOutputType(spec.Type),
}
if spec.JSONSchema != nil {
out.JSONSchema = &StructuredOutputJSONSpec{
Name: spec.JSONSchema.Name,
Strict: spec.JSONSchema.Strict,
Schema: copyAny(spec.JSONSchema.Schema),
}
}
return out
}
func copyStringMap(src map[string]string) map[string]string {
if src == nil {
return nil
}
out := make(map[string]string, len(src))
for k, v := range src {
out[k] = v
}
return out
}
func copyAnyMap(src map[string]any) map[string]any {
if src == nil {
return nil
}
out := make(map[string]any, len(src))
for k, v := range src {
out[k] = copyAny(v)
}
return out
}
func copyAny(value any) any {
if value == nil {
return nil
}
switch v := value.(type) {
case map[string]any:
return copyAnyMap(v)
case []any:
out := make([]any, len(v))
for i, item := range v {
out[i] = copyAny(item)
}
return out
case []string:
return copyStringSlice(v)
case []byte:
return copyBytes(v)
default:
return copyReflectValue(reflect.ValueOf(value)).Interface()
}
}
func copyReflectValue(value reflect.Value) reflect.Value {
if !value.IsValid() {
return value
}
switch value.Kind() {
case reflect.Interface:
if value.IsNil() {
return reflect.Zero(value.Type())
}
copied := copyReflectValue(value.Elem())
if copied.IsValid() && copied.Type().AssignableTo(value.Type()) {
return copied
}
out := reflect.New(value.Type()).Elem()
out.Set(copied)
return out
case reflect.Pointer:
if value.IsNil() {
return reflect.Zero(value.Type())
}
out := reflect.New(value.Type().Elem())
out.Elem().Set(copyReflectValue(value.Elem()))
return out
case reflect.Map:
if value.IsNil() {
return reflect.Zero(value.Type())
}
out := reflect.MakeMapWithSize(value.Type(), value.Len())
iter := value.MapRange()
for iter.Next() {
out.SetMapIndex(copyReflectValue(iter.Key()), copyReflectValue(iter.Value()))
}
return out
case reflect.Slice:
if value.IsNil() {
return reflect.Zero(value.Type())
}
out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap())
for i := 0; i < value.Len(); i++ {
out.Index(i).Set(copyReflectValue(value.Index(i)))
}
return out
case reflect.Array:
out := reflect.New(value.Type()).Elem()
for i := 0; i < value.Len(); i++ {
out.Index(i).Set(copyReflectValue(value.Index(i)))
}
return out
default:
return value
}
}
func copyStringSlice(src []string) []string {
if src == nil {
return nil
}
out := make([]string, len(src))
copy(out, src)
return out
}
func copyBytes(src []byte) []byte {
if src == nil {
return nil
}
out := make([]byte, len(src))
copy(out, src)
return out
}
func copyFloat64Ptr(src *float64) *float64 {
if src == nil {
return nil
}
v := *src
return &v
}
func copyStringPtr(src *string) *string {
if src == nil {
return nil
}
v := *src
return &v
}
func copyIntPtr(src *int) *int {
if src == nil {
return nil
}
v := *src
return &v
}

57
doc.go
View File

@@ -1,2 +1,57 @@
// Package promptkit defines the public package boundary for the Promptkit Go module.
// Package promptkit provides an embeddable engine for preparing and executing
// prompt-defined LLM workflows.
//
// Applications construct an [Engine] with [NewEngine], select filesystem or
// in-memory sources and optional engine-scoped [Backend] registrations, and
// call [Engine.Prepare] or [Engine.Run]. Concrete registries, repositories,
// validators, and the built-in OpenAI-compatible client remain internal
// implementation details.
//
// # Concurrency and ownership
//
// An Engine supports concurrent Prepare and Run calls. Engine-local backend
// policies bound admitted Run calls and model generations where configured,
// while different backend pools and unlimited backends continue independently.
// An injected [LLMClient] or [ArtifactReader] can therefore still receive
// concurrent calls and must be safe for that use.
//
// NewEngine copies in-memory profiles and backend definitions. Prepare and Run
// copy request maps, slices, pointer values, and JSON-compatible extra
// parameters before using them. Returned values and values passed to extension
// interfaces are likewise isolated from engine state. Callers own those copies
// and may mutate them after the call that supplied or returned them.
//
// # Security and sensitive data
//
// The default artifact reader treats [File] paths as caller-selected operating
// system paths. It does not restrict them to an application root or impose an
// inbound request-size policy. Promptkit is not an inbound request or
// untrusted-input security boundary. Applications must validate and restrict
// untrusted input before constructing a request, or install an [ArtifactReader]
// that enforces their filesystem, authorization, and size policies.
//
// Rendered messages, input and output [Artifact] bodies, [RunResult.RawOutput],
// and [ValidationResult.Errors] may contain sensitive data. Credential
// exclusion and redaction do not sanitize those values. Applications and
// injected collaborators are responsible for access control, retention,
// logging, and secret handling appropriate to their data.
//
// # JSON
//
// Stable JSON representations are provided for [PreparedRun], [RunResult],
// [Artifact], [ExecutionTarget], [OutputContract], [ValidationResult],
// [TokenUsage], [RenderedPrompt], [RenderedMessage], [CacheControl],
// [StructuredOutputSpec], [StructuredOutputJSONSpec], [GenerateRequest],
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
// used by those values.
//
// Construction values, including [Config], [Backend], [RunRequest],
// [ArtifactRef], [ExecutionTargetOverride], [Profile], and
// [OpenAICompatibleProfileConfig], do not have stable JSON representations.
// Direct API keys are nevertheless excluded from JSON for every public value.
//
// JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
// PreparedRun and RunResult durations are encoded as integer milliseconds in
// duration_ms and omitted when zero. Run IDs and all exposed hashes are opaque:
// their spelling, length, character set, and algorithm are not API contracts.
package promptkit

View File

@@ -0,0 +1,254 @@
# Package `promptkit`
## Purpose
This guide helps Go consumers assemble Promptkit and choose the main
preparation or execution workflow. The declarations and GoDoc in the
[root package](../../doc.go) own exact field, option, serialization,
concurrency, ownership, failure, and cancellation semantics. The
[framework format reference](../formats.md) owns prompt, profile, and schema
file contracts.
Import the package as:
```go
import "gitea.maximumdirect.net/eric/promptkit"
```
The following Go fragments are illustrative and omit surrounding package,
import, and error-handling code. Use the maintained examples for complete
programs.
## Construct An Engine
Create an engine with
[`NewEngine`](../../engine.go). A directory-backed setup supplies a prompt
directory and may supply profile and schema directories:
```go
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
ProfileDir: "profiles",
SchemaDir: "schemas",
})
```
Options support single-file or `fs.FS` sources, in-memory profiles,
engine-scoped backends, and injected artifact or model clients. Consult the
[constructor and option GoDoc](../../engine.go) for composition, precedence,
validation, and default transport behavior. Source discovery, format
validation, and profile precedence are defined by the
[framework format reference](../formats.md).
## Prepare Without Model Execution
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
loads inputs and any structured-output schema, and renders messages without
calling a model client:
```go
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
PromptID: "meeting.summary",
Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.Inline("Synthetic meeting notes"),
},
})
```
The maintained
[offline preparation example](../../examples/go-library/prepare/main.go)
shows a complete runnable setup with a prompt file, in-memory profile, and
inline input. Exact request requirements and prepared-result fields belong to
the [`RunRequest` and `PreparedRun` GoDoc](../../types.go).
## Execute And Validate
[`Engine.Run`](../../engine.go) performs the same preparation, invokes the
configured model client, classifies the generated artifact, and validates the
content. A completed content check may return `ValidationFailed` in the result;
an operational inability to validate returns an error.
The maintained
[offline execution example](../../examples/go-library/run/main.go) injects a
deterministic model client and exercises `Run` without credentials, network
access, or paid calls. It is intentionally separate from the preparation
example so each workflow and its small prompt fixture can be copied and run on
its own.
Use the [`RunResult` and `ValidationResult` GoDoc](../../types.go) for the
returned data and the `Engine.Run` GoDoc for failure and cancellation
semantics. The
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
owns the built-in client's outbound HTTP behavior.
## Inputs, Profiles, And Overrides
Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request
can select a profile explicitly or use the prompt's default profile, and can
replace execution settings or the complete output contract.
The [public value GoDoc](../../types.go) defines nil, empty, zero, replacement,
copy, and credential behavior. The
[framework format reference](../formats.md) defines how those request values
interact with prompt definitions, file-backed profiles, built-ins, schemas,
and framework defaults.
For programmatic profiles,
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
OpenAI-compatible settings into a value accepted by `WithProfiles`.
### Set A Per-Run Session And Reasoning
Supply a direct session ID when one prompt should be correlated with a
consumer-managed conversation or workflow without changing prompt variables:
```go
reasoning := "high"
result, err := engine.Run(ctx, promptkit.RunRequest{
PromptID: "meeting.summary",
SessionID: "conversation-42",
Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.Inline("Synthetic meeting notes"),
},
Execution: &promptkit.ExecutionTargetOverride{
ReasoningEffort: &reasoning,
},
})
```
A nil reasoning pointer inherits the selected profile, a pointer to a
nonblank string replaces it, and a pointer to a blank string disables
reasoning for that run. Session IDs are correlation metadata, not credentials;
use stable, non-secret values that are safe to expose to collaborators and
providers. The
[`RunRequest` and `ExecutionTargetOverride` GoDoc](../../types.go) owns the
exact normalization, precedence, error, copying, and exposure contract.
### Register A Custom Backend
Register a reusable OpenAI-compatible connection once, then select it from a
profile. This local backend limits model generation to two simultaneous calls;
because `QueueCapacity` is omitted, the engine admits up to 1024 additional
calls waiting behind them:
```go
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
},
promptkit.WithBackend(promptkit.Backend{
ID: "local",
Endpoint: "http://localhost:8000/v1",
APIKeyEnv: "LOCAL_LLM_API_KEY",
ConcurrencyLimit: 2,
}),
promptkit.WithProfiles(promptkit.Profile{
ID: "local-summary",
BackendID: "local",
Model: "example-model",
}),
)
```
Registrations belong to one engine and custom IDs cannot replace built-ins.
The [`Backend` and `WithBackend` GoDoc](../../backends.go) defines validation,
copying, uniqueness, exact concurrency-field semantics, and request-default
behavior.
Both file-backed and in-memory profiles select a registration through
`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain
that routing and capacity identity. `PreparedRun.SelectedBackendID`,
`RunResult.SelectedBackendID`, and the effective `ExecutionTarget.BackendID`
expose it to consumers and injected model clients. Endpoint-only profiles
remain supported and expose an empty backend ID.
### Limit Backend Concurrency
Set `Backend.ConcurrencyLimit` when a backend needs protection from too many
simultaneous model calls. Leaving `QueueCapacity` nil, as in the local-backend
example above, selects the default waiting capacity of 1024.
To accept no waiting backlog beyond the active calls, provide an explicit
zero:
```go
noWaiting := 0
backend := promptkit.Backend{
ID: "local",
Endpoint: "http://localhost:8000/v1",
ConcurrencyLimit: 2,
QueueCapacity: &noWaiting,
}
```
The pointer distinguishes an explicit zero from omission. Keep using keyed
`Backend` literals so additive configuration fields remain source-compatible.
Capacity belongs to one engine and the selected backend ID; endpoint-only
profiles and custom backends without a configured limit remain unrestricted.
Exact validation, defaulting, ownership, and concurrency semantics belong to
the [`Backend` GoDoc](../../backends.go).
## Credentials
File-backed profiles name an environment variable; in-memory profiles can
require a direct request key. Direct keys are request-scoped and are excluded
from supported JSON values and the package's `String` and `GoString`
summaries. The exact precedence and redaction guarantees belong to
[`RunRequest`, `GenerateRequest`, and the profile GoDoc](../../types.go).
## Protect Files And Generated Data
The default artifact reader opens a `File` reference as a caller-selected
operating-system path. It does not constrain paths to an application root,
impose an inbound request-size policy, or establish an untrusted-input security
boundary. Applications must validate and restrict untrusted paths and payloads
before constructing a request, or inject an artifact reader that enforces
their filesystem, authorization, and size policies.
Rendered messages, input and output artifact bodies, raw model output, and
validation diagnostics can contain sensitive data. API-key redaction does not
sanitize those values. Treat prepared values, results, collaborator requests,
errors, and logs according to the application's data-access, retention, and
secret-handling policies.
## Extension Interfaces
Inject an [`LLMClient` or `ArtifactReader`](../../types.go) when the built-in
behavior does not fit the application. Their GoDoc defines concurrent use,
context handling, ownership of copied values, nil responses, and preservation
of collaborator errors. Implementations must honor cancellation, safely manage
copies they retain, avoid unsafe logging of content or credentials, and enforce
the application policy that motivated the injection.
## Handle Errors
Use `errors.Is` with the
[public error sentinels and operation GoDoc](../../engine.go). The declarations
distinguish invalid construction, invalid requests, absent sources,
source-loading failures, collaborator failures, and operational validation
failures. Specific request conditions may also match the broader
`ErrInvalidRequest`, and injected collaborator identities are preserved where
documented. Invalid or duplicate backend registrations match
`ErrInvalidConfig`; selecting an unknown backend matches `ErrProfileLoad`.
When a limited backend has admitted all active and waiting calls, handle
`ErrCapacityExceeded` separately from request errors and provider failures:
```go
result, err := engine.Run(ctx, request)
if errors.Is(err, promptkit.ErrCapacityExceeded) {
// Apply application policy: shed work, report overload, or retry later.
}
```
A rejected call returns no partial result and does not invoke the model
client. Promptkit does not prescribe retries or map this error to an HTTP
status; those choices remain with the consuming application. The
[`Engine.Run` and error GoDoc](../../engine.go) owns exact error and
cancellation identities.
## Application Boundary
Promptkit is an importable library. It does not own a command, inbound HTTP
API, process configuration, or deployment policy. Applications map the root
package's results and errors into those concerns, including inbound size and
trust policy.

View File

@@ -31,11 +31,14 @@ Start with:
| Task | Read before changing |
| --- | --- |
| Documentation or examples | The [documentation policy](policy/documentation.md) and the canonical owner of the affected contract. |
| Tests or test fixtures | The [testing policy](policy/testing.md), the owning package, and any focused internal document listed by the component overview. |
| Root public API, once implemented | The [architecture policy](policy/architecture.md), [root package declaration](../doc.go), [testing policy](policy/testing.md), and existing GoDoc. |
| Internal package implementation, once introduced | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and any focused internal document that the overview lists for that package. |
| Integration behavior, once introduced | The [architecture policy](policy/architecture.md), [documentation policy](policy/documentation.md), and the integration's owning contract under `docs/integrations/`. |
| Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. |
| Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). |
| Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. |
| Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. |
| Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. |
| Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. |
| Maintained example | The [example](../examples/go-library/prepare/main.go), [consumer guide](consumers/pkg-promptkit.md), [framework format reference](formats.md), and [documentation policy](policy/documentation.md). |
| Documentation | The [documentation policy](policy/documentation.md) and canonical owner of every affected contract. |
| Release preparation or publication | The [release procedure](release.md). |
For cross-cutting changes, follow every applicable row. Do not create
@@ -50,8 +53,10 @@ validation from the Promptkit repository root:
```sh
go test ./...
go test -race ./...
go vet ./...
go build ./...
go run ./examples/go-library/prepare
```
Check formatting across every tracked Go file:
@@ -75,7 +80,7 @@ behavior checked by the module.
## Focused Validation
Use focused checks while iterating, then run the complete validation sequence
before accepting the change. The root package currently supports:
before accepting the change. The root package supports:
```sh
go test .
@@ -83,17 +88,16 @@ go vet .
go build .
```
Filter tests by name without assuming a future package layout:
Filter tests by name without assuming a fixed internal package layout:
```sh
go test ./... -run 'TestName'
```
Replace `TestName` with a useful regular expression. When internal packages are
introduced, target only paths that actually exist, such as
`go test ./path/to/package`, and consult the internal component overview for
their owning documentation. A filtered or package-specific run does not replace
the complete repository validation.
Replace `TestName` with a useful regular expression. Target only paths that
exist, and consult the internal component overview for their owning
documentation. A filtered or package-specific run does not replace the
complete repository validation.
## Coordinated Work With Scriptorium

300
docs/formats.md Normal file
View File

@@ -0,0 +1,300 @@
# Framework Format Reference
## Purpose
This document is the canonical contract for Promptkit prompt-definition,
profile, and schema files. The [Go package consumer guide](consumers/pkg-promptkit.md)
explains how to select these sources and invoke the engine. The
[OpenAI-compatible integration contract](integrations/openai-compatible-chat.md)
owns the resulting outbound wire behavior.
Prompt and profile sources recursively discover files ending in `.yaml` or
`.yml`. YAML decoding is strict: unknown fields are errors for the selected
definition. Definitions are selected by their YAML `id`, not their file name
or directory.
## Prompt Definitions
A prompt definition describes inputs, Go-template messages, an optional
default profile, and an output contract.
```yaml
id: meeting.summary
version: "1.0.0"
default_profile: local-summary
description: Summarize a synthetic meeting note.
session_id: '{{.session}}'
inputs:
- name: note
required: true
content_type: text/plain
description: Meeting note to summarize.
messages:
- role: system
content: Return a concise summary.
cache_control:
type: ephemeral
ttl: 1h
- role: user
content_file: ./summary.user.md
output:
format: markdown
validation_mode: basic
repair_attempts: 0
```
| Field | Required | Meaning |
| --- | --- | --- |
| `id` | yes | Non-empty prompt identifier used by `RunRequest.PromptID`. |
| `version` | yes | Non-empty version selected by an optional `RunRequest.PromptVersion`. |
| `default_profile` | no | Non-empty profile ID used when the request omits `ProfileID`. |
| `description` | no | Human-readable description. |
| `session_id` | no | Go template rendered from request variables and input helpers. |
| `inputs` | no | Declared input metadata. |
| `messages` | yes | One or more chat-message templates. |
| `output` | yes | Output format and validation settings. |
When a request omits a version, the selected prompt ID must identify exactly
one definition. When it supplies a version, the ID and version pair must be
unique.
### Inputs
Each `inputs` item has these fields:
| Field | Required | Meaning |
| --- | --- | --- |
| `name` | yes | Non-empty name used by the request input map and `input` template helper. Names must be unique. |
| `required` | no | When true, preparation fails if the request omits the input. The default is false. |
| `content_type` | no | Expected media-type metadata. |
| `description` | no | Human-readable input description. |
Requests supply inputs as inline or file-backed `ArtifactRef` values. Declared
required inputs must be present. A template reference also requires the named
input to exist, whether or not it was declared. Extra request inputs are
allowed.
### Messages And Templates
Each message has a non-empty `role` and exactly one of:
- `content`, containing an inline Go template; or
- `content_file`, naming a file whose contents are the Go template.
For directory and `fs.FS` prompt sources, `content_file` resolves relative to
the prompt file and remains within the source root. `WithPromptFile` also
resolves it relative to that file.
Request variables are the template data, so a variable named `audience` is
referenced as `{{.audience}}`. The `{{input "note"}}` helper renders the body
of a named input. Missing variables and input references are errors.
The optional `session_id` uses the same template data and input helper. Its
rendered value is trimmed, omitted when empty, and limited to 256 Unicode code
points. A nonblank direct request session ID bypasses this template completely;
a blank direct value leaves the template behavior unchanged.
### Cache Control
`cache_control` is optional and has these fields:
| Field | Required | Values |
| --- | --- | --- |
| `type` | yes | `ephemeral` |
| `ttl` | no | Empty or `1h` |
Promptkit preserves cache-control metadata on the rendered message. The
outbound integration determines its wire representation.
### Output Contract
| Field | Required | Values or behavior |
| --- | --- | --- |
| `format` | yes | `text`, `markdown`, or `json`. |
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema`. |
| `schema_path` | for `json_schema` | Path to a schema in the configured schema source. |
| `repair_attempts` | no | Integer zero or greater; omitted means zero. |
The validation modes behave as follows:
- `none` skips content validation;
- `basic` requires non-empty generated content;
- `json` requires valid JSON; and
- `json_schema` requires valid JSON that satisfies the selected schema.
`format` controls output artifact metadata. JSON Schema mode also supplies the
schema to compatible model clients as structured-output metadata. The public
engine does not install an output repairer, so its validation is single-pass
even when a positive `repair_attempts` value is present.
A request-level `OutputContract` replaces the complete prompt output contract.
It does not merge individual fields. If its format is empty, Promptkit uses
`text`.
## Profile Definitions
A profile supplies model execution settings:
```yaml
id: local-summary
backend: openrouter
model: example-model
temperature: 0.2
max_tokens: 500
top_p: 0.95
timeout_seconds: 90
service_tier: flex
reasoning_effort: medium
extra_params:
provider_option: enabled
```
| Field | Required | Meaning |
| --- | --- | --- |
| `id` | yes | Non-empty profile identifier. IDs must be unique within one source. |
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared. |
| `endpoint` | unless `backend` is present | Non-empty OpenAI-compatible base URL, including an API version path when required. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
| `model` | yes | Non-empty provider model name. |
| `temperature` | no | Number from 0 through 2. |
| `max_tokens` | no | Integer zero or greater. |
| `top_p` | no | Number from 0 through 1. |
| `timeout_seconds` | no | Per-generation deadline in whole seconds; integer zero or greater. |
| `service_tier` | no | Provider-specific request tier. |
| `reasoning_effort` | no | Provider-specific reasoning setting. |
| `api_key_env` | no | Name of an environment variable containing the API key. |
| `extra_params` | no | JSON-compatible provider-specific outbound fields. |
Raw `api_key` is prohibited in profile YAML. Store only an environment
variable name in `api_key_env`.
Promptkit does not infer a backend from a model or endpoint. Endpoint-only
profiles remain supported and have no effective backend ID.
The engine always provides the built-in `openrouter` ID. Consumers can add
engine-scoped IDs with
[`WithBackend`](../backends.go); exact registration validation belongs to its
GoDoc.
`extra_params` accepts null, booleans, finite numbers, strings, arrays, and
objects with string keys. Keys must be non-empty. With the built-in client,
they also cannot collide with the standard fields listed in the
[outbound request contract](integrations/openai-compatible-chat.md#request-body).
### Defaults And Overrides
Execution settings resolve in this order:
1. framework defaults;
2. the selected backend, when the profile names one;
3. the selected profile; and
4. request `ExecutionTargetOverride` values.
The framework defaults are:
| Setting | Default |
| --- | --- |
| `temperature` | `0` |
| `max_tokens` | `0` |
| `top_p` | `1` |
| `timeout_seconds` | `600` |
Numeric zero in a file or in-memory profile means that the profile does not
replace the framework default. Numeric request overrides use pointers, so an
explicit zero is preserved. In particular, an explicit request
`timeout_seconds` of zero disables the per-generation deadline while leaving
the caller context and transport timeout intact.
Non-empty profile strings replace backend defaults, and non-empty request
strings replace both. Request reasoning is the exception: a nil
`ReasoningEffort` pointer inherits the profile, a pointer to a nonblank string
trims and replaces it, and a pointer to a blank string clears it. Backend
identity is retained when either layer overrides the endpoint, so the override
also retains any engine-local capacity policy configured for that backend.
Capacity configuration belongs to the Go
[`Backend` API](../backends.go), not prompt or profile YAML. A non-empty
`extra_params` map at each layer replaces the entire lower-precedence map
rather than merging keys.
The [outbound integration contract](integrations/openai-compatible-chat.md)
defines how the effective settings are serialized.
### Source And Profile Precedence
An explicit request profile ID takes precedence over the prompt's
`default_profile`. If neither is present, preparation fails.
Profile sources resolve matching IDs in this order:
1. in-memory profiles supplied with `WithProfiles`;
2. a profile file, `fs.FS`, or configured profile directory; and
3. embedded built-in profiles.
A higher-precedence source falls back only when the profile is absent. An
invalid matching profile is an error and does not fall back. In-memory
`Profile` values follow the same ranges as YAML profiles. They use
`APIKeyRequired` for request-scoped credentials instead of `api_key_env`.
## Built-In Profile Catalog
Every built-in selects the `openrouter` backend. The engine's built-in backend
registry supplies `https://openrouter.ai/api/v1` and the environment-variable
name `OPENROUTER_API_KEY`, so individual profiles contain only model and
generation settings. Built-in profile files do not repeat those connection
values. A custom or in-memory profile with the same profile ID takes
precedence.
| Provider | ID | Model |
| --- | --- | --- |
| aion-labs | `aion-2` | `aion-labs/aion-2.0` |
| anthropic | `claude-fable-latest` | `~anthropic/claude-fable-latest` |
| anthropic | `claude-haiku-latest` | `~anthropic/claude-haiku-latest` |
| anthropic | `claude-opus-latest` | `~anthropic/claude-opus-latest` |
| anthropic | `claude-sonnet-latest` | `~anthropic/claude-sonnet-latest` |
| deepseek | `deepseek-3-2` | `deepseek/deepseek-v3.2` |
| deepseek | `deepseek-4-flash` | `deepseek/deepseek-v4-flash` |
| deepseek | `deepseek-4-pro` | `deepseek/deepseek-v4-pro` |
| google | `gemini-2-flash` | `google/gemini-2.5-flash` |
| google | `gemini-2-flash-lite` | `google/gemini-2.5-flash-lite` |
| google | `gemini-2-pro` | `google/gemini-2.5-pro` |
| google | `gemini-3-flash-lite` | `google/gemini-3.1-flash-lite` |
| google | `gemini-flash-latest` | `~google/gemini-flash-latest` |
| google | `gemini-pro-latest` | `~google/gemini-pro-latest` |
| google | `gemma-4-31b` | `google/gemma-4-31b-it:exacto` |
| minimax | `minimax-m2` | `minimax/minimax-m2.5` |
| minimax | `minimax-m3` | `minimax/minimax-m3` |
| mistral | `mistral-large-2512` | `mistralai/mistral-large-2512` |
| mistral | `mistral-medium-3-5` | `mistralai/mistral-medium-3-5` |
| mistral | `mistral-small-3` | `mistralai/mistral-small-3.2-24b-instruct` |
| mistral | `mistral-small-4` | `mistralai/mistral-small-2603` |
| nvidia | `nemotron-3-ultra` | `nvidia/nemotron-3-ultra-550b-a55b` |
| openai | `gpt-5-mini` | `openai/gpt-5.4-mini` |
| openai | `gpt-5-nano` | `openai/gpt-5.4-nano` |
## Schemas
Schemas are JSON documents selected by a prompt or request
`schema_path`. For a directory or `fs.FS` source, paths resolve within the
configured source root. Referenced nested schemas resolve relative to the
owning schema document. `WithSchemaFile` exposes one schema, addressed by its
base name.
An unreadable, invalid, or unresolvable schema produces an operational
validation error. Generated content that is valid JSON but does not satisfy the
schema produces a failed validation result.
## Credentials
Credential values belong at the request or environment boundary, never in
prompt, profile, schema, or example files:
- a file profile names an environment variable with `api_key_env`;
- an in-memory profile may set `APIKeyRequired`;
- a request can provide a direct `APIKey` or override `APIKeyEnv`; and
- a direct request key takes precedence over environment lookup.
After a direct request key, the credential-source precedence is request
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
profile with `APIKeyRequired` clears an inherited backend environment name and
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
Promptkit validates required credential availability during preparation.
Direct keys are excluded from JSON results and redacted by public string
formatters. Environment-variable names may appear in prepared metadata, but
their values do not.

View File

@@ -0,0 +1,106 @@
# OpenAI-Compatible Chat Integration
## Purpose
This document defines the outbound HTTP behavior implemented by Promptkit's
internal OpenAI-compatible model client. The
[internal model-client document](../internal/llm.md) owns implementation flow,
errors, and test ownership. The root Promptkit engine uses this client by
default unless a consumer injects another implementation. The
[framework format reference](../formats.md) owns the profile and prompt values
that produce these outbound settings.
## Endpoint And Method
Generation sends an HTTP `POST` with `Content-Type: application/json`.
Before the client is called, the engine resolves framework, backend, profile,
and request values into one execution target. A non-empty endpoint from that
target overrides the client's configured base URL. After trailing slashes are
removed, `/chat/completions` is appended. Generation fails before sending when
neither source supplies an endpoint.
The target's backend ID is routing metadata for prepared values, results, and
injected clients. The built-in client does not derive the URL from that ID and
does not serialize it in the provider request.
## Authentication
A non-empty API key supplied directly on the execution target takes
precedence. Otherwise, when an API-key environment-variable name is supplied,
the client reads that variable and requires a non-empty value. The selected
key is sent as `Authorization: Bearer <key>`. No authorization header is sent
when neither mechanism is configured.
The target contains the already resolved environment-variable name: an
explicit request override takes precedence over profile metadata, which takes
precedence over the backend default. Only the name reaches prepared metadata;
the environment value is read just before the provider call and is never added
to the JSON body.
## Request Body
The request body always contains `model` and `messages`. The execution
target's model takes precedence over the client's configured model, and one
must be available.
Each ordinary message contains its `role` and string `content`. A
cache-controlled message instead uses a text content block containing `type`,
`text`, and `cache_control`; an empty cache-control TTL is omitted.
The effective direct or prompt-rendered session ID is trimmed, limited to 256
Unicode code points, and sent when nonempty as top-level `session_id`. It is
never also sent as a session header.
The client conditionally includes:
- `temperature`, `max_tokens`, and `top_p` when non-zero or explicitly
present;
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly
disabled reasoning setting is empty and therefore omitted; and
- `response_format` for JSON Schema structured output, including its name,
strict flag, and schema document.
The engine resolves backend, profile, and request extra-parameter maps by
whole-map replacement rather than key merging. The resulting effective map is
then merged directly into the top-level body after JSON serialization is
verified. Empty keys and collisions with these reserved fields are rejected
before any provider call:
- `model`
- `session_id`
- `messages`
- `temperature`
- `max_tokens`
- `top_p`
- `service_tier`
- `reasoning_effort`
- `response_format`
`backend_id`, `api_key_env`, and resolved credential values are not provider
request fields.
## Response Handling
Any 2xx response is decoded as an OpenAI-compatible chat response. The client
returns the first choice's non-empty message content and maps prompt,
completion, total, cached, and cache-write token counts.
Invalid JSON, absent choices, and empty first-choice content are malformed
responses. For a non-2xx status, the error includes the status code but never
the provider response body.
## Timeout And Cancellation
Timeouts are layered:
- the caller context remains the outer cancellation boundary;
- a positive generation timeout adds a request context deadline;
- zero adds no generation-specific deadline;
- a negative generation timeout is invalid; and
- the cloned `http.Client` supplies the whole-request transport cap, retaining
a positive supplied-client timeout or applying the configured/default
timeout when the supplied value is not positive.
The earliest applicable caller, generation, or transport deadline controls the
request. Constructing the internal client does not mutate a supplied
`http.Client`.

100
docs/internal/capacity.md Normal file
View File

@@ -0,0 +1,100 @@
# Internal Capacity Management
## Purpose
This document describes the implemented engine-local capacity coordination in
`internal/capacity`. The [architecture policy](../policy/architecture.md) owns
component boundaries, the [backend GoDoc](../../backends.go) owns exact public
configuration semantics, and the
[internal runner document](runner.md) owns orchestration around admission.
Capacity scheduling is outside the provider wire contract. It does not add
fields to execution targets, generated requests, prompt or profile YAML, or
stable JSON values.
## Construction And Pool Lifecycle
Each root `NewEngine` call obtains a normalized capacity-policy snapshot from
its immutable backend registry and constructs a new `Manager`. The manager
creates one pool for each limited backend ID. It has no package-global mutable
state, background workers, shutdown protocol, or persistence, so engines with
the same registrations still have independent capacity.
Unlimited registered backends and endpoint-only profiles have no pool. Their
admission and generation calls take the unrestricted fast path. An endpoint
override does not change the selected backend ID and therefore does not change
the pool.
One pool owns immutable active and total limits plus mutex-protected admission
count, active count, and ordered waiter list. Pool state exists only for the
lifetime of its engine.
## Bounded Run Admission
The runner asks the manager to admit a run after resolving the prompt, profile,
selected backend, effective execution target, credentials, and output contract,
but before schema loading, artifact loading, or rendering. Admission is
immediate: a limited pool either reserves a slot or returns the internal
`ErrCapacityExceeded` identity. The root facade maps that identity to the
public error without treating it as an invalid request or generation failure.
The total admitted bound is the active-generation limit plus its configured
waiting capacity. The returned release function is idempotent. The runner
defers it as soon as admission succeeds and holds the lease across remaining
preparation, initial generation, validation, every repair attempt, and all
failure or cancellation exits. A repair is part of its original admission and
does not reserve another bounded slot.
## FIFO Generation Permits
`NewClient` wraps the engine's selected internal model client after public
client adaptation or built-in client construction. Initial generation and the
default repairer receive the same wrapper.
For each `Generate` call, the wrapper selects a pool from the request's
effective backend ID. An unlimited call passes directly to the next client. A
limited call acquires an active permit, invokes the next client, and defers
permit release so ordinary returns and panic unwinding both restore capacity.
Preparation and validation never hold an active permit.
When all active permits are occupied, calls join a mutex-protected FIFO waiter
list. Releasing a permit transfers it directly to the oldest remaining waiter
before making it generally available. Pools do not order work relative to
other backend IDs.
The wrapper passes generation requests, responses, and collaborator errors
through unchanged. It owns scheduling only; the concrete model client remains
responsible for provider transport behavior.
## Cancellation And Release
Admission checks the caller context before reserving a slot. A call canceled
while waiting for an active permit removes its waiter under the same pool lock
used to grant permits. If cancellation removes the waiter first, the wrapped
client is not invoked. If a concurrent grant wins first, the call owns the
permit and invokes the client with the original context, allowing the client
to observe cancellation normally.
This grant-or-cancel decision prevents lost and double-released permits.
Admission leases and active permits are released after success, collaborator
errors, validation failures, cancellation, and panic unwinding. Canceled
waiters are unlinked so their contexts and requests are not retained by the
pool.
## Test Ownership
The [manager tests](../../internal/capacity/manager_test.go) own policy
validation, bounded admission, idempotent release, context handling, and
unlimited admission. The
[client tests](../../internal/capacity/client_test.go) own peak enforcement,
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
unlimited calls, passthrough behavior, and panic release.
The [runner tests](../../internal/usecase/runner_test.go) own early admission,
lease lifetime, failure release, and shared initial/repair scheduling. The
[external package capacity tests](../../capacity_contract_test.go) own the
assembled public-engine behavior for configured limits, capacity errors,
endpoint identity, engine independence, and injected clients. The
[root error-boundary tests](../../errors_internal_test.go) own preservation of
the public generation category and context identity when generation is
canceled.

66
docs/internal/llm.md Normal file
View File

@@ -0,0 +1,66 @@
# Internal Model Client
## Purpose
This document describes Promptkit's internal model-client implementation. The
[architecture policy](../policy/architecture.md) owns the library boundary,
and the
[OpenAI-compatible chat integration](../integrations/openai-compatible-chat.md)
owns the observable outbound HTTP contract. The
[framework format reference](../formats.md) owns the profile and prompt
settings consumed by the client.
The concrete client remains under `internal/llm`. The root engine assembles it
as the default implementation behind Promptkit's public client boundary.
## Components And Flow
`Client` is the provider-neutral generation boundary consumed by later
orchestration. `OpenAICompatibleClient` is the built-in implementation. It
uses internal domain values for rendered prompts, execution targets,
structured output, responses, and token usage.
The runner supplies a fully resolved target after applying backend, profile,
and request precedence. The client uses its endpoint, credential metadata,
generation fields, and extra parameters. `BackendID` remains routing metadata
for the generation boundary and is not mapped into the provider payload.
Construction validates the configured base URL and clones any supplied
`http.Client` so Promptkit can apply its timeout default without mutating the
caller's client. Generation then:
1. validates request-level timeout and endpoint requirements;
2. maps the internal request into the OpenAI-compatible chat payload;
3. validates and merges extra parameters;
4. resolves authentication;
5. performs the outbound request under the applicable deadlines; and
6. decodes the first response choice and token usage.
`internal/llm` owns the set of reserved OpenAI-compatible request fields used
when validating extra parameters. Backend registration consumes the same rule
without making the model client depend on registry configuration.
The implementation has no retry loop, tool-call support, provider catalog,
inbound HTTP behavior, or durable session store.
## Failure Categories
The package preserves distinct error identities for invalid client
configuration, invalid generation requests, request execution failures,
non-success provider statuses, and malformed successful responses. Provider
response bodies are not included in non-success errors.
Caller cancellation and deadline failures during the outbound request are
reported as request execution failures. The runner classifies these identities
without depending on HTTP status mapping.
## Test Ownership
The
[OpenAI-compatible client tests](../../internal/llm/openai_compatible_client_test.go)
own configuration, client cloning, deterministic deadline precedence,
authentication, request and response mapping, malformed data, error identity,
cancellation, and response-body suppression. The root transport contract test
also verifies that resolved backend settings reach this client without
serializing backend identity. All use local test servers or test transports;
the default suite makes no live or paid provider requests.

View File

@@ -11,14 +11,30 @@ contributor workflow and validation.
| Component | Implemented responsibility | References |
| --- | --- | --- |
| Root `promptkit` package | Establishes the public package boundary for the Go module. It does not yet provide migrated framework behavior or exported APIs. | [Package declaration](../../doc.go) |
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request and result values, profile construction, extension interfaces, value conversion, redacted formatting, public error mapping, and engine-local assembly. | [Package GoDoc](../../doc.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
| `internal/backend` | Constructs each engine's immutable registry from the built-in OpenRouter definition and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
| `internal/capacity` | Owns engine-local bounded run admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation. | [Domain declarations](../../internal/domain/domain.go) |
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
| `internal/jsonvalue` | Validates and deeply copies JSON-compatible extra-parameter trees while preserving supported concrete value types. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter, and combines it with an optional primary repository. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
| `internal/usecase` | Resolves backend, profile, and request settings and coordinates preparation and execution across internal sources, rendering, artifact loading, generation, validation, and optional repair. | [Internal runner](runner.md) |
The root `promptkit` package is the sole implemented Go package. No internal
framework packages exist yet.
The root package assembles these internal components without exposing their
representations. Consumers depend only on the root facade.
## Maintenance
Update this inventory as framework extraction introduces packages or changes
component responsibilities. List only implemented components; proposed package
Update this inventory as implementation adds packages or changes component
responsibilities. List only implemented components; proposed package
boundaries belong in temporary planning documents until their implementation
lands.

159
docs/internal/runner.md Normal file
View File

@@ -0,0 +1,159 @@
# Internal Runner
## Purpose
This document describes Promptkit's implemented internal orchestration. The
[architecture policy](../policy/architecture.md) owns dependency and consumer
boundaries. The [source and validation document](sources.md) owns repository,
artifact, rendering, and validation behavior, while the
[model-client document](llm.md) owns generation behavior and failure
categories.
The runner remains under `internal/usecase` and is assembled by the root
Promptkit engine. Its concrete type is not part of the public API.
The [framework format reference](../formats.md) owns prompt, profile, schema,
and override semantics consumed by the runner.
## Collaborators
`Runner` coordinates narrow internal interfaces for prompt definitions,
profiles, backend resolution, artifacts, rendering, model generation, and
validation. The root engine supplies one immutable registry containing the
built-in backend and validated consumer additions, one engine-local run
admitter, and a model client wrapped by the same capacity manager. Schema
documents are loaded through the validator's optional schema-loader interface.
An output repairer can be injected internally, but the ordinary runner
constructor does not enable one.
Each invocation carries its state in request, prepared-run, and result values.
The runner has no durable run or session store.
## Shared Preparation Pipeline
`Prepare` and `Run` share one private preparation pipeline split at the point
where a run can be assigned to its selected backend pool. The resolution phase
performs only the work needed to validate routing and admission:
1. validate the required prompt selection and normalize any direct session ID;
2. load the prompt definition and hash the original definition;
3. select the request profile or the prompt's default profile;
4. resolve the profile's backend ID, when present;
5. resolve application-neutral defaults, backend defaults, profile values,
and explicit request overrides in that order;
6. validate endpoint, model, numeric overrides, and credential requirements;
7. resolve the effective output contract without loading its schema; and
8. retain the definition, source identities, effective settings, output
contract, and preparation start time in invocation-local state.
The completion phase consumes that state without reloading the prompt,
profile, or backend:
1. load structured-output schema metadata when required;
2. load and hash input artifacts;
3. render messages and the prompt-defined session;
4. apply any direct session ID;
5. hash the effective rendered prompt; and
6. construct the prepared value and preparation timing.
`Prepare` runs both phases consecutively and never performs capacity admission.
`Run` performs backend admission between the phases. This structure preserves
one execution-precedence and error-ordering implementation while allowing a
full backend pool to reject work before expensive schema, artifact, and
rendering operations.
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
out-of-range values fail as invalid requests. Endpoint overrides do not change
the selected backend identity. Non-empty extra-parameter maps replace whole
lower-precedence maps. A direct API key takes precedence over environment
lookup; otherwise request, profile, and backend environment-variable names
apply in that order. A profile requiring a direct key clears an inherited
backend environment name unless the request supplies its own name. Secret
values remain excluded from serialized metadata.
Reasoning overrides are tri-state: nil inherits the profile, a pointer to a
nonblank string trims and replaces it, and a pointer to a blank string clears
it. A nonblank direct session is normalized before source loading, bypasses
the prompt session template, and is applied after ordinary message rendering.
A blank direct value retains prompt-template behavior. The runner clears the
template only on a value copy of the definition, so the definition hash always
describes the original source while the rendered-prompt hash includes the
effective direct or rendered session.
The registry is read-only after engine construction. Concurrent `Prepare` and
`Run` calls resolve independent defensive backend values and keep all
invocation state local.
## Run Flow
`Run` records its start time, performs the shared resolution phase, and asks
its `RunAdmitter` to reserve capacity for the effective backend ID. A nil
admitter is an internal unlimited fallback. After successful admission, `Run`
immediately defers the returned release function, performs the completion
phase, makes one initial generation call, builds the named output artifact,
and validates that artifact. Invalid generated content remains a validation
result; an inability to perform validation is an operational error.
The admission lease covers completion-phase preparation, initial generation,
validation, every repair, and every exit. It bounds accepted work without
serializing preparation or validation behind the active-generation limit.
The wrapped model client separately acquires a FIFO active permit only around
each actual generation call.
When an internal repairer is present, a JSON or JSON Schema content failure can
trigger bounded repair attempts. Repair receives the effective execution
target and session ID, validation errors, prior output, and structured-output
specification. The default repairer uses the same wrapped client as initial
generation, so each repair reacquires the selected backend's active permit
while remaining inside its original admission lease. Repair never performs a
second bounded admission. This capability remains internal and is not a public
option.
A successful result includes the output artifact and raw output, validation
state, effective session ID, prompt and rendered-prompt hashes, selected
profile and backend, effective settings, input hashes, token usage, a generated
run identifier, and UTC timing. The same effective session reaches initial
generation and any repair attempt through the rendered prompt. The same
effective target, including backend identity, reaches generation and any
repair attempt.
## Failure Categories
Package errors distinguish invalid requests, required profile selection,
credential failures, and prompt, profile, artifact, rendering, generation, and
validation failures. Wrapping preserves the package identities mapped by the
public facade and retains collaborator identities where they are part of the
internal contract.
Admission capacity exhaustion retains the internal capacity identity and adds
the selected backend ID as context. It is not recategorized as an invalid
request or generation failure, and no partial result is returned. A context
already done at admission retains its context identity directly. Cancellation
while waiting for an active generation permit prevents client invocation when
it wins the grant race; the model-client boundary then preserves the context
error through the generation-failure category. Deferred release restores the
admission lease on preparation, generation, validation, repair, and
cancellation failures.
Other context cancellation propagates through the invoked collaborator and is
classified by the owning operation.
An overlong direct session is an invalid request before source loading, while
an invalid or overlong prompt session template remains a prompt-render failure.
An unknown selected backend, or a selected backend with no configured resolver,
is classified as a profile-load failure.
## Test Ownership And Changes
The [runner tests](../../internal/usecase/runner_test.go) own preparation order,
selection and override precedence, the two-phase boundary, early admission,
lease lifetime and release, direct-session resolution, schema-before-generation
behavior, hashing, generation and validation outcomes, backend propagation,
bounded repair, shared initial/repair capacity, credentials and redaction,
error categories, artifact metadata, usage, and timing. The
[capacity subsystem document](capacity.md) identifies the focused pool,
waiter, and wrapped-client tests.
Changes to orchestration should continue to use the existing package
interfaces, keep request state local to an invocation, and preserve the shared
resolution and completion pipeline. Source, renderer, validator, or
model-client contract changes belong first in their owning package and
document.

73
docs/internal/sources.md Normal file
View File

@@ -0,0 +1,73 @@
# Internal Sources And Validation
## Purpose
This document describes Promptkit's implemented internal source, artifact,
rendering, and output-validation behavior. The
[architecture policy](../policy/architecture.md) owns the library boundary and
dependency rules. None of these internal packages is a supported consumer API,
and the root engine assembles them behind its public source options and values.
The [framework format reference](../formats.md) owns the exact file fields,
validation modes, built-in catalog, and source precedence.
## Prompt Definitions
`internal/promptdef` discovers YAML deterministically, decodes and validates
definitions, selects an ID and optional version, and resolves file-backed
message content within the selected operating-system or `fs.FS` source.
Its package tests own prompt selection, strict decoding, definition validation,
duplicate detection, and source containment:
[prompt-definition repository tests](../../internal/promptdef/repository_test.go).
## Profiles And Built-Ins
`internal/profile` loads and validates execution profiles from an
operating-system filesystem or an `fs.FS`. It supports a primary repository
with fallback only when the primary reports that a profile is absent. Strict
YAML decoding recognizes the optional `backend` field, trims its value, and
requires a model plus at least one non-blank backend or endpoint. Loading does
not check registry membership because the available registry belongs to the
assembled engine; the runner checks membership during preparation.
`internal/profile/builtin` embeds the maintained built-in profile catalog and
can place a caller-selected repository ahead of that catalog. Every embedded
profile selects `openrouter` and inherits its endpoint and credential
environment-variable name from the built-in backend registry rather than
repeating those values. Profile behavior is owned by the
[profile repository tests](../../internal/profile/repository_test.go), while
catalog completeness, the backend-selection invariant, duplicate IDs, and
overlay behavior are owned by the
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
## Ordinary Artifacts
`internal/artifact` resolves inline references and unrestricted,
caller-selected file paths. It copies content into an artifact, records
metadata and a content hash, applies a content-type fallback, and honors
context cancellation.
This ordinary reader does not implement an inbound HTTP security boundary. In
particular, it does not constrain files to an application root or impose an
HTTP request-size policy. Scriptorium's restricted HTTP reader remains an
application concern outside Promptkit. The
[artifact reader tests](../../internal/artifact/reader_test.go) own the
implemented reader behavior and failures.
## Rendering
`internal/prompt` renders definition messages as Go templates using named
artifacts and variables. It carries message roles, session IDs, and cache
control into the rendered prompt. The
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
## Schemas And Output Validation
`internal/validate` provides validators backed by an operating-system
filesystem or an `fs.FS`. Invalid generated content is returned as a validation
result; inability to load, register, or compile a schema is an operational
error.
The [validator tests](../../internal/validate/standard_validator_test.go) own
basic, JSON, JSON Schema, source resolution, schema loading, compilation, and
content-failure behavior.

View File

@@ -9,15 +9,65 @@ implemented packages without redefining these rules.
## System Shape
Promptkit is an importable Go library. It does not provide a runnable command,
an HTTP service, or another application process.
Promptkit is an importable Go library. It does not ship a command, an HTTP
service, or another application process. Repository examples demonstrate
library use but are not Promptkit applications or release artifacts.
The module root contains package `promptkit`, which is the public facade and the
only implemented Go package in the current repository foundation. It declares
the module's public package boundary but does not yet provide migrated framework
behavior or exported APIs. No internal framework packages currently exist.
The module root contains package `promptkit`, which is the public facade. It
provides the supported engine, configuration and source options, requests,
results, public values, extension interfaces, profiles, and error sentinels.
Future framework extraction must follow this dependency direction:
The implemented internal components consist of:
- `internal/domain`, which owns framework data values shared by later internal
components;
- `internal/backend`, which owns validated immutable OpenAI-compatible backend
definitions and the built-in OpenRouter definition;
- `internal/capacity`, which owns engine-local bounded run admission and
model-generation scheduling for limited backends;
- `internal/defaults`, which owns application-neutral framework defaults and
constructs the default execution target;
- `internal/filecatalog`, which discovers YAML files and provides source-path
helpers for filesystem and `fs.FS` consumers;
- `internal/jsonvalue`, which validates and defensively copies JSON-compatible
extra-parameter trees;
- `internal/promptdef`, which loads and validates prompt definitions from
filesystem and `fs.FS` sources;
- `internal/profile`, which loads, validates, and overlays execution profiles
from filesystem and `fs.FS` sources;
- `internal/profile/builtin`, which embeds the built-in execution profile
catalog;
- `internal/prompt`, which renders prompt messages from Go templates;
- `internal/artifact`, which resolves ordinary inline and unrestricted
caller-selected file references;
- `internal/validate`, which validates basic, JSON, and JSON Schema output
using filesystem and `fs.FS` schema sources;
- `internal/llm`, which defines the provider-neutral generation boundary and
implements outbound OpenAI-compatible chat requests; and
- `internal/usecase`, which coordinates preparation and execution across the
internal framework components.
The `examples/go-library/prepare` and `examples/go-library/run` packages are
maintained downstream consumers of the root facade. They do not expose library
packages or participate in internal assembly.
The root facade assembles one immutable backend registry, one capacity manager,
the internal repositories, renderer, validator, outbound client, and use-case
runner while translating public values and errors at the library boundary. The
registry contains built-ins plus validated engine-scoped consumer additions.
The facade constructs the capacity manager from the registry's immutable
policy snapshot, wraps the selected built-in or injected model client, and
supplies bounded admission to the runner. The defaults and renderer depend on
the domain model. Prompt-definition and profile repositories use the domain
model, file catalog, and YAML decoder. The built-in profile repository supplies
an embedded `fs.FS` to the profile package. Artifact reading uses the domain
model and application-neutral defaults. Validation uses the domain model, file
catalog, and JSON Schema implementation. The model client uses the domain
model, application-neutral defaults, and an injected or standard-library HTTP
client. The use-case runner depends on the narrow interfaces owned by each
internal component, including backend lookup and run admission.
The current implementation follows this dependency direction:
```text
downstream consumers, including Scriptorium
@@ -32,14 +82,19 @@ downstream consumers, including Scriptorium
narrow injected abstractions
```
The facade may coordinate internal components. Internal components must depend
on narrow abstractions for behavior supplied from outside the library; they
must not depend on consumers or on Scriptorium. This diagram constrains future
work and does not assert that the internal components already exist.
The backend registry depends on the domain model and shared JSON-value
validation, has no mutation API after construction, and consumes the
OpenAI-compatible reserved request-field rule owned by the model client. The
capacity component depends on the domain model and the narrow internal
model-client boundary, not on provider transport implementation. The model
client does not depend on registry or capacity configuration. The facade
coordinates internal components and adapts the supported public extension
interfaces to narrow internal abstractions. Internal components must not depend
on consumers or on Scriptorium.
## Repository And Consumer Boundary
Scriptorium is a downstream application that will consume Promptkit through
Scriptorium is a downstream application that consumes Promptkit through
the supported public facade. It is not a Promptkit package and must not become
an internal dependency.
@@ -112,7 +167,6 @@ state.
## Current-State Maintenance
This policy distinguishes present implementation from constraints on future
framework extraction. Do not list planned packages as implemented components.
When extraction introduces a package, update the internal inventory and the
owning contract or subsystem document in the same change.
Do not list planned packages as implemented components. When implementation
introduces a package, update the internal inventory and the owning contract or
subsystem document in the same change.

View File

@@ -75,18 +75,20 @@ mechanisms, not secret values.
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression policy, and test maintenance. | Subsystem behavior, exact public contracts, subsystem-specific test inventories, and implementation plans. |
| Release procedure | `docs/release.md`, when present | Required release validation, version and tag procedure, release ordering, and post-publication checks. | General contributor workflow, public API semantics, and decision history. |
| Exact exported Go API | Go declarations and GoDoc, as APIs are implemented | Exported names, signatures, types, values, errors, and exact behavioral contracts. | Task-oriented consumer walkthroughs, implementation details, and future API proposals. |
| Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, built-in profiles, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. |
| Consumer guidance | `docs/consumers/`, when consumer workflows require dedicated guidance | Task-oriented use of implemented public APIs, minimal examples, and consumer responsibilities. | Exact exported declarations and internal mechanics. |
| Durable integration contracts | `docs/integrations/`, when integrations exist | External formats and protocols, compatibility behavior, and upstream or downstream responsibilities. | Internal transformations and public Go declarations. |
| Supplemental release guidance | None. `docs/releases/` may be used when a release benefits from a changelog or migration guide. | No canonical content. These files may briefly summarize release-specific changes, compatibility, and consumer migration paths, and may be corrected, consolidated, archived, or removed when no longer useful. | Public API and behavior contracts, formats, integrations, architecture, release procedure, and the authoritative annotated-tag release record. |
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal documents. | Normative architecture, contributor workflow, external contracts, and proposed components. |
| Internal subsystem behavior | Other files under `docs/internal/`, when a subsystem needs durable detail | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, public API definitions, and future package plans. |
| Architectural decision history | `docs/adr/`, when repository-local decisions require records | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, and task sequencing. |
| Temporary feature roadmaps | `docs/roadmap/`, while planned work needs coordination | Proposed or accepted scope, sequencing, gates, and implementation status. | Implemented behavior reference and durable decision rationale. |
| Complete copyable artifacts | `examples/`, when maintained examples exist | Valid inputs, Go programs, and other files intended to be copied or run. | Field-by-field reference, exact API declarations, and prose explanation. |
| Complete copyable artifacts | `examples/` | Valid inputs, Go programs, and other files intended to be copied or run. | Field-by-field reference, exact API declarations, and prose explanation. |
Conditional owners do not require placeholder files or directories. Create a
consumer, integration, subsystem, ADR, roadmap, or example document only when
the corresponding implemented interface, decision, planned effort, or
maintained artifact exists.
consumer, integration, release, subsystem, ADR, roadmap, or example document
only when the corresponding implemented interface, release, decision, planned
effort, or maintained artifact exists.
## Boundary Rules
@@ -105,6 +107,26 @@ for a task. Internal documents explain how Promptkit implements them. Internal
documentation may identify a public type or external format as a dependency,
but must link to its canonical definition rather than restate it.
The [framework format reference](../formats.md) owns exact prompt, profile, and
schema-file contracts. Integration documents own external wire formats.
### Supplemental Release Guidance
Files under `docs/releases/` may provide changelog-style summaries and
migration guidance for a particular release. They are navigation and
orientation aids, not canonical owners of public APIs, behavior, formats,
integrations, architecture, release procedure, or other durable facts. When a
reader needs detail beyond a short release-specific note, the release document
must link to the applicable canonical documentation rather than reproduce its
contract.
The annotated tag message required by the
[release procedure](../release.md#write-the-release-note) remains the
authoritative release record. Supplemental release documents may be corrected,
consolidated, archived, or removed at any time when they are no longer useful,
provided maintained documentation does not depend on them and the annotated
tag record remains intact.
### Security Topics
This policy owns what documentation and examples may contain. Architecture owns
@@ -156,6 +178,10 @@ durable owners, update incoming links, and archive or remove the roadmap
according to repository practice. Do not preserve completed roadmaps as a
second current-state reference.
Supplemental release documents may likewise be removed without preserving a
replacement. Before removal, update maintained incoming links so current
documentation does not depend on an optional historical guide.
Before completing documentation work:
- verify affected behavior and examples;

View File

@@ -55,6 +55,15 @@ documentation-link, and repository-hygiene checks before accepting changes.
Introducing hosted CI later would supplement, not silently redefine, this
documented validation model.
The complete test sequence includes ordinary and race-enabled package tests.
The maintained offline consumer workflow is also run from the repository root:
```sh
go test ./...
go test -race ./...
go run ./examples/go-library/prepare
```
Tests in the default suite must be deterministic, offline, and independent of
real credentials. They must not invoke paid APIs, use live network
dependencies, or depend on mutable external services. Tests that require live
@@ -77,6 +86,10 @@ Use each test type where it protects a distinct risk:
relied upon by consumers.
- Integration tests use real collaborators when correctness depends on their
interaction, while replacing live or nondeterministic external boundaries.
- External-package root tests exercise the public facade as a Go consumer,
while internal package tests own focused implementation behavior.
- The maintained offline preparation example protects one representative
assembled consumer workflow without contacting a model provider.
- Fixtures should be minimal, synthetic, versioned with the behavior they
exercise, and free of credentials or private data.
- Golden files are appropriate only when the complete output is intentionally
@@ -154,7 +167,8 @@ A test failing is not the same as a test needing to be edited. Many tests may co
Configurable thresholds and defaults must not be duplicated throughout the test suite.
For example, do not encode an internal concurrency limit indirectly:
The following fragments are illustrative rather than standalone Go programs.
Do not encode an internal concurrency limit indirectly:
```go
// Production policy:

View File

@@ -7,26 +7,91 @@ tags. It does not publish runnable binaries or binary packages and does not
currently use hosted CI. The release maintainer performs and records the
required validation.
The first planned release is `v0.1.0`. Do not create that tag until the
framework has been extracted and the resulting public library has passed this
procedure. Later tags use the `vMAJOR.MINOR.PATCH` form. While Promptkit remains
pre-`v1`, release notes must identify intentional public API changes and any
consumer migration required by them.
`v0.1.0` is the initial published release. Later releases use semantic
`vMAJOR.MINOR.PATCH` tags. Before `v1`, minor releases may change the public
API and patch releases preserve compatibility within their minor line. Every
pre-`v1` release note must summarize compatibility, identify public API
changes, and state any action required of consumers.
## Prepare The Release
Promptkit releases are source-only. The annotated tag message is the release
note; there is no separate hosted release or binary packaging step.
Work from a clean checkout of the intended release commit, outside any Go
workspace and without a local module replacement. Confirm the source commit is
already published through the normal branch workflow.
## Establish The Candidate
From the Promptkit repository root, verify the checkout:
Choose a version that has not been published and export it as
`RELEASE_VERSION`. Run every command in this procedure from the Promptkit
repository root in the same POSIX shell. Do not reuse `v0.1.0` or another
existing version.
The following guard derives the release commit from `HEAD` and stops on a
missing or malformed version, a checkout other than synchronized `main`,
uncommitted changes, an active Go workspace, a module replacement, a vendor
tree, or an existing local or remote tag:
```sh
test -z "$(go env GOWORK)"
test -z "$(git status --short)"
git fetch --tags origin
set -eu
: "${RELEASE_VERSION:?export an unpublished vMAJOR.MINOR.PATCH version}"
if ! printf '%s\n' "$RELEASE_VERSION" |
grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$'
then
printf '%s\n' "invalid release version: $RELEASE_VERSION" >&2
exit 1
fi
RELEASE_COMMIT=$(git rev-parse --verify 'HEAD^{commit}')
export RELEASE_COMMIT
check_release_candidate() {
test "$(git branch --show-current)" = main
test -z "$(git status --porcelain)"
gowork_value=$(go env GOWORK)
case "$gowork_value" in
''|off) ;;
*)
printf '%s\n' "active Go workspace: $gowork_value" >&2
return 1
;;
esac
test -z "$(git ls-files go.work go.work.sum)"
test ! -e vendor
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
then
printf '%s\n' 'go.mod contains a replacement' >&2
return 1
fi
git fetch origin main --tags
test "$RELEASE_COMMIT" = \
"$(git rev-parse --verify 'refs/remotes/origin/main^{commit}')"
if git show-ref --verify --quiet "refs/tags/$RELEASE_VERSION"
then
printf '%s\n' "local tag already exists: $RELEASE_VERSION" >&2
return 1
fi
if test -n "$(
git ls-remote --tags origin \
"refs/tags/$RELEASE_VERSION" \
"refs/tags/$RELEASE_VERSION^{}"
)"
then
printf '%s\n' "remote tag already exists: $RELEASE_VERSION" >&2
return 1
fi
}
check_release_candidate
```
Do not continue unless the guard completes successfully. In particular, push
the intended commit through the normal `main` branch workflow before release;
the tag procedure is not a substitute for publishing the source commit.
## Validate The Candidate
Confirm the module and root package metadata:
```sh
@@ -41,93 +106,169 @@ gitea.maximumdirect.net/eric/promptkit 1.25.5
promptkit gitea.maximumdirect.net/eric/promptkit
```
Run the same default Go validation required by the
Run the complete maintainer validation required by the
[development guide](development.md):
```sh
go test ./...
go test -race ./...
go vet ./...
go build ./...
go run ./examples/go-library/prepare
```
Check every tracked Go file and repository whitespace:
Check every tracked Go file. This command must produce no output:
```sh
gofmt -l $(git ls-files '*.go')
unformatted=$(
git ls-files '*.go' |
while IFS= read -r go_file
do
gofmt -l "$go_file"
done
)
test -z "$unformatted"
```
Follow every maintained Markdown link and confirm that its local or published
target exists. Review the repository for generated binaries, test or coverage
output, credentials, template residue, downloaded assets, and other files that
do not belong in source control.
Recheck module and repository hygiene, whitespace, and the clean checkout:
```sh
test -z "$(git ls-files go.work go.work.sum)"
test ! -e vendor
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
then
printf '%s\n' 'go.mod contains a replacement' >&2
exit 1
fi
git diff --check
test -z "$(git status --porcelain)"
```
The formatting command must produce no paths. Follow every maintained Markdown
link and confirm its target exists. Review the repository for generated
binaries, test or coverage output, credentials, template residue, and other
files that do not belong in source control.
## Write The Release Note
Confirm that no workspace override is tracked and that `go.mod` contains no
`replace` directive:
Prepare a plain-text annotated-tag message outside the repository and export
its path as `RELEASE_NOTES_FILE`. Use this form, replacing each summary with
release-specific text; write `None.` when there are no public API changes or
consumer actions:
```sh
git ls-files go.work go.work.sum
rg -n '^replace\b' go.mod
```text
Promptkit vMAJOR.MINOR.PATCH
Validated commit: full commit ID
Compatibility: compatibility summary
Public API changes: changes or None.
Consumer action: required action or None.
```
Both commands must produce no output. Re-run `git status --short` and require a
clean result after every validation and review check.
## Create And Publish The Tag
Choose the semantic version from the intended compatibility change. Record the
release commit before tagging:
After writing it, require all release-note fields, the selected version, and
the validated commit to be present:
```sh
release_version=v0.1.0
release_commit=$(git rev-parse HEAD)
: "${RELEASE_NOTES_FILE:?export the path to the release-note file}"
test -f "$RELEASE_NOTES_FILE"
test -s "$RELEASE_NOTES_FILE"
grep -F "Promptkit $RELEASE_VERSION" "$RELEASE_NOTES_FILE"
grep -F "Validated commit: $RELEASE_COMMIT" "$RELEASE_NOTES_FILE"
grep -F 'Compatibility:' "$RELEASE_NOTES_FILE"
grep -F 'Public API changes:' "$RELEASE_NOTES_FILE"
grep -F 'Consumer action:' "$RELEASE_NOTES_FILE"
```
Replace the example version for later releases and keep both values in the same
shell for the remaining commands. Confirm the tag does not already exist
locally or remotely:
Inspect the complete message and confirm that it accurately records the
compatibility impact, public API changes, and required consumer action.
## Create And Inspect The Tag
Run the candidate guard again immediately before tag creation. This ensures
that validation or release-note preparation did not change the checkout and
that the commit is still published and untagged:
```sh
test -z "$(git tag --list "$release_version")"
test -z "$(git ls-remote --tags origin "refs/tags/$release_version")"
check_release_candidate
```
Create an annotated tag whose message identifies the release and records that
the documented validation passed for the tagged commit:
Create the annotated tag from the prepared release note and bind it explicitly
to the validated commit:
```sh
git tag --annotate "$release_version" \
--message "Promptkit $release_version; documented validation passed for $release_commit"
git tag --annotate "$RELEASE_VERSION" \
--file "$RELEASE_NOTES_FILE" \
"$RELEASE_COMMIT"
```
Inspect the tag before publication:
Inspect both the tag message and its source commit before publication:
```sh
git show --no-patch --decorate "$release_version"
test "$(git rev-list -n 1 "$release_version")" = "$release_commit"
test "$(git cat-file -t "refs/tags/$RELEASE_VERSION")" = tag
git show --no-patch --decorate "refs/tags/$RELEASE_VERSION"
test "$(
git rev-parse --verify "refs/tags/$RELEASE_VERSION^{commit}"
)" = "$RELEASE_COMMIT"
```
Publish the tag without relying on a hosting-provider-specific release
interface:
If inspection finds an error, delete the unpublished local tag, correct the
release note or candidate, and repeat the guards. Never move or recreate a tag
that has been published.
## Publish The Selected Tag
Push only the selected tag ref. Do not use `git push --tags`:
```sh
git push origin "refs/tags/$release_version"
git push origin \
"refs/tags/$RELEASE_VERSION:refs/tags/$RELEASE_VERSION"
```
## Verify Publication
Confirm that the remote tag object matches the local annotated tag and still
resolves to the intended source commit:
Compare the remote annotated-tag object with the local object, then compare the
remote peeled source commit with the validated commit:
```sh
remote_tag=$(git ls-remote --tags origin "refs/tags/$release_version" | awk '{print $1}')
test "$remote_tag" = "$(git rev-parse "refs/tags/$release_version")"
test "$(git rev-list -n 1 "refs/tags/$release_version")" = "$release_commit"
remote_tag=$(
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" |
awk 'NR == 1 { print $1 }'
)
remote_commit=$(
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION^{}" |
awk 'NR == 1 { print $1 }'
)
test -n "$remote_tag"
test "$remote_tag" = \
"$(git rev-parse --verify "refs/tags/$RELEASE_VERSION")"
test "$remote_commit" = "$RELEASE_COMMIT"
```
Promptkit must publish the required tag before Scriptorium or another consumer
publishes a release that depends on that version. Released consumer modules
must not use a local replacement or unpublished Promptkit revision.
Finally, resolve the version as an ordinary Go module in a temporary module
outside this repository and without a workspace or replacement:
```sh
resolution_dir=$(mktemp -d)
(
trap 'rm -rf "$resolution_dir"' 0 1 2 15
cd "$resolution_dir"
GOWORK=off go mod init example.com/promptkit-release-check
GOWORK=off go mod download \
"gitea.maximumdirect.net/eric/promptkit@$RELEASE_VERSION"
resolved_version=$(
GOWORK=off go list -m -f '{{.Version}}' \
"gitea.maximumdirect.net/eric/promptkit@$RELEASE_VERSION"
)
test "$resolved_version" = "$RELEASE_VERSION"
)
```
Promptkit must publish and verify the required version before Scriptorium or
another consumer publishes a release that depends on it. This ordering does
not replace the consumer project's own release procedure. Released consumers
must select the published Promptkit tag through ordinary module resolution,
without a workspace, replacement, vendored Promptkit source, or unpublished
revision.
## Policy Changes

243
docs/releases/v0.2.0.md Normal file
View File

@@ -0,0 +1,243 @@
# Promptkit v0.2.0
This supplemental changelog and migration guide summarizes the consumer-facing
changes from `v0.1.0` to `v0.2.0`. The annotated `v0.2.0` tag is the
authoritative release record. Exact current contracts belong to the linked
GoDoc and durable documentation.
## Summary
`v0.2.0` adds three major capabilities:
- an engine-scoped registry for reusable OpenAI-compatible backend
definitions;
- bounded, backend-specific run admission and model-generation concurrency;
and
- direct per-run session IDs and tri-state reasoning-effort overrides.
Existing endpoint-only profiles remain supported. Consumers can adopt backend
registration and runtime overrides incrementally rather than rewriting all
profiles during the upgrade.
## Compatibility At A Glance
Promptkit remains pre-`v1`, and this minor release includes source-level and
behavioral changes that deserve review.
| Area | `v0.1.0` consumer impact |
| --- | --- |
| Endpoint-only profiles | Continue to work without migration. |
| Built-in profiles | Continue to use OpenRouter and `OPENROUTER_API_KEY`; they now select the built-in `openrouter` backend. |
| Custom backends | Registration is optional. Existing profiles may keep their endpoint and credential configuration. |
| Reasoning overrides | String assignments must migrate to the new pointer field. |
| `RunRequest.Metadata` | Removed; delete assignments to this field. |
| OpenRouter concurrency | Now limited to 16 active generations with waiting capacity of 1024 per engine. |
| Public JSON | `v0.2.0` formalizes supported JSON representations; consumers relying on `v0.1.0` encodings should review the notes below. |
| Unkeyed public struct literals | May require updates because fields were added. Keyed literals are recommended. |
## Upgrade
After the `v0.2.0` tag is published, update the module dependency with:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.2.0
go mod tidy
```
Run the consuming project's ordinary tests and race-enabled tests after the
upgrade, especially if it calls one engine concurrently or persists Promptkit
JSON values.
## Backend Registry
Consumers may now register reusable OpenAI-compatible backend definitions with
`WithBackend`, then select them by ID from file-backed or in-memory profiles.
A backend can supply its endpoint, API-key environment-variable name,
request-wide extra parameters, and optional capacity policy.
Registrations are immutable and belong to one engine. Consumer registrations
can add new IDs but cannot replace Promptkit's reserved `openrouter` backend.
Profiles that select a backend may still override its endpoint without losing
the backend's routing or capacity identity.
An existing endpoint-only in-memory profile remains valid:
```go
promptkit.Profile{
ID: "local",
Endpoint: "http://localhost:8000/v1",
Model: "example-model",
}
```
Adopting the registry is optional and can be done when several profiles should
share connection or capacity settings:
```go
engine, err := promptkit.NewEngine(
promptkit.Config{PromptDir: "prompts"},
promptkit.WithBackend(promptkit.Backend{
ID: "local",
Endpoint: "http://localhost:8000/v1",
APIKeyEnv: "LOCAL_LLM_API_KEY",
}),
promptkit.WithProfiles(promptkit.Profile{
ID: "local-summary",
BackendID: "local",
Model: "example-model",
}),
)
```
See the
[custom-backend consumer guide](../consumers/pkg-promptkit.md#register-a-custom-backend)
for task-oriented usage. The
[`Backend` and `WithBackend` GoDoc](../../backends.go) owns exact registration,
validation, copying, defaulting, and uniqueness semantics. The
[framework format reference](../formats.md) owns the profile `backend` field
and execution precedence.
## Backend-Specific Concurrency
Each registered backend may now define:
- an active model-generation limit; and
- a bounded number of additional admitted `Run` calls.
Promptkit owns scheduling for both its built-in model client and an injected
`LLMClient`. `Run` remains synchronous: an admitted caller waits for its
ordinary result, while a call beyond the bounded admission capacity returns
`ErrCapacityExceeded`. Capacity is engine-local and keyed by backend ID.
Endpoint-only profiles and custom backends without a configured limit remain
unlimited.
The built-in OpenRouter backend now permits 16 active generations and 1024
additional admitted calls per engine. Applications that can exceed this bound
should handle capacity exhaustion separately from provider and request
failures:
```go
result, err := engine.Run(ctx, request)
if errors.Is(err, promptkit.ErrCapacityExceeded) {
// Apply application-specific overload or retry policy.
}
```
Promptkit does not prescribe retries or map this error to an HTTP status. See
the
[concurrency consumer guidance](../consumers/pkg-promptkit.md#limit-backend-concurrency)
and the [`Backend` GoDoc](../../backends.go) for the canonical configuration
contract. Runtime behavior and public error identities belong to the
[`Engine.Run` GoDoc](../../engine.go).
## Per-Run Session IDs
`RunRequest.SessionID` can now supply a consumer-managed correlation ID for one
`Prepare` or `Run` invocation. A nonblank direct value overrides the prompt's
session template and is exposed in prepared values, results, injected-client
requests, and provider observability. Session IDs should therefore be stable,
non-secret values.
```go
result, err := engine.Run(ctx, promptkit.RunRequest{
PromptID: "meeting.summary",
SessionID: "conversation-42",
})
```
The built-in OpenAI-compatible client sends a nonempty effective session as the
top-level `session_id` request-body field, not as an `x-session-id` header. See
the
[session and reasoning consumer guide](../consumers/pkg-promptkit.md#set-a-per-run-session-and-reasoning),
the [`RunRequest` GoDoc](../../types.go), and the
[OpenAI-compatible request contract](../integrations/openai-compatible-chat.md#request-body)
for exact normalization, length, exposure, and wire behavior.
## Per-Run Reasoning Effort
`ExecutionTargetOverride.ReasoningEffort` changed from `string` to `*string` so
one request can distinguish inheritance, replacement, and explicit clearing.
Update a `v0.1.0` override like this:
```go
// v0.1.0
Execution: &promptkit.ExecutionTargetOverride{
ReasoningEffort: "high",
}
```
to:
```go
// v0.2.0
reasoning := "high"
Execution: &promptkit.ExecutionTargetOverride{
ReasoningEffort: &reasoning,
}
```
The three states are:
- `nil` inherits the selected profile's value;
- a pointer to a nonblank string replaces it for that invocation; and
- a pointer to an empty or whitespace-only string clears it for that
invocation.
This allows consumers to consolidate profiles that differed only by reasoning
effort. The [`ExecutionTargetOverride` GoDoc](../../types.go) owns the exact
override contract.
## Other Migration Notes
### Remove `RunRequest.Metadata`
`RunRequest.Metadata` is no longer part of the public request. Remove any
assignment to that field. Use application-owned state keyed by `RunResult.RunID`
or a direct `SessionID` when correlation is needed; these identifiers have
different purposes, so choose according to the application's lifecycle.
### Review Persisted JSON
`v0.2.0` defines stable JSON representations for the public result, artifact,
execution, validation, and model-client values listed in the
[package documentation](../../doc.go). Consumers that treated `v0.1.0`
reflection-derived encodings as stable should update fixtures and stored-data
adapters.
In particular:
- `RunResult` encodes elapsed time as integer milliseconds in `duration_ms`
instead of encoding `time.Duration` under `duration`;
- result JSON can include the new `session_id` and `selected_backend_id`
fields;
- execution-target JSON can include `backend_id`; and
- artifact and target-presence fields now use their documented lower-case
names.
The `v0.2.0` `RunResult` decoder reads `duration_ms`; it does not translate a
persisted `v0.1.0` `duration` field. Transform old payloads before decoding
when preserving their elapsed duration matters.
### Prefer Keyed Struct Literals
New fields were added to several public structs. Replace positional composite
literals with keyed literals so future additive fields do not cause another
source migration.
## Migration Checklist
- Update the module dependency and run the consumer's tests.
- Change reasoning overrides from strings to pointers.
- Remove uses of `RunRequest.Metadata`.
- Review unkeyed Promptkit struct literals.
- Decide whether shared endpoints should move into registered backends.
- If using built-in OpenRouter profiles at high concurrency, handle
`ErrCapacityExceeded` and review the new engine-local bound.
- Review stored JSON, fixtures, and downstream decoders.
- Optionally replace profile-specific session or reasoning variants with
per-run overrides.
For complete consumer workflows, use the
[package consumer guide](../consumers/pkg-promptkit.md) and maintained
[offline execution example](../../examples/go-library/run/main.go).

236
docs/roadmap/concurrency.md Normal file
View File

@@ -0,0 +1,236 @@
# Backend-Specific Concurrency Management
**Status:** Complete.
## Purpose
This roadmap defines the scope and target end state for engine-local,
backend-specific concurrency management. It records the intended capability,
consumer value, and important policy choices.
This document is planning material, not a description of current behavior.
Current exported contracts remain owned by Go declarations and GoDoc, backend
registration guidance by the
[consumer guide](../consumers/pkg-promptkit.md#register-a-custom-backend), and
implemented orchestration by the
[internal runner document](../internal/runner.md).
## Motivation
Different model backends can sustain very different request loads. A local
network endpoint may need a small concurrency limit, while OpenRouter can
usually accept substantially more simultaneous work. Requiring every consumer
to build its own semaphores and queues would duplicate routing knowledge,
create inconsistent cancellation behavior, and make it easy for one caller to
bypass the intended backend limit.
Promptkit should own this coordination because it already resolves each run to
an engine-scoped backend identity and owns every model-generation call made by
the runner. Consumers should continue submitting ready-to-run requests through
the synchronous API, including concurrently from multiple goroutines, without
implementing their own backend scheduler.
The buffered queue is a safety boundary, not an ordinary throughput
restriction. Its primary purpose is to prevent a bug or unintended submission
loop from creating an unbounded in-memory backlog.
## Scope
The feature will add optional concurrency policy to registered backends and
coordinate `Run` calls against independent per-backend capacity pools.
Each policy has two distinct controls:
- an active-generation limit, which protects the backend from too many
simultaneous model requests; and
- a bounded waiting capacity, which protects the process from admitting an
unbounded backlog.
Concurrency policy belongs to a backend registration. It is not a profile
model parameter and cannot be overridden per run. Profiles select the policy
through their backend ID, while a profile or request endpoint override remains
in the selected backend's pool.
`Prepare` does not call a model and will remain outside concurrency admission.
## Defaults And Configuration
The built-in OpenRouter backend will use:
- an active-generation limit of 16; and
- a waiting capacity of 1024.
The waiting default is intentionally generous. Reaching it should indicate
abnormal submission pressure rather than normal application behavior.
Consumer-registered backends will remain unlimited unless the consumer
configures an active-generation limit. When a consumer enables a limit and
does not specify waiting capacity, the waiting capacity will default to 1024.
Consumers may configure a different bounded capacity, including zero when
they want no admitted backlog beyond the active-limit-sized run set.
The public representation must distinguish an omitted waiting capacity from
an explicit zero.
Endpoint-only profiles have no backend registration from which to obtain
policy and will remain unlimited. A future engine-wide or endpoint-keyed
policy can be considered separately if consumers demonstrate that need.
Invalid limits or capacities will fail engine construction as invalid
configuration. Policy values will be copied into engine-owned immutable state
along with the rest of the backend registration.
## Admission And Execution Behavior
`Run` remains a synchronous, wait-for-result operation. Concurrent callers may
block inside `Run` while waiting for their selected backend, then receive the
ordinary result or error from that invocation.
For a configured pool, the active-generation limit plus the waiting capacity
defines the maximum number of concurrent `Run` invocations that Promptkit will
accept for that backend. A waiting capacity of zero therefore accepts no more
runs than the active limit. Admission is immediate: a call either reserves one
of those bounded slots or receives the capacity error. An accepted run may
then wait internally for active-generation capacity.
For a limited backend, Promptkit will bound the number of accepted runs before
expensive artifact loading, prompt rendering, and large defensive copies where
practical. Lightweight prompt, profile, and backend resolution may occur first
when it is required to identify the correct capacity pool. This pre-admission
resolution must not become a second execution-precedence path with behavior
that can drift from `Prepare`.
An accepted run retains its admission until it completes or fails. Every
actual model-generation call for that run must separately observe the
backend's active-generation limit. This includes:
- the initial generation;
- every output-repair generation; and
- calls made through either the built-in or an injected model client.
Preparation and output validation should not hold an active-generation permit.
A repair remains part of its already-admitted run, but reacquires active
generation capacity so repairs cannot exceed the backend limit. It must not be
rejected merely because new runs filled the waiting queue after its initial
generation.
Within one backend pool, waiting generation calls should be served in FIFO
order, subject to canceled calls being removed. Different backend pools make
progress independently; a saturated local backend must not consume
OpenRouter's active or waiting capacity.
The feature will not promise ordering across backend pools or completion order
among admitted runs.
## Capacity Failure And Cancellation
When a backend's bounded waiting capacity is full, a new `Run` call will fail
promptly rather than waiting outside the bounded admission system. The public
API will expose a recognizable capacity-exhaustion error identity distinct
from invalid configuration, invalid requests, and model-client failures.
Rejected calls return no partial result and do not invoke the model client.
Waiting within the admitted backlog or for active-generation capacity must
honor the caller's context. Cancellation or deadline expiry will:
- stop waiting promptly;
- release any admission or generation capacity held by that invocation;
- preserve the applicable context error identity; and
- avoid invoking the model client if cancellation wins before generation
starts.
Capacity must also be released after preparation, generation, validation,
repair, or collaborator failure. One failed or canceled run must not reduce
the backend's future usable capacity.
Elapsed `Run` timing will include time spent waiting after the call is
accepted. `PreparedRun` timing will continue to describe preparation rather
than queue waiting.
## Engine And Client Boundaries
All pools and queued state belong to one `Engine`. Separate engines do not
share capacity, even when they register the same backend ID or endpoint. The
feature introduces no process-global scheduler.
The engine will apply policy consistently to the built-in model client and an
injected `LLMClient`. Consumers calling their own client outside Promptkit are
outside this boundary. Injected clients remain responsible for their internal
thread safety and cancellation behavior.
Backend policy is keyed by the resolved backend ID rather than endpoint text.
This preserves stable routing when a selected backend's endpoint is overridden
and avoids accidentally combining unrelated registrations that happen to use
the same URL.
## Queue Lifetime And Observability
Admission state is buffered, ephemeral, and in-process. It is not persisted
and has no survival guarantee across engine disposal or process termination.
Promptkit will not introduce background job ownership or require consumers to
start or stop workers.
The initial feature does not require public queue-depth metrics, callbacks, or
inspection APIs. Capacity errors and ordinary call timing provide the
consumer-visible behavior. Operational observability can be added later
without coupling the scheduling mechanism to an application logging or
metrics system.
## Compatibility
Consumer-registered backends and endpoint-only profiles remain unlimited
unless concurrency is explicitly configured, preserving their existing
behavior.
The built-in OpenRouter backend will change from unlimited concurrency to a
limit of 16 with a bounded waiting capacity of 1024. Ordinary synchronous
calls remain unchanged, while unusually high concurrent use may now wait or
return the capacity error. This behavioral change must be identified in the
release notes for the version that publishes it.
Adding backend policy fields and a public capacity error is otherwise
additive. The change will use a pre-`v1` minor release under Promptkit's
[release policy](../release.md#release-model).
## Non-Goals
This scope does not include:
- asynchronous job handles, polling, or detached result delivery;
- durable or cross-process queues;
- persistence or recovery across engine or process shutdown;
- priorities, scheduling weights, or consumer-defined fairness classes;
- automatic retries, backoff, rate-limit interpretation, or provider quota
discovery;
- token-per-minute or request-per-minute rate limiting;
- dynamic reconfiguration after engine construction;
- per-profile or per-run concurrency overrides;
- endpoint-keyed pooling for profiles without a backend ID;
- process-global coordination across engines;
- application worker lifecycle, logging, tracing, or metrics policy; or
- changes to prompt, profile, schema, or model-provider wire formats.
## Target End State
This roadmap reaches its target end state when:
- each engine independently coordinates configured backend capacity;
- the built-in OpenRouter backend allows 16 active generations and up to 1024
waiting runs;
- consumer backends can opt into their own active and waiting limits while
remaining unlimited by default;
- endpoint overrides retain the selected backend's capacity pool and
endpoint-only profiles remain unlimited;
- synchronous `Run` callers wait for and receive their ordinary result;
- admission is bounded before expensive preparation work where practical;
- every initial and repair generation observes the backend's active limit
without serializing preparation or validation;
- a full waiting queue returns a recognizable capacity error without invoking
the model client;
- cancellation and all failure paths promptly release capacity and preserve
context error identity;
- built-in and injected model clients receive the same scheduling behavior;
- pools remain ephemeral, engine-scoped, and independent across backend IDs;
and
- current-state GoDoc, consumer, internal, and release documentation describe
the implemented behavior once it lands.

53
docs/roadmap/future.md Normal file
View File

@@ -0,0 +1,53 @@
# Future Feature Ideas
## Purpose
This document catalogs reasonably specific ideas that may be useful in future
Promptkit development. It is an idea pool, not a commitment, schedule, or
description of current behavior.
Ideas belong here while they are worth retaining but have not been selected
for active development. Keep each entry at the level of intended capability,
consumer value, and important scope boundaries. Defer API design,
implementation details, sequencing, and acceptance criteria until an idea is
selected.
## Using This Catalog
- Add an idea when its purpose and likely value can be stated clearly.
- Keep entries independent enough that maintainers can evaluate and select
them individually.
- Note significant dependencies or boundary concerns, but do not turn entries
into implementation plans.
- Treat inclusion as an invitation to evaluate, not as approval or priority.
- When an idea is selected, move its active planning to a focused roadmap or,
when it requires a durable architectural decision, an ADR. Update
current-state documentation only when implementation lands.
- Remove ideas that are no longer relevant. Retain a rejected idea only when
its rationale is likely to prevent repeated reconsideration.
Future capabilities must continue to respect the
[architecture policy](../policy/architecture.md), particularly Promptkit's
role as an application-neutral library and its boundary with downstream
consumers.
## Ideas
No ideas are currently cataloged. Backend-specific concurrency management has
been selected for active planning in the
[focused concurrency roadmap](concurrency.md).
## Entry Format
Use a short heading followed by a concise summary. Add focused bullets when
they help preserve important scope boundaries without becoming an
implementation plan:
```markdown
### Idea name
Describe the intended capability, who benefits, and the most important scope
boundary or dependency.
- Optionally record an important behavior or boundary.
```

View File

@@ -0,0 +1,841 @@
# Backend-Specific Concurrency Management Implementation Plan
**Status:** Complete.
## Purpose
This document is the decision-complete implementation plan for
[backend-specific concurrency management](concurrency.md). It is written for a
coding agent that will implement each stage in order.
The feature roadmap owns the intended capability, consumer value, policy
choices, compatibility decision, and target end state. This document owns the
concrete API, internal representation, scheduling architecture, implementation
sequence, test ownership, documentation updates, and completion gates.
## Implementation Rules
- Complete the stages in order. Keep the repository compiling and the focused
tests passing at every stage boundary.
- Preserve unrelated working-tree changes. In particular, `concurrency.md` and
the removal of its source idea from `future.md` may already be uncommitted
when implementation begins; retain both.
- Follow every policy under `docs/policy/`, the task-specific reading guide in
`docs/development.md`, and the target behavior in `concurrency.md`.
- Keep the public API in the root `promptkit` package and implementation
details under `internal/`. Do not expose scheduler types or create another
public package.
- Use only the Go standard library for scheduling. Do not add a queue,
semaphore, worker-pool, or metrics dependency.
- Preserve synchronous, wait-for-result `Run`, unrestricted `Prepare`,
engine-local state, endpoint-only profiles, backend-selected profiles,
backend identity through endpoint overrides, and injected `LLMClient`
behavior.
- Do not broaden the work into asynchronous jobs, durable queues, retries,
rate limiting, dynamic configuration, priorities, worker lifecycle,
endpoint-keyed pools, or public queue observability.
- Keep all tests deterministic, bounded, offline, and race-safe. Coordinate
concurrent tests with channels and barriers rather than timing assumptions
or live providers.
- Update exact GoDoc with each exported declaration change. Update durable
current-state documents only after the corresponding behavior is
implemented.
- Test configurable mechanisms with small test-owned limits. Assert the exact
OpenRouter `16` and default queue `1024` values only at the registry contract
that owns those operational defaults.
- Do not create a release, change a module version, or tag a commit. The final
implementation handoff must identify the built-in OpenRouter behavior change
for the next pre-`v1` minor release.
## Fixed Design
### Public Backend Configuration
Append these fields to the existing root `Backend` type in `backends.go`:
```go
type Backend struct {
// Existing fields remain unchanged and in their current order.
ConcurrencyLimit int
QueueCapacity *int
}
```
Use these exact semantics:
| Public values | Meaning |
| --- | --- |
| `ConcurrencyLimit == 0`, `QueueCapacity == nil` | Unlimited backend; preserve current behavior. |
| `ConcurrencyLimit > 0`, `QueueCapacity == nil` | Limit active generations and use the default waiting capacity of 1024. |
| `ConcurrencyLimit > 0`, `QueueCapacity != nil` | Limit active generations and use the pointed-to capacity exactly, including zero. |
| `ConcurrencyLimit < 0` | Invalid engine configuration. |
| `QueueCapacity != nil` and `*QueueCapacity < 0` | Invalid engine configuration. |
| `ConcurrencyLimit == 0` and `QueueCapacity != nil` | Invalid engine configuration because a queue without an active limit has no defined consumer value. |
`ConcurrencyLimit` counts simultaneous calls to the engine-owned internal
model-client boundary for this backend. `QueueCapacity` controls additional
accepted `Run` invocations beyond that limit. The maximum admitted runs for a
limited backend is therefore:
```text
ConcurrencyLimit + effective QueueCapacity
```
Guard that addition against integer overflow during backend validation.
Do not impose an arbitrary upper bound beyond non-negativity and overflow
safety.
The `QueueCapacity` pointer exists only to distinguish omission from explicit
zero. `WithBackend` and `NewEngine` must not retain the caller's pointer.
`Backend` continues to have no stable JSON representation, and consumers
remain directed to keyed literals.
Do not add concurrency fields to `Profile`, `ExecutionTarget`,
`ExecutionTargetOverride`, `RunRequest`, prompt or profile files, or stable
prepared/result JSON.
### Built-In And Custom Defaults
The backend registry owns these exact operational defaults:
```go
const (
openRouterConcurrencyLimit = 16
defaultQueueCapacity = 1024
)
```
The built-in `openrouter` definition has a normalized concurrency limit of 16
and queue capacity of 1024.
Consumer registrations remain unlimited when concurrency is omitted. For a
consumer backend with a positive limit and omitted queue capacity, normalize
the queue capacity to 1024. Preserve an explicitly configured zero.
Consumers still cannot replace the reserved `openrouter` registration.
Endpoint-only profiles have no backend policy and remain unlimited. A selected
backend retains its pool when a profile or request overrides only its endpoint.
### Internal Backend Representation
Extend `internal/domain.Backend` with scalar policy values and explicit
presence rather than retaining a pointer:
```go
type Backend struct {
// Existing fields...
ConcurrencyLimit int
QueueCapacity int
QueueCapacitySet bool
}
type BackendCapacityPolicy struct {
ConcurrencyLimit int
QueueCapacity int
}
```
`WithBackend` converts the public pointer into `QueueCapacity` plus
`QueueCapacitySet`. Registry normalization validates the combinations above,
fills the default, and leaves every limited stored backend with
`QueueCapacitySet == true`. Unlimited stored backends retain zero values and
`QueueCapacitySet == false`.
Add this internal registry method:
```go
func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy
```
It returns a newly allocated map containing only limited backends. Values are
scalars, so callers cannot mutate registry state. The built-in OpenRouter
policy is included. `GetBackend` continues returning a defensive backend copy,
now including normalized scalar capacity metadata.
Capacity policy is operational registry metadata. Do not merge it into an
execution target or expose it to injected model clients.
### Public Capacity Error
Add this root sentinel beside the other run errors in `engine.go`:
```go
var ErrCapacityExceeded = errors.New("backend capacity exceeded")
```
Its GoDoc must state that it identifies a `Run` rejected because the selected
backend has already admitted `ConcurrencyLimit + QueueCapacity` runs. It is
not an invalid request, an LLM/provider rate-limit response, or an
`ErrLLMGenerate` failure.
The internal capacity component owns a corresponding internal
`ErrCapacityExceeded`. Add its mapping in `publicErrorFor` before the broader
generation and invalid-request cases. The public error must preserve the
internal error through wrapping while matching `ErrCapacityExceeded` with
`errors.Is`.
A capacity rejection returns no partial result and must not invoke the
artifact reader, renderer, schema loader, validator, or model client. Prompt,
profile, and backend loading needed to select the pool may already have
occurred.
### Internal Capacity Component
Add `internal/capacity` as the single owner of engine-local run admission and
active-generation permits.
Use these package-level boundaries:
```go
var ErrCapacityExceeded error
type Manager struct {
// Private immutable pool map.
}
func NewManager(
policies map[string]domain.BackendCapacityPolicy,
) (*Manager, error)
func (m *Manager) Admit(
ctx context.Context,
backendID string,
) (release func(), err error)
func NewClient(m *Manager, next llm.Client) llm.Client
```
`NewManager` copies the supplied map and creates one independent pool per
limited backend. Defensively reject blank IDs, non-positive concurrency
limits, negative queue capacities, or total-capacity overflow even though the
registry normally supplies normalized values. Construction creates no worker
goroutines.
An absent manager, blank backend ID, or ID absent from the policy map is
unlimited:
- `Admit` succeeds with a non-nil no-op release function; and
- the client wrapper calls the next client directly.
For a limited pool, `Admit` is immediate and context-aware:
1. return `ctx.Err()` if the context is already done;
2. under the pool lock, compare admitted runs with
`ConcurrencyLimit + QueueCapacity`;
3. return an error matching internal `ErrCapacityExceeded` when full; or
4. increment admitted runs and return an idempotent release function.
The release function decrements admission exactly once, even if accidentally
called more than once. It does not release an active-generation permit; those
permits have their own lifetime.
### FIFO Generation Permits
`NewClient` returns an internal `llm.Client` wrapper around either the built-in
client or the public-client adapter. It must preserve requests, successful
responses, nil responses, and collaborator error identities exactly.
`next` must be non-nil; `NewEngine` and internal runner construction maintain
that invariant. A nil manager returns `next` unchanged.
For a configured backend ID, the wrapper:
1. acquires one active-generation permit from the matching pool;
2. waits in FIFO order when the active count equals `ConcurrencyLimit`;
3. removes a canceled waiter and returns `ctx.Err()` when cancellation wins
before the permit is granted;
4. invokes the next client only after a permit is granted; and
5. releases the permit with `defer` after every success, nil response,
collaborator error, panic unwinding, or context outcome.
Implement FIFO and cancellation explicitly with a mutex and an ordered waiter
list. A channel used only as a counting semaphore is insufficient because it
does not define FIFO ordering or safe removal of canceled waiters.
Permit grant and cancellation must have one lock-protected linearization
point. If cancellation removes the waiter first, do not invoke the next
client. If grant wins first, invoke the next client with the caller's context;
the next client may then observe cancellation normally. Never lose or
double-release a permit in this race.
Releasing a permit transfers it to the oldest non-canceled waiter before
making it generally available. Different backend pools never share admission
or active counts.
The active wrapper enforces its limit even if an internal caller invokes it
without a run admission lease. Bounded backlog is guaranteed for ordinary
engine `Run` calls by the runner admission path; no public API exposes the
wrapped internal client directly.
### Engine Assembly
In `NewEngine`, after constructing the validated backend registry:
1. obtain `backendRegistry.CapacityPolicies()`;
2. construct one `capacity.Manager`;
3. construct the selected base internal LLM client exactly as today;
4. wrap that base client with `capacity.NewClient`; and
5. pass both the wrapped client and manager-as-admitter to the runner.
Every `NewEngine` call constructs a distinct manager. Do not cache managers,
pools, or policies in package globals. The wrapper must be applied after a
public injected client is adapted to `internal/llm.Client`, so built-in and
injected clients receive identical scheduling behavior.
If `NewManager` reports a defensive configuration error, make `NewEngine`
return an error matching `ErrInvalidConfig`.
`Prepare` does not use the manager. An injected client remains required to be
safe for concurrent calls because different backend pools and unlimited
backends may still invoke it concurrently.
### Shared Two-Phase Preparation
Refactor `internal/usecase.Runner` so `Prepare` and `Run` share one preparation
pipeline with two private phases. Do not duplicate prompt/profile/backend
selection or execution precedence.
The first phase resolves only the state required before admission:
1. validate `PromptID`;
2. normalize the direct session ID;
3. load the prompt definition;
4. hash the original prompt definition at its existing error-order position;
5. select and load the execution profile;
6. resolve the selected backend;
7. resolve and validate the effective execution target and credentials; and
8. resolve the effective output contract without loading its schema.
Return a private state value containing the loaded definition, normalized
direct session, prompt-definition hash, selected profile ID, effective target,
numeric-presence metadata, effective output contract, and preparation start
time. Keep this value private to `internal/usecase`.
The second phase consumes that state and performs:
1. structured-output schema loading;
2. artifact loading and input hashing;
3. message and prompt-session rendering;
4. direct-session application;
5. rendered-prompt hashing; and
6. `PreparedRun` construction and timing.
Preserve every existing precedence rule, error identity, direct-session
template bypass, hash input, selected identity, copy guarantee, and timing
field. Do not reload the prompt, profile, or backend between phases.
`Runner.Prepare` records its start time, runs both phases consecutively, and
never calls admission. Its behavior and error ordering remain unchanged.
`Runner.Run` records its existing run start time, runs the first preparation
phase, and then calls:
```go
release, err := r.admitter.Admit(ctx, effectiveBackendID)
```
Use a narrow use-case-owned interface with the same signature:
```go
type RunAdmitter interface {
Admit(context.Context, string) (func(), error)
}
```
A nil admitter means unlimited behavior for internal constructors and tests.
On successful admission, immediately `defer release()` around the remainder of
the run. Then run the second preparation phase, initial generation,
validation, and all repair attempts.
If admission returns internal `capacity.ErrCapacityExceeded`, add useful
backend context without changing its identity. If it returns `ctx.Err()`,
preserve that identity directly rather than recategorizing it as invalid
request or generation failure.
This refactor intentionally replaces the current literal `Run`-calls-`Prepare`
implementation with shared private phases. Update current-state documentation
to describe one shared pipeline rather than retaining an inaccurate call-graph
claim.
### Generation And Repair Lifetime
The admission lease covers the entire accepted run:
- second-phase preparation;
- initial generation;
- validation;
- every repair; and
- all failure and cancellation exits.
Preparation and validation do not hold an active-generation permit. The
wrapped client acquires a permit only around each actual `Generate` call.
The runner's initial generation already carries the effective backend ID in
`GenerateRequest.Target`. Preserve that value. `RepairRequest.Target` and the
default repairer's generated request must continue carrying the same backend
ID, allowing each repair to reacquire the same pool's active permit.
When testing or constructing `NewRunnerWithRepairer`, pass the same wrapped
client to both the runner and `NewDefaultOutputRepairer`. Do not add capacity
state to `RepairRequest`, `ExecutionTarget`, or public generation values.
A repair remains within its existing admission lease. It waits for a FIFO
active permit but never performs a second bounded admission and therefore
cannot fail merely because later runs filled the admission capacity.
### Error And Cancellation Semantics
The required public outcomes are:
| Situation | Required error identity |
| --- | --- |
| Admission capacity is full | `ErrCapacityExceeded` only; not `ErrInvalidRequest` or `ErrLLMGenerate`. |
| Context is done before admission succeeds | Preserve `ctx.Err()`; do not return capacity exhaustion. |
| Context cancels while waiting for an active permit | Preserve `ctx.Err()` through the existing `ErrLLMGenerate` generation category. |
| Wrapped client fails after permit acquisition | Preserve existing `ErrLLMGenerate` and collaborator identities. |
| Preparation or validation fails after admission | Preserve its existing category and release admission. |
Maintain the existing rule that `Run` returns no partial result on any
operational error. Do not add queue status to errors or results.
`RunResult.Duration` continues to start at runner entry and therefore includes
pre-admission resolution, accepted preparation, and active-permit waiting.
`PreparedRun.DurationMS` continues to cover only its shared preparation phases;
it does not include later generation waiting. Capacity-rejected calls have no
result or timing value.
### Ownership And Concurrency Safety
The registry, capacity policy map, pool map, and per-pool limits are immutable
after engine construction. Only admission counts, active counts, and waiter
lists are mutable and must be protected by the owning pool mutex.
Do not retain public queue pointers, caller request values, contexts, or
generation requests after their call completes. A canceled waiter must be
unlinked so its context and request cannot remain reachable from the pool.
Do not hold a pool mutex while:
- loading or rendering prompts;
- reading artifacts or schemas;
- invoking a model client;
- validating output;
- closing a waiter notification channel if the implementation could re-enter
pool code; or
- calling consumer code.
No scheduler operation may spawn a goroutine whose lifetime outlasts the
calling `Run`. The zero steady-state goroutine count is part of the
in-process/no-worker-lifecycle design.
## Test Ownership
Use this ownership split and avoid repeating the full policy matrix at every
layer:
- `internal/backend/registry_test.go` owns normalization, validation, the exact
OpenRouter policy, the custom default queue, explicit zero, unlimited
omission, and policy-map copying.
- `internal/capacity/manager_test.go` owns admission bounds, idempotent release,
FIFO active permits, cancellation races, capacity recovery, independent
pools, unlimited IDs, and observed peak concurrency.
- `internal/capacity/client_test.go` owns wrapper request/response/error
transparency and the rule that cancellation before grant does not invoke the
next client. Combine these with manager tests if one coherent package test
expresses the behavior more clearly.
- `internal/usecase/runner_test.go` owns two-phase preparation parity, pool
selection, admission before expensive work, admission release across run
exits, `Prepare` bypass, and repair reuse of the admitted backend.
- Root external-package tests own public configuration conversion, assembled
engine-local behavior, endpoint-override routing, injected-client limiting,
and public capacity/context error identities.
- Existing model-client HTTP tests remain unchanged because scheduling does
not alter the OpenAI-compatible wire contract.
Concurrency tests must use test-owned limits such as one or two and
channel-controlled blocking clients. Record observed active and peak counts
under a mutex or atomics. Do not use `time.Sleep` to infer queue state.
Package-internal tests may inspect a waiter list under its mutex through a
small test helper when necessary to establish deterministic FIFO ordering; do
not add production metrics or hooks solely for tests.
Do not add separate tests for trivial scalar copies when registry or assembled
behavior already protects them.
## Stage 1 — Backend Policy And Public Configuration
**Status:** Complete.
### Goal
Add the public and internal backend policy representation, normalize all
configured states, and expose immutable normalized policies without changing
runtime scheduling yet.
### Work
1. Add `ConcurrencyLimit` and `QueueCapacity` to `Backend` in `backends.go`
with exact GoDoc for unlimited, defaulted, explicit-zero, invalid, and
engine-scoped behavior.
2. Convert the public queue pointer into scalar value plus presence in
`WithBackend`; do not retain the pointer.
3. Add the internal backend policy fields and
`BackendCapacityPolicy` to `internal/domain/domain.go`.
4. Add the two registry-owned constants and configure the built-in OpenRouter
definition with 16 and 1024.
5. Extend `normalizeBackend` with the fixed validation, defaulting, explicit
zero, and overflow rules.
6. Add `Registry.CapacityPolicies`, returning only limited policies in a fresh
map.
7. Update existing backend composite literals and assertions only where the
new fields are relevant. Continue using keyed literals.
### Tests
1. Extend the exact built-in registry test with the OpenRouter limit and queue.
2. Add one coherent table covering unlimited omission, default queue,
explicit-zero queue, negative values, queue-without-limit, and total
overflow.
3. Extend the registry copy/normalization test to prove returned policy maps
cannot mutate registry state.
4. Add root coverage only if needed to prove the public pointer/presence
conversion; do not reproduce registry validation cases at the facade.
### Focused Validation
Run:
```sh
gofmt -w backends.go internal/domain/domain.go \
internal/backend/registry.go internal/backend/registry_test.go
go test . ./internal/backend
go vet . ./internal/backend
git diff --check
```
Include another touched Go test file in `gofmt` only if it actually changed.
### Completion Gate
This stage is complete when every public configuration state has one normalized
internal meaning, OpenRouter exposes exactly 16/1024, custom backends remain
unlimited by omission, and no runtime call is scheduled yet.
## Stage 2 — Engine-Local Capacity Manager
**Status:** Complete.
### Goal
Implement and prove the bounded admission mechanism and FIFO active-generation
client wrapper independently of runner orchestration.
### Work
1. Add `internal/capacity/manager.go` with the manager, immutable policy copy,
per-backend pools, internal error, immediate admission, idempotent release,
and FIFO context-aware active permits.
2. Add `internal/capacity/client.go` with the transparent `llm.Client` wrapper.
3. Use mutex-protected waiter state and an ordered list; explicitly resolve
grant-versus-cancel races.
4. Ensure unlimited and independent-pool fast paths avoid queue allocation.
5. Do not start workers, timers, cleanup goroutines, or process-global state.
### Tests
1. Add a compact constructor-validation table for blank IDs, non-positive
limits, negative queues, and total-capacity overflow.
2. With a small configured policy, prove that exactly
`limit + queueCapacity` admissions succeed, the next matches
`ErrCapacityExceeded`, and a release permits another admission.
3. Prove release is idempotent.
4. Drive more blocked client calls than the active limit and assert observed
peak concurrency never exceeds that limit.
5. Prove FIFO order with deterministic queue-entry synchronization.
6. Cancel the first and a middle waiter and prove they are removed, never call
the wrapped client, and do not block later waiters.
7. Exercise the grant/cancel race repeatedly under `go test -race`, asserting
no permit leak or double invocation.
8. Prove different backend IDs proceed independently and blank, unknown, or
nil-manager paths remain unlimited.
9. Prove request values, successful and nil responses, and collaborator errors
pass through unchanged after permit acquisition.
### Focused Validation
Run:
```sh
gofmt -w internal/capacity/manager.go \
internal/capacity/manager_test.go \
internal/capacity/client.go \
internal/capacity/client_test.go
go test ./internal/capacity
go test -race ./internal/capacity
go vet ./internal/capacity
git diff --check
```
If tests are combined into one file, omit the nonexistent file from `gofmt`.
### Completion Gate
This stage is complete when the standalone component enforces relational
admission and active limits, FIFO cancellation is race-safe, separate pools
are independent, and the wrapper is transparent apart from waiting.
## Stage 3 — Shared Preparation And Early Run Admission
**Status:** Complete.
### Goal
Refactor runner preparation into one shared two-phase pipeline and place
bounded admission after backend resolution but before schema, artifact, and
rendering work.
### Work
1. Add the private pre-admission preparation state and split the existing
`Prepare` logic according to the fixed design.
2. Make `Runner.Prepare` call both phases without an admitter.
3. Add the `RunAdmitter` interface and runner field.
4. Update `NewRunner` and `NewRunnerWithRepairer` to accept the optional
admitter; update internal call sites with nil until root assembly is wired.
5. Change `Runner.Run` to use the first phase, admit by effective backend ID,
defer the returned release, and then use the second phase.
6. Preserve all existing error precedence, target resolution, hashes,
metadata, session behavior, and timing.
7. Return capacity and context errors with the fixed identities. Do not invoke
later collaborators after rejection.
### Tests
1. Keep the existing `Run`/`Prepare` parity coverage passing to prove the
shared phases do not drift.
2. Add a fake admitter that records backend IDs and release calls.
3. Prove a backend-selected run admits with the selected ID even when the
endpoint is overridden.
4. Prove an endpoint-only run uses the unlimited/blank identity and that
`Prepare` never calls admission.
5. Reject admission and assert schema, artifact, renderer, validator, repairer,
and LLM collaborators are not invoked.
6. Prove admission is released after one successful run and representative
second-phase, generation, and validation errors. Prefer a small table around
the single `defer` invariant rather than duplicating every error test.
7. Retain direct-session, backend precedence, credential, hashing, and repair
tests unchanged except for constructor arguments.
### Focused Validation
Run:
```sh
gofmt -w internal/usecase/runner.go \
internal/usecase/runner_test.go
go test ./internal/usecase
go test -race ./internal/usecase
go vet ./internal/usecase
git diff --check
```
### Completion Gate
This stage is complete when `Prepare` remains unrestricted, `Run` admits after
one canonical routing phase and before expensive completion work, every exit
releases admission, and existing preparation semantics remain unchanged.
## Stage 4 — Engine Assembly And Public Runtime Contract
**Status:** Complete.
### Goal
Wire one manager into each engine, schedule built-in and injected clients,
expose the capacity error, and prove assembled runtime behavior.
### Work
1. Add public `ErrCapacityExceeded` and its exact GoDoc in `engine.go`.
2. Map internal capacity exhaustion in `errors.go`.
3. Construct the manager from the registry policy snapshot in `NewEngine`.
4. Wrap the selected internal client after built-in or injected-client
selection and pass the manager and wrapped client to the runner.
5. Update `Engine`, `NewEngine`, `Run`, `WithLLMClient`, and `LLMClient` GoDoc
only where concurrency, capacity, or cancellation statements change.
6. Ensure manager-construction errors match `ErrInvalidConfig`.
7. For internal repair coverage, construct the default repairer with the same
wrapped client used by its runner and confirm repair target backend identity
remains intact.
### Tests
1. Add an external-package assembled test with a small custom limit and a
blocking injected client; assert peak generation equals or remains below
the configured limit.
2. With queue capacity zero, block one accepted run before generation and
assert the next matching-backend run returns `ErrCapacityExceeded`, does not
match `ErrInvalidRequest` or `ErrLLMGenerate`, returns no result, and never
reaches expensive collaborators or the client.
3. In the same or another focused workflow, prove an endpoint override remains
in the selected backend's pool.
4. Prove two engines with the same backend ID have independent capacity.
5. Prove an unlimited custom backend and an endpoint-only profile preserve
concurrent behavior.
6. Cancel a call waiting for an active permit; assert it matches both
`context.Canceled` and `ErrLLMGenerate`, never invokes the injected client,
and leaves capacity reusable.
7. Add one internal repair workflow with concurrent runs or controlled permits
showing initial and repair generations never exceed the same backend limit
and repairs do not perform a second admission.
8. Extend the public error sentinel contract test with
`ErrCapacityExceeded`.
Avoid a second HTTP-level concurrency suite: the capacity client tests and one
assembled injected-client workflow already protect the shared wrapper used by
the built-in client.
### Focused Validation
Run:
```sh
gofmt -w engine.go errors.go backends.go \
internal/usecase/runner.go internal/usecase/runner_test.go \
engine_test.go public_contract_test.go
go test . ./internal/backend ./internal/capacity ./internal/usecase
go test -race . ./internal/capacity ./internal/usecase
go vet . ./internal/backend ./internal/capacity ./internal/usecase
git diff --check
```
Add any newly created capacity files to `gofmt` when they changed in this
stage.
### Completion Gate
This stage is complete when every engine has independent pools, limited runs
are bounded and FIFO at generation, endpoint routing is correct, capacity and
context errors are stable, repairs reuse admission, and both client kinds pass
through the same wrapper.
## Stage 5 — Durable Documentation And Final Validation
**Status:** Complete.
### Goal
Move implemented contracts into their durable owners, record compatibility
impact, and validate the complete repository.
### Work
1. Review every changed exported declaration. Ensure GoDoc is the canonical
owner of exact field types, nil/zero semantics, defaulting, error identity,
engine scope, concurrency safety, cancellation, and source compatibility.
2. Update `doc.go` so its concurrency summary acknowledges backend scheduling
while continuing to require injected collaborators to be concurrency-safe.
3. Update `docs/consumers/pkg-promptkit.md` with task-oriented examples for:
- a limited local backend;
- omitted queue capacity selecting 1024;
- explicit zero queue capacity; and
- handling `ErrCapacityExceeded`.
Keep exact field semantics in GoDoc rather than duplicating a full table.
4. Add `docs/internal/capacity.md` as the durable owner of pool lifecycle,
admission, FIFO active permits, cancellation, client wrapping, and test
ownership.
5. Add `internal/capacity` to `docs/internal/overview.md`.
6. Update `docs/policy/architecture.md` to include the implemented component
and root assembly dependency without turning policy into an API reference.
7. Update `docs/internal/runner.md` to describe the shared two-phase
preparation pipeline, early bounded admission, lease lifetime, generation
permits, repairs, capacity failures, and cancellation.
8. Review `docs/formats.md`; add only a concise link or clarification if needed
to explain that endpoint overrides preserve backend capacity identity.
Do not add concurrency fields to YAML.
9. Do not change the OpenAI-compatible integration contract or
`docs/internal/llm.md` unless implementation changes their current
statements; scheduling is outside the provider wire contract and concrete
model-client implementation.
10. Record in the implementation handoff that built-in OpenRouter now limits
active generations to 16 with queue capacity 1024 and that the release
must be a pre-`v1` minor release. Do not edit the release procedure or
create a tag.
11. After every check passes, set `concurrency.md`, this implementation plan,
and each stage status to `Complete`. Do not remove the roadmaps in the
implementation change; lifecycle retirement follows review.
### Full Validation
Run the complete sequence from `docs/development.md`:
```sh
go test ./...
go test -race ./...
go vet ./...
go build ./...
go run ./examples/go-library/prepare
gofmt -l $(git ls-files '*.go')
git diff --check
```
The formatting command must produce no paths. Follow every added or changed
Markdown link and confirm its target and heading exist.
Also inspect:
```sh
git status --short
git diff --stat
git diff
```
Confirm that:
- only intended backend, capacity, runner, facade, test, documentation, and
roadmap files changed;
- no `go.work`, `go.work.sum`, local module replacement, credential,
generated binary, coverage output, or unrelated change was introduced;
- the built-in OpenRouter policy is exactly 16/1024;
- custom and endpoint-only backends remain unlimited by omission;
- explicit queue zero is distinguishable from omission;
- no capacity value enters execution targets, generated requests, stable JSON,
prompt/profile YAML, or provider payloads;
- every engine owns distinct pools with no package-global mutable state;
- every initial and repair generation uses the active permit wrapper;
- capacity and waiter state is released on success, error, panic unwinding,
and cancellation;
- concurrency tests use deterministic coordination rather than sleeps;
- current-state documentation describes implemented behavior rather than
referring readers to the roadmaps; and
- the feature and implementation roadmaps contain no unresolved work marked
complete.
### Completion Gate
The implementation is complete only when every target-end-state item in
`concurrency.md` is implemented, race-enabled tests demonstrate the configured
limits and cancellation safety, durable contracts no longer depend on roadmap
prose, and the OpenRouter compatibility change is clearly reported for the
next minor release.
## Implementation Handoff
Backend-specific capacity management is implemented and has passed the complete
repository validation sequence. The built-in OpenRouter backend now permits 16
active generations and a waiting capacity of 1024. Custom backends remain
unlimited when their limit is omitted, and endpoint-only profiles remain
unlimited.
Publishing this behavior requires a pre-`v1` minor release. Its release notes
must identify that unusually high concurrent OpenRouter use can now wait or
return `ErrCapacityExceeded`. This implementation does not change a module
version or create a tag.
## Open Questions
None. The feature roadmap and this plan fix the public representation,
registry defaults, admission bound, FIFO generation behavior, early-routing
refactor, cancellation races, error identities, engine and repair lifetimes,
test ownership, compatibility treatment, and non-goals required for
implementation.

493
engine.go Normal file
View File

@@ -0,0 +1,493 @@
package promptkit
import (
"context"
"errors"
"fmt"
"io/fs"
"net/http"
"os"
"path/filepath"
"strings"
"time"
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
"gitea.maximumdirect.net/eric/promptkit/internal/profile/builtin"
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
)
// ErrInvalidConfig identifies invalid engine construction, including missing
// required configuration, invalid options or backend registrations, and a nil
// Engine receiver.
var ErrInvalidConfig = errors.New("invalid engine configuration")
var (
// ErrInvalidRequest identifies a request whose required values, overrides,
// credentials, or effective settings are invalid.
ErrInvalidRequest = errors.New("invalid run request")
// ErrPromptNotFound identifies a requested prompt ID or version that is not
// present in the selected prompt source. It does not also match
// ErrPromptLoad.
ErrPromptNotFound = errors.New("prompt not found")
// ErrProfileNotFound identifies a selected profile ID that is absent from
// every configured profile source. It does not also match ErrProfileLoad.
ErrProfileNotFound = errors.New("profile not found")
// ErrProfileRequired identifies a request for which neither RunRequest.ProfileID
// nor the selected prompt's default profile is present. Such an error also
// matches ErrInvalidRequest.
ErrProfileRequired = errors.New("profile selection is required")
// ErrPromptLoad identifies a failure to read, decode, validate, select, or
// hash a prompt definition, except for the not-found case represented by
// ErrPromptNotFound.
ErrPromptLoad = errors.New("failed to load prompt definition")
// ErrProfileLoad identifies a failure to read, decode, validate, or select
// an execution profile or resolve its backend, except for the profile
// not-found case represented by ErrProfileNotFound.
ErrProfileLoad = errors.New("failed to load execution profile")
// ErrAPIKeyEnvMissing identifies an APIKeyEnv whose environment variable is
// unset or empty when no direct RunRequest.APIKey takes precedence. Such an
// error also matches ErrInvalidRequest.
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
// ErrArtifactLoad identifies a failure to resolve an input artifact. Errors
// returned by an injected ArtifactReader remain available through errors.Is.
ErrArtifactLoad = errors.New("failed to load artifact")
// ErrPromptRender identifies a failure to render prompt messages or the
// session ID from the resolved inputs and variables.
ErrPromptRender = errors.New("failed to render prompt")
// ErrCapacityExceeded identifies a Run rejected because the selected backend
// already admitted ConcurrencyLimit + QueueCapacity calls. It is not an
// invalid request, an LLM or provider rate-limit response, or ErrLLMGenerate.
ErrCapacityExceeded = errors.New("backend capacity exceeded")
// ErrLLMGenerate identifies a model-client failure or a nil successful
// response. Errors returned by an injected LLMClient remain available
// through errors.Is.
ErrLLMGenerate = errors.New("failed to generate output")
// ErrValidation identifies an operational failure to load or compile a
// schema or validate output. A completed validation whose Status is
// ValidationFailed is returned in RunResult without this error.
ErrValidation = errors.New("failed to validate output")
)
// Engine prepares and runs Promptkit prompt requests.
//
// An Engine is safe for concurrent calls to [Engine.Prepare] and [Engine.Run].
// Each Engine owns independent backend-capacity pools that coordinate Run
// admission and model generation. Injected collaborators may still be invoked
// concurrently across different backend pools or for unlimited backends.
type Engine struct {
runner *usecase.Runner
}
// Config selects the directory-backed sources and built-in model-client
// transport used by [NewEngine]. Config has no stable JSON representation.
type Config struct {
// PromptDir is the directory searched recursively for prompt definitions.
// It is required unless a WithPromptFS or WithPromptFile option supplies the
// prompt source.
PromptDir string
// ProfileDir is an optional directory whose profiles take precedence over
// embedded built-in profiles. An empty value selects only built-ins unless
// profile options are also supplied.
ProfileDir string
// SchemaDir is the root for JSON Schema files. An empty value uses the
// current directory. WithSchemaFS or WithSchemaFile replaces this source.
SchemaDir string
// Timeout is the transport-wide safety cap for the built-in LLM client
// when HTTPClient is absent or has a non-positive timeout. A zero or negative
// value selects the 10-minute default.
Timeout time.Duration
// HTTPClient is cloned for the built-in LLM client. Its positive Timeout
// takes precedence over Timeout. A zero or negative client Timeout inherits
// Timeout or the 10-minute default. The supplied client is not mutated. This
// field is ignored when WithLLMClient is used.
HTTPClient *http.Client
}
// Option customizes engine construction.
//
// NewEngine applies options in argument order and ignores nil options. Within
// each prompt-source, profile-source, in-memory-profile, schema-source,
// model-client, and artifact-reader category, the last non-nil valid option
// replaces earlier options in that category. WithBackend is the additive
// exception: unique registrations accumulate, and a repeated backend ID is an
// error rather than a replacement. An invalid option fails construction even
// if a later option would replace it.
type Option interface {
apply(*engineOptions) error
}
type optionFunc func(*engineOptions) error
func (f optionFunc) apply(options *engineOptions) error {
return f(options)
}
type engineOptions struct {
llmClient llm.Client
artifactReader artifactadapter.Reader
promptDefs promptdef.Repository
profiles profile.Repository
memoryProfiles profile.Repository
backends []domain.Backend
validator validate.Validator
promptSource bool
profileSource bool
memorySource bool
validatorSource bool
artifactSource bool
}
// WithLLMClient replaces the built-in model client used by [Engine.Run].
//
// A nil client makes NewEngine fail with ErrInvalidConfig. The Engine schedules
// Generate calls according to the selected backend's capacity policy, but the
// client may still be called concurrently across different backend pools or for
// unlimited backends. The client is not used by [Engine.Prepare].
func WithLLMClient(client LLMClient) Option {
return optionFunc(func(options *engineOptions) error {
if client == nil {
return ErrInvalidConfig
}
options.llmClient = publicLLMClientAdapter{client: client}
return nil
})
}
// WithArtifactReader replaces the default reader for every input artifact
// reference, regardless of its ArtifactRef.Type.
//
// A nil reader makes NewEngine fail with ErrInvalidConfig. The reader may be
// called concurrently.
func WithArtifactReader(reader ArtifactReader) Option {
return optionFunc(func(options *engineOptions) error {
if reader == nil {
return ErrInvalidConfig
}
options.artifactReader = publicArtifactReaderAdapter{reader: reader}
options.artifactSource = true
return nil
})
}
// WithPromptFS loads prompt definitions from fsys under root.
//
// The source uses the same strict prompt YAML rules as configured prompt
// directories, and prompt content_file paths resolve within this source.
// fsys must be non-nil and root must be non-empty; otherwise NewEngine fails
// with ErrInvalidConfig. This option replaces Config.PromptDir and earlier
// prompt-source options.
func WithPromptFS(fsys fs.FS, root string) Option {
return optionFunc(func(options *engineOptions) error {
if fsys == nil {
return ErrInvalidConfig
}
if strings.TrimSpace(root) == "" {
return ErrInvalidConfig
}
options.promptDefs = promptdef.NewFSRepository(fsys, root)
options.promptSource = true
return nil
})
}
// WithPromptFile loads prompt definitions from the single prompt file at path.
//
// Relative prompt content_file paths resolve from the file's directory. path
// must name an existing non-directory file when NewEngine applies the option.
// This option replaces Config.PromptDir and earlier prompt-source options.
func WithPromptFile(path string) Option {
return optionFunc(func(options *engineOptions) error {
fsys, root, err := fileSource(path)
if err != nil {
return err
}
options.promptDefs = promptdef.NewFSRepository(fsys, root)
options.promptSource = true
return nil
})
}
// WithProfileFS loads execution profiles from fsys under root.
//
// Profiles from this source overlay built-in profiles. Profile YAML must use
// api_key_env for environment-based credentials; raw API keys are rejected.
// fsys must be non-nil and root must be non-empty; otherwise NewEngine fails
// with ErrInvalidConfig. This option replaces Config.ProfileDir and earlier
// file or FS profile-source options, but remains below WithProfiles in
// precedence.
func WithProfileFS(fsys fs.FS, root string) Option {
return optionFunc(func(options *engineOptions) error {
if fsys == nil {
return ErrInvalidConfig
}
if strings.TrimSpace(root) == "" {
return ErrInvalidConfig
}
options.profiles = profile.NewFSRepository(fsys, root)
options.profileSource = true
return nil
})
}
// WithProfileFile loads execution profiles from the single profile file at path.
//
// The profile overlays built-in profiles. Profile YAML must use api_key_env for
// environment-based credentials; raw API keys are rejected. path must name an
// existing non-directory file when NewEngine applies the option. This option
// replaces Config.ProfileDir and earlier file or FS profile-source options,
// but remains below WithProfiles in precedence.
func WithProfileFile(path string) Option {
return optionFunc(func(options *engineOptions) error {
fsys, root, err := fileSource(path)
if err != nil {
return err
}
options.profiles = profile.NewFSRepository(fsys, root)
options.profileSource = true
return nil
})
}
// WithProfiles configures in-memory profiles that take precedence over
// configured profile files and built-in profiles.
//
// NewEngine validates and copies every profile. IDs must be unique within one
// call. An invalid profile, duplicate ID, or unsupported ExtraParams value
// makes construction fail with ErrInvalidConfig. Repeating WithProfiles
// replaces the complete earlier in-memory set rather than merging it.
func WithProfiles(profiles ...Profile) Option {
return optionFunc(func(options *engineOptions) error {
repo, err := newMemoryProfileRepository(profiles)
if err != nil {
return err
}
options.memoryProfiles = repo
options.memorySource = true
return nil
})
}
// WithSchemaFS loads JSON Schema documents from fsys under root.
//
// Prompt schema_path values resolve within this source when schema validation
// or structured output is requested. fsys must be non-nil and root must be
// non-empty; otherwise NewEngine fails with ErrInvalidConfig. This option
// replaces Config.SchemaDir and earlier schema-source options.
func WithSchemaFS(fsys fs.FS, root string) Option {
return optionFunc(func(options *engineOptions) error {
if fsys == nil {
return ErrInvalidConfig
}
if strings.TrimSpace(root) == "" {
return ErrInvalidConfig
}
options.validator = validate.NewFSValidator(fsys, root)
options.validatorSource = true
return nil
})
}
// WithSchemaFile loads JSON Schema documents from the single schema file at path.
//
// Prompt schema_path values refer to the file's base name. path must name an
// existing non-directory file when NewEngine applies the option. This option
// replaces Config.SchemaDir and earlier schema-source options.
func WithSchemaFile(path string) Option {
return optionFunc(func(options *engineOptions) error {
fsys, root, err := fileSource(path)
if err != nil {
return err
}
options.validator = validate.NewFSValidator(fsys, root)
options.validatorSource = true
return nil
})
}
// NewEngine constructs an Engine from configuration and options.
//
// Options are applied in order according to [Option]. PromptDir is required
// unless a prompt-source option is present. Construction validates option
// arguments, in-memory profiles, and backend registrations but defers reading
// and validating prompt, file-backed profile, and schema contents until Prepare
// or Run needs them.
//
// NewEngine returns an error matching ErrInvalidConfig for invalid
// configuration, options, or backend-capacity policies. Each constructed
// Engine has independent backend-capacity pools. Construction does not perform
// model requests or require credentials.
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
var options engineOptions
for _, opt := range opts {
if opt == nil {
continue
}
if err := opt.apply(&options); err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
}
}
promptDefs := options.promptDefs
if !options.promptSource {
if strings.TrimSpace(cfg.PromptDir) == "" {
return nil, fmt.Errorf("%w: prompt directory is required", ErrInvalidConfig)
}
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
}
profiles := builtin.NewRepositoryWithDirectory(cfg.ProfileDir)
if options.profileSource {
profiles = builtin.NewRepositoryWithPrimary(options.profiles)
}
if options.memorySource {
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
}
backendRegistry, err := backend.NewRegistry(options.backends)
if err != nil {
return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err)
}
capacityManager, err := capacity.NewManager(backendRegistry.CapacityPolicies())
if err != nil {
return nil, fmt.Errorf("%w: failed to construct backend capacity manager: %v", ErrInvalidConfig, err)
}
validator := options.validator
if !options.validatorSource {
schemaDir := cfg.SchemaDir
if strings.TrimSpace(schemaDir) == "" {
schemaDir = defaults.SchemaDirDefault
}
validator = validate.NewStandardValidator(schemaDir)
}
llmClient := options.llmClient
if llmClient == nil {
var err error
llmClient, err = llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
Timeout: cfg.Timeout,
HTTPClient: cfg.HTTPClient,
})
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
}
}
llmClient = capacity.NewClient(capacityManager, llmClient)
artifacts := options.artifactReader
if !options.artifactSource {
artifacts = artifactadapter.NewCompositeReader()
}
return &Engine{
runner: usecase.NewRunner(
promptDefs,
profiles,
backendRegistry,
artifacts,
prompt.NewGoRenderer(),
llmClient,
validator,
capacityManager,
),
}, nil
}
func fileSource(name string) (fs.FS, string, error) {
cleanName := strings.TrimSpace(name)
if cleanName == "" {
return nil, "", ErrInvalidConfig
}
dir := filepath.Dir(cleanName)
base := filepath.Base(cleanName)
if base == "." || base == string(filepath.Separator) || strings.TrimSpace(base) == "" {
return nil, "", ErrInvalidConfig
}
info, err := os.Stat(cleanName)
if err != nil {
return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, cleanName, err)
}
if info.IsDir() {
return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, cleanName)
}
return os.DirFS(dir), filepath.ToSlash(base), nil
}
// Prepare resolves and renders a prompt request without calling an LLM.
//
// Prepare selects the prompt and profile, resolves any selected backend and
// effective execution settings, resolves the output contract, loads and hashes
// inputs, loads structured-output schema metadata when required, and renders
// the session ID and messages. The returned PreparedRun is owned by the caller
// and never contains a resolved API-key value, model output, or validation
// result.
//
// A nil Engine returns an error matching ErrInvalidConfig. Request and
// preparation failures may match ErrInvalidRequest, ErrPromptNotFound,
// ErrPromptLoad, ErrProfileNotFound, ErrProfileLoad, ErrProfileRequired,
// ErrAPIKeyEnvMissing, ErrArtifactLoad, ErrPromptRender, or ErrValidation as
// applicable. Cancellation is passed to the active collaborator and is
// reported in the applicable operation category; no general errors.Is
// relationship to ctx.Err is promised. Prepare returns no partial result on
// error.
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
domainReq, err := toDomainRunRequest(req)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
}
prepared, err := e.runner.Prepare(ctx, domainReq)
if err != nil {
return nil, mapPublicError(err)
}
return fromDomainPreparedRun(prepared), nil
}
// Run prepares a request, invokes the configured LLMClient, and validates the
// generated output.
//
// A content-validation failure is a successful run whose
// RunResult.Validation has Status ValidationFailed. An inability to perform
// validation returns an error matching ErrValidation and no partial result.
// The public Engine does not perform output repair, so validation is
// single-pass even when OutputContract.RepairAttempts is positive.
//
// Run can return every error category documented by [Engine.Prepare], plus
// ErrCapacityExceeded and ErrLLMGenerate. ErrCapacityExceeded identifies
// rejection before artifacts, schemas, rendering, or model generation because
// the selected backend's admission capacity is full; it does not match
// ErrInvalidRequest or ErrLLMGenerate. Errors from injected clients remain
// available through errors.Is. Cancellation while waiting for model-generation
// capacity matches both ErrLLMGenerate and the context error. Cancellation
// otherwise follows the active collaborator's documented behavior. A nil
// Engine returns ErrInvalidConfig. Run returns no partial result on error.
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
domainReq, err := toDomainRunRequest(req)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
}
result, err := e.runner.Run(ctx, domainReq)
if err != nil {
return nil, mapPublicError(err)
}
return fromDomainRunResult(result), nil
}

2706
engine_test.go Normal file

File diff suppressed because it is too large Load Diff

63
errors.go Normal file
View File

@@ -0,0 +1,63 @@
package promptkit
import (
"errors"
"fmt"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
)
func mapPublicError(err error) error {
if err == nil {
return nil
}
publicErr := publicErrorFor(err)
if publicErr == nil {
return err
}
return fmt.Errorf("%w: %w", publicErr, err)
}
func publicErrorFor(err error) error {
switch {
case errors.Is(err, promptdef.ErrPromptDefinitionNotFound):
return ErrPromptNotFound
case errors.Is(err, profile.ErrProfileNotFound):
return ErrProfileNotFound
case errors.Is(err, usecase.ErrProfileRequired):
return errors.Join(ErrInvalidRequest, ErrProfileRequired)
case errors.Is(err, usecase.ErrPromptLoad):
return ErrPromptLoad
case errors.Is(err, usecase.ErrProfileLoad):
return ErrProfileLoad
case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition):
return ErrPromptLoad
case isProfileLoadCause(err):
return ErrProfileLoad
case errors.Is(err, usecase.ErrAPIKeyEnvMissing):
return errors.Join(ErrInvalidRequest, ErrAPIKeyEnvMissing)
case errors.Is(err, capacity.ErrCapacityExceeded):
return ErrCapacityExceeded
case errors.Is(err, usecase.ErrArtifactLoad):
return ErrArtifactLoad
case errors.Is(err, usecase.ErrPromptRender):
return ErrPromptRender
case errors.Is(err, usecase.ErrLLMGenerate):
return ErrLLMGenerate
case errors.Is(err, usecase.ErrValidation):
return ErrValidation
case errors.Is(err, usecase.ErrInvalidRequest):
return ErrInvalidRequest
default:
return nil
}
}
func isProfileLoadCause(err error) bool {
return errors.Is(err, profile.ErrInvalidYAML) ||
errors.Is(err, profile.ErrInvalidProfile) ||
errors.Is(err, profile.ErrRawAPIKeyNotAllowed)
}

22
errors_internal_test.go Normal file
View File

@@ -0,0 +1,22 @@
package promptkit
import (
"context"
"errors"
"fmt"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
)
func TestMapPublicErrorPreservesGenerationCancellation(t *testing.T) {
internalErr := fmt.Errorf("%w: %w", usecase.ErrLLMGenerate, context.Canceled)
err := mapPublicError(internalErr)
if !errors.Is(err, ErrLLMGenerate) {
t.Fatalf("mapped error=%v, want ErrLLMGenerate", err)
}
if !errors.Is(err, context.Canceled) {
t.Fatalf("mapped error=%v, want context.Canceled", err)
}
}

View File

@@ -0,0 +1,60 @@
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"gitea.maximumdirect.net/eric/promptkit"
)
type summary struct {
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version"`
SelectedProfile string `json:"selected_profile"`
Model string `json:"model"`
MessageCount int `json:"message_count"`
}
func main() {
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFile("examples/go-library/prepare/prompt.yaml"),
promptkit.WithProfiles(promptkit.Profile{
ID: "offline-example",
Endpoint: "https://example.invalid/v1",
Model: "offline-model",
}),
)
if err != nil {
exit(err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "example.prepare",
Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.Inline("Ada finished the migration review."),
},
})
if err != nil {
exit(err)
}
encoder := json.NewEncoder(os.Stdout)
encoder.SetIndent("", " ")
if err := encoder.Encode(summary{
PromptID: prepared.PromptID,
PromptVersion: prepared.PromptVersion,
SelectedProfile: prepared.SelectedProfileID,
Model: prepared.EffectiveModelParams.Model,
MessageCount: len(prepared.Messages),
}); err != nil {
exit(err)
}
}
func exit(err error) {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}

View File

@@ -0,0 +1,16 @@
id: example.prepare
version: "1.0.0"
default_profile: offline-example
description: Prepare a prompt without contacting a model provider.
inputs:
- name: note
required: true
content_type: text/plain
messages:
- role: system
content: Summarize the note in one sentence.
- role: user
content: '{{input "note"}}'
output:
format: text
validation_mode: basic

View File

@@ -0,0 +1,81 @@
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"gitea.maximumdirect.net/eric/promptkit"
)
type deterministicClient struct{}
func (deterministicClient) Generate(
ctx context.Context,
_ promptkit.GenerateRequest,
) (*promptkit.GenerateResponse, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
return &promptkit.GenerateResponse{
Content: "Ada finished the migration review.",
Usage: promptkit.TokenUsage{
PromptTokens: 12,
CompletionTokens: 6,
TotalTokens: 18,
},
}, nil
}
type summary struct {
Output string `json:"output"`
ValidationStatus promptkit.ValidationStatus `json:"validation_status"`
IsValid bool `json:"is_valid"`
Model string `json:"model"`
TotalTokens int `json:"total_tokens"`
}
func main() {
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFile("examples/go-library/run/prompt.yaml"),
promptkit.WithProfiles(promptkit.Profile{
ID: "offline-example",
Endpoint: "https://example.invalid/v1",
Model: "offline-model",
}),
promptkit.WithLLMClient(deterministicClient{}),
)
if err != nil {
exit(err)
}
result, err := engine.Run(context.Background(), promptkit.RunRequest{
PromptID: "example.run",
Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.Inline("Ada finished the migration review."),
},
})
if err != nil {
exit(err)
}
encoder := json.NewEncoder(os.Stdout)
encoder.SetIndent("", " ")
if err := encoder.Encode(summary{
Output: result.RawOutput,
ValidationStatus: result.Validation.Status,
IsValid: result.Validation.IsValid,
Model: result.ModelName,
TotalTokens: result.Usage.TotalTokens,
}); err != nil {
exit(err)
}
}
func exit(err error) {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}

View File

@@ -0,0 +1,16 @@
id: example.run
version: "1.0.0"
default_profile: offline-example
description: Run a prompt with a deterministic injected model client.
inputs:
- name: note
required: true
content_type: text/plain
messages:
- role: system
content: Summarize the note in one sentence.
- role: user
content: '{{input "note"}}'
output:
format: text
validation_mode: basic

56
formatting.go Normal file
View File

@@ -0,0 +1,56 @@
package promptkit
import "fmt"
// String returns a concise request summary without exposing the direct API key
// or input and variable contents. Reflection-based formatting does not carry
// this guarantee.
func (r RunRequest) String() string {
return r.redactedString()
}
// GoString returns a concise request summary without exposing the direct API
// key or input and variable contents. Reflection-based formatting does not
// carry this guarantee.
func (r RunRequest) GoString() string {
return r.redactedString()
}
func (r RunRequest) redactedString() string {
return fmt.Sprintf(
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t}",
r.PromptID,
r.PromptVersion,
r.ProfileID,
r.APIKey != "",
len(r.Inputs),
len(r.Vars),
r.Execution != nil,
r.Validation != nil,
)
}
// String returns a concise request summary without exposing direct API keys or
// rendered prompt content. Reflection-based formatting does not carry this
// guarantee.
func (r GenerateRequest) String() string {
return r.redactedString()
}
// GoString returns a concise request summary without exposing direct API keys or
// rendered prompt content. Reflection-based formatting does not carry this
// guarantee.
func (r GenerateRequest) GoString() string {
return r.redactedString()
}
func (r GenerateRequest) redactedString() string {
return fmt.Sprintf(
"promptkit.GenerateRequest{Messages:%d Model:%q APIKeySet:%t StructuredOutputSet:%t ExtraParams:%d}",
len(r.Prompt.Messages),
r.Target.Model,
r.APIKey != "",
r.StructuredOutput != nil,
len(r.Target.ExtraParams),
)
}

7
go.mod
View File

@@ -1,3 +1,10 @@
module gitea.maximumdirect.net/eric/promptkit
go 1.25.5
require (
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
gopkg.in/yaml.v3 v3.0.1
)
require golang.org/x/text v0.14.0 // indirect

10
go.sum Normal file
View File

@@ -0,0 +1,10 @@
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

122
internal/artifact/reader.go Normal file
View File

@@ -0,0 +1,122 @@
package artifact
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"io"
"mime"
"os"
"path/filepath"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
var (
ErrUnsupportedRefType = errors.New("unsupported artifact reference type")
ErrMissingInlineBody = errors.New("missing body for inline artifact")
ErrMissingFilePath = errors.New("missing file path for file artifact")
)
// Reader resolves artifact references into actual artifacts.
type Reader interface {
Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error)
}
// CompositeReader routes artifact resolution based on the reference type.
type CompositeReader struct {
inlineReader *inlineReader
fileReader Reader
}
func NewCompositeReader() Reader {
return &CompositeReader{
inlineReader: &inlineReader{},
fileReader: &fileReader{},
}
}
func (c *CompositeReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
switch ref.Type {
case domain.ArtifactRefInline:
return c.inlineReader.Read(ctx, ref)
case domain.ArtifactRefFile:
return c.fileReader.Read(ctx, ref)
default:
return nil, fmt.Errorf("%w: %s", ErrUnsupportedRefType, ref.Type)
}
}
type inlineReader struct{}
func (r *inlineReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
if ref.Body == "" {
return nil, ErrMissingInlineBody
}
body := []byte(ref.Body)
return &domain.Artifact{
ContentType: defaults.ContentTypeTextPlain,
Body: body,
Size: int64(len(body)),
Hash: fmt.Sprintf("%x", sha256.Sum256(body)),
URI: ref.URI,
}, nil
}
type fileReader struct{}
func (r *fileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
if ref.URI == "" {
return nil, ErrMissingFilePath
}
return readFileArtifact(ref.URI)
}
func readFileArtifact(path string) (*domain.Artifact, error) {
file, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
}
defer file.Close()
data, err := io.ReadAll(file)
if err != nil {
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
}
contentType := mime.TypeByExtension(filepath.Ext(path))
if contentType == "" {
contentType = defaults.ContentTypeTextPlain
}
return &domain.Artifact{
Name: filepath.Base(path),
ContentType: contentType,
Body: data,
URI: path,
Size: int64(len(data)),
Hash: fmt.Sprintf("%x", sha256.Sum256(data)),
}, nil
}

View File

@@ -0,0 +1,180 @@
package artifact
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestCompositeReader_Read(t *testing.T) {
reader := NewCompositeReader()
ctx := context.Background()
t.Run("inline artifact", func(t *testing.T) {
ref := domain.ArtifactRef{
Type: domain.ArtifactRefInline,
Body: "hello world",
}
art, err := reader.Read(ctx, ref)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if string(art.Body) != "hello world" {
t.Errorf("expected 'hello world', got %s", string(art.Body))
}
if art.ContentType != "text/plain" {
t.Errorf("expected text/plain content type, got %q", art.ContentType)
}
if art.Hash != "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" {
t.Errorf("unexpected hash: %s", art.Hash)
}
if art.Size != int64(len(ref.Body)) {
t.Errorf("expected size %d, got %d", len(ref.Body), art.Size)
}
})
t.Run("inline artifact missing body", func(t *testing.T) {
ref := domain.ArtifactRef{
Type: domain.ArtifactRefInline,
Body: "",
}
_, err := reader.Read(ctx, ref)
if !errors.Is(err, ErrMissingInlineBody) {
t.Errorf("expected ErrMissingInlineBody, got %v", err)
}
})
t.Run("unsupported ref type", func(t *testing.T) {
ref := domain.ArtifactRef{
Type: domain.ArtifactRefType("unsupported"),
URI: "unsupported://bucket/key",
}
_, err := reader.Read(ctx, ref)
if !errors.Is(err, ErrUnsupportedRefType) {
t.Error("expected error for unsupported type")
}
})
}
func TestCompositeReaderCopiesInlineData(t *testing.T) {
reader := NewCompositeReader()
ref := domain.ArtifactRef{
Type: domain.ArtifactRefInline,
Body: "hello",
URI: "inline:greeting",
}
first, err := reader.Read(context.Background(), ref)
if err != nil {
t.Fatalf("read first artifact: %v", err)
}
first.Body[0] = 'j'
second, err := reader.Read(context.Background(), ref)
if err != nil {
t.Fatalf("read second artifact: %v", err)
}
if got := string(second.Body); got != ref.Body {
t.Fatalf("expected an independent body %q, got %q", ref.Body, got)
}
if second.URI != ref.URI {
t.Fatalf("expected URI %q, got %q", ref.URI, second.URI)
}
}
func TestCompositeReaderHonorsCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := NewCompositeReader().Read(ctx, domain.ArtifactRef{
Type: domain.ArtifactRefInline,
Body: "ignored",
})
if !errors.Is(err, context.Canceled) {
t.Fatalf("expected context cancellation, got %v", err)
}
}
func TestFileReader_Read(t *testing.T) {
content := []byte("test file content")
filePath := filepath.Join(t.TempDir(), "artifact.txt")
if err := os.WriteFile(filePath, content, 0o600); err != nil {
t.Fatal(err)
}
reader := NewCompositeReader()
ctx := context.Background()
t.Run("file artifact loading", func(t *testing.T) {
ref := domain.ArtifactRef{
Type: domain.ArtifactRefFile,
URI: filePath,
}
art, err := reader.Read(ctx, ref)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if string(art.Body) != string(content) {
t.Errorf("expected %s, got %s", string(content), string(art.Body))
}
if art.Name != filepath.Base(filePath) {
t.Errorf("expected name %q, got %q", filepath.Base(filePath), art.Name)
}
if !strings.HasPrefix(art.ContentType, "text/plain") {
t.Errorf("expected text content type, got %q", art.ContentType)
}
if art.URI != filePath {
t.Errorf("expected URI %q, got %q", filePath, art.URI)
}
if art.Size != int64(len(content)) {
t.Errorf("expected size %d, got %d", len(content), art.Size)
}
if art.Hash != "60f5237ed4049f0382661ef009d2bc42e48c3ceb3edb6600f7024e7ab3b838f3" {
t.Errorf("unexpected hash: %s", art.Hash)
}
})
t.Run("missing file path", func(t *testing.T) {
ref := domain.ArtifactRef{
Type: domain.ArtifactRefFile,
URI: "",
}
_, err := reader.Read(ctx, ref)
if !errors.Is(err, ErrMissingFilePath) {
t.Errorf("expected ErrMissingFilePath, got %v", err)
}
})
t.Run("missing file", func(t *testing.T) {
ref := domain.ArtifactRef{
Type: domain.ArtifactRefFile,
URI: filepath.Join(t.TempDir(), "missing.txt"),
}
if _, err := reader.Read(ctx, ref); err == nil {
t.Fatal("expected missing file error")
}
})
t.Run("unknown extension uses text fallback", func(t *testing.T) {
path := filepath.Join(t.TempDir(), "artifact.unknownextension")
if err := os.WriteFile(path, content, 0o600); err != nil {
t.Fatal(err)
}
art, err := reader.Read(ctx, domain.ArtifactRef{
Type: domain.ArtifactRefFile,
URI: path,
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if art.ContentType != "text/plain" {
t.Errorf("expected text/plain fallback, got %q", art.ContentType)
}
})
}

View File

@@ -0,0 +1,212 @@
// Package backend owns validated, immutable OpenAI-compatible backend
// definitions.
package backend
import (
"errors"
"fmt"
"net/url"
"regexp"
"sort"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
)
const (
// OpenRouterID is the reserved ID of Promptkit's built-in OpenRouter
// backend.
OpenRouterID = "openrouter"
openRouterEndpoint = "https://openrouter.ai/api/v1"
openRouterAPIKeyEnv = "OPENROUTER_API_KEY"
openRouterConcurrencyLimit = 16
defaultQueueCapacity = 1024
)
// ErrBackendNotFound identifies a registry lookup for an unknown backend ID.
var ErrBackendNotFound = errors.New("backend not found")
var environmentVariableName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
// Registry is an immutable collection of validated backend definitions.
type Registry struct {
backends map[string]domain.Backend
}
// NewRegistry constructs a registry containing the built-in OpenRouter
// definition followed by the supplied additions. Every ID must be unique.
func NewRegistry(additions []domain.Backend) (*Registry, error) {
registry := &Registry{
backends: make(map[string]domain.Backend, len(additions)+1),
}
definitions := make([]domain.Backend, 0, len(additions)+1)
definitions = append(definitions, domain.Backend{
ID: OpenRouterID,
Endpoint: openRouterEndpoint,
APIKeyEnv: openRouterAPIKeyEnv,
ConcurrencyLimit: openRouterConcurrencyLimit,
})
definitions = append(definitions, additions...)
for _, definition := range definitions {
definition.ID = strings.TrimSpace(definition.ID)
if definition.ID == "" {
return nil, errors.New("backend ID must not be blank")
}
if _, exists := registry.backends[definition.ID]; exists {
return nil, fmt.Errorf("backend ID %q is already registered", definition.ID)
}
normalized, err := normalizeBackend(definition)
if err != nil {
return nil, err
}
registry.backends[normalized.ID] = normalized
}
return registry, nil
}
// GetBackend returns a defensive copy of the backend registered with id.
func (r *Registry) GetBackend(id string) (domain.Backend, error) {
if r == nil {
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
}
definition, ok := r.backends[id]
if !ok {
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
}
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
}
definition.ExtraParams = extraParams
return definition, nil
}
// CapacityPolicies returns a copy of the normalized policies for limited
// backends.
func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy {
policies := make(map[string]domain.BackendCapacityPolicy)
if r == nil {
return policies
}
for id, definition := range r.backends {
if definition.ConcurrencyLimit == 0 {
continue
}
policies[id] = domain.BackendCapacityPolicy{
ConcurrencyLimit: definition.ConcurrencyLimit,
QueueCapacity: definition.QueueCapacity,
}
}
return policies
}
func normalizeBackend(definition domain.Backend) (domain.Backend, error) {
definition.Endpoint = strings.TrimSpace(definition.Endpoint)
if err := validateEndpoint(definition.Endpoint); err != nil {
return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err)
}
definition.APIKeyEnv = strings.TrimSpace(definition.APIKeyEnv)
if definition.APIKeyEnv != "" && !environmentVariableName.MatchString(definition.APIKeyEnv) {
return domain.Backend{}, fmt.Errorf(
"backend %q api key environment variable %q is invalid",
definition.ID,
definition.APIKeyEnv,
)
}
if definition.ConcurrencyLimit < 0 {
return domain.Backend{}, fmt.Errorf(
"backend %q concurrency limit must not be negative",
definition.ID,
)
}
if definition.QueueCapacity < 0 {
return domain.Backend{}, fmt.Errorf(
"backend %q queue capacity must not be negative",
definition.ID,
)
}
if definition.ConcurrencyLimit == 0 {
if definition.QueueCapacitySet {
return domain.Backend{}, fmt.Errorf(
"backend %q queue capacity requires a positive concurrency limit",
definition.ID,
)
}
definition.QueueCapacity = 0
} else {
if !definition.QueueCapacitySet {
definition.QueueCapacity = defaultQueueCapacity
definition.QueueCapacitySet = true
}
maxInt := int(^uint(0) >> 1)
if definition.QueueCapacity > maxInt-definition.ConcurrencyLimit {
return domain.Backend{}, fmt.Errorf(
"backend %q total capacity overflows int",
definition.ID,
)
}
}
keys := make([]string, 0, len(definition.ExtraParams))
for key := range definition.ExtraParams {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
if key == "" {
return domain.Backend{}, fmt.Errorf("backend %q extra parameter key must not be empty", definition.ID)
}
if llm.IsReservedOpenAIChatRequestField(key) {
return domain.Backend{}, fmt.Errorf(
"backend %q extra parameter %q collides with a reserved request field",
definition.ID,
key,
)
}
}
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return domain.Backend{}, fmt.Errorf("backend %q extra parameters: %w", definition.ID, err)
}
definition.ExtraParams = extraParams
return definition, nil
}
func validateEndpoint(endpoint string) error {
if endpoint == "" {
return errors.New("must not be blank")
}
if strings.Contains(endpoint, "#") {
return errors.New("must not contain a fragment")
}
parsed, err := url.Parse(endpoint)
if err != nil {
return fmt.Errorf("must be a valid URL: %w", err)
}
scheme := strings.ToLower(parsed.Scheme)
if scheme != "http" && scheme != "https" {
return errors.New("must use http or https")
}
if !parsed.IsAbs() || parsed.Hostname() == "" {
return errors.New("must be absolute and include a host")
}
if parsed.User != nil {
return errors.New("must not contain user information")
}
if parsed.RawQuery != "" || parsed.ForceQuery {
return errors.New("must not contain a query string")
}
return nil
}

View File

@@ -0,0 +1,368 @@
package backend_test
import (
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
const validEndpoint = "https://backend.example/v1"
func TestRegistryIncludesExactOpenRouterDefinition(t *testing.T) {
registry, err := backend.NewRegistry(nil)
if err != nil {
t.Fatalf("construct registry: %v", err)
}
definition, err := registry.GetBackend(backend.OpenRouterID)
if err != nil {
t.Fatalf("look up OpenRouter: %v", err)
}
if definition.ID != "openrouter" ||
definition.Endpoint != "https://openrouter.ai/api/v1" ||
definition.APIKeyEnv != "OPENROUTER_API_KEY" ||
definition.ConcurrencyLimit != 16 ||
definition.QueueCapacity != 1024 ||
!definition.QueueCapacitySet ||
definition.ExtraParams != nil {
t.Fatalf("unexpected OpenRouter definition: %#v", definition)
}
policies := registry.CapacityPolicies()
if len(policies) != 1 ||
policies["openrouter"] != (domain.BackendCapacityPolicy{
ConcurrencyLimit: 16,
QueueCapacity: 1024,
}) {
t.Fatalf("unexpected OpenRouter capacity policies: %#v", policies)
}
}
func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
nested := map[string]int{"limit": 2}
extraParams := map[string]any{
"count": int64(7),
"nested": nested,
}
registry, err := backend.NewRegistry([]domain.Backend{
{
ID: " custom ",
Endpoint: " https://custom.example/openai/v1 ",
APIKeyEnv: " CUSTOM_API_KEY ",
ExtraParams: extraParams,
ConcurrencyLimit: 3,
QueueCapacity: 2,
QueueCapacitySet: true,
},
{
ID: "Custom",
Endpoint: validEndpoint,
},
})
if err != nil {
t.Fatalf("construct registry: %v", err)
}
nested["limit"] = 99
extraParams["added"] = true
got, err := registry.GetBackend("custom")
if err != nil {
t.Fatalf("look up custom backend: %v", err)
}
if got.ID != "custom" ||
got.Endpoint != "https://custom.example/openai/v1" ||
got.APIKeyEnv != "CUSTOM_API_KEY" ||
got.ConcurrencyLimit != 3 ||
got.QueueCapacity != 2 ||
!got.QueueCapacitySet {
t.Fatalf("unexpected normalized definition: %#v", got)
}
if count, ok := got.ExtraParams["count"].(int64); !ok || count != 7 {
t.Fatalf("integer type or value changed: %#v", got.ExtraParams["count"])
}
gotNested, ok := got.ExtraParams["nested"].(map[string]int)
if !ok || gotNested["limit"] != 2 {
t.Fatalf("container type or value changed: %#v", got.ExtraParams["nested"])
}
if _, exists := got.ExtraParams["added"]; exists {
t.Fatalf("registry retained caller map: %#v", got.ExtraParams)
}
gotNested["limit"] = 100
got.ExtraParams["added"] = true
again, err := registry.GetBackend("custom")
if err != nil {
t.Fatalf("look up custom backend again: %v", err)
}
if again.ExtraParams["nested"].(map[string]int)["limit"] != 2 {
t.Fatalf("lookup exposed registry nested map: %#v", again.ExtraParams)
}
if _, exists := again.ExtraParams["added"]; exists {
t.Fatalf("lookup exposed registry map: %#v", again.ExtraParams)
}
if _, err := registry.GetBackend("Custom"); err != nil {
t.Fatalf("backend IDs should be case-sensitive: %v", err)
}
policies := registry.CapacityPolicies()
if len(policies) != 2 {
t.Fatalf("unexpected capacity policy count: %#v", policies)
}
policies["custom"] = domain.BackendCapacityPolicy{}
delete(policies, backend.OpenRouterID)
againPolicies := registry.CapacityPolicies()
if againPolicies["custom"] != (domain.BackendCapacityPolicy{
ConcurrencyLimit: 3,
QueueCapacity: 2,
}) {
t.Fatalf("capacity policy map mutated registry state: %#v", againPolicies)
}
if _, ok := againPolicies[backend.OpenRouterID]; !ok {
t.Fatalf("capacity policy deletion mutated registry state: %#v", againPolicies)
}
}
func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) {
maxInt := int(^uint(0) >> 1)
tests := []struct {
name string
definition domain.Backend
want domain.BackendCapacityPolicy
wantSet bool
wantError bool
}{
{
name: "unlimited when omitted",
definition: domain.Backend{},
},
{
name: "default queue",
definition: domain.Backend{
ConcurrencyLimit: 2,
},
want: domain.BackendCapacityPolicy{
ConcurrencyLimit: 2,
QueueCapacity: 1024,
},
wantSet: true,
},
{
name: "explicit zero queue",
definition: domain.Backend{
ConcurrencyLimit: 2,
QueueCapacitySet: true,
},
want: domain.BackendCapacityPolicy{
ConcurrencyLimit: 2,
},
wantSet: true,
},
{
name: "negative concurrency limit",
definition: domain.Backend{
ConcurrencyLimit: -1,
},
wantError: true,
},
{
name: "negative queue capacity",
definition: domain.Backend{
ConcurrencyLimit: 1,
QueueCapacity: -1,
QueueCapacitySet: true,
},
wantError: true,
},
{
name: "queue without limit",
definition: domain.Backend{
QueueCapacitySet: true,
},
wantError: true,
},
{
name: "total overflow",
definition: domain.Backend{
ConcurrencyLimit: maxInt,
QueueCapacity: 1,
QueueCapacitySet: true,
},
wantError: true,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
tc.definition.ID = "custom"
tc.definition.Endpoint = validEndpoint
registry, err := backend.NewRegistry([]domain.Backend{tc.definition})
if tc.wantError {
if err == nil {
t.Fatal("expected invalid capacity policy error")
}
return
}
if err != nil {
t.Fatalf("construct registry: %v", err)
}
definition, err := registry.GetBackend("custom")
if err != nil {
t.Fatalf("look up custom backend: %v", err)
}
if definition.ConcurrencyLimit != tc.want.ConcurrencyLimit ||
definition.QueueCapacity != tc.want.QueueCapacity ||
definition.QueueCapacitySet != tc.wantSet {
t.Fatalf("normalized capacity=(%d, %d, %t), want (%d, %d, %t)",
definition.ConcurrencyLimit,
definition.QueueCapacity,
definition.QueueCapacitySet,
tc.want.ConcurrencyLimit,
tc.want.QueueCapacity,
tc.wantSet,
)
}
policies := registry.CapacityPolicies()
got, ok := policies["custom"]
if ok != tc.wantSet || got != tc.want {
t.Fatalf("capacity policy=(%#v, %t), want (%#v, %t)", got, ok, tc.want, tc.wantSet)
}
})
}
}
func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
tests := []struct {
name string
additions []domain.Backend
wantID string
}{
{
name: "built-in collision after normalization",
additions: []domain.Backend{{
ID: " openrouter ",
}},
wantID: "openrouter",
},
{
name: "consumer collision after normalization",
additions: []domain.Backend{
{ID: "custom", Endpoint: validEndpoint},
{ID: " custom ", Endpoint: "https://other.example/v1"},
},
wantID: "custom",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry(tc.additions)
if err == nil {
t.Fatal("expected duplicate ID error")
}
if !strings.Contains(err.Error(), tc.wantID) {
t.Fatalf("expected error to identify %q, got %v", tc.wantID, err)
}
})
}
}
func TestNewRegistryValidatesIDs(t *testing.T) {
for _, id := range []string{"", " \t\n "} {
t.Run(id, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{
ID: id,
Endpoint: validEndpoint,
}})
if err == nil {
t.Fatal("expected blank ID error")
}
})
}
}
func TestNewRegistryValidatesEndpoints(t *testing.T) {
tests := []struct {
name string
endpoint string
}{
{name: "blank", endpoint: ""},
{name: "relative", endpoint: "/v1"},
{name: "missing host", endpoint: "https:///v1"},
{name: "unsupported scheme", endpoint: "ftp://backend.example/v1"},
{name: "user information", endpoint: "https://user@backend.example/v1"},
{name: "query", endpoint: "https://backend.example/v1?mode=chat"},
{name: "empty query", endpoint: "https://backend.example/v1?"},
{name: "fragment", endpoint: "https://backend.example/v1#chat"},
{name: "empty fragment", endpoint: "https://backend.example/v1#"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{
ID: "custom",
Endpoint: tc.endpoint,
}})
if err == nil {
t.Fatal("expected invalid endpoint error")
}
})
}
}
func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) {
for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} {
t.Run(name, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{
ID: "custom",
Endpoint: validEndpoint,
APIKeyEnv: name,
}})
if err == nil {
t.Fatal("expected invalid environment-variable name error")
}
})
}
}
func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
tests := []struct {
name string
extraParams map[string]any
}{
{name: "unsupported value", extraParams: map[string]any{"value": make(chan int)}},
{name: "reserved key", extraParams: map[string]any{"model": "override"}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{
ID: "custom",
Endpoint: validEndpoint,
ExtraParams: tc.extraParams,
}})
if err == nil {
t.Fatal("expected invalid extra parameters error")
}
})
}
}
func TestRegistryLookupReportsNotFound(t *testing.T) {
registry, err := backend.NewRegistry(nil)
if err != nil {
t.Fatalf("construct registry: %v", err)
}
_, err = registry.GetBackend("missing")
if !errors.Is(err, backend.ErrBackendNotFound) {
t.Fatalf("expected ErrBackendNotFound, got %v", err)
}
if !strings.Contains(err.Error(), "missing") {
t.Fatalf("expected error to identify backend, got %v", err)
}
}

View File

@@ -0,0 +1,40 @@
package capacity
import (
"context"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
)
type client struct {
manager *Manager
next llm.Client
}
// NewClient wraps next with configured active-generation limits. A nil manager
// leaves next unchanged.
func NewClient(manager *Manager, next llm.Client) llm.Client {
if manager == nil {
return next
}
return &client{
manager: manager,
next: next,
}
}
func (c *client) Generate(
ctx context.Context,
req domain.GenerateRequest,
) (*domain.GenerateResponse, error) {
pool := c.manager.getPool(req.Target.BackendID)
if pool == nil {
return c.next.Generate(ctx, req)
}
if err := pool.acquire(ctx); err != nil {
return nil, err
}
defer pool.releaseActive()
return c.next.Generate(ctx, req)
}

View File

@@ -0,0 +1,517 @@
package capacity
import (
"context"
"errors"
"reflect"
"runtime"
"sync"
"sync/atomic"
"testing"
"time"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
)
type generateResult struct {
response *domain.GenerateResponse
err error
}
type clientFunc func(
context.Context,
domain.GenerateRequest,
) (*domain.GenerateResponse, error)
func (f clientFunc) Generate(
ctx context.Context,
req domain.GenerateRequest,
) (*domain.GenerateResponse, error) {
return f(ctx, req)
}
type blockingClient struct {
mu sync.Mutex
active int
peak int
calls map[string]int
started chan string
releases map[string]chan struct{}
}
func newBlockingClient(releases map[string]chan struct{}) *blockingClient {
return &blockingClient{
calls: make(map[string]int),
started: make(chan string, 64),
releases: releases,
}
}
func (c *blockingClient) Generate(
ctx context.Context,
req domain.GenerateRequest,
) (*domain.GenerateResponse, error) {
id := req.Prompt.SessionID
c.mu.Lock()
c.active++
if c.active > c.peak {
c.peak = c.active
}
c.calls[id]++
c.mu.Unlock()
defer func() {
c.mu.Lock()
c.active--
c.mu.Unlock()
}()
c.started <- id
if release := c.releases[id]; release != nil {
select {
case <-release:
case <-ctx.Done():
return nil, ctx.Err()
}
}
return &domain.GenerateResponse{Content: id}, nil
}
func (c *blockingClient) callCount(id string) int {
c.mu.Lock()
defer c.mu.Unlock()
return c.calls[id]
}
func (c *blockingClient) peakConcurrency() int {
c.mu.Lock()
defer c.mu.Unlock()
return c.peak
}
func generateAsync(
client llm.Client,
ctx context.Context,
backendID string,
id string,
) <-chan generateResult {
result := make(chan generateResult, 1)
go func() {
response, err := client.Generate(ctx, domain.GenerateRequest{
Prompt: domain.RenderedPrompt{SessionID: id},
Target: domain.ExecutionTarget{BackendID: backendID},
})
result <- generateResult{response: response, err: err}
}()
return result
}
func waitForWaiterCount(t *testing.T, manager *Manager, backendID string, want int) {
t.Helper()
pool := manager.pools[backendID]
deadline := time.Now().Add(2 * time.Second)
for {
pool.mu.Lock()
got := pool.waiters.Len()
pool.mu.Unlock()
if got == want {
return
}
if time.Now().After(deadline) {
t.Fatalf("waiter count=%d, want %d", got, want)
}
runtime.Gosched()
}
}
func receiveStarted(t *testing.T, started <-chan string) string {
t.Helper()
select {
case id := <-started:
return id
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for wrapped client invocation")
return ""
}
}
func receiveResult(t *testing.T, result <-chan generateResult) generateResult {
t.Helper()
select {
case got := <-result:
return got
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for generation result")
return generateResult{}
}
}
func newTestManager(t *testing.T, policies map[string]domain.BackendCapacityPolicy) *Manager {
t.Helper()
manager, err := NewManager(policies)
if err != nil {
t.Fatalf("construct manager: %v", err)
}
return manager
}
func TestClientLimitsPeakConcurrencyAndServesWaitersFIFO(t *testing.T) {
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
"limited": {ConcurrencyLimit: 1},
})
firstRelease := make(chan struct{})
secondRelease := make(chan struct{})
thirdRelease := make(chan struct{})
next := newBlockingClient(map[string]chan struct{}{
"first": firstRelease,
"second": secondRelease,
"third": thirdRelease,
})
client := NewClient(manager, next)
first := generateAsync(client, context.Background(), "limited", "first")
if got := receiveStarted(t, next.started); got != "first" {
t.Fatalf("first invocation=%q, want first", got)
}
second := generateAsync(client, context.Background(), "limited", "second")
waitForWaiterCount(t, manager, "limited", 1)
third := generateAsync(client, context.Background(), "limited", "third")
waitForWaiterCount(t, manager, "limited", 2)
close(firstRelease)
if got := receiveResult(t, first); got.err != nil {
t.Fatalf("first generation: %v", got.err)
}
if got := receiveStarted(t, next.started); got != "second" {
t.Fatalf("second invocation=%q, want second", got)
}
close(secondRelease)
if got := receiveResult(t, second); got.err != nil {
t.Fatalf("second generation: %v", got.err)
}
if got := receiveStarted(t, next.started); got != "third" {
t.Fatalf("third invocation=%q, want third", got)
}
close(thirdRelease)
if got := receiveResult(t, third); got.err != nil {
t.Fatalf("third generation: %v", got.err)
}
if peak := next.peakConcurrency(); peak != 1 {
t.Fatalf("peak concurrency=%d, want 1", peak)
}
}
func TestClientPeakConcurrencyDoesNotExceedConfiguredLimit(t *testing.T) {
const limit = 2
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
"limited": {ConcurrencyLimit: limit},
})
gate := make(chan struct{})
releases := make(map[string]chan struct{})
for i := range 5 {
releases[string(rune('a'+i))] = gate
}
next := newBlockingClient(releases)
client := NewClient(manager, next)
results := make([]<-chan generateResult, 0, len(releases))
for id := range releases {
results = append(results, generateAsync(client, context.Background(), "limited", id))
}
for range limit {
receiveStarted(t, next.started)
}
waitForWaiterCount(t, manager, "limited", len(releases)-limit)
close(gate)
for _, result := range results {
if got := receiveResult(t, result); got.err != nil {
t.Fatalf("generation: %v", got.err)
}
}
if peak := next.peakConcurrency(); peak != limit {
t.Fatalf("peak concurrency=%d, want %d", peak, limit)
}
}
func TestClientRemovesCanceledWaiters(t *testing.T) {
tests := []struct {
name string
cancelID string
wantOrder []string
}{
{name: "first waiter", cancelID: "one", wantOrder: []string{"two", "three"}},
{name: "middle waiter", cancelID: "two", wantOrder: []string{"one", "three"}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
"limited": {ConcurrencyLimit: 1},
})
holderRelease := make(chan struct{})
releases := map[string]chan struct{}{
"holder": holderRelease,
"one": make(chan struct{}),
"two": make(chan struct{}),
"three": make(chan struct{}),
}
next := newBlockingClient(releases)
client := NewClient(manager, next)
holder := generateAsync(client, context.Background(), "limited", "holder")
if got := receiveStarted(t, next.started); got != "holder" {
t.Fatalf("initial invocation=%q, want holder", got)
}
contexts := make(map[string]context.Context)
cancels := make(map[string]context.CancelFunc)
results := make(map[string]<-chan generateResult)
for _, id := range []string{"one", "two", "three"} {
contexts[id], cancels[id] = context.WithCancel(context.Background())
results[id] = generateAsync(client, contexts[id], "limited", id)
waitForWaiterCount(t, manager, "limited", len(results))
}
cancels[tc.cancelID]()
if got := receiveResult(t, results[tc.cancelID]); !errors.Is(got.err, context.Canceled) {
t.Fatalf("canceled waiter error=%v, want context.Canceled", got.err)
}
waitForWaiterCount(t, manager, "limited", 2)
close(holderRelease)
if got := receiveResult(t, holder); got.err != nil {
t.Fatalf("holder generation: %v", got.err)
}
for _, id := range tc.wantOrder {
if got := receiveStarted(t, next.started); got != id {
t.Fatalf("next invocation=%q, want %q", got, id)
}
close(releases[id])
if got := receiveResult(t, results[id]); got.err != nil {
t.Fatalf("%s generation: %v", id, got.err)
}
}
if calls := next.callCount(tc.cancelID); calls != 0 {
t.Fatalf("canceled waiter invoked wrapped client %d times", calls)
}
for _, cancel := range cancels {
cancel()
}
})
}
}
func TestClientGrantCancellationRaceDoesNotLeakPermit(t *testing.T) {
const iterations = 200
for i := range iterations {
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
"limited": {ConcurrencyLimit: 1},
})
holderRelease := make(chan struct{})
var waiterCalls atomic.Int64
next := clientFunc(func(
_ context.Context,
req domain.GenerateRequest,
) (*domain.GenerateResponse, error) {
if req.Prompt.SessionID == "holder" {
<-holderRelease
} else if req.Prompt.SessionID == "waiter" {
waiterCalls.Add(1)
}
return &domain.GenerateResponse{Content: req.Prompt.SessionID}, nil
})
client := NewClient(manager, next)
holder := generateAsync(client, context.Background(), "limited", "holder")
waitForActiveCount(t, manager, "limited", 1)
ctx, cancel := context.WithCancel(context.Background())
waiterResult := generateAsync(client, ctx, "limited", "waiter")
waitForWaiterCount(t, manager, "limited", 1)
start := make(chan struct{})
var race sync.WaitGroup
race.Add(2)
go func() {
defer race.Done()
<-start
cancel()
}()
go func() {
defer race.Done()
<-start
close(holderRelease)
}()
close(start)
race.Wait()
if got := receiveResult(t, holder); got.err != nil {
t.Fatalf("iteration %d holder generation: %v", i, got.err)
}
got := receiveResult(t, waiterResult)
switch calls := waiterCalls.Load(); {
case calls == 0 && errors.Is(got.err, context.Canceled):
case calls == 1 && got.err == nil:
default:
t.Fatalf("iteration %d waiter calls=%d error=%v", i, calls, got.err)
}
probe := generateAsync(client, context.Background(), "limited", "probe")
if got := receiveResult(t, probe); got.err != nil {
t.Fatalf("iteration %d probe generation: %v", i, got.err)
}
waitForActiveCount(t, manager, "limited", 0)
waitForWaiterCount(t, manager, "limited", 0)
}
}
func waitForActiveCount(t *testing.T, manager *Manager, backendID string, want int) {
t.Helper()
pool := manager.pools[backendID]
deadline := time.Now().Add(2 * time.Second)
for {
pool.mu.Lock()
got := pool.active
pool.mu.Unlock()
if got == want {
return
}
if time.Now().After(deadline) {
t.Fatalf("active count=%d, want %d", got, want)
}
runtime.Gosched()
}
}
func TestClientUsesIndependentPoolsAndUnlimitedFastPaths(t *testing.T) {
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
"alpha": {ConcurrencyLimit: 1},
"beta": {ConcurrencyLimit: 1},
})
alphaRelease := make(chan struct{})
betaRelease := make(chan struct{})
next := newBlockingClient(map[string]chan struct{}{
"alpha": alphaRelease,
"beta": betaRelease,
})
client := NewClient(manager, next)
alpha := generateAsync(client, context.Background(), "alpha", "alpha")
beta := generateAsync(client, context.Background(), "beta", "beta")
started := map[string]bool{
receiveStarted(t, next.started): true,
receiveStarted(t, next.started): true,
}
if !started["alpha"] || !started["beta"] {
t.Fatalf("independent pools did not both start: %#v", started)
}
close(alphaRelease)
close(betaRelease)
if got := receiveResult(t, alpha); got.err != nil {
t.Fatalf("alpha generation: %v", got.err)
}
if got := receiveResult(t, beta); got.err != nil {
t.Fatalf("beta generation: %v", got.err)
}
for _, backendID := range []string{"", "unknown"} {
response, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{SessionID: backendID},
Target: domain.ExecutionTarget{BackendID: backendID},
})
if err != nil || response == nil {
t.Fatalf("unlimited backend %q response=(%#v, %v)", backendID, response, err)
}
}
if got := NewClient(nil, next); got != next {
t.Fatal("nil manager did not return the wrapped client unchanged")
}
}
func TestClientPreservesRequestsResponsesAndErrors(t *testing.T) {
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
"limited": {ConcurrencyLimit: 1},
})
request := domain.GenerateRequest{
Prompt: domain.RenderedPrompt{
SessionID: "session",
Messages: []domain.RenderedMessage{
{Role: "user", Content: "content"},
},
},
Target: domain.ExecutionTarget{
BackendID: "limited",
Model: "model",
ExtraParams: map[string]any{"key": "value"},
},
}
response := &domain.GenerateResponse{
Content: "output",
Usage: domain.TokenUsage{TotalTokens: 7},
}
collaboratorErr := errors.New("collaborator failure")
tests := []struct {
name string
response *domain.GenerateResponse
err error
}{
{name: "successful response", response: response},
{name: "nil response"},
{name: "collaborator error", response: response, err: collaboratorErr},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
var captured domain.GenerateRequest
next := clientFunc(func(
_ context.Context,
req domain.GenerateRequest,
) (*domain.GenerateResponse, error) {
captured = req
return tc.response, tc.err
})
gotResponse, gotErr := NewClient(manager, next).Generate(context.Background(), request)
if !reflect.DeepEqual(captured, request) {
t.Fatalf("request changed: %#v", captured)
}
if gotResponse != tc.response || gotErr != tc.err {
t.Fatalf("response=(%p, %v), want (%p, %v)",
gotResponse, gotErr, tc.response, tc.err)
}
})
}
}
func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
"limited": {ConcurrencyLimit: 1},
})
var calls atomic.Int64
next := clientFunc(func(
_ context.Context,
_ domain.GenerateRequest,
) (*domain.GenerateResponse, error) {
if calls.Add(1) == 1 {
panic("test panic")
}
return &domain.GenerateResponse{Content: "recovered"}, nil
})
client := NewClient(manager, next)
request := domain.GenerateRequest{
Target: domain.ExecutionTarget{BackendID: "limited"},
}
func() {
defer func() {
if recover() == nil {
t.Fatal("expected wrapped client panic")
}
}()
_, _ = client.Generate(context.Background(), request)
}()
response, err := client.Generate(context.Background(), request)
if err != nil || response == nil || response.Content != "recovered" {
t.Fatalf("generation after panic=(%#v, %v)", response, err)
}
}

View File

@@ -0,0 +1,160 @@
// Package capacity coordinates engine-local run admission and model-generation
// concurrency for configured backends.
package capacity
import (
"container/list"
"context"
"errors"
"fmt"
"strings"
"sync"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
// ErrCapacityExceeded identifies an admission rejected because a backend's
// configured run capacity is full.
var ErrCapacityExceeded = errors.New("backend capacity exceeded")
// Manager owns independent backend capacity pools with immutable limits.
type Manager struct {
pools map[string]*pool
}
type pool struct {
mu sync.Mutex
concurrencyLimit int
totalCapacity int
admitted int
active int
waiters list.List
}
type waiter struct {
ready chan struct{}
element *list.Element
granted bool
}
// NewManager constructs independent pools from normalized backend policies.
func NewManager(policies map[string]domain.BackendCapacityPolicy) (*Manager, error) {
manager := &Manager{
pools: make(map[string]*pool, len(policies)),
}
maxInt := int(^uint(0) >> 1)
for id, policy := range policies {
if strings.TrimSpace(id) == "" {
return nil, errors.New("backend capacity policy ID must not be blank")
}
if policy.ConcurrencyLimit <= 0 {
return nil, fmt.Errorf(
"backend %q concurrency limit must be positive",
id,
)
}
if policy.QueueCapacity < 0 {
return nil, fmt.Errorf(
"backend %q queue capacity must not be negative",
id,
)
}
if policy.QueueCapacity > maxInt-policy.ConcurrencyLimit {
return nil, fmt.Errorf("backend %q total capacity overflows int", id)
}
manager.pools[id] = &pool{
concurrencyLimit: policy.ConcurrencyLimit,
totalCapacity: policy.ConcurrencyLimit + policy.QueueCapacity,
}
}
return manager, nil
}
// Admit immediately reserves one configured backend run slot. Backends without
// a configured pool are unlimited.
func (m *Manager) Admit(ctx context.Context, backendID string) (func(), error) {
pool := m.getPool(backendID)
if pool == nil {
return releaseNothing, nil
}
pool.mu.Lock()
defer pool.mu.Unlock()
if err := ctx.Err(); err != nil {
return nil, err
}
if pool.admitted >= pool.totalCapacity {
return nil, ErrCapacityExceeded
}
pool.admitted++
var once sync.Once
return func() {
once.Do(func() {
pool.mu.Lock()
pool.admitted--
pool.mu.Unlock()
})
}, nil
}
func releaseNothing() {}
func (m *Manager) getPool(backendID string) *pool {
if m == nil || backendID == "" {
return nil
}
return m.pools[backendID]
}
func (p *pool) acquire(ctx context.Context) error {
p.mu.Lock()
if err := ctx.Err(); err != nil {
p.mu.Unlock()
return err
}
if p.active < p.concurrencyLimit && p.waiters.Len() == 0 {
p.active++
p.mu.Unlock()
return nil
}
waiter := &waiter{ready: make(chan struct{})}
waiter.element = p.waiters.PushBack(waiter)
p.mu.Unlock()
select {
case <-waiter.ready:
return nil
case <-ctx.Done():
p.mu.Lock()
if !waiter.granted {
p.waiters.Remove(waiter.element)
waiter.element = nil
p.mu.Unlock()
return ctx.Err()
}
p.mu.Unlock()
return nil
}
}
func (p *pool) releaseActive() {
var ready chan struct{}
p.mu.Lock()
if element := p.waiters.Front(); element != nil {
waiter := element.Value.(*waiter)
p.waiters.Remove(element)
waiter.element = nil
waiter.granted = true
ready = waiter.ready
} else {
p.active--
}
p.mu.Unlock()
if ready != nil {
close(ready)
}
}

View File

@@ -0,0 +1,158 @@
package capacity
import (
"context"
"errors"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestNewManagerRejectsInvalidPolicies(t *testing.T) {
maxInt := int(^uint(0) >> 1)
tests := []struct {
name string
id string
policy domain.BackendCapacityPolicy
}{
{
name: "blank ID",
id: " \t ",
policy: domain.BackendCapacityPolicy{ConcurrencyLimit: 1},
},
{
name: "zero concurrency",
id: "backend",
policy: domain.BackendCapacityPolicy{},
},
{
name: "negative concurrency",
id: "backend",
policy: domain.BackendCapacityPolicy{ConcurrencyLimit: -1},
},
{
name: "negative queue",
id: "backend",
policy: domain.BackendCapacityPolicy{
ConcurrencyLimit: 1,
QueueCapacity: -1,
},
},
{
name: "total overflow",
id: "backend",
policy: domain.BackendCapacityPolicy{
ConcurrencyLimit: maxInt,
QueueCapacity: 1,
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := NewManager(map[string]domain.BackendCapacityPolicy{
tc.id: tc.policy,
})
if err == nil {
t.Fatal("expected invalid policy error")
}
})
}
}
func TestManagerAdmissionIsBoundedAndReleaseIsIdempotent(t *testing.T) {
policies := map[string]domain.BackendCapacityPolicy{
"limited": {
ConcurrencyLimit: 2,
QueueCapacity: 1,
},
"independent": {
ConcurrencyLimit: 1,
},
}
manager, err := NewManager(policies)
if err != nil {
t.Fatalf("construct manager: %v", err)
}
policies["limited"] = domain.BackendCapacityPolicy{
ConcurrencyLimit: 100,
QueueCapacity: 100,
}
releases := make([]func(), 0, 3)
for range 3 {
release, err := manager.Admit(context.Background(), "limited")
if err != nil {
t.Fatalf("admit within configured capacity: %v", err)
}
releases = append(releases, release)
}
if release, err := manager.Admit(context.Background(), "limited"); release != nil ||
!errors.Is(err, ErrCapacityExceeded) {
t.Fatalf("admission beyond capacity=(release=%t, err=%v), want ErrCapacityExceeded",
release != nil, err)
}
independentRelease, err := manager.Admit(context.Background(), "independent")
if err != nil {
t.Fatalf("admit independent backend while first is full: %v", err)
}
independentRelease()
releases[0]()
releases[0]()
replacement, err := manager.Admit(context.Background(), "limited")
if err != nil {
t.Fatalf("admit after release: %v", err)
}
replacement()
releases[1]()
releases[2]()
pool := manager.pools["limited"]
pool.mu.Lock()
admitted := pool.admitted
pool.mu.Unlock()
if admitted != 0 {
t.Fatalf("admitted runs after releases=%d, want 0", admitted)
}
}
func TestManagerAdmissionHonorsContextAndUnlimitedBackends(t *testing.T) {
manager, err := NewManager(map[string]domain.BackendCapacityPolicy{
"limited": {ConcurrencyLimit: 1},
})
if err != nil {
t.Fatalf("construct manager: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if release, err := manager.Admit(ctx, "limited"); release != nil ||
!errors.Is(err, context.Canceled) {
t.Fatalf("canceled limited admission=(release=%t, err=%v), want context cancellation",
release != nil, err)
}
var nilManager *Manager
for _, tc := range []struct {
name string
manager *Manager
backendID string
}{
{name: "nil manager", manager: nilManager, backendID: "limited"},
{name: "blank ID", manager: manager},
{name: "unknown ID", manager: manager, backendID: "unknown"},
} {
t.Run(tc.name, func(t *testing.T) {
release, err := tc.manager.Admit(ctx, tc.backendID)
if err != nil {
t.Fatalf("unlimited admission: %v", err)
}
if release == nil {
t.Fatal("unlimited admission returned nil release")
}
release()
release()
})
}
}

View File

@@ -0,0 +1,34 @@
package defaults
import (
"time"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
const (
SchemaDirDefault = "."
OutputArtifactName = "output"
ContentTypeTextPlain = "text/plain"
ContentTypeTextMarkdown = "text/markdown"
ContentTypeApplicationJSON = "application/json"
OpenAIChatCompletionsPath = "/chat/completions"
ExecutionDefaultTemperature = 0.0
ExecutionDefaultMaxTokens = 0
ExecutionDefaultTopP = 1.0
ExecutionDefaultTimeoutSeconds = 600
)
var (
LLMRequestTimeoutDefault = 10 * time.Minute
)
func ExecutionTargetDefault() domain.ExecutionTarget {
return domain.ExecutionTarget{
Temperature: ExecutionDefaultTemperature,
MaxTokens: ExecutionDefaultMaxTokens,
TopP: ExecutionDefaultTopP,
TimeoutSeconds: ExecutionDefaultTimeoutSeconds,
}
}

311
internal/domain/domain.go Normal file
View File

@@ -0,0 +1,311 @@
package domain
import (
"time"
)
// ArtifactRefType defines how an artifact is referenced.
type ArtifactRefType string
const (
ArtifactRefInline ArtifactRefType = "inline"
ArtifactRefFile ArtifactRefType = "file"
)
// OutputFormat defines the desired format of the generated artifact.
type OutputFormat string
const (
FormatText OutputFormat = "text"
FormatMarkdown OutputFormat = "markdown"
FormatJSON OutputFormat = "json"
)
// ValidationMode defines how the output should be validated.
type ValidationMode string
const (
ValidationNone ValidationMode = "none"
ValidationBasic ValidationMode = "basic"
ValidationJSON ValidationMode = "json"
ValidationJSONSchema ValidationMode = "json_schema"
)
// ValidationStatus defines the result of a validation check.
type ValidationStatus string
const (
ValidationPassed ValidationStatus = "passed"
ValidationFailed ValidationStatus = "failed"
ValidationSkipped ValidationStatus = "skipped"
)
// CacheControlType defines provider cache behavior for prompt content.
type CacheControlType string
const (
CacheControlEphemeral CacheControlType = "ephemeral"
)
const (
// SessionIDMaxLength is OpenRouter's documented maximum session_id length.
SessionIDMaxLength = 256
)
// CacheControl describes provider cache metadata attached to prompt content.
type CacheControl struct {
Type CacheControlType `yaml:"type" json:"type"`
TTL string `yaml:"ttl,omitempty" json:"ttl,omitempty"`
}
// RunRequest represents a request to generate a single artifact.
type RunRequest struct {
PromptID string
PromptVersion string
ProfileID string
SessionID string
APIKey string `json:"-" yaml:"-"`
Inputs map[string]ArtifactRef
Vars map[string]string
Execution *ExecutionTargetOverride
Validation *OutputContract
}
// RunResult represents the complete result of a prompt execution run.
type RunResult struct {
RunID string
Artifact Artifact
RawOutput string
Validation ValidationResult
PromptID string
PromptVersion string
PromptHash string
SessionID string
RenderedPromptHash string
SelectedProfileID string
SelectedBackendID string
ModelName string
Endpoint string
EffectiveModelParams ExecutionTarget
InputHashes map[string]string
Usage TokenUsage
StartTime time.Time
EndTime time.Time
Duration time.Duration
}
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
// It must never include resolved API key values, model output, or validation data.
type PreparedRun struct {
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id"`
SelectedBackendID string `json:"selected_backend_id,omitempty"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
TargetPresence ExecutionTargetPresence `json:"-"`
OutputContract OutputContract `json:"output_contract"`
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
InputHashes map[string]string `json:"input_hashes,omitempty"`
SessionID string `json:"session_id,omitempty"`
RenderedPromptHash string `json:"rendered_prompt_hash"`
Messages []RenderedMessage `json:"messages"`
StartTime time.Time `json:"start_time,omitempty"`
EndTime time.Time `json:"end_time,omitempty"`
DurationMS int64 `json:"duration_ms,omitempty"`
}
// ArtifactRef represents a reference to an input artifact.
type ArtifactRef struct {
Type ArtifactRefType
URI string
Body string // Used for inline
}
// Artifact represents the actual loaded content of a reference.
type Artifact struct {
Name string
ContentType string
Body []byte
URI string
Size int64
Hash string
}
// PromptDefinition represents a configured prompt execution definition.
type PromptDefinition struct {
ID string `yaml:"id"`
Version string `yaml:"version"`
DefaultProfile string `yaml:"default_profile"`
Description string `yaml:"description"`
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
Inputs []PromptInput `yaml:"inputs"`
Templates []PromptMessageTemplate `yaml:"templates"`
OutputFormat OutputFormat `yaml:"output_format"`
Validation OutputContract `yaml:"validation"`
}
// PromptInput describes one named input expected by a prompt definition.
type PromptInput struct {
Name string `yaml:"name"`
Required bool `yaml:"required"`
ContentType string `yaml:"content_type"`
Description string `yaml:"description"`
}
// PromptMessageTemplate defines a template for a chat message.
type PromptMessageTemplate struct {
Role string `yaml:"role"`
Content string `yaml:"content"`
ContentFile string `yaml:"content_file"`
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
}
// Backend describes reusable OpenAI-compatible connection defaults.
type Backend struct {
ID string
Endpoint string
APIKeyEnv string
ExtraParams map[string]any
ConcurrencyLimit int
QueueCapacity int
QueueCapacitySet bool
}
// BackendCapacityPolicy describes normalized run and generation capacity for
// one limited backend.
type BackendCapacityPolicy struct {
ConcurrencyLimit int
QueueCapacity int
}
// ExecutionProfile describes how and where to execute a model.
type ExecutionProfile struct {
ID string `yaml:"id"`
BackendID string `yaml:"backend"`
Endpoint string `yaml:"endpoint"`
Model string `yaml:"model"`
Temperature float64 `yaml:"temperature"`
MaxTokens int `yaml:"max_tokens"`
TopP float64 `yaml:"top_p"`
TimeoutSeconds int `yaml:"timeout_seconds"`
ServiceTier string `yaml:"service_tier"`
ReasoningEffort string `yaml:"reasoning_effort"`
APIKeyEnv string `yaml:"api_key_env"`
APIKeyRequired bool `yaml:"-" json:"-"`
ExtraParams map[string]any `yaml:"extra_params"`
}
// ExecutionTargetOverride represents per-request runtime setting overrides.
type ExecutionTargetOverride struct {
Endpoint string `json:"endpoint,omitempty"`
Model string `json:"model,omitempty"`
Temperature *float64 `json:"temperature,omitempty"`
MaxTokens *int `json:"max_tokens,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
ServiceTier string `json:"service_tier,omitempty"`
ReasoningEffort *string `json:"reasoning_effort,omitempty"`
APIKeyEnv string `json:"api_key_env,omitempty"`
ExtraParams map[string]any `json:"extra_params,omitempty"`
}
// ExecutionTargetPresence tracks which effective runtime fields came from an
// explicit request override even when the resolved value is a zero value.
type ExecutionTargetPresence struct {
Temperature bool
MaxTokens bool
TopP bool
TimeoutSeconds bool
}
// ExecutionTarget represents effective model runtime settings for a run.
type ExecutionTarget struct {
BackendID string `yaml:"backend" json:"backend_id,omitempty"`
Endpoint string `yaml:"endpoint" json:"endpoint"`
Model string `yaml:"model" json:"model"`
Temperature float64 `yaml:"temperature" json:"temperature"`
MaxTokens int `yaml:"max_tokens" json:"max_tokens"`
TopP float64 `yaml:"top_p" json:"top_p"`
TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"`
ServiceTier string `yaml:"service_tier" json:"service_tier"`
ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"`
APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"`
APIKey string `yaml:"-" json:"-"`
APIKeyRequired bool `yaml:"-" json:"-"`
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
}
// OutputContract defines the requirements for the output artifact.
type OutputContract struct {
Format OutputFormat `yaml:"format"`
ValidationMode ValidationMode `yaml:"validation_mode"`
SchemaPath string `yaml:"schema_path"`
RepairAttempts int `yaml:"repair_attempts"`
}
// RenderedPrompt represents the prompt after template application.
type RenderedPrompt struct {
SessionID string `json:"session_id,omitempty"`
Messages []RenderedMessage `json:"messages"`
}
// RenderedMessage is a single message in a rendered prompt.
type RenderedMessage struct {
Role string `json:"role"`
Content string `json:"content"`
CacheControl *CacheControl `json:"cache_control,omitempty"`
}
// GenerateRequest is the internal request passed to the LLM client.
type GenerateRequest struct {
Prompt RenderedPrompt
Target ExecutionTarget
TargetPresence ExecutionTargetPresence
StructuredOutput *StructuredOutputSpec
}
// StructuredOutputType indicates which provider-level output mode is requested.
type StructuredOutputType string
const (
StructuredOutputJSONSchema StructuredOutputType = "json_schema"
)
// StructuredOutputSpec describes provider-level structured output requirements.
type StructuredOutputSpec struct {
Type StructuredOutputType `json:"type"`
JSONSchema *StructuredOutputJSONSpec `json:"json_schema,omitempty"`
}
// StructuredOutputJSONSpec contains json_schema output constraints.
type StructuredOutputJSONSpec struct {
Name string `json:"name"`
Strict bool `json:"strict"`
Schema any `json:"schema"`
}
// GenerateResponse is the response received from the LLM client.
type GenerateResponse struct {
Content string
Usage TokenUsage
}
// TokenUsage tracks token consumption.
type TokenUsage struct {
PromptTokens int
CompletionTokens int
TotalTokens int
CachedTokens int
CacheWriteTokens int
}
// ValidationResult represents the outcome of an output validation.
type ValidationResult struct {
Status ValidationStatus
Mode ValidationMode
Errors []string
SchemaPath string
RepairAttempts int
IsValid bool
}

View File

@@ -0,0 +1,141 @@
package domain
import (
"encoding/json"
"strings"
"testing"
)
func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
const envName = "PROMPTKIT_TEST_API_KEY"
const secret = "super-secret-value"
t.Setenv(envName, secret)
prepared := PreparedRun{
PromptID: "prompt.id",
PromptVersion: "v1",
PromptHash: "prompt-hash",
SelectedProfileID: "local-fast",
EffectiveModelParams: ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "gpt-test",
APIKeyEnv: envName,
APIKey: secret,
},
InputHashes: map[string]string{"transcript": "hash-1"},
RenderedPromptHash: "rendered-hash",
Messages: []RenderedMessage{
{Role: "system", Content: "You are helpful."},
{Role: "user", Content: "Summarize this."},
},
}
b, err := json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal failed: %v", err)
}
out := string(b)
if strings.Contains(out, secret) {
t.Fatalf("prepared run JSON unexpectedly contains secret value: %s", out)
}
if !strings.Contains(out, `"api_key_env":"`+envName+`"`) {
t.Fatalf("prepared run JSON should include api_key_env name: %s", out)
}
var top map[string]any
if err := json.Unmarshal(b, &top); err != nil {
t.Fatalf("unmarshal failed: %v", err)
}
for _, forbidden := range []string{"raw_output", "validation", "artifact"} {
if _, ok := top[forbidden]; ok {
t.Fatalf("prepared run JSON should not include %q", forbidden)
}
}
}
func TestPreparedRunJSONIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
prepared := PreparedRun{
PromptID: "prompt.id",
SelectedProfileID: "local-fast",
EffectiveModelParams: ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "gpt-test",
},
RenderedPromptHash: "rendered-hash",
Messages: []RenderedMessage{
{
Role: "system",
Content: "You are helpful.",
CacheControl: &CacheControl{
Type: CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "Summarize this."},
},
}
b, err := json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal failed: %v", err)
}
var decoded struct {
Messages []map[string]any `json:"messages"`
}
if err := json.Unmarshal(b, &decoded); err != nil {
t.Fatalf("unmarshal failed: %v", err)
}
if len(decoded.Messages) != 2 {
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
}
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
if !ok {
t.Fatalf("expected cache_control on first message, got %#v", decoded.Messages[0])
}
if cacheControl["type"] != string(CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
}
if _, ok := decoded.Messages[1]["cache_control"]; ok {
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
}
}
func TestPreparedRunJSONIncludesSessionIDOnlyWhenPresent(t *testing.T) {
prepared := PreparedRun{
PromptID: "prompt.id",
SelectedProfileID: "local-fast",
EffectiveModelParams: ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "gpt-test",
},
SessionID: "session-123",
RenderedPromptHash: "rendered-hash",
Messages: []RenderedMessage{{Role: "user", Content: "Summarize this."}},
}
b, err := json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal failed: %v", err)
}
var decoded map[string]any
if err := json.Unmarshal(b, &decoded); err != nil {
t.Fatalf("unmarshal failed: %v", err)
}
if decoded["session_id"] != "session-123" {
t.Fatalf("expected session_id in prepared run JSON, got %#v", decoded["session_id"])
}
prepared.SessionID = ""
b, err = json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal failed: %v", err)
}
if strings.Contains(string(b), "session_id") {
t.Fatalf("expected empty session_id to be omitted, got %s", b)
}
}

View File

@@ -0,0 +1,19 @@
package domain
import (
"fmt"
"strings"
"unicode/utf8"
)
// NormalizeSessionID applies the shared session identifier rule.
func NormalizeSessionID(raw string) (string, error) {
normalized := strings.TrimSpace(raw)
if normalized == "" {
return "", nil
}
if length := utf8.RuneCountInString(normalized); length > SessionIDMaxLength {
return "", fmt.Errorf("session_id length %d exceeds maximum %d", length, SessionIDMaxLength)
}
return normalized, nil
}

View File

@@ -0,0 +1,57 @@
package domain
import (
"strings"
"testing"
)
func TestNormalizeSessionID(t *testing.T) {
tests := []struct {
name string
raw string
want string
wantErr bool
}{
{
name: "trims surrounding Unicode whitespace",
raw: "\u2003 session-123 \u2003",
want: "session-123",
},
{
name: "blank input is omitted",
raw: " \t\u2003 ",
want: "",
},
{
name: "maximum Unicode length is accepted",
raw: strings.Repeat("界", SessionIDMaxLength),
want: strings.Repeat("界", SessionIDMaxLength),
},
{
name: "one Unicode code point over maximum is rejected",
raw: strings.Repeat("界", SessionIDMaxLength+1),
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := NormalizeSessionID(tt.raw)
if tt.wantErr {
if err == nil {
t.Fatal("expected normalization error")
}
if !strings.Contains(err.Error(), "exceeds maximum") {
t.Fatalf("expected useful length diagnostic, got %v", err)
}
return
}
if err != nil {
t.Fatalf("normalize session id: %v", err)
}
if got != tt.want {
t.Fatalf("normalized session id = %q, want %q", got, tt.want)
}
})
}
}

View File

@@ -0,0 +1,142 @@
package filecatalog
import (
"context"
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
"sort"
"strings"
)
// FindYAMLFiles returns sorted full paths for .yaml and .yml files under root.
func FindYAMLFiles(ctx context.Context, root string) ([]string, error) {
var files []string
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if d.IsDir() {
return nil
}
if !IsYAMLFile(d.Name()) {
return nil
}
files = append(files, path)
return nil
})
sort.Strings(files)
return files, err
}
// FindFSYAMLFiles returns sorted paths for .yaml and .yml files under root in fsys.
func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, error) {
cleanRoot := CleanFSRoot(root)
var files []string
err := fs.WalkDir(fsys, cleanRoot, func(name string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if d.IsDir() {
return nil
}
if !IsYAMLFile(d.Name()) {
return nil
}
files = append(files, name)
return nil
})
sort.Strings(files)
return files, err
}
// RelativePath computes a clean relative path from root to path.
func RelativePath(root string, filePath string) string {
rel, err := filepath.Rel(root, filePath)
if err != nil {
return filepath.Clean(filePath)
}
return filepath.Clean(rel)
}
// CleanFSRoot normalizes a root path for use with fs.FS.
func CleanFSRoot(root string) string {
root = strings.TrimSpace(root)
if root == "" || root == "." {
return "."
}
return path.Clean(root)
}
// DisplayPath returns name relative to root for messages about fs.FS paths.
func DisplayPath(root string, name string) string {
cleanRoot := CleanFSRoot(root)
cleanName := path.Clean(name)
if cleanRoot == "." {
return cleanName
}
prefix := strings.TrimSuffix(cleanRoot, "/") + "/"
if strings.HasPrefix(cleanName, prefix) {
return strings.TrimPrefix(cleanName, prefix)
}
return cleanName
}
// ResolveFSPath resolves userPath from baseDir and keeps it inside root.
func ResolveFSPath(root string, baseDir string, userPath string) (string, string, error) {
cleanRoot := CleanFSRoot(root)
cleanBase := path.Clean(strings.TrimSpace(baseDir))
if cleanBase == "" {
cleanBase = cleanRoot
}
if !containsFSPath(cleanRoot, cleanBase) {
return "", "", fmt.Errorf("base path %q is outside source root %q", cleanBase, cleanRoot)
}
cleanUserPath := strings.TrimSpace(userPath)
if cleanUserPath == "" {
return "", "", fmt.Errorf("path is required")
}
cleanUserPath = path.Clean(cleanUserPath)
if path.IsAbs(cleanUserPath) {
return "", "", fmt.Errorf("path %q must be relative", userPath)
}
resolved := path.Clean(path.Join(cleanBase, cleanUserPath))
if !containsFSPath(cleanRoot, resolved) {
return "", "", fmt.Errorf("path %q escapes source root %q", userPath, cleanRoot)
}
return resolved, DisplayPath(cleanRoot, resolved), nil
}
func containsFSPath(root string, name string) bool {
root = CleanFSRoot(root)
name = path.Clean(name)
if root == "." {
return name == "." || (name != ".." && !strings.HasPrefix(name, "../"))
}
return name == root || strings.HasPrefix(name, strings.TrimSuffix(root, "/")+"/")
}
// Stem strips .yaml or .yml from a file name.
func Stem(name string) string {
name = strings.TrimSuffix(name, ".yaml")
name = strings.TrimSuffix(name, ".yml")
return name
}
func IsYAMLFile(name string) bool {
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
}

View File

@@ -0,0 +1,270 @@
package filecatalog
import (
"context"
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"testing/fstest"
)
func TestFindYAMLFilesNestedSortedAndFiltered(t *testing.T) {
root := t.TempDir()
mustWriteFile(t, filepath.Join(root, "z", "prompt.yml"), "id: z")
mustWriteFile(t, filepath.Join(root, "a", "profile.yaml"), "id: a")
mustWriteFile(t, filepath.Join(root, "a", "ignore.txt"), "not yaml")
mustWriteFile(t, filepath.Join(root, "b", "ignore.yaml.bak"), "not yaml")
got, err := FindYAMLFiles(context.Background(), root)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
want := []string{
filepath.Join(root, "a", "profile.yaml"),
filepath.Join(root, "z", "prompt.yml"),
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("expected sorted YAML files %v, got %v", want, got)
}
}
func TestFindYAMLFilesHonorsContextCancellation(t *testing.T) {
root := t.TempDir()
mustWriteFile(t, filepath.Join(root, "one.yaml"), "id: one")
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := FindYAMLFiles(ctx, root)
if !errors.Is(err, context.Canceled) {
t.Fatalf("expected context.Canceled, got %v", err)
}
}
func TestFindFSYAMLFilesNestedSortedAndFiltered(t *testing.T) {
fsys := fstest.MapFS{
"prompts/z/prompt.yml": &fstest.MapFile{Data: []byte("id: z")},
"prompts/a/profile.yaml": &fstest.MapFile{Data: []byte("id: a")},
"prompts/a/ignore.txt": &fstest.MapFile{Data: []byte("not yaml")},
"prompts/b/ignore.yaml.bak": &fstest.MapFile{Data: []byte("not yaml")},
"other/ignored.yaml": &fstest.MapFile{Data: []byte("id: ignored")},
}
got, err := FindFSYAMLFiles(context.Background(), fsys, " prompts ")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
want := []string{
"prompts/a/profile.yaml",
"prompts/z/prompt.yml",
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("expected sorted YAML files %v, got %v", want, got)
}
}
func TestFindFSYAMLFilesHonorsContextCancellation(t *testing.T) {
fsys := fstest.MapFS{
"one.yaml": &fstest.MapFile{Data: []byte("id: one")},
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := FindFSYAMLFiles(ctx, fsys, ".")
if !errors.Is(err, context.Canceled) {
t.Fatalf("expected context.Canceled, got %v", err)
}
}
func TestRelativePathNested(t *testing.T) {
root := t.TempDir()
path := filepath.Join(root, "nested", "profiles", "local.yaml")
got := RelativePath(root, path)
want := filepath.Join("nested", "profiles", "local.yaml")
if got != want {
t.Fatalf("expected relative path %q, got %q", want, got)
}
}
func TestCleanFSRoot(t *testing.T) {
tests := []struct {
name string
root string
want string
}{
{name: "empty", root: "", want: "."},
{name: "dot", root: ".", want: "."},
{name: "trimmed", root: " prompts/../profiles ", want: "profiles"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if got := CleanFSRoot(tc.root); got != tc.want {
t.Fatalf("expected %q, got %q", tc.want, got)
}
})
}
}
func TestDisplayPath(t *testing.T) {
tests := []struct {
name string
root string
path string
want string
}{
{name: "root dot", root: ".", path: "profiles/local.yaml", want: "profiles/local.yaml"},
{name: "nested root", root: "profiles", path: "profiles/local.yaml", want: "local.yaml"},
{name: "outside root", root: "profiles", path: "other/local.yaml", want: "other/local.yaml"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if got := DisplayPath(tc.root, tc.path); got != tc.want {
t.Fatalf("expected %q, got %q", tc.want, got)
}
})
}
}
func TestResolveFSPath(t *testing.T) {
tests := []struct {
name string
root string
baseDir string
userPath string
wantPath string
wantDisplay string
wantErr string
}{
{
name: "sibling inside root",
root: "prompts",
baseDir: "prompts/nested",
userPath: "./messages/user.tmpl",
wantPath: "prompts/nested/messages/user.tmpl",
wantDisplay: "nested/messages/user.tmpl",
},
{
name: "parent inside root",
root: "prompts",
baseDir: "prompts/nested",
userPath: "../shared/user.tmpl",
wantPath: "prompts/shared/user.tmpl",
wantDisplay: "shared/user.tmpl",
},
{
name: "escape rejected",
root: "prompts",
baseDir: "prompts/nested",
userPath: "../../outside.tmpl",
wantErr: "escapes source root",
},
{
name: "absolute path rejected",
root: "prompts",
baseDir: "prompts/nested",
userPath: "/outside.tmpl",
wantErr: "must be relative",
},
{
name: "empty path rejected",
root: "prompts",
baseDir: "prompts/nested",
userPath: " ",
wantErr: "path is required",
},
{
name: "dot root allows normal relative path",
root: ".",
baseDir: ".",
userPath: "schemas/events.schema.json",
wantPath: "schemas/events.schema.json",
wantDisplay: "schemas/events.schema.json",
},
{
name: "dot root rejects parent escape",
root: ".",
baseDir: ".",
userPath: "../outside.tmpl",
wantErr: "escapes source root",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
gotPath, gotDisplay, err := ResolveFSPath(tc.root, tc.baseDir, tc.userPath)
if tc.wantErr != "" {
if err == nil {
t.Fatalf("expected error containing %q", tc.wantErr)
}
if !strings.Contains(err.Error(), tc.wantErr) {
t.Fatalf("expected error to contain %q, got %v", tc.wantErr, err)
}
return
}
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if gotPath != tc.wantPath || gotDisplay != tc.wantDisplay {
t.Fatalf("expected path/display %q/%q, got %q/%q", tc.wantPath, tc.wantDisplay, gotPath, gotDisplay)
}
})
}
}
func TestStemStripsYAMLExtensions(t *testing.T) {
tests := []struct {
name string
in string
want string
}{
{name: "yaml", in: "prompt.yaml", want: "prompt"},
{name: "yml", in: "profile.yml", want: "profile"},
{name: "other", in: "file.txt", want: "file.txt"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if got := Stem(tc.in); got != tc.want {
t.Fatalf("expected %q, got %q", tc.want, got)
}
})
}
}
func TestIsYAMLFile(t *testing.T) {
tests := []struct {
name string
in string
want bool
}{
{name: "yaml", in: "prompt.yaml", want: true},
{name: "yml", in: "profile.yml", want: true},
{name: "backup", in: "profile.yaml.bak", want: false},
{name: "uppercase", in: "profile.YAML", want: false},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if got := IsYAMLFile(tc.in); got != tc.want {
t.Fatalf("expected %v, got %v", tc.want, got)
}
})
}
}
func mustWriteFile(t *testing.T, path string, content string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("failed to create directory: %v", err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatalf("failed to write file %q: %v", path, err)
}
}

View File

@@ -0,0 +1,232 @@
// Package jsonvalue validates and defensively copies JSON-compatible value
// trees used by public configuration and request boundaries.
package jsonvalue
import (
"encoding/json"
"fmt"
"math"
"reflect"
"sort"
"strconv"
)
const maxSafeJSONInteger = 1<<53 - 1
type visit struct {
typ reflect.Type
ptr uintptr
}
// CopyMap validates and deeply copies an extra-parameter map while preserving
// compatible concrete map, slice, array, scalar, and number types.
func CopyMap(src map[string]any) (map[string]any, error) {
if src == nil {
return nil, nil
}
copied, err := copyValue(reflect.ValueOf(src), "extra_params", make(map[visit]struct{}))
if err != nil {
return nil, err
}
out, ok := copied.(map[string]any)
if !ok {
return nil, fmt.Errorf("extra_params: expected object")
}
return out, nil
}
func copyValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
if !value.IsValid() {
return nil, nil
}
if value.Kind() == reflect.Interface {
if value.IsNil() {
return nil, nil
}
return copyValue(value.Elem(), path, seen)
}
if !value.CanInterface() {
return nil, fmt.Errorf("%s: value cannot be copied", path)
}
if number, ok := value.Interface().(json.Number); ok {
if _, err := json.Marshal(number); err != nil {
return nil, fmt.Errorf("%s: invalid JSON number", path)
}
f, err := strconv.ParseFloat(number.String(), 64)
if err != nil || math.IsNaN(f) || math.IsInf(f, 0) {
return nil, fmt.Errorf("%s: invalid JSON number", path)
}
return number, nil
}
switch value.Kind() {
case reflect.Bool, reflect.String:
return value.Interface(), nil
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
if value.Int() < -maxSafeJSONInteger || value.Int() > maxSafeJSONInteger {
return nil, fmt.Errorf("%s: integer is outside the JSON-safe range", path)
}
return value.Interface(), nil
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
if value.Uint() > maxSafeJSONInteger {
return nil, fmt.Errorf("%s: integer is outside the JSON-safe range", path)
}
return value.Interface(), nil
case reflect.Float32, reflect.Float64:
number := value.Convert(reflect.TypeOf(float64(0))).Float()
if math.IsNaN(number) || math.IsInf(number, 0) {
return nil, fmt.Errorf("%s: floating-point value must be finite", path)
}
return value.Interface(), nil
case reflect.Pointer:
if value.IsNil() {
return nil, nil
}
current := visit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[current]; ok {
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
}
seen[current] = struct{}{}
defer delete(seen, current)
return copyValue(value.Elem(), path, seen)
case reflect.Map:
return copyMapValue(value, path, seen)
case reflect.Slice:
if value.IsNil() {
return nil, nil
}
return copySequenceValue(value, path, seen)
case reflect.Array:
return copySequenceValue(value, path, seen)
default:
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
}
}
func copyMapValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
if value.IsNil() {
return nil, nil
}
if value.Type().Key().Kind() != reflect.String {
return nil, fmt.Errorf("%s: map key type %s is not supported", path, value.Type().Key())
}
current := visit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[current]; ok {
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
}
seen[current] = struct{}{}
defer delete(seen, current)
keys := value.MapKeys()
sort.Slice(keys, func(i, j int) bool {
return keys[i].String() < keys[j].String()
})
type entry struct {
key reflect.Value
name string
value any
}
entries := make([]entry, 0, len(keys))
preserveType := true
elementType := value.Type().Elem()
for _, key := range keys {
name := key.String()
if name == "" {
return nil, fmt.Errorf("%s: map key must not be empty", path)
}
copied, err := copyValue(value.MapIndex(key), path+"."+name, seen)
if err != nil {
return nil, err
}
entries = append(entries, entry{key: key, name: name, value: copied})
if copied == nil {
if !canAssignNil(elementType) {
preserveType = false
}
continue
}
if !reflect.TypeOf(copied).AssignableTo(elementType) {
preserveType = false
}
}
if preserveType {
out := reflect.MakeMapWithSize(value.Type(), len(entries))
for _, entry := range entries {
if entry.value == nil {
out.SetMapIndex(entry.key, reflect.Zero(elementType))
continue
}
out.SetMapIndex(entry.key, reflect.ValueOf(entry.value))
}
return out.Interface(), nil
}
out := make(map[string]any, len(entries))
for _, entry := range entries {
out[entry.name] = entry.value
}
return out, nil
}
func copySequenceValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
var current visit
if value.Kind() == reflect.Slice {
current = visit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[current]; ok {
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
}
seen[current] = struct{}{}
defer delete(seen, current)
}
values := make([]any, value.Len())
preserveType := true
elementType := value.Type().Elem()
for i := 0; i < value.Len(); i++ {
copied, err := copyValue(value.Index(i), fmt.Sprintf("%s[%d]", path, i), seen)
if err != nil {
return nil, err
}
values[i] = copied
if copied == nil {
if !canAssignNil(elementType) {
preserveType = false
}
continue
}
if !reflect.TypeOf(copied).AssignableTo(elementType) {
preserveType = false
}
}
if preserveType {
out := reflect.New(value.Type()).Elem()
if value.Kind() == reflect.Slice {
out = reflect.MakeSlice(value.Type(), value.Len(), value.Len())
}
for i, copied := range values {
if copied == nil {
out.Index(i).Set(reflect.Zero(elementType))
continue
}
out.Index(i).Set(reflect.ValueOf(copied))
}
return out.Interface(), nil
}
out := make([]any, len(values))
copy(out, values)
return out, nil
}
func canAssignNil(typ reflect.Type) bool {
switch typ.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
return true
default:
return false
}
}

View File

@@ -0,0 +1,97 @@
package jsonvalue_test
import (
"encoding/json"
"math"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
)
func TestCopyMapPreservesTypesAndIsolatesMutations(t *testing.T) {
nested := map[string]int{"limit": 2}
sequence := []string{"one", "two"}
input := map[string]any{
"count": int64(7),
"number": json.Number("-1.25e+2"),
"nested": nested,
"sequence": sequence,
}
copied, err := jsonvalue.CopyMap(input)
if err != nil {
t.Fatalf("copy map: %v", err)
}
nested["limit"] = 99
sequence[0] = "changed"
input["added"] = true
if got, ok := copied["count"].(int64); !ok || got != 7 {
t.Fatalf("integer type or value changed: %#v", copied["count"])
}
if got, ok := copied["number"].(json.Number); !ok || got != "-1.25e+2" {
t.Fatalf("JSON number type or value changed: %#v", copied["number"])
}
if got := copied["nested"].(map[string]int)["limit"]; got != 2 {
t.Fatalf("nested map was not isolated: %d", got)
}
if got := copied["sequence"].([]string)[0]; got != "one" {
t.Fatalf("sequence was not isolated: %q", got)
}
if _, ok := copied["added"]; ok {
t.Fatalf("top-level map was not isolated: %#v", copied)
}
}
func TestCopyMapRejectsInvalidValues(t *testing.T) {
cyclicMap := map[string]any{}
cyclicMap["self"] = cyclicMap
cyclicSlice := []any{nil}
cyclicSlice[0] = cyclicSlice
tests := []struct {
name string
value any
}{
{name: "empty nested key", value: map[string]int{"": 1}},
{name: "non-string map key", value: map[int]string{1: "one"}},
{name: "unsupported value", value: make(chan int)},
{name: "cyclic map", value: cyclicMap},
{name: "cyclic slice", value: cyclicSlice},
{name: "NaN", value: math.NaN()},
{name: "positive infinity", value: math.Inf(1)},
{name: "unsafe signed integer", value: int64(1 << 53)},
{name: "unsafe unsigned integer", value: uint64(1 << 53)},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if _, err := jsonvalue.CopyMap(map[string]any{"value": tc.value}); err == nil {
t.Fatal("expected validation error")
}
})
}
}
func TestCopyMapValidatesJSONNumberSyntaxAndRange(t *testing.T) {
for _, number := range []json.Number{"0", "-1", "1.25", "-1.25e+2"} {
t.Run("valid "+number.String(), func(t *testing.T) {
got, err := jsonvalue.CopyMap(map[string]any{"value": number})
if err != nil {
t.Fatalf("copy valid JSON number: %v", err)
}
if !reflect.DeepEqual(got["value"], number) {
t.Fatalf("JSON number changed: got %#v want %#v", got["value"], number)
}
})
}
for _, number := range []json.Number{"", "01", "+1", "1.", ".1", "1e9999", "not-a-number"} {
t.Run("invalid "+number.String(), func(t *testing.T) {
if _, err := jsonvalue.CopyMap(map[string]any{"value": number}); err == nil {
t.Fatal("expected invalid JSON number error")
}
})
}
}

11
internal/llm/client.go Normal file
View File

@@ -0,0 +1,11 @@
package llm
import (
"context"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
// Client executes a rendered prompt against an LLM endpoint.
type Client interface {
Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error)
}

View File

@@ -0,0 +1,390 @@
package llm
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
"time"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
var (
ErrInvalidConfig = errors.New("invalid llm client configuration")
ErrInvalidRequest = errors.New("invalid generate request")
ErrRequestFailed = errors.New("llm request failed")
ErrUnexpectedStatus = errors.New("llm returned non-success status")
ErrMalformedResponse = errors.New("malformed llm response")
)
type OpenAICompatibleConfig struct {
BaseURL string
Model string
Timeout time.Duration
HTTPClient *http.Client
}
type OpenAICompatibleClient struct {
baseURL string
defaultModel string
httpClient *http.Client
}
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) {
baseURL := strings.TrimSpace(cfg.BaseURL)
if baseURL != "" {
if _, err := url.ParseRequestURI(baseURL); err != nil {
return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err)
}
}
timeout := cfg.Timeout
if timeout <= 0 {
timeout = defaults.LLMRequestTimeoutDefault
}
var client *http.Client
if cfg.HTTPClient != nil {
cloned := *cfg.HTTPClient
if cloned.Timeout <= 0 {
cloned.Timeout = timeout
}
client = &cloned
} else {
client = &http.Client{Timeout: timeout}
}
return &OpenAICompatibleClient{
baseURL: strings.TrimRight(baseURL, "/"),
defaultModel: cfg.Model,
httpClient: client,
}, nil
}
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
if req.Target.TimeoutSeconds < 0 {
return nil, fmt.Errorf("%w: timeout_seconds must be greater than or equal to 0", ErrInvalidRequest)
}
endpoint := strings.TrimSpace(req.Target.Endpoint)
if endpoint == "" {
endpoint = c.baseURL
}
if endpoint == "" {
return nil, fmt.Errorf("%w: endpoint is required", ErrInvalidRequest)
}
endpoint = strings.TrimRight(endpoint, "/") + defaults.OpenAIChatCompletionsPath
wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
}
wirePayload, err := openAIChatRequestPayload(wireReq)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
}
payload, err := json.Marshal(wirePayload)
if err != nil {
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
}
requestContext := ctx
if req.Target.TimeoutSeconds > 0 {
var cancel context.CancelFunc
requestContext, cancel = context.WithTimeout(
ctx,
time.Duration(req.Target.TimeoutSeconds)*time.Second,
)
defer cancel()
}
httpReq, err := http.NewRequestWithContext(requestContext, http.MethodPost, endpoint, bytes.NewReader(payload))
if err != nil {
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
}
httpReq.Header.Set("Content-Type", "application/json")
if apiKey := strings.TrimSpace(req.Target.APIKey); apiKey != "" {
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
} else if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" {
apiKey := strings.TrimSpace(os.Getenv(envName))
if apiKey == "" {
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
}
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
}
httpClient := c.httpClient
if httpClient == nil {
httpClient = &http.Client{Timeout: defaults.LLMRequestTimeoutDefault}
}
httpResp, err := httpClient.Do(httpReq)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrRequestFailed, err)
}
defer httpResp.Body.Close()
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
_, _ = io.Copy(io.Discard, io.LimitReader(httpResp.Body, 4096))
return nil, fmt.Errorf("%w: status=%d", ErrUnexpectedStatus, httpResp.StatusCode)
}
var wireResp openAIChatResponse
if err := json.NewDecoder(httpResp.Body).Decode(&wireResp); err != nil {
return nil, fmt.Errorf("%w: failed to decode response: %v", ErrMalformedResponse, err)
}
if len(wireResp.Choices) == 0 {
return nil, fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
}
content := wireResp.Choices[0].Message.Content
if content == "" {
return nil, fmt.Errorf("%w: first choice has empty message content", ErrMalformedResponse)
}
return &domain.GenerateResponse{
Content: content,
Usage: domain.TokenUsage{
PromptTokens: wireResp.Usage.PromptTokens,
CompletionTokens: wireResp.Usage.CompletionTokens,
TotalTokens: wireResp.Usage.TotalTokens,
CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens,
CacheWriteTokens: wireResp.Usage.CacheWriteTokens,
},
}, nil
}
func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) {
model := strings.TrimSpace(req.Target.Model)
if model == "" {
model = strings.TrimSpace(defaultModel)
}
if model == "" {
return openAIChatRequest{}, errors.New("model is required")
}
wireReq := openAIChatRequest{
Model: model,
}
sessionID, err := domain.NormalizeSessionID(req.Prompt.SessionID)
if err != nil {
return openAIChatRequest{}, err
}
wireReq.SessionID = sessionID
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
for _, msg := range req.Prompt.Messages {
wireReq.Messages = append(wireReq.Messages, openAIChatRequestMessageFromRenderedMessage(msg))
}
if req.Target.Temperature != 0 || req.TargetPresence.Temperature {
wireReq.Temperature = &req.Target.Temperature
}
if req.Target.MaxTokens != 0 || req.TargetPresence.MaxTokens {
wireReq.MaxTokens = &req.Target.MaxTokens
}
if req.Target.TopP != 0 || req.TargetPresence.TopP {
wireReq.TopP = &req.Target.TopP
}
if strings.TrimSpace(req.Target.ServiceTier) != "" {
wireReq.ServiceTier = req.Target.ServiceTier
}
if strings.TrimSpace(req.Target.ReasoningEffort) != "" {
wireReq.ReasoningEffort = req.Target.ReasoningEffort
}
if len(req.Target.ExtraParams) > 0 {
wireReq.ExtraParams = req.Target.ExtraParams
}
if req.StructuredOutput != nil {
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
if err != nil {
return openAIChatRequest{}, err
}
wireReq.ResponseFormat = responseFormat
}
return wireReq, nil
}
type openAIChatRequest struct {
Model string `json:"model"`
SessionID string `json:"session_id,omitempty"`
Messages []openAIChatRequestMessage `json:"messages"`
Temperature *float64 `json:"temperature,omitempty"`
MaxTokens *int `json:"max_tokens,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
ServiceTier string `json:"service_tier,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"`
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"`
ExtraParams map[string]any `json:"-"`
}
func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
out := map[string]any{
"model": req.Model,
"messages": req.Messages,
}
if req.SessionID != "" {
out["session_id"] = req.SessionID
}
if req.Temperature != nil {
out["temperature"] = *req.Temperature
}
if req.MaxTokens != nil {
out["max_tokens"] = *req.MaxTokens
}
if req.TopP != nil {
out["top_p"] = *req.TopP
}
if req.ServiceTier != "" {
out["service_tier"] = req.ServiceTier
}
if req.ReasoningEffort != "" {
out["reasoning_effort"] = req.ReasoningEffort
}
if req.ResponseFormat != nil {
out["response_format"] = req.ResponseFormat
}
for key, value := range req.ExtraParams {
if key == "" {
return nil, errors.New("extra_params key must not be empty")
}
if IsReservedOpenAIChatRequestField(key) {
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
}
if _, err := json.Marshal(value); err != nil {
return nil, fmt.Errorf("extra_params.%s must be JSON-serializable: %w", key, err)
}
out[key] = value
}
return out, nil
}
// IsReservedOpenAIChatRequestField reports whether name is owned by the
// standard OpenAI-compatible chat request rather than extra parameters.
func IsReservedOpenAIChatRequestField(name string) bool {
switch name {
case "model",
"session_id",
"messages",
"temperature",
"max_tokens",
"top_p",
"service_tier",
"reasoning_effort",
"response_format":
return true
default:
return false
}
}
type openAIChatRequestMessage struct {
Role string `json:"role"`
Content any `json:"content"`
}
type openAIChatTextContentBlock struct {
Type string `json:"type"`
Text string `json:"text"`
CacheControl *openAICacheControl `json:"cache_control,omitempty"`
}
type openAICacheControl struct {
Type string `json:"type"`
TTL string `json:"ttl,omitempty"`
}
type openAIChatResponseMessage struct {
Role string `json:"role"`
Content string `json:"content"`
}
type openAIChatResponse struct {
Choices []struct {
Message openAIChatResponseMessage `json:"message"`
} `json:"choices"`
Usage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
PromptTokensDetails struct {
CachedTokens int `json:"cached_tokens"`
} `json:"prompt_tokens_details"`
CacheWriteTokens int `json:"cache_write_tokens"`
} `json:"usage"`
}
type openAIResponseFormat struct {
Type string `json:"type"`
JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"`
}
type openAIJSONSchemaEnvelope struct {
Name string `json:"name"`
Strict bool `json:"strict"`
Schema any `json:"schema"`
}
func openAIChatRequestMessageFromRenderedMessage(msg domain.RenderedMessage) openAIChatRequestMessage {
wireMsg := openAIChatRequestMessage{
Role: msg.Role,
Content: msg.Content,
}
if msg.CacheControl == nil {
return wireMsg
}
wireMsg.Content = []openAIChatTextContentBlock{
{
Type: "text",
Text: msg.Content,
CacheControl: &openAICacheControl{
Type: string(msg.CacheControl.Type),
TTL: msg.CacheControl.TTL,
},
},
}
return wireMsg
}
func toOpenAIResponseFormat(spec *domain.StructuredOutputSpec) (*openAIResponseFormat, error) {
if spec == nil {
return nil, nil
}
switch spec.Type {
case domain.StructuredOutputJSONSchema:
if spec.JSONSchema == nil {
return nil, errors.New("json_schema structured output requires schema payload")
}
if strings.TrimSpace(spec.JSONSchema.Name) == "" {
return nil, errors.New("json_schema structured output requires non-empty schema name")
}
if spec.JSONSchema.Schema == nil {
return nil, errors.New("json_schema structured output requires schema document")
}
return &openAIResponseFormat{
Type: "json_schema",
JSONSchema: &openAIJSONSchemaEnvelope{
Name: spec.JSONSchema.Name,
Strict: spec.JSONSchema.Strict,
Schema: spec.JSONSchema.Schema,
},
}, nil
default:
return nil, fmt.Errorf("unsupported structured output type %q", spec.Type)
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,8 @@
id: aion-2
backend: openrouter
model: aion-labs/aion-2.0
temperature: 0.72
reasoning_effort: high
top_p: 0.95
timeout_seconds: 180
service_tier: flex

View File

@@ -0,0 +1,6 @@
id: claude-fable-latest
backend: openrouter
model: "~anthropic/claude-fable-latest"
reasoning_effort: high
timeout_seconds: 600
service_tier: flex

View File

@@ -0,0 +1,6 @@
id: claude-haiku-latest
backend: openrouter
model: "~anthropic/claude-haiku-latest"
reasoning_effort: medium
timeout_seconds: 240
service_tier: flex

View File

@@ -0,0 +1,6 @@
id: claude-opus-latest
backend: openrouter
model: "~anthropic/claude-opus-latest"
reasoning_effort: high
timeout_seconds: 240
service_tier: flex

View File

@@ -0,0 +1,6 @@
id: claude-sonnet-latest
backend: openrouter
model: "~anthropic/claude-sonnet-latest"
reasoning_effort: high
timeout_seconds: 240
service_tier: flex

View File

@@ -0,0 +1,6 @@
id: deepseek-3-2
backend: openrouter
model: deepseek/deepseek-v3.2
reasoning_effort: high
timeout_seconds: 180
service_tier: flex

View File

@@ -0,0 +1,6 @@
id: deepseek-4-flash
backend: openrouter
model: deepseek/deepseek-v4-flash
#reasoning_effort: medium
timeout_seconds: 180
service_tier: flex

View File

@@ -0,0 +1,6 @@
id: deepseek-4-pro
backend: openrouter
model: deepseek/deepseek-v4-pro
reasoning_effort: high
timeout_seconds: 180
service_tier: flex

View File

@@ -0,0 +1,8 @@
id: gemini-2-flash-lite
backend: openrouter
model: "google/gemini-2.5-flash-lite"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -0,0 +1,8 @@
id: gemini-2-flash
backend: openrouter
model: "google/gemini-2.5-flash"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -0,0 +1,8 @@
id: gemini-2-pro
backend: openrouter
model: "google/gemini-2.5-pro"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -0,0 +1,8 @@
id: gemini-3-flash-lite
backend: openrouter
model: "google/gemini-3.1-flash-lite"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -0,0 +1,8 @@
id: gemini-flash-latest
backend: openrouter
model: "~google/gemini-flash-latest"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -0,0 +1,8 @@
id: gemini-pro-latest
backend: openrouter
model: "~google/gemini-pro-latest"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -0,0 +1,8 @@
id: gemma-4-31b
backend: openrouter
model: google/gemma-4-31b-it:exacto
temperature: 0.15
reasoning_effort: high
top_p: 0.98
timeout_seconds: 240
service_tier: flex

View File

@@ -0,0 +1,8 @@
id: minimax-m2
backend: openrouter
model: minimax/minimax-m2.5
temperature: 0.5
reasoning_effort: high
top_p: 0.95
timeout_seconds: 180
service_tier: flex

View File

@@ -0,0 +1,8 @@
id: minimax-m3
backend: openrouter
model: minimax/minimax-m3
#temperature: 0.5
reasoning_effort: high
#top_p: 0.95
timeout_seconds: 180
service_tier: flex

View File

@@ -0,0 +1,6 @@
id: mistral-large-2512
backend: openrouter
model: mistralai/mistral-large-2512
temperature: 0.15
top_p: 0.98
timeout_seconds: 180

View File

@@ -0,0 +1,7 @@
id: mistral-medium-3-5
backend: openrouter
model: mistralai/mistral-medium-3-5
temperature: 0.15
reasoning_effort: high
top_p: 0.98
timeout_seconds: 180

View File

@@ -0,0 +1,6 @@
id: mistral-small-3
backend: openrouter
model: mistralai/mistral-small-3.2-24b-instruct
temperature: 0.05
top_p: 1.0
timeout_seconds: 180

View File

@@ -0,0 +1,7 @@
id: mistral-small-4
backend: openrouter
model: mistralai/mistral-small-2603
temperature: 0.1
reasoning_effort: high
top_p: 0.98
timeout_seconds: 180

View File

@@ -0,0 +1,6 @@
id: nemotron-3-ultra
backend: openrouter
model: nvidia/nemotron-3-ultra-550b-a55b
reasoning_effort: high
timeout_seconds: 180
service_tier: flex

View File

@@ -0,0 +1,6 @@
id: gpt-5-mini
backend: openrouter
model: "openai/gpt-5.4-mini"
reasoning_effort: high
timeout_seconds: 240
service_tier: flex

View File

@@ -0,0 +1,6 @@
id: gpt-5-nano
backend: openrouter
model: "openai/gpt-5.4-nano"
reasoning_effort: high
timeout_seconds: 240
service_tier: flex

View File

@@ -0,0 +1,31 @@
package builtin
import (
"embed"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
const assetRoot = "assets"
//go:embed assets/**/*.yml
var assets embed.FS
func NewRepository() profile.Repository {
return profile.NewFSRepository(assets, assetRoot)
}
func NewRepositoryWithPrimary(primary profile.Repository) profile.Repository {
if primary == nil {
return NewRepository()
}
return profile.NewOverlayRepository(primary, NewRepository())
}
func NewRepositoryWithDirectory(dir string) profile.Repository {
if strings.TrimSpace(dir) == "" {
return NewRepository()
}
return NewRepositoryWithPrimary(profile.NewFilesystemRepository(dir))
}

View File

@@ -0,0 +1,143 @@
package builtin
import (
"context"
"errors"
"io/fs"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
"gopkg.in/yaml.v3"
)
func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
repo := NewRepository()
ids := loadBuiltInProfileIDs(t)
if len(ids) == 0 {
t.Fatal("expected built-in profiles")
}
for id := range ids {
t.Run(id, func(t *testing.T) {
p, err := repo.GetProfile(context.Background(), id)
if err != nil {
t.Fatalf("expected built-in profile %q to load, got %v", id, err)
}
if p.ID != id {
t.Fatalf("expected profile id %q, got %q", id, p.ID)
}
if p.BackendID != backend.OpenRouterID {
t.Fatalf("expected profile %q to select %q, got %q", id, backend.OpenRouterID, p.BackendID)
}
if p.Endpoint != "" || p.APIKeyEnv != "" {
t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv)
}
})
}
}
func TestBuiltInProfilesDoNotContainDuplicateIDsOrRawAPIKeys(t *testing.T) {
loadBuiltInProfileIDs(t)
}
func loadBuiltInProfileIDs(t *testing.T) map[string]string {
t.Helper()
ids := map[string]string{}
err := fs.WalkDir(assets, assetRoot, func(name string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() || !strings.HasSuffix(name, ".yml") {
return nil
}
data, err := assets.ReadFile(name)
if err != nil {
t.Fatalf("failed to read built-in profile %s: %v", name, err)
}
var raw map[string]any
if err := yaml.Unmarshal(data, &raw); err != nil {
t.Fatalf("failed to decode built-in profile %s: %v", name, err)
}
if _, ok := raw["api_key"]; ok {
t.Fatalf("built-in profile %s contains raw api_key", name)
}
if raw["backend"] != backend.OpenRouterID {
t.Fatalf("built-in profile %s does not select %q", name, backend.OpenRouterID)
}
if _, ok := raw["endpoint"]; ok {
t.Fatalf("built-in profile %s repeats endpoint", name)
}
if _, ok := raw["api_key_env"]; ok {
t.Fatalf("built-in profile %s repeats api_key_env", name)
}
id, ok := raw["id"].(string)
if !ok || strings.TrimSpace(id) == "" {
t.Fatalf("built-in profile %s has missing id", name)
}
if previous, ok := ids[id]; ok {
t.Fatalf("duplicate built-in profile id %q in %s and %s", id, previous, name)
}
ids[id] = name
return nil
})
if err != nil {
t.Fatalf("failed to walk built-in profiles: %v", err)
}
return ids
}
func TestRepositoryWithPrimaryUsesPrimaryBeforeBuiltIns(t *testing.T) {
repo := NewRepositoryWithPrimary(staticProfileRepo{
profiles: map[string]string{"mistral-small-3": "custom-model"},
})
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
if err != nil {
t.Fatalf("expected profile to load, got %v", err)
}
if p.Model != "custom-model" {
t.Fatalf("expected primary profile to override built-in, got %+v", p)
}
}
func TestRepositoryWithPrimaryFallsBackToBuiltIns(t *testing.T) {
repo := NewRepositoryWithPrimary(staticProfileRepo{})
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
if err != nil {
t.Fatalf("expected built-in profile to load, got %v", err)
}
if p.ID != "mistral-small-3" {
t.Fatalf("unexpected profile: %+v", p)
}
}
func TestRepositoryWithPrimaryDoesNotFallBackAfterPrimaryError(t *testing.T) {
repo := NewRepositoryWithPrimary(staticProfileRepo{err: profile.ErrInvalidProfile})
_, err := repo.GetProfile(context.Background(), "mistral-small-3")
if !errors.Is(err, profile.ErrInvalidProfile) {
t.Fatalf("expected primary error, got %v", err)
}
}
type staticProfileRepo struct {
profiles map[string]string
err error
}
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
if r.err != nil {
return nil, r.err
}
if model, ok := r.profiles[id]; ok {
return &domain.ExecutionProfile{ID: id, Endpoint: "http://primary/v1", Model: model}, nil
}
return nil, profile.ErrProfileNotFound
}

View File

@@ -0,0 +1,214 @@
package profile
import (
"bytes"
"context"
"errors"
"fmt"
"io/fs"
"os"
"path"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
"gopkg.in/yaml.v3"
)
var (
ErrProfileNotFound = errors.New("execution profile not found")
ErrInvalidYAML = errors.New("invalid YAML format")
ErrInvalidProfile = errors.New("invalid execution profile configuration")
ErrRawAPIKeyNotAllowed = errors.New("raw api_key is not allowed; use api_key_env")
)
type filesystemRepository struct {
dir string
}
func NewFilesystemRepository(dir string) Repository {
return &filesystemRepository{dir: dir}
}
func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
return loadProfile(ctx, os.DirFS(r.dir), ".", id)
}
type fsRepository struct {
fsys fs.FS
root string
}
func NewFSRepository(fsys fs.FS, root string) Repository {
return &fsRepository{fsys: fsys, root: root}
}
func (r *fsRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
return loadProfile(ctx, r.fsys, r.root, id)
}
type overlayRepository struct {
primary Repository
fallback Repository
}
func NewOverlayRepository(primary, fallback Repository) Repository {
return &overlayRepository{primary: primary, fallback: fallback}
}
func (r *overlayRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
if r.primary != nil {
prof, err := r.primary.GetProfile(ctx, id)
if err == nil {
return prof, nil
}
if !errors.Is(err, ErrProfileNotFound) {
return nil, err
}
}
if r.fallback == nil {
return nil, ErrProfileNotFound
}
return r.fallback.GetProfile(ctx, id)
}
func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*domain.ExecutionProfile, error) {
if strings.TrimSpace(id) == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
}
if fsys == nil {
return nil, fmt.Errorf("failed to read profile directory: filesystem is nil")
}
files, err := filecatalog.FindFSYAMLFiles(ctx, fsys, root)
if err != nil {
return nil, fmt.Errorf("failed to read profile directory: %w", err)
}
var matches []profileMatch
for _, fullPath := range files {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
relPath := filecatalog.DisplayPath(root, fullPath)
fileMatch := filecatalog.Stem(path.Base(fullPath)) == id
data, err := fs.ReadFile(fsys, fullPath)
if err != nil {
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
}
metadata := readProfileFileMetadata(data)
idMatch := fileMatch || metadata.id == id
if metadata.hasRawAPIKey {
if idMatch {
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath)
}
continue
}
var prof domain.ExecutionProfile
decoder := yaml.NewDecoder(bytes.NewReader(data))
decoder.KnownFields(true)
if err := decoder.Decode(&prof); err != nil {
if idMatch {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
}
continue
}
if prof.ID != id {
continue
}
prof.BackendID = strings.TrimSpace(prof.BackendID)
if err := validateProfile(&prof); err != nil {
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
return nil, fmt.Errorf("%w: %s", err, relPath)
}
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
}
matches = append(matches, profileMatch{
profile: &prof,
path: relPath,
})
}
if len(matches) > 1 {
paths := make([]string, 0, len(matches))
for _, match := range matches {
paths = append(paths, match.path)
}
return nil, fmt.Errorf("%w: duplicate execution profile id %q found in: %s", ErrInvalidProfile, id, strings.Join(paths, ", "))
}
if len(matches) == 1 {
return matches[0].profile, nil
}
return nil, ErrProfileNotFound
}
type profileMatch struct {
profile *domain.ExecutionProfile
path string
}
type profileFileMetadata struct {
id string
hasRawAPIKey bool
}
func readProfileFileMetadata(data []byte) profileFileMetadata {
var node yaml.Node
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&node); err != nil {
return profileFileMetadata{}
}
if node.Kind != yaml.DocumentNode || len(node.Content) == 0 {
return profileFileMetadata{}
}
mapping := node.Content[0]
if mapping.Kind != yaml.MappingNode {
return profileFileMetadata{}
}
var metadata profileFileMetadata
for i := 0; i+1 < len(mapping.Content); i += 2 {
key := mapping.Content[i]
value := mapping.Content[i+1]
switch key.Value {
case "id":
metadata.id = strings.TrimSpace(value.Value)
case "api_key":
metadata.hasRawAPIKey = true
}
}
return metadata
}
func validateProfile(p *domain.ExecutionProfile) error {
if strings.TrimSpace(p.ID) == "" {
return errors.New("id is required")
}
if strings.TrimSpace(p.BackendID) == "" && strings.TrimSpace(p.Endpoint) == "" {
return errors.New("backend or endpoint is required")
}
if strings.TrimSpace(p.Model) == "" {
return errors.New("model is required")
}
if p.Temperature < 0 || p.Temperature > 2 {
return errors.New("temperature must be between 0 and 2")
}
if p.MaxTokens < 0 {
return errors.New("max_tokens must be greater than or equal to 0")
}
if p.TopP < 0 || p.TopP > 1 {
return errors.New("top_p must be between 0 and 1")
}
if p.TimeoutSeconds < 0 {
return errors.New("timeout_seconds must be greater than or equal to 0")
}
return nil
}

View File

@@ -0,0 +1,12 @@
package profile
import (
"context"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
// Repository loads execution profiles.
type Repository interface {
GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error)
}

View File

@@ -0,0 +1,516 @@
package profile
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestFilesystemRepository_GetProfile(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "execution_profile_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)
files, err := os.ReadDir("testdata")
if err != nil {
t.Fatalf("failed to read testdata: %v", err)
}
for _, f := range files {
src := filepath.Join("testdata", f.Name())
dst := filepath.Join(tmpDir, f.Name())
data, err := os.ReadFile(src)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(dst, data, 0644); err != nil {
t.Fatal(err)
}
}
repo := NewFilesystemRepository(tmpDir)
ctx := context.Background()
t.Run("valid local profile", func(t *testing.T) {
p, err := repo.GetProfile(ctx, "local-default")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.ID != "local-default" {
t.Fatalf("unexpected id: %q", p.ID)
}
if p.Endpoint == "" || p.Model == "" {
t.Fatalf("expected endpoint/model to be set: %+v", p)
}
})
t.Run("backend and endpoint connection matrix", func(t *testing.T) {
tests := []struct {
name string
connection string
wantBackend string
wantEndpoint string
wantErr bool
}{
{name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"},
{name: "endpoint only", connection: "endpoint: http://localhost:8000/v1", wantEndpoint: "http://localhost:8000/v1"},
{name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"},
{name: "neither", wantErr: true},
{name: "blank backend", connection: "backend: ' '", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
id := "connection-" + strings.ReplaceAll(tt.name, " ", "-")
writeProfileTestFile(t, filepath.Join(tmpDir, id+".yaml"), "id: "+id+"\nmodel: model\n"+tt.connection+"\n")
p, err := repo.GetProfile(ctx, id)
if tt.wantErr {
if !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("expected ErrInvalidProfile, got %v", err)
}
return
}
if err != nil {
t.Fatalf("expected profile to load, got %v", err)
}
if p.BackendID != tt.wantBackend || p.Endpoint != tt.wantEndpoint {
t.Fatalf("unexpected connection values: backend=%q endpoint=%q", p.BackendID, p.Endpoint)
}
})
}
})
t.Run("valid profile with api_key_env", func(t *testing.T) {
p, err := repo.GetProfile(ctx, "local-secure")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.APIKeyEnv != "PROMPTKIT_API_KEY" {
t.Fatalf("unexpected api_key_env: %q", p.APIKeyEnv)
}
if p.ReasoningEffort != "medium" {
t.Fatalf("unexpected reasoning_effort: %q", p.ReasoningEffort)
}
if p.ServiceTier != "priority" {
t.Fatalf("unexpected service_tier: %q", p.ServiceTier)
}
})
t.Run("valid nested profile", func(t *testing.T) {
nestedDir := filepath.Join(tmpDir, "local")
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
t.Fatal(err)
}
writeProfileTestFile(t, filepath.Join(nestedDir, "nested-local.yaml"), `
id: nested-local
endpoint: http://localhost:8000/v1
model: nested-model
temperature: 0.1
`)
p, err := repo.GetProfile(ctx, "nested-local")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.Model != "nested-model" {
t.Fatalf("unexpected model: %q", p.Model)
}
})
t.Run("valid profile with JSON-compatible extra params", func(t *testing.T) {
writeProfileTestFile(t, filepath.Join(tmpDir, "json-extra-params.yaml"), `
id: json-extra-params
endpoint: http://localhost:8000/v1
model: nested-model
extra_params:
string_value: enabled
number_value: 42
boolean_value: true
object_value:
nested: value
count: 2
array_value:
- first
- 3
- false
`)
p, err := repo.GetProfile(ctx, "json-extra-params")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
var got map[string]any
encoded, err := json.Marshal(p.ExtraParams)
if err != nil {
t.Fatalf("expected extra_params to marshal as JSON, got %v", err)
}
if err := json.Unmarshal(encoded, &got); err != nil {
t.Fatalf("expected extra_params JSON to decode, got %v", err)
}
if got["string_value"] != "enabled" {
t.Fatalf("unexpected string extra param: %#v", got["string_value"])
}
if got["number_value"] != float64(42) {
t.Fatalf("unexpected number extra param: %#v", got["number_value"])
}
if got["boolean_value"] != true {
t.Fatalf("unexpected boolean extra param: %#v", got["boolean_value"])
}
objectValue, ok := got["object_value"].(map[string]any)
if !ok {
t.Fatalf("expected object extra param, got %#v", got["object_value"])
}
if objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
t.Fatalf("unexpected object extra param: %#v", objectValue)
}
arrayValue, ok := got["array_value"].([]any)
if !ok {
t.Fatalf("expected array extra param, got %#v", got["array_value"])
}
if len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
t.Fatalf("unexpected array extra param: %#v", arrayValue)
}
})
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
id: duplicate-profile
endpoint: http://localhost:8000/v1
model: first-model
`)
nestedDir := filepath.Join(tmpDir, "duplicates")
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
t.Fatal(err)
}
writeProfileTestFile(t, filepath.Join(nestedDir, "duplicate-profile-b.yaml"), `
id: duplicate-profile
endpoint: http://localhost:8000/v1
model: second-model
`)
_, err := repo.GetProfile(ctx, "duplicate-profile")
if !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("expected duplicate profile to return ErrInvalidProfile, got %v", err)
}
for _, want := range []string{"duplicate execution profile id", "duplicate-profile-a.yaml", filepath.Join("duplicates", "duplicate-profile-b.yaml")} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("expected error to contain %q, got %v", want, err)
}
}
})
t.Run("nested raw api_key rejected for likely target file", func(t *testing.T) {
nestedDir := filepath.Join(tmpDir, "secure")
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
t.Fatal(err)
}
writeProfileTestFile(t, filepath.Join(nestedDir, "not_named_like_id.yaml"), `
id: nested_raw_api_key
endpoint: http://localhost:8000/v1
model: m
api_key: secret
`)
_, err := repo.GetProfile(ctx, "nested_raw_api_key")
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
}
if !strings.Contains(err.Error(), filepath.Join("secure", "not_named_like_id.yaml")) {
t.Fatalf("expected nested path in error, got %v", err)
}
})
t.Run("raw api_key in non-target profile is ignored", func(t *testing.T) {
writeProfileTestFile(t, filepath.Join(tmpDir, "raw-api-key-non-target.yaml"), `
id: raw-api-key-non-target
endpoint: http://localhost:8000/v1
model: m
api_key: secret
`)
_, err := repo.GetProfile(ctx, "does-not-exist-with-raw-key-nearby")
if !errors.Is(err, ErrProfileNotFound) {
t.Fatalf("expected ErrProfileNotFound for non-target raw api_key file, got %v", err)
}
})
t.Run("invalid yaml", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "invalid_yaml")
if !errors.Is(err, ErrInvalidYAML) {
t.Fatalf("expected ErrInvalidYAML, got %v", err)
}
})
t.Run("missing id", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "missing_id")
if !errors.Is(err, ErrProfileNotFound) {
t.Fatalf("expected ErrProfileNotFound, got %v", err)
}
})
t.Run("missing endpoint", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "missing-endpoint")
if !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("expected ErrInvalidProfile, got %v", err)
}
})
t.Run("missing model", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "missing-model")
if !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("expected ErrInvalidProfile, got %v", err)
}
})
t.Run("unknown field", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "unknown_field")
if !errors.Is(err, ErrInvalidYAML) {
t.Fatalf("expected ErrInvalidYAML for strict decode unknown field, got %v", err)
}
})
t.Run("raw api_key rejected", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "raw_api_key")
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
}
})
t.Run("profile not found", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "does-not-exist")
if !errors.Is(err, ErrProfileNotFound) {
t.Fatalf("expected ErrProfileNotFound, got %v", err)
}
})
}
func writeProfileTestFile(t *testing.T, path string, content string) {
t.Helper()
if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil {
t.Fatalf("failed to write profile test file %q: %v", path, err)
}
}
func TestFSRepository(t *testing.T) {
ctx := context.Background()
t.Run("loads valid profiles from nested directories", func(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"profiles/provider/nested.yaml": profileMapFile(`
id: nested-profile
endpoint: http://localhost:8000/v1
model: nested-model
temperature: 0.1
`),
}, "profiles")
p, err := repo.GetProfile(ctx, "nested-profile")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.ID != "nested-profile" || p.Model != "nested-model" {
t.Fatalf("unexpected profile: %+v", p)
}
})
t.Run("rejects unknown YAML fields", func(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"profiles/unknown.yaml": profileMapFile(`
id: unknown-profile
endpoint: http://localhost:8000/v1
model: model
unknown: value
`),
}, "profiles")
_, err := repo.GetProfile(ctx, "unknown-profile")
if !errors.Is(err, ErrInvalidYAML) {
t.Fatalf("expected ErrInvalidYAML, got %v", err)
}
})
t.Run("rejects raw api_key in selected profile", func(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"profiles/raw.yaml": profileMapFile(`
id: raw-profile
endpoint: http://localhost:8000/v1
model: model
api_key: secret
`),
}, "profiles")
_, err := repo.GetProfile(ctx, "raw-profile")
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
}
})
t.Run("ignores raw api_key in non-selected profiles", func(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"profiles/raw.yaml": profileMapFile(`
id: raw-profile
endpoint: http://localhost:8000/v1
model: model
api_key: secret
`),
"profiles/valid.yaml": profileMapFile(`
id: valid-profile
endpoint: http://localhost:8000/v1
model: model
`),
}, "profiles")
p, err := repo.GetProfile(ctx, "valid-profile")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.ID != "valid-profile" {
t.Fatalf("unexpected profile: %+v", p)
}
})
t.Run("rejects duplicate IDs within one source", func(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"profiles/a.yaml": profileMapFile(`
id: duplicate-profile
endpoint: http://localhost:8000/v1
model: first
`),
"profiles/nested/b.yaml": profileMapFile(`
id: duplicate-profile
endpoint: http://localhost:8000/v1
model: second
`),
}, "profiles")
_, err := repo.GetProfile(ctx, "duplicate-profile")
if !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("expected ErrInvalidProfile, got %v", err)
}
for _, want := range []string{"duplicate execution profile id", "a.yaml", "nested/b.yaml"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("expected error to contain %q, got %v", want, err)
}
}
})
}
func TestOverlayRepository(t *testing.T) {
ctx := context.Background()
primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"}
fallbackProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://fallback", Model: "fallback"}
t.Run("returns primary matches before fallback matches", func(t *testing.T) {
repo := NewOverlayRepository(
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": primaryProfile}},
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
)
p, err := repo.GetProfile(ctx, "shared")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.Model != "primary" {
t.Fatalf("expected primary profile, got %+v", p)
}
})
t.Run("falls back on primary not found", func(t *testing.T) {
repo := NewOverlayRepository(
staticProfileRepo{},
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
)
p, err := repo.GetProfile(ctx, "shared")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.Model != "fallback" {
t.Fatalf("expected fallback profile, got %+v", p)
}
})
t.Run("does not fall back after primary load errors", func(t *testing.T) {
for _, tc := range []struct {
name string
err error
}{
{name: "invalid yaml", err: ErrInvalidYAML},
{name: "invalid profile", err: ErrInvalidProfile},
{name: "raw api key", err: ErrRawAPIKeyNotAllowed},
} {
t.Run(tc.name, func(t *testing.T) {
repo := NewOverlayRepository(
staticProfileRepo{err: tc.err},
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
)
_, err := repo.GetProfile(ctx, "shared")
if !errors.Is(err, tc.err) {
t.Fatalf("expected %v, got %v", tc.err, err)
}
})
}
})
t.Run("returns not found when both sources miss", func(t *testing.T) {
repo := NewOverlayRepository(staticProfileRepo{}, staticProfileRepo{})
_, err := repo.GetProfile(ctx, "missing")
if !errors.Is(err, ErrProfileNotFound) {
t.Fatalf("expected ErrProfileNotFound, got %v", err)
}
})
t.Run("nil primary uses fallback", func(t *testing.T) {
repo := NewOverlayRepository(nil, staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}})
p, err := repo.GetProfile(ctx, "shared")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.Model != "fallback" {
t.Fatalf("expected fallback profile, got %+v", p)
}
})
t.Run("nil fallback returns not found after primary miss", func(t *testing.T) {
repo := NewOverlayRepository(staticProfileRepo{}, nil)
_, err := repo.GetProfile(ctx, "missing")
if !errors.Is(err, ErrProfileNotFound) {
t.Fatalf("expected ErrProfileNotFound, got %v", err)
}
})
}
func profileMapFile(content string) *fstest.MapFile {
return &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))}
}
type staticProfileRepo struct {
profiles map[string]*domain.ExecutionProfile
err error
}
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
if r.err != nil {
return nil, r.err
}
if p, ok := r.profiles[id]; ok {
cp := *p
return &cp, nil
}
return nil, ErrProfileNotFound
}

View File

@@ -0,0 +1,3 @@
id: invalid_yaml
endpoint: http://localhost:8000/v1
model: [broken

View File

@@ -0,0 +1,2 @@
id: missing-endpoint
model: gpt-4o-mini

View File

@@ -0,0 +1,2 @@
endpoint: http://localhost:8000/v1
model: gpt-4o-mini

View File

@@ -0,0 +1,2 @@
id: missing-model
endpoint: http://localhost:8000/v1

View File

@@ -0,0 +1,4 @@
id: raw-api-key
endpoint: http://localhost:8000/v1
model: gpt-4o-mini
api_key: super-secret-should-not-be-here

View File

@@ -0,0 +1,4 @@
id: unknown-field
endpoint: http://localhost:8000/v1
model: gpt-4o-mini
foo: bar

View File

@@ -0,0 +1,7 @@
id: local-default
endpoint: http://localhost:8000/v1
model: gpt-4o-mini
temperature: 0.2
max_tokens: 700
top_p: 1.0
timeout_seconds: 120

View File

@@ -0,0 +1,8 @@
id: local-secure
endpoint: http://localhost:8000/v1
model: gpt-4o-mini
api_key_env: PROMPTKIT_API_KEY
service_tier: priority
reasoning_effort: medium
extra_params:
provider: local

View File

@@ -0,0 +1,120 @@
package prompt
import (
"bytes"
"context"
"errors"
"fmt"
"text/template"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
var (
ErrMissingRequiredInput = errors.New("missing required input artifact")
ErrUnknownInput = errors.New("referenced unknown input artifact")
ErrInvalidTemplate = errors.New("invalid prompt template")
ErrRenderFailure = errors.New("prompt render failure")
ErrInvalidMessageRole = errors.New("invalid or empty message role")
)
type goRenderer struct{}
func NewGoRenderer() Renderer {
return &goRenderer{}
}
func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
if definition == nil {
return nil, fmt.Errorf("%w: nil prompt definition", ErrRenderFailure)
}
// 1. Verify required inputs
for _, in := range definition.Inputs {
if !in.Required {
continue
}
art, ok := inputs[in.Name]
if !ok || art == nil {
return nil, fmt.Errorf("%w: %s", ErrMissingRequiredInput, in.Name)
}
}
// 2. Setup template functions
funcs := template.FuncMap{
"input": func(name string) (string, error) {
art, ok := inputs[name]
if !ok || art == nil {
return "", fmt.Errorf("%w: %s", ErrUnknownInput, name)
}
return string(art.Body), nil
},
}
sessionID, err := renderSessionID(definition.SessionID, funcs, vars)
if err != nil {
return nil, err
}
var renderedMessages []domain.RenderedMessage
for i, tmplMsg := range definition.Templates {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
if tmplMsg.Role == "" {
return nil, fmt.Errorf("%w: message %d", ErrInvalidMessageRole, i)
}
// Parse and execute template
tmpl, err := template.New(fmt.Sprintf("msg_%d", i)).Funcs(funcs).Option("missingkey=error").Parse(tmplMsg.Content)
if err != nil {
return nil, fmt.Errorf("%w: message %d: %v", ErrInvalidTemplate, i, err)
}
var buf bytes.Buffer
if err := tmpl.Execute(&buf, vars); err != nil {
return nil, fmt.Errorf("%w: message %d: %w", ErrRenderFailure, i, err)
}
renderedMessages = append(renderedMessages, domain.RenderedMessage{
Role: tmplMsg.Role,
Content: buf.String(),
CacheControl: cloneCacheControl(tmplMsg.CacheControl),
})
}
return &domain.RenderedPrompt{
SessionID: sessionID,
Messages: renderedMessages,
}, nil
}
func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
tmpl, err := template.New("session_id").Funcs(funcs).Option("missingkey=error").Parse(raw)
if err != nil {
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, err)
}
var buf bytes.Buffer
if err := tmpl.Execute(&buf, vars); err != nil {
return "", fmt.Errorf("%w: session_id: %w", ErrRenderFailure, err)
}
sessionID, err := domain.NormalizeSessionID(buf.String())
if err != nil {
return "", fmt.Errorf("%w: session_id: %v", ErrRenderFailure, err)
}
return sessionID, nil
}
func cloneCacheControl(in *domain.CacheControl) *domain.CacheControl {
if in == nil {
return nil
}
out := *in
return &out
}

View File

@@ -0,0 +1,11 @@
package prompt
import (
"context"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
// Renderer renders prompt templates using named artifacts and variables.
type Renderer interface {
Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error)
}

View File

@@ -0,0 +1,345 @@
package prompt
import (
"context"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestGoRenderer_Render(t *testing.T) {
renderer := NewGoRenderer()
ctx := context.Background()
inputs := map[string]*domain.Artifact{
"transcript": {Body: []byte("The quick brown fox.")},
}
vars := map[string]string{
"role": "helpful assistant",
"tone": "concise",
}
t.Run("rendering inline message content", func(t *testing.T) {
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "user", Content: "Analyze this: {{input \"transcript\"}}"},
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(res.Messages) != 1 {
t.Fatalf("expected 1 message, got %d", len(res.Messages))
}
if res.Messages[0].Content != "Analyze this: The quick brown fox." {
t.Fatalf("unexpected rendered content: %q", res.Messages[0].Content)
}
})
t.Run("rendering file-backed message content loaded into prompt definition", func(t *testing.T) {
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "user", Content: "From file: {{input \"transcript\"}}", ContentFile: "/tmp/user.tmpl"},
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := res.Messages[0].Content; got != "From file: The quick brown fox." {
t.Fatalf("unexpected file-backed render result: %q", got)
}
})
t.Run("rendering system and user messages", func(t *testing.T) {
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "You are a {{.role}}."},
{Role: "user", Content: "Analyze this: {{input \"transcript\"}}"},
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(res.Messages) != 2 {
t.Fatalf("expected 2 messages, got %d", len(res.Messages))
}
if res.Messages[0].Role != "system" || res.Messages[1].Role != "user" {
t.Fatalf("unexpected roles: %#v", res.Messages)
}
})
t.Run("copying cache control to rendered messages", func(t *testing.T) {
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{
Role: "system",
Content: "You are concise.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "Analyze this: {{input \"transcript\"}}"},
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(res.Messages) != 2 {
t.Fatalf("expected 2 messages, got %d", len(res.Messages))
}
if res.Messages[0].CacheControl == nil {
t.Fatal("expected rendered cache control")
}
if res.Messages[0].CacheControl.Type != domain.CacheControlEphemeral {
t.Fatalf("unexpected cache control type: %q", res.Messages[0].CacheControl.Type)
}
if res.Messages[0].CacheControl.TTL != "1h" {
t.Fatalf("unexpected cache control ttl: %q", res.Messages[0].CacheControl.TTL)
}
if res.Messages[1].CacheControl != nil {
t.Fatalf("expected no cache control on second message, got %#v", res.Messages[1].CacheControl)
}
})
t.Run("rendered cache control does not alias source template", func(t *testing.T) {
source := &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "You are concise.", CacheControl: source},
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if res.Messages[0].CacheControl == source {
t.Fatal("expected rendered cache control to be cloned")
}
res.Messages[0].CacheControl.TTL = ""
if source.TTL != "1h" {
t.Fatalf("source cache control was mutated, ttl=%q", source.TTL)
}
})
t.Run("accessing vars", func(t *testing.T) {
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "Speak in a {{.tone}} tone."},
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if res.Messages[0].Content != "Speak in a concise tone." {
t.Fatalf("unexpected vars rendering: %q", res.Messages[0].Content)
}
})
t.Run("rendering session id from vars", func(t *testing.T) {
def := &domain.PromptDefinition{
SessionID: " {{ .session_id }} ",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "Speak in a {{.tone}} tone."},
},
}
res, err := renderer.Render(ctx, def, inputs, map[string]string{
"tone": "concise",
"session_id": "agent-session-123",
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if res.SessionID != "agent-session-123" {
t.Fatalf("unexpected session id: %q", res.SessionID)
}
})
t.Run("empty rendered session id is omitted", func(t *testing.T) {
def := &domain.PromptDefinition{
SessionID: " ",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "Speak in a {{.tone}} tone."},
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if res.SessionID != "" {
t.Fatalf("expected empty session id, got %q", res.SessionID)
}
})
t.Run("missing session id var fails rendering", func(t *testing.T) {
def := &domain.PromptDefinition{
SessionID: "{{ .session_id }}",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "Speak in a {{.tone}} tone."},
},
}
_, err := renderer.Render(ctx, def, inputs, vars)
if !errors.Is(err, ErrRenderFailure) {
t.Fatalf("expected ErrRenderFailure, got %v", err)
}
})
t.Run("too long rendered session id fails rendering", func(t *testing.T) {
def := &domain.PromptDefinition{
SessionID: "{{ .session_id }}",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "Speak in a {{.tone}} tone."},
},
}
_, err := renderer.Render(ctx, def, inputs, map[string]string{
"tone": "concise",
"session_id": strings.Repeat("x", domain.SessionIDMaxLength+1),
})
if !errors.Is(err, ErrRenderFailure) {
t.Fatalf("expected ErrRenderFailure, got %v", err)
}
})
t.Run("inserting required input artifact", func(t *testing.T) {
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "user", Content: "{{input \"transcript\"}}"},
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if res.Messages[0].Content != "The quick brown fox." {
t.Fatalf("unexpected required input rendering: %q", res.Messages[0].Content)
}
})
t.Run("optional input absent and not referenced", func(t *testing.T) {
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{
{Name: "transcript", Required: true},
{Name: "glossary", Required: false},
},
Templates: []domain.PromptMessageTemplate{
{Role: "user", Content: "Transcript: {{input \"transcript\"}}"},
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(res.Messages) != 1 {
t.Fatalf("expected one rendered message, got %d", len(res.Messages))
}
})
t.Run("optional input absent but referenced, expecting failure", func(t *testing.T) {
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{
{Name: "transcript", Required: true},
{Name: "glossary", Required: false},
},
Templates: []domain.PromptMessageTemplate{
{Role: "user", Content: "Glossary: {{input \"glossary\"}}"},
},
}
_, err := renderer.Render(ctx, def, inputs, vars)
if !errors.Is(err, ErrRenderFailure) {
t.Fatalf("expected ErrRenderFailure, got %v", err)
}
if !errors.Is(err, ErrUnknownInput) {
t.Fatalf("expected ErrUnknownInput, got %v", err)
}
})
t.Run("required input missing, expecting failure", func(t *testing.T) {
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "user", Content: "Analyze this: {{input \"transcript\"}}"},
},
}
_, err := renderer.Render(ctx, def, map[string]*domain.Artifact{}, vars)
if !errors.Is(err, ErrMissingRequiredInput) {
t.Fatalf("expected ErrMissingRequiredInput, got %v", err)
}
})
t.Run("invalid template syntax", func(t *testing.T) {
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "user", Content: "Hello {{.unclosed"},
},
}
_, err := renderer.Render(ctx, def, inputs, vars)
if !errors.Is(err, ErrInvalidTemplate) {
t.Fatalf("expected ErrInvalidTemplate, got %v", err)
}
})
t.Run("unknown input reference", func(t *testing.T) {
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "user", Content: "Hello {{input \"ghost\"}}"},
},
}
_, err := renderer.Render(ctx, def, inputs, vars)
if !errors.Is(err, ErrRenderFailure) {
t.Fatalf("expected ErrRenderFailure, got %v", err)
}
if !errors.Is(err, ErrUnknownInput) {
t.Fatalf("expected ErrUnknownInput, got %v", err)
}
})
t.Run("empty message role", func(t *testing.T) {
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "", Content: "Hello"},
},
}
_, err := renderer.Render(ctx, def, inputs, vars)
if !errors.Is(err, ErrInvalidMessageRole) {
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
}
})
}

View File

@@ -0,0 +1,484 @@
package promptdef
import (
"bytes"
"context"
"errors"
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
"gopkg.in/yaml.v3"
)
var (
ErrPromptDefinitionNotFound = errors.New("prompt definition not found")
ErrInvalidYAML = errors.New("invalid YAML format")
ErrInvalidPromptDefinition = errors.New("invalid prompt definition configuration")
)
type filesystemRepository struct {
dir string
}
type fsRepository struct {
fsys fs.FS
root string
}
type promptDefinitionFile struct {
ID string `yaml:"id"`
Version string `yaml:"version"`
DefaultProfile *string `yaml:"default_profile"`
Description string `yaml:"description"`
SessionID string `yaml:"session_id"`
Inputs []promptInputFile `yaml:"inputs"`
Messages []promptMessageFile `yaml:"messages"`
Output promptOutputContractFile `yaml:"output"`
}
type promptInputFile struct {
Name string `yaml:"name"`
Required bool `yaml:"required"`
ContentType string `yaml:"content_type"`
Description string `yaml:"description"`
}
type promptMessageFile struct {
Role string `yaml:"role"`
Content string `yaml:"content"`
ContentFile string `yaml:"content_file"`
CacheControl *cacheControlFile `yaml:"cache_control"`
}
type cacheControlFile struct {
Type string `yaml:"type"`
TTL string `yaml:"ttl"`
}
type promptOutputContractFile struct {
Format domain.OutputFormat `yaml:"format"`
ValidationMode domain.ValidationMode `yaml:"validation_mode"`
SchemaPath string `yaml:"schema_path"`
RepairAttempts int `yaml:"repair_attempts"`
}
func NewFilesystemRepository(dir string) Repository {
return &filesystemRepository{dir: dir}
}
func NewFSRepository(fsys fs.FS, root string) Repository {
return &fsRepository{fsys: fsys, root: root}
}
func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
if strings.TrimSpace(id) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
}
files, err := filecatalog.FindYAMLFiles(ctx, r.dir)
if err != nil {
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
}
var matches []promptDefinitionMatch
for _, fullPath := range files {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
relPath := filecatalog.RelativePath(r.dir, fullPath)
fileMatch := filecatalog.Stem(filepath.Base(fullPath)) == id
raw, err := loadPromptDefinitionFile(fullPath)
if err != nil {
if fileMatch || promptDefinitionFileHasID(fullPath, id) {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
}
continue
}
def, err := normalizePromptDefinition(raw, fullPath)
if err != nil {
if fileMatch || strings.TrimSpace(raw.ID) == id {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, relPath, err)
}
continue
}
if def.ID != id {
continue
}
if version != "" && def.Version != version {
continue
}
matches = append(matches, promptDefinitionMatch{
def: def,
path: relPath,
})
}
if len(matches) > 1 {
paths := make([]string, 0, len(matches))
for _, match := range matches {
paths = append(paths, match.path)
}
if version != "" {
return nil, fmt.Errorf("%w: duplicate prompt definition id %q version %q found in: %s", ErrInvalidPromptDefinition, id, version, strings.Join(paths, ", "))
}
return nil, fmt.Errorf("%w: duplicate prompt definition id %q found in: %s", ErrInvalidPromptDefinition, id, strings.Join(paths, ", "))
}
if len(matches) == 1 {
return matches[0].def, nil
}
return nil, ErrPromptDefinitionNotFound
}
func (r *fsRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
return loadPromptDefinition(ctx, r.fsys, r.root, id, version)
}
type promptDefinitionMatch struct {
def *domain.PromptDefinition
path string
}
func loadPromptDefinitionFile(path string) (*promptDefinitionFile, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("failed to read prompt definition file: %w", err)
}
var raw promptDefinitionFile
decoder := yaml.NewDecoder(bytes.NewReader(data))
decoder.KnownFields(true)
if err := decoder.Decode(&raw); err != nil {
return nil, err
}
return &raw, nil
}
func promptDefinitionFileHasID(path string, id string) bool {
data, err := os.ReadFile(path)
if err != nil {
return false
}
var raw struct {
ID string `yaml:"id"`
}
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&raw); err != nil {
return false
}
return strings.TrimSpace(raw.ID) == id
}
func loadPromptDefinition(ctx context.Context, fsys fs.FS, root string, id string, version string) (*domain.PromptDefinition, error) {
if strings.TrimSpace(id) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
}
if fsys == nil {
return nil, fmt.Errorf("failed to read prompt definition directory: filesystem is nil")
}
files, err := filecatalog.FindFSYAMLFiles(ctx, fsys, root)
if err != nil {
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
}
cleanRoot := filecatalog.CleanFSRoot(root)
rootInfo, err := fs.Stat(fsys, cleanRoot)
if err != nil {
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
}
var matches []promptDefinitionMatch
for _, fullPath := range files {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
relPath := filecatalog.DisplayPath(root, fullPath)
fileMatch := filecatalog.Stem(path.Base(fullPath)) == id
data, err := fs.ReadFile(fsys, fullPath)
if err != nil {
if fileMatch {
return nil, fmt.Errorf("%w: %s: failed to read prompt definition file: %v", ErrInvalidYAML, relPath, err)
}
continue
}
raw, err := decodePromptDefinition(data)
if err != nil {
if fileMatch || promptDefinitionDataHasID(data, id) {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
}
continue
}
def, err := normalizePromptDefinitionFromFS(raw, fsys, root, fullPath, rootInfo.IsDir())
if err != nil {
if fileMatch || strings.TrimSpace(raw.ID) == id {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, relPath, err)
}
continue
}
if def.ID != id {
continue
}
if version != "" && def.Version != version {
continue
}
matches = append(matches, promptDefinitionMatch{
def: def,
path: relPath,
})
}
if len(matches) > 1 {
paths := make([]string, 0, len(matches))
for _, match := range matches {
paths = append(paths, match.path)
}
if version != "" {
return nil, fmt.Errorf("%w: duplicate prompt definition id %q version %q found in: %s", ErrInvalidPromptDefinition, id, version, strings.Join(paths, ", "))
}
return nil, fmt.Errorf("%w: duplicate prompt definition id %q found in: %s", ErrInvalidPromptDefinition, id, strings.Join(paths, ", "))
}
if len(matches) == 1 {
return matches[0].def, nil
}
return nil, ErrPromptDefinitionNotFound
}
func decodePromptDefinition(data []byte) (*promptDefinitionFile, error) {
var raw promptDefinitionFile
decoder := yaml.NewDecoder(bytes.NewReader(data))
decoder.KnownFields(true)
if err := decoder.Decode(&raw); err != nil {
return nil, err
}
return &raw, nil
}
func promptDefinitionDataHasID(data []byte, id string) bool {
var raw struct {
ID string `yaml:"id"`
}
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&raw); err != nil {
return false
}
return strings.TrimSpace(raw.ID) == id
}
func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*domain.PromptDefinition, error) {
promptDir := filepath.Dir(sourcePath)
return normalizePromptDefinitionWithContent(raw, func(contentFile string) (string, string, error) {
resolvedPath := strings.TrimSpace(contentFile)
if !filepath.IsAbs(resolvedPath) {
resolvedPath = filepath.Join(promptDir, resolvedPath)
}
resolvedPath = filepath.Clean(resolvedPath)
body, err := os.ReadFile(resolvedPath)
if err != nil {
return "", "", err
}
return string(body), resolvedPath, nil
})
}
func normalizePromptDefinitionFromFS(raw *promptDefinitionFile, fsys fs.FS, root string, sourcePath string, rootIsDir bool) (*domain.PromptDefinition, error) {
promptDir := path.Dir(sourcePath)
return normalizePromptDefinitionWithContent(raw, func(contentFile string) (string, string, error) {
var resolvedPath string
if rootIsDir {
var err error
resolvedPath, _, err = filecatalog.ResolveFSPath(root, promptDir, contentFile)
if err != nil {
return "", "", err
}
} else {
resolvedPath = strings.TrimSpace(contentFile)
if !path.IsAbs(resolvedPath) {
resolvedPath = path.Join(promptDir, resolvedPath)
}
resolvedPath = strings.TrimPrefix(path.Clean(resolvedPath), "/")
}
body, err := fs.ReadFile(fsys, resolvedPath)
if err != nil {
return "", "", err
}
return string(body), resolvedPath, nil
})
}
func normalizePromptDefinitionWithContent(raw *promptDefinitionFile, readContentFile func(string) (string, string, error)) (*domain.PromptDefinition, error) {
if raw == nil {
return nil, errors.New("prompt definition is nil")
}
id := strings.TrimSpace(raw.ID)
if id == "" {
return nil, errors.New("id is required")
}
version := strings.TrimSpace(raw.Version)
if version == "" {
return nil, errors.New("version is required")
}
if len(raw.Messages) == 0 {
return nil, errors.New("at least one message is required")
}
inputs := make([]domain.PromptInput, 0, len(raw.Inputs))
seenInputNames := make(map[string]struct{}, len(raw.Inputs))
for i, in := range raw.Inputs {
name := strings.TrimSpace(in.Name)
if name == "" {
return nil, fmt.Errorf("input %d has empty name", i)
}
if _, exists := seenInputNames[name]; exists {
return nil, fmt.Errorf("duplicate input name %q", name)
}
seenInputNames[name] = struct{}{}
inputs = append(inputs, domain.PromptInput{
Name: name,
Required: in.Required,
ContentType: strings.TrimSpace(in.ContentType),
Description: strings.TrimSpace(in.Description),
})
}
templates := make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
for i, msg := range raw.Messages {
role := strings.TrimSpace(msg.Role)
if role == "" {
return nil, fmt.Errorf("message %d role is required", i)
}
hasContent := strings.TrimSpace(msg.Content) != ""
hasContentFile := strings.TrimSpace(msg.ContentFile) != ""
if hasContent == hasContentFile {
return nil, fmt.Errorf("message %d (%s) must set exactly one of content or content_file", i, role)
}
cacheControl, err := normalizeCacheControl(msg.CacheControl)
if err != nil {
return nil, fmt.Errorf("message %d (%s) cache_control: %w", i, role, err)
}
templateContent := msg.Content
resolvedContentFile := ""
if hasContentFile {
body, resolvedPath, err := readContentFile(msg.ContentFile)
if err != nil {
return nil, fmt.Errorf("prompt %q message %d (%s): failed to read content_file %q: %w", id, i, role, msg.ContentFile, err)
}
templateContent = body
resolvedContentFile = resolvedPath
}
templates = append(templates, domain.PromptMessageTemplate{
Role: role,
Content: templateContent,
ContentFile: resolvedContentFile,
CacheControl: cacheControl,
})
}
if !isValidOutputFormat(raw.Output.Format) {
return nil, fmt.Errorf("invalid output format: %q", raw.Output.Format)
}
if !isValidValidationMode(raw.Output.ValidationMode) {
return nil, fmt.Errorf("invalid validation mode: %q", raw.Output.ValidationMode)
}
if raw.Output.ValidationMode == domain.ValidationJSONSchema && strings.TrimSpace(raw.Output.SchemaPath) == "" {
return nil, errors.New("output.schema_path is required when output.validation_mode is json_schema")
}
if raw.Output.RepairAttempts < 0 {
return nil, errors.New("output.repair_attempts must be greater than or equal to 0")
}
defaultProfile := ""
if raw.DefaultProfile != nil {
defaultProfile = strings.TrimSpace(*raw.DefaultProfile)
if defaultProfile == "" {
return nil, errors.New("default_profile must be a non-empty string when set")
}
}
return &domain.PromptDefinition{
ID: id,
Version: version,
DefaultProfile: defaultProfile,
Description: strings.TrimSpace(raw.Description),
SessionID: strings.TrimSpace(raw.SessionID),
Inputs: inputs,
Templates: templates,
OutputFormat: raw.Output.Format,
Validation: domain.OutputContract{
Format: raw.Output.Format,
ValidationMode: raw.Output.ValidationMode,
SchemaPath: strings.TrimSpace(raw.Output.SchemaPath),
RepairAttempts: raw.Output.RepairAttempts,
},
}, nil
}
func normalizeCacheControl(raw *cacheControlFile) (*domain.CacheControl, error) {
if raw == nil {
return nil, nil
}
cacheType := strings.TrimSpace(raw.Type)
if cacheType == "" {
return nil, errors.New("type is required")
}
if domain.CacheControlType(cacheType) != domain.CacheControlEphemeral {
return nil, fmt.Errorf("unsupported type %q", cacheType)
}
ttl := strings.TrimSpace(raw.TTL)
if ttl != "" && ttl != "1h" {
return nil, fmt.Errorf("unsupported ttl %q", ttl)
}
return &domain.CacheControl{
Type: domain.CacheControlType(cacheType),
TTL: ttl,
}, nil
}
func isValidOutputFormat(f domain.OutputFormat) bool {
switch f {
case domain.FormatText, domain.FormatMarkdown, domain.FormatJSON:
return true
default:
return false
}
}
func isValidValidationMode(m domain.ValidationMode) bool {
switch m {
case domain.ValidationNone, domain.ValidationBasic, domain.ValidationJSON, domain.ValidationJSONSchema:
return true
default:
return false
}
}

View File

@@ -0,0 +1,12 @@
package promptdef
import (
"context"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
// Repository loads prompt definitions.
type Repository interface {
GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error)
}

View File

@@ -0,0 +1,526 @@
package promptdef
import (
"context"
"errors"
"io/fs"
"os"
"path/filepath"
"strings"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
tmpDir := t.TempDir()
if err := copyTree("testdata", tmpDir); err != nil {
t.Fatalf("failed to copy testdata: %v", err)
}
repo := NewFilesystemRepository(tmpDir)
ctx := context.Background()
t.Run("valid inline prompt", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "valid-inline", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.ID != "valid-inline" {
t.Fatalf("unexpected id: %q", p.ID)
}
if p.Version != "1.0.0" {
t.Fatalf("unexpected version: %q", p.Version)
}
if p.OutputFormat != domain.FormatMarkdown {
t.Fatalf("unexpected output format: %q", p.OutputFormat)
}
if p.Validation.ValidationMode != domain.ValidationBasic {
t.Fatalf("unexpected validation mode: %q", p.Validation.ValidationMode)
}
if len(p.Templates) != 2 {
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
}
if len(p.Inputs) != 1 {
t.Fatalf("expected 1 input, got %d", len(p.Inputs))
}
if p.Inputs[0].ContentType != "text/markdown" {
t.Fatalf("expected input content_type to be preserved, got %q", p.Inputs[0].ContentType)
}
})
t.Run("valid file-backed prompt", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "valid-file-backed", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(p.Templates) != 2 {
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
}
if !strings.Contains(p.Templates[1].Content, "{{input \"transcript\"}}") {
t.Fatalf("expected content_file template body to be loaded, got %q", p.Templates[1].Content)
}
if p.Templates[1].ContentFile == "" {
t.Fatal("expected ContentFile source metadata to be preserved")
}
if !filepath.IsAbs(p.Templates[1].ContentFile) {
t.Fatalf("expected resolved content_file path to be absolute, got %q", p.Templates[1].ContentFile)
}
})
t.Run("valid cache control with ttl", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "valid-cache-control-ttl", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(p.Templates) != 2 {
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
}
assertCacheControl(t, p.Templates[0].CacheControl, domain.CacheControlEphemeral, "1h")
if p.Templates[1].CacheControl != nil {
t.Fatalf("expected second message cache control to be nil, got %#v", p.Templates[1].CacheControl)
}
})
t.Run("valid cache control without ttl", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "valid-cache-control-without-ttl", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(p.Templates) != 2 {
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
}
assertCacheControl(t, p.Templates[0].CacheControl, domain.CacheControlEphemeral, "")
if p.Templates[1].CacheControl != nil {
t.Fatalf("expected second message cache control to be nil, got %#v", p.Templates[1].CacheControl)
}
})
t.Run("valid session id template", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "valid-session-id", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.SessionID != "{{ .session_id }}" {
t.Fatalf("expected trimmed session_id template, got %q", p.SessionID)
}
})
t.Run("valid nested file-backed prompt resolves content file relative to nested YAML", func(t *testing.T) {
nestedDir := filepath.Join(tmpDir, "dnd", "recap")
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
t.Fatal(err)
}
writePromptTestFile(t, filepath.Join(nestedDir, "nested_recap.yaml"), `
id: nested-recap
version: "1.0.0"
messages:
- role: user
content_file: ./nested_recap.user.tmpl
output:
format: markdown
validation_mode: basic
repair_attempts: 0
`)
writePromptTestFile(t, filepath.Join(nestedDir, "nested_recap.user.tmpl"), `Nested recap: {{input "transcript"}}`)
p, err := repo.GetPromptDefinition(ctx, "nested-recap", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(p.Templates) != 1 {
t.Fatalf("expected one template, got %d", len(p.Templates))
}
if !strings.Contains(p.Templates[0].Content, "Nested recap") {
t.Fatalf("expected nested content file body, got %q", p.Templates[0].Content)
}
if !strings.Contains(p.Templates[0].ContentFile, filepath.Join("dnd", "recap", "nested_recap.user.tmpl")) {
t.Fatalf("expected nested content file path, got %q", p.Templates[0].ContentFile)
}
})
t.Run("prompt with default_profile", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "with-default-profile", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.DefaultProfile != "local-default" {
t.Fatalf("unexpected default profile: %q", p.DefaultProfile)
}
if len(p.Inputs) != 1 {
t.Fatalf("expected one input, got %d", len(p.Inputs))
}
if p.Inputs[0].ContentType != "" {
t.Fatalf("expected missing content_type to remain empty, got %q", p.Inputs[0].ContentType)
}
})
t.Run("duplicate prompt IDs fail as ambiguous", func(t *testing.T) {
writePromptTestFile(t, filepath.Join(tmpDir, "duplicate_a.yaml"), `
id: duplicate-prompt
version: "1.0.0"
messages:
- role: user
content: First duplicate.
output:
format: markdown
validation_mode: basic
repair_attempts: 0
`)
nestedDir := filepath.Join(tmpDir, "nested")
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
t.Fatal(err)
}
writePromptTestFile(t, filepath.Join(nestedDir, "duplicate_b.yaml"), `
id: duplicate-prompt
version: "2.0.0"
messages:
- role: user
content: Second duplicate.
output:
format: markdown
validation_mode: basic
repair_attempts: 0
`)
_, err := repo.GetPromptDefinition(ctx, "duplicate-prompt", "")
if !errors.Is(err, ErrInvalidPromptDefinition) {
t.Fatalf("expected duplicate prompt to return ErrInvalidPromptDefinition, got %v", err)
}
for _, want := range []string{"duplicate prompt definition id", "duplicate_a.yaml", filepath.Join("nested", "duplicate_b.yaml")} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("expected error to contain %q, got %v", want, err)
}
}
})
t.Run("duplicate prompt ID and requested version fails as ambiguous", func(t *testing.T) {
writePromptTestFile(t, filepath.Join(tmpDir, "version_duplicate_a.yaml"), `
id: duplicate-version-prompt
version: "1.0.0"
messages:
- role: user
content: First duplicate version.
output:
format: markdown
validation_mode: basic
repair_attempts: 0
`)
nestedDir := filepath.Join(tmpDir, "versioned")
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
t.Fatal(err)
}
writePromptTestFile(t, filepath.Join(nestedDir, "version_duplicate_b.yaml"), `
id: duplicate-version-prompt
version: "1.0.0"
messages:
- role: user
content: Second duplicate version.
output:
format: markdown
validation_mode: basic
repair_attempts: 0
`)
_, err := repo.GetPromptDefinition(ctx, "duplicate-version-prompt", "1.0.0")
if !errors.Is(err, ErrInvalidPromptDefinition) {
t.Fatalf("expected duplicate prompt version to return ErrInvalidPromptDefinition, got %v", err)
}
for _, want := range []string{"duplicate prompt definition id", "version \"1.0.0\"", "version_duplicate_a.yaml", filepath.Join("versioned", "version_duplicate_b.yaml")} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("expected error to contain %q, got %v", want, err)
}
}
})
t.Run("non-matching malformed nested prompt is ignored for not found lookup", func(t *testing.T) {
nestedDir := filepath.Join(tmpDir, "broken")
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
t.Fatal(err)
}
writePromptTestFile(t, filepath.Join(nestedDir, "unrelated.yaml"), "id: [")
_, err := repo.GetPromptDefinition(ctx, "does-not-exist-even-with-broken-nested-file", "")
if !errors.Is(err, ErrPromptDefinitionNotFound) {
t.Fatalf("expected ErrPromptDefinitionNotFound, got %v", err)
}
})
t.Run("strict decode failure in nested prompt matches by YAML ID", func(t *testing.T) {
nestedDir := filepath.Join(tmpDir, "strict")
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
t.Fatal(err)
}
writePromptTestFile(t, filepath.Join(nestedDir, "not_named_like_id.yaml"), `
id: nested-strict-error
version: "1.0.0"
unknown_field: true
messages:
- role: user
content: Invalid because of unknown field.
output:
format: markdown
validation_mode: basic
repair_attempts: 0
`)
_, err := repo.GetPromptDefinition(ctx, "nested-strict-error", "")
if !errors.Is(err, ErrInvalidYAML) {
t.Fatalf("expected ErrInvalidYAML, got %v", err)
}
if !strings.Contains(err.Error(), filepath.Join("strict", "not_named_like_id.yaml")) {
t.Fatalf("expected nested path in error, got %v", err)
}
})
t.Run("version lookup", func(t *testing.T) {
_, err := repo.GetPromptDefinition(ctx, "valid-inline", "9.9.9")
if !errors.Is(err, ErrPromptDefinitionNotFound) {
t.Fatalf("expected ErrPromptDefinitionNotFound, got %v", err)
}
})
cases := []struct {
name string
id string
targetErr error
errSubstrs []string
}{
{name: "invalid YAML", id: "invalid_yaml", targetErr: ErrInvalidYAML},
{name: "missing id", id: "missing_id", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"id is required"}},
{name: "no messages", id: "no_messages", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"at least one message is required"}},
{name: "both content and content_file", id: "both_content_and_content_file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"exactly one"}},
{name: "neither content nor content_file", id: "neither_content_nor_content_file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"exactly one"}},
{name: "missing content_file", id: "missing_content_file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"failed to read content_file"}},
{name: "duplicate input names", id: "duplicate_input_names", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"duplicate input name"}},
{name: "invalid validation mode", id: "invalid_validation_mode", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"invalid validation mode"}},
{name: "json_schema without schema_path", id: "json_schema_without_schema_path", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"schema_path"}},
{name: "unknown input field", id: "unknown_input_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unknown_input_setting not found"}},
{name: "empty cache control type", id: "empty_cache_control_type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "type is required"}},
{name: "unsupported cache control type", id: "unsupported_cache_control_type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported type"}},
{name: "unsupported cache control ttl", id: "unsupported_cache_control_ttl", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported ttl"}},
{name: "unknown cache control field", id: "unknown_cache_control_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unexpected not found"}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
_, err := repo.GetPromptDefinition(ctx, tc.id, "")
if !errors.Is(err, tc.targetErr) {
t.Fatalf("expected %v, got %v", tc.targetErr, err)
}
for _, sub := range tc.errSubstrs {
if !strings.Contains(err.Error(), sub) {
t.Fatalf("expected error to contain %q, got %v", sub, err)
}
}
})
}
t.Run("prompt definition not found", func(t *testing.T) {
_, err := repo.GetPromptDefinition(ctx, "does-not-exist", "")
if !errors.Is(err, ErrPromptDefinitionNotFound) {
t.Fatalf("expected ErrPromptDefinitionNotFound, got %v", err)
}
})
}
func TestFSRepositoryGetPromptDefinition(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"prompts/nested/prompt.yaml": &fstest.MapFile{Data: []byte(`
id: fs-prompt
version: "1.0.0"
inputs:
- name: transcript
required: true
messages:
- role: user
content_file: ./messages/user.tmpl
output:
format: markdown
validation_mode: basic
repair_attempts: 0
`)},
"prompts/nested/messages/user.tmpl": &fstest.MapFile{Data: []byte(`Summarize {{input "transcript"}}.`)},
}, "prompts")
got, err := repo.GetPromptDefinition(context.Background(), "fs-prompt", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if got.ID != "fs-prompt" {
t.Fatalf("unexpected prompt id: %q", got.ID)
}
if len(got.Templates) != 1 || !strings.Contains(got.Templates[0].Content, `{{input "transcript"}}`) {
t.Fatalf("expected content_file body to be loaded, got %+v", got.Templates)
}
if got.Templates[0].ContentFile != "prompts/nested/messages/user.tmpl" {
t.Fatalf("unexpected content file path: %q", got.Templates[0].ContentFile)
}
}
func TestFSRepositoryContentFileContainment(t *testing.T) {
t.Run("nested prompt can reference file inside root", func(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"prompts/nested/prompt.yaml": &fstest.MapFile{Data: []byte(`
id: fs-contained-prompt
version: "1.0.0"
messages:
- role: user
content_file: ../shared/user.tmpl
output:
format: markdown
validation_mode: basic
repair_attempts: 0
`)},
"prompts/shared/user.tmpl": &fstest.MapFile{Data: []byte(`Inside root.`)},
}, "prompts")
got, err := repo.GetPromptDefinition(context.Background(), "fs-contained-prompt", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(got.Templates) != 1 || got.Templates[0].Content != "Inside root." {
t.Fatalf("expected contained content file, got %+v", got.Templates)
}
})
tests := []struct {
name string
contentFile string
wantErr string
}{
{name: "parent escape rejected", contentFile: "../outside.tmpl", wantErr: "escapes source root"},
{name: "absolute path rejected", contentFile: "/outside.tmpl", wantErr: "must be relative"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"prompts/prompt.yaml": &fstest.MapFile{Data: []byte(`
id: fs-escaped-prompt
version: "1.0.0"
messages:
- role: user
content_file: ` + tc.contentFile + `
output:
format: markdown
validation_mode: basic
repair_attempts: 0
`)},
"outside.tmpl": &fstest.MapFile{Data: []byte(`Outside root.`)},
}, "prompts")
_, err := repo.GetPromptDefinition(context.Background(), "fs-escaped-prompt", "")
if !errors.Is(err, ErrInvalidPromptDefinition) {
t.Fatalf("expected ErrInvalidPromptDefinition, got %v", err)
}
if !strings.Contains(err.Error(), tc.wantErr) {
t.Fatalf("expected error to contain %q, got %v", tc.wantErr, err)
}
})
}
}
func TestFSRepositoryRejectsDuplicatePromptIDs(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"one.yaml": &fstest.MapFile{Data: []byte(`
id: duplicate-fs-prompt
version: "1.0.0"
messages:
- role: user
content: First.
output:
format: text
validation_mode: none
repair_attempts: 0
`)},
"nested/two.yaml": &fstest.MapFile{Data: []byte(`
id: duplicate-fs-prompt
version: "1.0.0"
messages:
- role: user
content: Second.
output:
format: text
validation_mode: none
repair_attempts: 0
`)},
}, ".")
_, err := repo.GetPromptDefinition(context.Background(), "duplicate-fs-prompt", "")
if !errors.Is(err, ErrInvalidPromptDefinition) {
t.Fatalf("expected ErrInvalidPromptDefinition, got %v", err)
}
if !strings.Contains(err.Error(), "one.yaml") || !strings.Contains(err.Error(), "nested/two.yaml") {
t.Fatalf("expected duplicate paths in error, got %v", err)
}
}
func TestFSRepositoryRejectsUnknownYAMLFields(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"not_named_like_id.yaml": &fstest.MapFile{Data: []byte(`
id: strict-fs-prompt
version: "1.0.0"
unknown: true
messages:
- role: user
content: Invalid.
output:
format: text
validation_mode: none
repair_attempts: 0
`)},
}, ".")
_, err := repo.GetPromptDefinition(context.Background(), "strict-fs-prompt", "")
if !errors.Is(err, ErrInvalidYAML) {
t.Fatalf("expected ErrInvalidYAML, got %v", err)
}
}
func assertCacheControl(t *testing.T, got *domain.CacheControl, wantType domain.CacheControlType, wantTTL string) {
t.Helper()
if got == nil {
t.Fatal("expected cache control, got nil")
}
if got.Type != wantType {
t.Fatalf("unexpected cache control type: got %q want %q", got.Type, wantType)
}
if got.TTL != wantTTL {
t.Fatalf("unexpected cache control ttl: got %q want %q", got.TTL, wantTTL)
}
}
func writePromptTestFile(t *testing.T, path string, content string) {
t.Helper()
if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil {
t.Fatalf("failed to write prompt test file %q: %v", path, err)
}
}
func copyTree(src, dst string) error {
return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
rel, err := filepath.Rel(src, path)
if err != nil {
return err
}
if rel == "." {
return nil
}
target := filepath.Join(dst, rel)
if d.IsDir() {
return os.MkdirAll(target, 0o755)
}
data, err := os.ReadFile(path)
if err != nil {
return err
}
return os.WriteFile(target, data, 0o644)
})
}

View File

@@ -0,0 +1,10 @@
id: both-content-and-content-file
version: "1.0.0"
messages:
- role: user
content: "Hi"
content_file: ./messages/user_prompt.tmpl
output:
format: text
validation_mode: none
repair_attempts: 0

Some files were not shown because too many files have changed in this diff Show More