Compare commits

7 Commits

16 changed files with 768 additions and 49 deletions

View File

@@ -6,7 +6,11 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
)
func toDomainRunRequest(req RunRequest) domain.RunRequest {
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
execution, err := toDomainExecutionTargetOverride(req.Execution)
if err != nil {
return domain.RunRequest{}, err
}
return domain.RunRequest{
PromptID: req.PromptID,
PromptVersion: req.PromptVersion,
@@ -14,10 +18,10 @@ func toDomainRunRequest(req RunRequest) domain.RunRequest {
APIKey: req.APIKey,
Inputs: toDomainArtifactRefMap(req.Inputs),
Vars: copyStringMap(req.Vars),
Execution: toDomainExecutionTargetOverride(req.Execution),
Execution: execution,
Validation: toDomainOutputContractPtr(req.Validation),
Metadata: copyStringMap(req.Metadata),
}
}, nil
}
func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
@@ -124,9 +128,13 @@ func fromDomainArtifact(artifact domain.Artifact) Artifact {
}
}
func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) *domain.ExecutionTargetOverride {
func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain.ExecutionTargetOverride, error) {
if override == nil {
return nil
return nil, nil
}
extraParams, err := copyPublicJSONMap(override.ExtraParams)
if err != nil {
return nil, err
}
return &domain.ExecutionTargetOverride{
Endpoint: override.Endpoint,
@@ -138,8 +146,8 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) *domain.
ServiceTier: override.ServiceTier,
ReasoningEffort: override.ReasoningEffort,
APIKeyEnv: override.APIKeyEnv,
ExtraParams: copyAnyMap(override.ExtraParams),
}
ExtraParams: extraParams,
}, nil
}
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {

View File

@@ -50,7 +50,7 @@ engine, err := scriptorium.NewEngine(cfg, scriptorium.WithProfiles(profile))
In-memory profiles have highest precedence, followed by configured profile file/FS/directory sources, then built-in profiles. Duplicate IDs in one `WithProfiles` call return `ErrInvalidConfig`.
`Profile` and `OpenAICompatibleProfileConfig` include endpoint, model, numeric defaults, service tier, reasoning effort, `APIKeyRequired`, and JSON-compatible `ExtraParams`. They do not accept raw API-key fields. When `APIKeyRequired` is true, pass the secret with `RunRequest.APIKey`.
`Profile` and `OpenAICompatibleProfileConfig` include endpoint, model, numeric defaults, service tier, reasoning effort, `APIKeyRequired`, and JSON-compatible `ExtraParams`. `WithProfiles` validates `ExtraParams` and returns `ErrInvalidConfig` for unsupported values such as functions, channels, non-string map keys, non-finite floats, or cyclic values. Raw API-key fields are not accepted. When `APIKeyRequired` is true, pass the secret with `RunRequest.APIKey`.
## Prepare A Prompt
@@ -97,7 +97,9 @@ _ = result.Artifact
`RunResult` includes the run ID, output artifact, raw output, validation result, prompt/profile/model metadata, effective model parameters, input hashes, token/cache usage, and timing fields. Validation content failures return a successful `RunResult` with failed validation status. Runtime validation errors return `ErrValidation`.
For the public Go API, pass provider credentials with `RunRequest.APIKey`. The value is request-scoped, uses `json:"-"`, is preferred over profile `api_key_env` by the default OpenAI-compatible client, and is not included in `PreparedRun` or `RunResult` JSON. Do not store raw keys in config, prompt files, or profile YAML.
For the public Go API, pass provider credentials with `RunRequest.APIKey`. The value is request-scoped, uses `json:"-"`, is preferred over profile `api_key_env` by the default OpenAI-compatible client, and is not included in `PreparedRun` or `RunResult` JSON. Normal Go string formatting of `RunRequest` reports only whether a direct key is set. Do not store raw keys in config, prompt files, or profile YAML.
Avoid logging raw request structs with reflection-based debug dumpers; exported fields remain visible to tools that bypass `String` and `GoString` methods.
## Inject An LLM Client
@@ -116,7 +118,7 @@ func (fakeLLM) Generate(ctx context.Context, req scriptorium.GenerateRequest) (*
engine, err := scriptorium.NewEngine(cfg, scriptorium.WithLLMClient(fakeLLM{}))
```
The injected client receives the rendered prompt, effective execution target, target presence metadata for explicit numeric overrides, structured-output spec, and request API key when provided. `GenerateRequest.APIKey` also uses `json:"-"`; custom and fake clients should avoid logging or serializing it. `WithLLMClient(nil)` returns `ErrInvalidConfig`.
The injected client receives the rendered prompt, effective execution target, target presence metadata for explicit numeric overrides, structured-output spec, and request API key when provided. `GenerateRequest.APIKey` also uses `json:"-"`, and normal Go string formatting reports only whether a direct key is set. Custom and fake clients should avoid logging or serializing it. `WithLLMClient(nil)` returns `ErrInvalidConfig`.
## Request Overrides
@@ -129,6 +131,8 @@ req.Execution = &scriptorium.ExecutionTargetOverride{
}
```
`ExecutionTargetOverride.ExtraParams` accepts JSON-compatible values and copies typed maps/slices so later caller mutation does not affect the run. Unsupported values, non-string map keys, non-finite floats, and cycles return `ErrInvalidRequest`.
## Errors
Public methods wrap context while preserving stable sentinel checks with `errors.Is`:

View File

@@ -71,7 +71,8 @@ Primary runner error classes:
- `ErrInvalidRequest`: invalid run request envelope.
- `ErrProfileRequired`: specific invalid-request reason when neither request `profile_id` nor prompt `default_profile` is available.
- `ErrAPIKeyEnvMissing`: specific invalid-request reason when `api_key_env` is set but the named environment variable is unset/empty.
- `ErrProfileLoad`: prompt/profile repository load failures.
- `ErrPromptLoad`: prompt-definition repository load failures.
- `ErrProfileLoad`: execution-profile repository load failures.
- `ErrArtifactLoad`: artifact read failures.
- `ErrPromptRender`: template render failures.
- `ErrLLMGenerate`: outbound model request failures.

View File

@@ -52,7 +52,15 @@ type Config struct {
}
// Option customizes engine construction.
type Option func(*engineOptions) error
type Option interface {
apply(*engineOptions) error
}
type optionFunc func(*engineOptions) error
func (f optionFunc) apply(options *engineOptions) error {
return f(options)
}
type engineOptions struct {
llmClient llm.Client
@@ -68,17 +76,21 @@ type engineOptions struct {
// WithLLMClient injects a custom LLM client for execution.
func WithLLMClient(client LLMClient) Option {
return func(options *engineOptions) error {
return optionFunc(func(options *engineOptions) error {
if client == nil {
return ErrInvalidConfig
}
options.llmClient = publicLLMClientAdapter{client: client}
return nil
}
})
}
// WithPromptFS loads prompt definitions from fsys under root.
//
// The source uses the same strict prompt YAML rules as configured prompt
// directories, and prompt content_file paths resolve within this source.
func WithPromptFS(fsys fs.FS, root string) Option {
return func(options *engineOptions) error {
return optionFunc(func(options *engineOptions) error {
if fsys == nil {
return ErrInvalidConfig
}
@@ -88,11 +100,14 @@ func WithPromptFS(fsys fs.FS, root string) Option {
options.promptDefs = promptdef.NewFSRepository(fsys, root)
options.promptSource = true
return nil
}
})
}
// WithPromptFile loads prompt definitions from the single prompt file at path.
//
// Relative prompt content_file paths resolve from the file's directory.
func WithPromptFile(path string) Option {
return func(options *engineOptions) error {
return optionFunc(func(options *engineOptions) error {
fsys, root, err := fileSource(path)
if err != nil {
return err
@@ -100,11 +115,15 @@ func WithPromptFile(path string) Option {
options.promptDefs = promptdef.NewFSRepository(fsys, root)
options.promptSource = true
return nil
}
})
}
// WithProfileFS loads execution profiles from fsys under root.
//
// Profiles from this source overlay built-in profiles. Profile YAML must use
// api_key_env for environment-based credentials; raw API keys are rejected.
func WithProfileFS(fsys fs.FS, root string) Option {
return func(options *engineOptions) error {
return optionFunc(func(options *engineOptions) error {
if fsys == nil {
return ErrInvalidConfig
}
@@ -114,11 +133,15 @@ func WithProfileFS(fsys fs.FS, root string) Option {
options.profiles = profile.NewFSRepository(fsys, root)
options.profileSource = true
return nil
}
})
}
// WithProfileFile loads execution profiles from the single profile file at path.
//
// The profile overlays built-in profiles. Profile YAML must use api_key_env for
// environment-based credentials; raw API keys are rejected.
func WithProfileFile(path string) Option {
return func(options *engineOptions) error {
return optionFunc(func(options *engineOptions) error {
fsys, root, err := fileSource(path)
if err != nil {
return err
@@ -126,13 +149,13 @@ func WithProfileFile(path string) Option {
options.profiles = profile.NewFSRepository(fsys, root)
options.profileSource = true
return nil
}
})
}
// WithProfiles configures in-memory profiles that take precedence over
// configured profile files and built-in profiles.
func WithProfiles(profiles ...Profile) Option {
return func(options *engineOptions) error {
return optionFunc(func(options *engineOptions) error {
repo, err := newMemoryProfileRepository(profiles)
if err != nil {
return err
@@ -140,11 +163,15 @@ func WithProfiles(profiles ...Profile) Option {
options.memoryProfiles = repo
options.memorySource = true
return nil
}
})
}
// WithSchemaFS loads JSON Schema documents from fsys under root.
//
// Prompt schema_path values resolve within this source when schema validation
// or structured output is requested.
func WithSchemaFS(fsys fs.FS, root string) Option {
return func(options *engineOptions) error {
return optionFunc(func(options *engineOptions) error {
if fsys == nil {
return ErrInvalidConfig
}
@@ -154,11 +181,14 @@ func WithSchemaFS(fsys fs.FS, root string) Option {
options.validator = validate.NewFSValidator(fsys, root)
options.validatorSource = true
return nil
}
})
}
// WithSchemaFile loads JSON Schema documents from the single schema file at path.
//
// Prompt schema_path values refer to the file's base name.
func WithSchemaFile(path string) Option {
return func(options *engineOptions) error {
return optionFunc(func(options *engineOptions) error {
fsys, root, err := fileSource(path)
if err != nil {
return err
@@ -166,7 +196,7 @@ func WithSchemaFile(path string) Option {
options.validator = validate.NewFSValidator(fsys, root)
options.validatorSource = true
return nil
}
})
}
// NewEngine constructs an Engine using the same default internal components as
@@ -177,7 +207,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
if opt == nil {
continue
}
if err := opt(&options); err != nil {
if err := opt.apply(&options); err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
}
}
@@ -257,7 +287,12 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
prepared, err := e.runner.Prepare(ctx, toDomainRunRequest(req))
domainReq, err := toDomainRunRequest(req)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
}
prepared, err := e.runner.Prepare(ctx, domainReq)
if err != nil {
return nil, mapPublicError(err)
}
@@ -270,7 +305,12 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
result, err := e.runner.Run(ctx, toDomainRunRequest(req))
domainReq, err := toDomainRunRequest(req)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
}
result, err := e.runner.Run(ctx, domainReq)
if err != nil {
return nil, mapPublicError(err)
}

View File

@@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"math"
"net/http"
"net/http/httptest"
"os"
@@ -117,6 +119,79 @@ func TestPreparedRunJSONDoesNotExposeSecretOrTargetPresence(t *testing.T) {
}
}
func TestRunRequestFormattingRedactsDirectAPIKey(t *testing.T) {
const secret = "run-request-secret"
req := scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
ProfileID: "local-fast",
APIKey: secret,
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
},
}
for _, formatted := range []string{
fmt.Sprint(req),
fmt.Sprintf("%+v", req),
fmt.Sprintf("%#v", req),
} {
if strings.Contains(formatted, secret) {
t.Fatalf("formatted RunRequest leaked API key: %s", formatted)
}
if !strings.Contains(formatted, "APIKeySet:true") {
t.Fatalf("formatted RunRequest should indicate an API key is set, got %s", formatted)
}
}
payload, err := json.Marshal(req)
if err != nil {
t.Fatalf("expected RunRequest to marshal, got %v", err)
}
if strings.Contains(string(payload), secret) {
t.Fatalf("RunRequest JSON leaked API key: %s", payload)
}
}
func TestGenerateRequestFormattingRedactsDirectAPIKey(t *testing.T) {
const secret = "generate-request-secret"
req := scriptorium.GenerateRequest{
Prompt: scriptorium.RenderedPrompt{Messages: []scriptorium.RenderedMessage{
{Role: "user", Content: "secret prompt content"},
}},
Target: scriptorium.ExecutionTarget{
Model: "test-model",
ExtraParams: map[string]any{
"provider_option": "on",
},
},
APIKey: secret,
}
for _, formatted := range []string{
fmt.Sprint(req),
fmt.Sprintf("%+v", req),
fmt.Sprintf("%#v", req),
} {
if strings.Contains(formatted, secret) {
t.Fatalf("formatted GenerateRequest leaked API key: %s", formatted)
}
if strings.Contains(formatted, "secret prompt content") {
t.Fatalf("formatted GenerateRequest leaked prompt content: %s", formatted)
}
if !strings.Contains(formatted, "APIKeySet:true") {
t.Fatalf("formatted GenerateRequest should indicate an API key is set, got %s", formatted)
}
}
payload, err := json.Marshal(req)
if err != nil {
t.Fatalf("expected GenerateRequest to marshal, got %v", err)
}
if strings.Contains(string(payload), secret) {
t.Fatalf("GenerateRequest JSON leaked API key: %s", payload)
}
}
func TestPreparePreservesExplicitZeroExecutionOverrides(t *testing.T) {
engine := newExampleEngine(t)
zeroFloat := 0.0
@@ -593,6 +668,59 @@ unknown_field: true
}
}
func TestPromptRepositoryReadFailureMapsToPromptLoad(t *testing.T) {
missingPromptDir := filepath.Join(t.TempDir(), "missing-prompts")
engine, err := scriptorium.NewEngine(scriptorium.Config{
PromptDir: missingPromptDir,
ProfileDir: "./examples/profiles",
SchemaDir: "./examples/schemas",
})
if err != nil {
t.Fatalf("expected engine construction to succeed, got %v", err)
}
_, err = engine.Prepare(context.Background(), scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
"glossary": scriptorium.Inline("gate: A guarded passage."),
},
})
if !errors.Is(err, scriptorium.ErrPromptLoad) {
t.Fatalf("expected ErrPromptLoad, got %v", err)
}
if errors.Is(err, scriptorium.ErrProfileLoad) {
t.Fatalf("did not expect ErrProfileLoad, got %v", err)
}
}
func TestSelectedProfileRepositoryReadFailureMapsToProfileLoad(t *testing.T) {
missingProfileDir := filepath.Join(t.TempDir(), "missing-profiles")
engine, err := scriptorium.NewEngine(scriptorium.Config{
PromptDir: "./examples/prompts",
ProfileDir: missingProfileDir,
SchemaDir: "./examples/schemas",
})
if err != nil {
t.Fatalf("expected engine construction to succeed, got %v", err)
}
_, err = engine.Prepare(context.Background(), scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
ProfileID: "local-fast",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
"glossary": scriptorium.Inline("gate: A guarded passage."),
},
})
if !errors.Is(err, scriptorium.ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
if errors.Is(err, scriptorium.ErrPromptLoad) {
t.Fatalf("did not expect ErrPromptLoad, got %v", err)
}
}
func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
t.Setenv("OPENROUTER_API_KEY", "test-key")
engine, err := scriptorium.NewEngine(scriptorium.Config{
@@ -1094,6 +1222,66 @@ func TestInMemoryProfileExtraParamsAreCopiedAcrossPublicBoundary(t *testing.T) {
}
}
func TestWithProfilesRejectsInvalidExtraParams(t *testing.T) {
tests := []struct {
name string
extraParams map[string]any
}{
{name: "function", extraParams: map[string]any{"bad": func() {}}},
{name: "channel", extraParams: map[string]any{"bad": make(chan struct{})}},
{name: "struct", extraParams: map[string]any{"bad": struct{ Name string }{Name: "bad"}}},
{name: "non string map key", extraParams: map[string]any{"bad": map[int]string{1: "one"}}},
{name: "nan", extraParams: map[string]any{"bad": math.NaN()}},
{name: "positive infinity", extraParams: map[string]any{"bad": math.Inf(1)}},
{name: "negative infinity", extraParams: map[string]any{"bad": math.Inf(-1)}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: "./examples/prompts"},
scriptorium.WithProfiles(scriptorium.Profile{
ID: "invalid-extra-params",
Endpoint: "http://invalid/v1",
Model: "invalid-model",
ExtraParams: tc.extraParams,
}),
)
if !errors.Is(err, scriptorium.ErrInvalidConfig) {
t.Fatalf("expected ErrInvalidConfig, got %v", err)
}
})
}
}
func TestWithProfilesRejectsCyclicExtraParams(t *testing.T) {
cyclicMap := map[string]any{}
cyclicMap["self"] = cyclicMap
cyclicSlice := []any{nil}
cyclicSlice[0] = cyclicSlice
tests := []struct {
name string
extraParams map[string]any
}{
{name: "map", extraParams: cyclicMap},
{name: "slice", extraParams: map[string]any{"cycle": cyclicSlice}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: "./examples/prompts"},
scriptorium.WithProfiles(scriptorium.Profile{
ID: "cyclic-extra-params",
Endpoint: "http://cyclic/v1",
Model: "cyclic-model",
ExtraParams: tc.extraParams,
}),
)
if !errors.Is(err, scriptorium.ErrInvalidConfig) {
t.Fatalf("expected ErrInvalidConfig, got %v", err)
}
})
}
}
func TestRunStructuredOutputWorksWithSchemaFS(t *testing.T) {
fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: `{"events":[]}`}}
engine, err := scriptorium.NewEngine(scriptorium.Config{
@@ -1190,6 +1378,45 @@ func TestSourceOptionsRejectInvalidInputs(t *testing.T) {
}
}
func TestPackageOptionsComposeFromSlice(t *testing.T) {
fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}
options := []scriptorium.Option{
nil,
scriptorium.WithProfiles(scriptorium.Profile{
ID: "slice-profile",
Endpoint: "http://slice/v1",
Model: "slice-model",
}),
scriptorium.WithLLMClient(fake),
}
engine, err := scriptorium.NewEngine(scriptorium.Config{
PromptDir: "./examples/prompts",
SchemaDir: "./examples/schemas",
}, options...)
if err != nil {
t.Fatalf("expected package-provided options to compose, got %v", err)
}
_, err = engine.Run(context.Background(), scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
ProfileID: "slice-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
"glossary": scriptorium.Inline("gate: A guarded passage."),
},
})
if err != nil {
t.Fatalf("expected run with composed options to succeed, got %v", err)
}
if len(fake.requests) != 1 {
t.Fatalf("expected one generate request, got %d", len(fake.requests))
}
if fake.requests[0].Target.Model != "slice-model" {
t.Fatalf("expected profile from composed options, got %q", fake.requests[0].Target.Model)
}
}
func TestExtraParamsTypedNestedValuesAreCopiedAcrossPublicBoundary(t *testing.T) {
fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}
engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake))
@@ -1248,6 +1475,78 @@ func TestExtraParamsTypedNestedValuesAreCopiedAcrossPublicBoundary(t *testing.T)
}
}
func TestRunRejectsInvalidExtraParams(t *testing.T) {
tests := []struct {
name string
extraParams map[string]any
}{
{name: "function", extraParams: map[string]any{"bad": func() {}}},
{name: "channel", extraParams: map[string]any{"bad": make(chan struct{})}},
{name: "struct", extraParams: map[string]any{"bad": struct{ Name string }{Name: "bad"}}},
{name: "non string map key", extraParams: map[string]any{"bad": map[int]string{1: "one"}}},
{name: "nan", extraParams: map[string]any{"bad": math.NaN()}},
{name: "positive infinity", extraParams: map[string]any{"bad": math.Inf(1)}},
{name: "negative infinity", extraParams: map[string]any{"bad": math.Inf(-1)}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}
engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake))
_, err := engine.Run(context.Background(), scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
"glossary": scriptorium.Inline("gate: A guarded passage."),
},
Execution: &scriptorium.ExecutionTargetOverride{ExtraParams: tc.extraParams},
})
if !errors.Is(err, scriptorium.ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
if len(fake.requests) != 0 {
t.Fatalf("expected invalid request to fail before LLM call, got %d requests", len(fake.requests))
}
})
}
}
func TestRunRejectsCyclicExtraParams(t *testing.T) {
cyclicMap := map[string]any{}
cyclicMap["self"] = cyclicMap
cyclicSlice := []any{nil}
cyclicSlice[0] = cyclicSlice
tests := []struct {
name string
extraParams map[string]any
}{
{name: "map", extraParams: cyclicMap},
{name: "slice", extraParams: map[string]any{"cycle": cyclicSlice}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}
engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake))
_, err := engine.Run(context.Background(), scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
"glossary": scriptorium.Inline("gate: A guarded passage."),
},
Execution: &scriptorium.ExecutionTargetOverride{ExtraParams: tc.extraParams},
})
if !errors.Is(err, scriptorium.ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
if len(fake.requests) != 0 {
t.Fatalf("expected invalid request to fail before LLM call, got %d requests", len(fake.requests))
}
})
}
}
func TestWithLLMClientRejectsNilClient(t *testing.T) {
_, err := scriptorium.NewEngine(exampleConfig("./examples/schemas"), scriptorium.WithLLMClient(nil))
if !errors.Is(err, scriptorium.ErrInvalidConfig) {

View File

@@ -49,6 +49,10 @@ func publicErrorFor(err error) error {
return ErrPromptNotFound
case errors.Is(err, profile.ErrProfileNotFound):
return ErrProfileNotFound
case errors.Is(err, usecase.ErrPromptLoad):
return ErrPromptLoad
case errors.Is(err, usecase.ErrProfileLoad):
return ErrProfileLoad
case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition):
return ErrPromptLoad
case isProfileLoadCause(err):
@@ -63,8 +67,6 @@ func publicErrorFor(err error) error {
return ErrValidation
case errors.Is(err, usecase.ErrInvalidRequest):
return ErrInvalidRequest
case errors.Is(err, usecase.ErrProfileLoad):
return ErrPromptLoad
default:
return nil
}

51
formatting.go Normal file
View File

@@ -0,0 +1,51 @@
package scriptorium
import "fmt"
// String returns a concise request summary without exposing direct API keys.
func (r RunRequest) String() string {
return r.redactedString()
}
// GoString returns a concise request summary without exposing direct API keys.
func (r RunRequest) GoString() string {
return r.redactedString()
}
func (r RunRequest) redactedString() string {
return fmt.Sprintf(
"scriptorium.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t Metadata:%d}",
r.PromptID,
r.PromptVersion,
r.ProfileID,
r.APIKey != "",
len(r.Inputs),
len(r.Vars),
r.Execution != nil,
r.Validation != nil,
len(r.Metadata),
)
}
// String returns a concise request summary without exposing direct API keys or
// rendered prompt content.
func (r GenerateRequest) String() string {
return r.redactedString()
}
// GoString returns a concise request summary without exposing direct API keys or
// rendered prompt content.
func (r GenerateRequest) GoString() string {
return r.redactedString()
}
func (r GenerateRequest) redactedString() string {
return fmt.Sprintf(
"scriptorium.GenerateRequest{Messages:%d Model:%q APIKeySet:%t StructuredOutputSet:%t ExtraParams:%d}",
len(r.Prompt.Messages),
r.Target.Model,
r.APIKey != "",
r.StructuredOutput != nil,
len(r.Target.ExtraParams),
)
}

View File

@@ -175,7 +175,7 @@ func mapRunError(err error) (int, string, string) {
return http.StatusNotFound, "profile_not_found", "execution profile not found"
case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition):
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
case errors.Is(err, profile.ErrInvalidYAML), errors.Is(err, profile.ErrInvalidProfile):
case errors.Is(err, profile.ErrInvalidYAML), errors.Is(err, profile.ErrInvalidProfile), errors.Is(err, profile.ErrRawAPIKeyNotAllowed):
return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile"
case errors.Is(err, usecase.ErrProfileRequired):
return http.StatusBadRequest, "profile_required", "profile_id is required when prompt default_profile is not set"
@@ -183,8 +183,10 @@ func mapRunError(err error) (int, string, string) {
return http.StatusBadRequest, "api_key_env_missing", "api_key_env is set but the environment variable is missing"
case errors.Is(err, usecase.ErrInvalidRequest):
return http.StatusBadRequest, "invalid_request", "invalid run request"
case errors.Is(err, usecase.ErrProfileLoad):
case errors.Is(err, usecase.ErrPromptLoad):
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
case errors.Is(err, usecase.ErrProfileLoad):
return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile"
case errors.Is(err, usecase.ErrArtifactLoad):
return http.StatusBadRequest, "artifact_read_failed", "failed to read input artifact"
case errors.Is(err, usecase.ErrPromptRender):

View File

@@ -563,11 +563,13 @@ func TestHandlerUsecaseErrorMapping(t *testing.T) {
message string
avoidCause string
}{
{name: "prompt not found", err: wrap(usecase.ErrProfileLoad, promptdef.ErrPromptDefinitionNotFound), status: http.StatusNotFound, code: "prompt_not_found", message: "prompt definition not found"},
{name: "prompt load invalid", err: wrap(usecase.ErrProfileLoad, promptdef.ErrInvalidPromptDefinition), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition"},
{name: "prompt not found", err: wrap(usecase.ErrPromptLoad, promptdef.ErrPromptDefinitionNotFound), status: http.StatusNotFound, code: "prompt_not_found", message: "prompt definition not found"},
{name: "prompt load invalid", err: wrap(usecase.ErrPromptLoad, promptdef.ErrInvalidPromptDefinition), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition"},
{name: "prompt load generic", err: wrap(usecase.ErrPromptLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition", avoidCause: "read failed"},
{name: "missing profile/default", err: wrap(usecase.ErrInvalidRequest, usecase.ErrProfileRequired), status: http.StatusBadRequest, code: "profile_required", message: "profile_id is required when prompt default_profile is not set"},
{name: "profile not found", err: wrap(usecase.ErrProfileLoad, profile.ErrProfileNotFound), status: http.StatusNotFound, code: "profile_not_found", message: "execution profile not found"},
{name: "profile invalid", err: wrap(usecase.ErrProfileLoad, profile.ErrInvalidProfile), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile"},
{name: "profile load generic", err: wrap(usecase.ErrProfileLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile", avoidCause: "read failed"},
{name: "api key env missing", err: wrap(usecase.ErrInvalidRequest, usecase.ErrAPIKeyEnvMissing), status: http.StatusBadRequest, code: "api_key_env_missing", message: "api_key_env is set but the environment variable is missing"},
{name: "artifact", err: wrap(usecase.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"},
{name: "prompt render", err: wrap(usecase.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"},

View File

@@ -55,10 +55,11 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
var client *http.Client
if cfg.HTTPClient != nil {
client = cfg.HTTPClient
if client.Timeout == 0 {
client.Timeout = timeout
cloned := *cfg.HTTPClient
if cloned.Timeout == 0 {
cloned.Timeout = timeout
}
client = &cloned
} else {
client = &http.Client{Timeout: timeout}
}

View File

@@ -14,6 +14,64 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
)
func TestNewOpenAICompatibleClientDoesNotMutateSuppliedZeroTimeoutClient(t *testing.T) {
transport := http.DefaultTransport
supplied := &http.Client{Transport: transport}
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
HTTPClient: supplied,
})
if err != nil {
t.Fatalf("unexpected constructor error: %v", err)
}
if supplied.Timeout != 0 {
t.Fatalf("expected supplied client timeout to remain zero, got %v", supplied.Timeout)
}
if client.httpClient == supplied {
t.Fatal("expected constructed client to use a cloned HTTP client")
}
if client.httpClient.Timeout != client.timeout {
t.Fatalf("expected cloned client timeout %v, got %v", client.timeout, client.httpClient.Timeout)
}
if client.httpClient.Timeout <= 0 {
t.Fatalf("expected constructed client to use a positive default timeout, got %v", client.httpClient.Timeout)
}
if client.httpClient.Transport != transport {
t.Fatal("expected cloned client to preserve the supplied transport")
}
}
func TestNewOpenAICompatibleClientDoesNotMutateSuppliedNonzeroTimeoutClient(t *testing.T) {
transport := http.DefaultTransport
suppliedTimeout := 37 * time.Second
supplied := &http.Client{
Timeout: suppliedTimeout,
Transport: transport,
}
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
Timeout: 2 * time.Second,
HTTPClient: supplied,
})
if err != nil {
t.Fatalf("unexpected constructor error: %v", err)
}
if supplied.Timeout != suppliedTimeout {
t.Fatalf("expected supplied client timeout to remain %v, got %v", suppliedTimeout, supplied.Timeout)
}
if client.httpClient == supplied {
t.Fatal("expected constructed client to use a cloned HTTP client")
}
if client.httpClient.Timeout != suppliedTimeout {
t.Fatalf("expected cloned client timeout %v, got %v", suppliedTimeout, client.httpClient.Timeout)
}
if client.httpClient.Transport != transport {
t.Fatal("expected cloned client to preserve the supplied transport")
}
}
func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
type observedRequest struct {
Authorization string

View File

@@ -28,7 +28,8 @@ var (
ErrProfileRequired = errors.New("profile selection is required")
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
ErrAPIKeyRequired = errors.New("api key is required")
ErrProfileLoad = errors.New("failed to load prompt definition")
ErrPromptLoad = errors.New("failed to load prompt definition")
ErrProfileLoad = errors.New("failed to load execution profile")
ErrArtifactLoad = errors.New("failed to load artifact")
ErrPromptRender = errors.New("failed to render prompt")
ErrLLMGenerate = errors.New("failed to generate output")
@@ -172,11 +173,11 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
def, err := r.promptDefs.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
}
promptDefinitionHash, err := hashPromptDefinition(def)
if err != nil {
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrProfileLoad, err)
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err)
}
selectedProfileID := strings.TrimSpace(req.ProfileID)

View File

@@ -253,6 +253,17 @@ func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) {
}
}
func TestRunnerPreparePromptLoadFailure(t *testing.T) {
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrPromptLoad) {
t.Fatalf("expected ErrPromptLoad, got %v", err)
}
if errors.Is(err, ErrProfileLoad) {
t.Fatalf("did not expect ErrProfileLoad, got %v", err)
}
}
func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
@@ -1323,8 +1334,11 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
func TestRunnerRunPromptLoadFailure(t *testing.T) {
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
if !errors.Is(err, ErrPromptLoad) {
t.Fatalf("expected ErrPromptLoad, got %v", err)
}
if errors.Is(err, ErrProfileLoad) {
t.Fatalf("did not expect ErrProfileLoad, got %v", err)
}
}

218
json_copy.go Normal file
View File

@@ -0,0 +1,218 @@
package scriptorium
import (
"encoding/json"
"fmt"
"math"
"reflect"
"strconv"
)
const maxSafeJSONInteger = 1<<53 - 1
type jsonVisit struct {
typ reflect.Type
ptr uintptr
}
func copyPublicJSONMap(src map[string]any) (map[string]any, error) {
if src == nil {
return nil, nil
}
copied, err := copyPublicJSONValue(reflect.ValueOf(src), "extra_params", make(map[jsonVisit]struct{}))
if err != nil {
return nil, err
}
out, ok := copied.(map[string]any)
if !ok {
return nil, fmt.Errorf("extra_params: expected object")
}
return out, nil
}
func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
if !value.IsValid() {
return nil, nil
}
if value.Kind() == reflect.Interface {
if value.IsNil() {
return nil, nil
}
return copyPublicJSONValue(value.Elem(), path, seen)
}
if !value.CanInterface() {
return nil, fmt.Errorf("%s: value cannot be copied", path)
}
if number, ok := value.Interface().(json.Number); ok {
f, err := strconv.ParseFloat(number.String(), 64)
if err != nil || math.IsNaN(f) || math.IsInf(f, 0) {
return nil, fmt.Errorf("%s: invalid JSON number", path)
}
return number, nil
}
switch value.Kind() {
case reflect.Bool, reflect.String:
return value.Interface(), nil
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
if value.Int() < -maxSafeJSONInteger || value.Int() > maxSafeJSONInteger {
return nil, fmt.Errorf("%s: integer is outside the JSON-safe range", path)
}
return value.Interface(), nil
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
if value.Uint() > maxSafeJSONInteger {
return nil, fmt.Errorf("%s: integer is outside the JSON-safe range", path)
}
return value.Interface(), nil
case reflect.Float32, reflect.Float64:
f := value.Convert(reflect.TypeOf(float64(0))).Float()
if math.IsNaN(f) || math.IsInf(f, 0) {
return nil, fmt.Errorf("%s: floating-point value must be finite", path)
}
return value.Interface(), nil
case reflect.Pointer:
if value.IsNil() {
return nil, nil
}
visit := jsonVisit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[visit]; ok {
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
}
seen[visit] = struct{}{}
defer delete(seen, visit)
return copyPublicJSONValue(value.Elem(), path, seen)
case reflect.Map:
return copyPublicJSONMapValue(value, path, seen)
case reflect.Slice:
if value.IsNil() {
return nil, nil
}
return copyPublicJSONSequenceValue(value, path, seen)
case reflect.Array:
return copyPublicJSONSequenceValue(value, path, seen)
default:
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
}
}
func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
if value.IsNil() {
return nil, nil
}
if value.Type().Key().Kind() != reflect.String {
return nil, fmt.Errorf("%s: map key type %s is not supported", path, value.Type().Key())
}
visit := jsonVisit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[visit]; ok {
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
}
seen[visit] = struct{}{}
defer delete(seen, visit)
type entry struct {
key reflect.Value
name string
value any
}
entries := make([]entry, 0, value.Len())
preserveType := true
elemType := value.Type().Elem()
iter := value.MapRange()
for iter.Next() {
key := iter.Key()
name := key.String()
copied, err := copyPublicJSONValue(iter.Value(), path+"."+name, seen)
if err != nil {
return nil, err
}
entries = append(entries, entry{key: key, name: name, value: copied})
if copied == nil {
if !canAssignNil(elemType) {
preserveType = false
}
continue
}
if !reflect.TypeOf(copied).AssignableTo(elemType) {
preserveType = false
}
}
if preserveType {
out := reflect.MakeMapWithSize(value.Type(), len(entries))
for _, entry := range entries {
if entry.value == nil {
out.SetMapIndex(entry.key, reflect.Zero(elemType))
continue
}
out.SetMapIndex(entry.key, reflect.ValueOf(entry.value))
}
return out.Interface(), nil
}
out := make(map[string]any, len(entries))
for _, entry := range entries {
out[entry.name] = entry.value
}
return out, nil
}
func copyPublicJSONSequenceValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
var visit jsonVisit
if value.Kind() == reflect.Slice {
visit = jsonVisit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[visit]; ok {
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
}
seen[visit] = struct{}{}
defer delete(seen, visit)
}
values := make([]any, value.Len())
preserveType := true
elemType := value.Type().Elem()
for i := 0; i < value.Len(); i++ {
copied, err := copyPublicJSONValue(value.Index(i), fmt.Sprintf("%s[%d]", path, i), seen)
if err != nil {
return nil, err
}
values[i] = copied
if copied == nil {
if !canAssignNil(elemType) {
preserveType = false
}
continue
}
if !reflect.TypeOf(copied).AssignableTo(elemType) {
preserveType = false
}
}
if preserveType {
out := reflect.New(value.Type()).Elem()
if value.Kind() == reflect.Slice {
out = reflect.MakeSlice(value.Type(), value.Len(), value.Len())
}
for i, copied := range values {
if copied == nil {
out.Index(i).Set(reflect.Zero(elemType))
continue
}
out.Index(i).Set(reflect.ValueOf(copied))
}
return out.Interface(), nil
}
out := make([]any, len(values))
copy(out, values)
return out, nil
}
func canAssignNil(typ reflect.Type) bool {
switch typ.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
return true
default:
return false
}
}

View File

@@ -10,8 +10,12 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
)
// OpenAICompatibleProfile returns an in-memory profile for an OpenAI-compatible
// chat-completions endpoint.
// OpenAICompatibleProfile returns an ordinary in-memory Profile for an
// OpenAI-compatible chat-completions endpoint.
//
// It does not register global state, maintain a model catalog, or resolve
// credentials. If APIKeyRequired is true, callers satisfy it with
// RunRequest.APIKey. Raw API keys do not belong in profiles.
func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile {
return Profile{
ID: cfg.ID,
@@ -60,6 +64,10 @@ func (r *memoryProfileRepository) GetProfile(_ context.Context, id string) (*dom
}
func toDomainProfile(publicProfile Profile) (domain.ExecutionProfile, error) {
extraParams, err := copyPublicJSONMap(publicProfile.ExtraParams)
if err != nil {
return domain.ExecutionProfile{}, err
}
prof := domain.ExecutionProfile{
ID: strings.TrimSpace(publicProfile.ID),
Endpoint: publicProfile.Endpoint,
@@ -71,7 +79,7 @@ func toDomainProfile(publicProfile Profile) (domain.ExecutionProfile, error) {
ServiceTier: publicProfile.ServiceTier,
ReasoningEffort: publicProfile.ReasoningEffort,
APIKeyRequired: publicProfile.APIKeyRequired,
ExtraParams: copyAnyMap(publicProfile.ExtraParams),
ExtraParams: extraParams,
}
if err := validatePublicProfile(prof); err != nil {
return domain.ExecutionProfile{}, err

View File

@@ -155,6 +155,11 @@ type ExecutionTargetOverride struct {
}
// Profile is an in-memory execution profile for library consumers.
//
// It is equivalent to a loaded profile file after validation. Raw API keys do
// not belong in profiles; use APIKeyRequired to require callers to provide
// RunRequest.APIKey for each request, or use profile YAML api_key_env with file
// and FS profile sources.
type Profile struct {
ID string
Endpoint string
@@ -169,7 +174,12 @@ type Profile struct {
ExtraParams map[string]any
}
// OpenAICompatibleProfileConfig configures an OpenAI-compatible in-memory profile.
// OpenAICompatibleProfileConfig configures an OpenAI-compatible in-memory
// profile.
//
// It contains ordinary profile fields for OpenAI-compatible chat-completions
// endpoints. APIKeyRequired is satisfied by RunRequest.APIKey. Raw API keys do
// not belong in this config.
type OpenAICompatibleProfileConfig struct {
ID string
Endpoint string