Implement a library profile API and built-in profile docs
This commit is contained in:
@@ -32,6 +32,7 @@ Integration references:
|
|||||||
- an effective `prompt_dir` from flags or config
|
- an effective `prompt_dir` from flags or config
|
||||||
- `serve` requires an effective `prompt_dir` from flags or config.
|
- `serve` requires an effective `prompt_dir` from flags or config.
|
||||||
- `profile_dir` is optional. If omitted, only built-in profiles are available; if provided, custom profiles override built-ins with the same ID.
|
- `profile_dir` is optional. If omitted, only built-in profiles are available; if provided, custom profiles override built-ins with the same ID.
|
||||||
|
- Built-in profile IDs are listed in the [configuration reference](config.md#profile-definition-files).
|
||||||
- Positional arguments are rejected.
|
- Positional arguments are rejected.
|
||||||
- Prompt cache control is configured in prompt YAML (`messages[].cache_control`), not with CLI flags.
|
- Prompt cache control is configured in prompt YAML (`messages[].cache_control`), not with CLI flags.
|
||||||
- Provider-specific `reasoning_effort` and `extra_params` are configured in profile YAML or HTTP model overrides, not with CLI flags.
|
- Provider-specific `reasoning_effort` and `extra_params` are configured in profile YAML or HTTP model overrides, not with CLI flags.
|
||||||
|
|||||||
@@ -220,6 +220,34 @@ Profile rules:
|
|||||||
- Duplicate profile IDs are invalid. If multiple files declare the requested profile ID, Scriptorium fails instead of choosing one.
|
- Duplicate profile IDs are invalid. If multiple files declare the requested profile ID, Scriptorium fails instead of choosing one.
|
||||||
- `extra_params` keys must not be empty and must not collide with reserved outbound request fields: `model`, `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, or `response_format`.
|
- `extra_params` keys must not be empty and must not collide with reserved outbound request fields: `model`, `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, or `response_format`.
|
||||||
|
|
||||||
|
Built-in profile IDs:
|
||||||
|
|
||||||
|
| Provider | ID | Model | API key env |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| aion-labs | `aion-2` | `aion-labs/aion-2.0` | `OPENROUTER_API_KEY` |
|
||||||
|
| anthropic | `claude-fable-latest` | `~anthropic/claude-fable-latest` | `OPENROUTER_API_KEY` |
|
||||||
|
| anthropic | `claude-haiku-latest` | `~anthropic/claude-haiku-latest` | `OPENROUTER_API_KEY` |
|
||||||
|
| anthropic | `claude-opus-latest` | `~anthropic/claude-opus-latest` | `OPENROUTER_API_KEY` |
|
||||||
|
| anthropic | `claude-sonnet-latest` | `~anthropic/claude-sonnet-latest` | `OPENROUTER_API_KEY` |
|
||||||
|
| deepseek | `deepseek-3-2` | `deepseek/deepseek-v3.2` | `OPENROUTER_API_KEY` |
|
||||||
|
| deepseek | `deepseek-4-pro` | `deepseek/deepseek-v4-pro` | `OPENROUTER_API_KEY` |
|
||||||
|
| google | `gemini-2-flash` | `google/gemini-2.5-flash` | `OPENROUTER_API_KEY` |
|
||||||
|
| google | `gemini-2-flash-lite` | `google/gemini-2.5-flash-lite` | `OPENROUTER_API_KEY` |
|
||||||
|
| google | `gemini-2-pro` | `google/gemini-2.5-pro` | `OPENROUTER_API_KEY` |
|
||||||
|
| google | `gemini-3-flash-lite` | `google/gemini-3.1-flash-lite` | `OPENROUTER_API_KEY` |
|
||||||
|
| google | `gemini-flash-latest` | `~google/gemini-flash-latest` | `OPENROUTER_API_KEY` |
|
||||||
|
| google | `gemini-pro-latest` | `~google/gemini-pro-latest` | `OPENROUTER_API_KEY` |
|
||||||
|
| google | `gemma-4-31b` | `google/gemma-4-31b-it:exacto` | `OPENROUTER_API_KEY` |
|
||||||
|
| minimax | `minimax-m2` | `minimax/minimax-m2.5` | `OPENROUTER_API_KEY` |
|
||||||
|
| minimax | `minimax-m3` | `minimax/minimax-m3` | `OPENROUTER_API_KEY` |
|
||||||
|
| mistral | `mistral-large-2512` | `mistralai/mistral-large-2512` | `OPENROUTER_API_KEY` |
|
||||||
|
| mistral | `mistral-medium-3-5` | `mistralai/mistral-medium-3-5` | `OPENROUTER_API_KEY` |
|
||||||
|
| mistral | `mistral-small-3` | `mistralai/mistral-small-3.2-24b-instruct` | `OPENROUTER_API_KEY` |
|
||||||
|
| mistral | `mistral-small-4` | `mistralai/mistral-small-2603` | `OPENROUTER_API_KEY` |
|
||||||
|
| nvidia | `nemotron-3-ultra` | `nvidia/nemotron-3-ultra-550b-a55b` | `OPENROUTER_API_KEY` |
|
||||||
|
| openai | `gpt-5-mini` | `openai/gpt-5.4-mini` | `OPENROUTER_API_KEY` |
|
||||||
|
| openai | `gpt-5-nano` | `openai/gpt-5.4-nano` | `OPENROUTER_API_KEY` |
|
||||||
|
|
||||||
Current outbound request behavior:
|
Current outbound request behavior:
|
||||||
|
|
||||||
- The OpenAI-compatible client currently serializes: `model`, optional `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, optional `response_format` for `json_schema` prompts, and `extra_params`.
|
- The OpenAI-compatible client currently serializes: `model`, optional `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, optional `response_format` for `json_schema` prompts, and `extra_params`.
|
||||||
|
|||||||
@@ -33,6 +33,25 @@ Directory fields on `Config` remain the compatibility path. Explicit source opti
|
|||||||
|
|
||||||
Prompt and profile sources load standard Scriptorium YAML with the same strict validation as directory sources. Prompt `content_file` paths resolve relative to the prompt file in the same source. Profile options overlay custom profiles above built-ins. Schema `fs.FS` sources preserve prompt `schema_path` semantics; schema file options expose the file by its base name.
|
Prompt and profile sources load standard Scriptorium YAML with the same strict validation as directory sources. Prompt `content_file` paths resolve relative to the prompt file in the same source. Profile options overlay custom profiles above built-ins. Schema `fs.FS` sources preserve prompt `schema_path` semantics; schema file options expose the file by its base name.
|
||||||
|
|
||||||
|
## In-Memory Profiles
|
||||||
|
|
||||||
|
Use `WithProfiles` when the consuming application already has profile settings in typed Go configuration:
|
||||||
|
|
||||||
|
```go
|
||||||
|
profile := scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||||
|
ID: "app.default",
|
||||||
|
Endpoint: "https://openrouter.ai/api/v1",
|
||||||
|
Model: "mistralai/mistral-small-3.2-24b-instruct",
|
||||||
|
APIKeyRequired: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
engine, err := scriptorium.NewEngine(cfg, scriptorium.WithProfiles(profile))
|
||||||
|
```
|
||||||
|
|
||||||
|
In-memory profiles have highest precedence, followed by configured profile file/FS/directory sources, then built-in profiles. Duplicate IDs in one `WithProfiles` call return `ErrInvalidConfig`.
|
||||||
|
|
||||||
|
`Profile` and `OpenAICompatibleProfileConfig` include endpoint, model, numeric defaults, service tier, reasoning effort, `APIKeyRequired`, and JSON-compatible `ExtraParams`. They do not accept raw API-key fields. When `APIKeyRequired` is true, pass the secret with `RunRequest.APIKey`.
|
||||||
|
|
||||||
## Prepare A Prompt
|
## Prepare A Prompt
|
||||||
|
|
||||||
`Prepare` resolves the prompt definition, profile, inputs, variables, output contract, structured-output metadata, and rendered messages without calling an LLM.
|
`Prepare` resolves the prompt definition, profile, inputs, variables, output contract, structured-output metadata, and rendered messages without calling an LLM.
|
||||||
|
|||||||
19
engine.go
19
engine.go
@@ -58,9 +58,11 @@ type engineOptions struct {
|
|||||||
llmClient llm.Client
|
llmClient llm.Client
|
||||||
promptDefs promptdef.Repository
|
promptDefs promptdef.Repository
|
||||||
profiles profile.Repository
|
profiles profile.Repository
|
||||||
|
memoryProfiles profile.Repository
|
||||||
validator validate.Validator
|
validator validate.Validator
|
||||||
promptSource bool
|
promptSource bool
|
||||||
profileSource bool
|
profileSource bool
|
||||||
|
memorySource bool
|
||||||
validatorSource bool
|
validatorSource bool
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -127,6 +129,20 @@ func WithProfileFile(path string) Option {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WithProfiles configures in-memory profiles that take precedence over
|
||||||
|
// configured profile files and built-in profiles.
|
||||||
|
func WithProfiles(profiles ...Profile) Option {
|
||||||
|
return func(options *engineOptions) error {
|
||||||
|
repo, err := newMemoryProfileRepository(profiles)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
options.memoryProfiles = repo
|
||||||
|
options.memorySource = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func WithSchemaFS(fsys fs.FS, root string) Option {
|
func WithSchemaFS(fsys fs.FS, root string) Option {
|
||||||
return func(options *engineOptions) error {
|
return func(options *engineOptions) error {
|
||||||
if fsys == nil {
|
if fsys == nil {
|
||||||
@@ -178,6 +194,9 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
if options.profileSource {
|
if options.profileSource {
|
||||||
profiles = builtin.NewRepositoryWithPrimary(options.profiles)
|
profiles = builtin.NewRepositoryWithPrimary(options.profiles)
|
||||||
}
|
}
|
||||||
|
if options.memorySource {
|
||||||
|
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
|
||||||
|
}
|
||||||
|
|
||||||
validator := options.validator
|
validator := options.validator
|
||||||
if !options.validatorSource {
|
if !options.validatorSource {
|
||||||
|
|||||||
230
engine_test.go
230
engine_test.go
@@ -864,6 +864,236 @@ model: profile-file-model
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPrepareWorksWithInMemoryProfilesWithoutProfileFiles(t *testing.T) {
|
||||||
|
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||||
|
PromptDir: "./examples/prompts",
|
||||||
|
SchemaDir: "./examples/schemas",
|
||||||
|
}, scriptorium.WithProfiles(scriptorium.Profile{
|
||||||
|
ID: "memory-profile",
|
||||||
|
Endpoint: "http://memory-profile/v1",
|
||||||
|
Model: "memory-model",
|
||||||
|
}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected engine construction to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
ProfileID: "memory-profile",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||||
|
"glossary": scriptorium.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected prepare to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.Model != "memory-model" {
|
||||||
|
t.Fatalf("expected in-memory profile model, got %q", prepared.EffectiveModelParams.Model)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInMemoryProfilesOverrideBuiltInsAndProfileSources(t *testing.T) {
|
||||||
|
profileFS := fstest.MapFS{
|
||||||
|
"profiles/mistral-small-3.yaml": &fstest.MapFile{Data: []byte(`
|
||||||
|
id: mistral-small-3
|
||||||
|
endpoint: http://profile-fs/v1
|
||||||
|
model: profile-fs-model
|
||||||
|
`)},
|
||||||
|
}
|
||||||
|
|
||||||
|
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||||
|
PromptDir: "./examples/prompts",
|
||||||
|
SchemaDir: "./examples/schemas",
|
||||||
|
},
|
||||||
|
scriptorium.WithProfileFS(profileFS, "profiles"),
|
||||||
|
scriptorium.WithProfiles(scriptorium.Profile{
|
||||||
|
ID: "mistral-small-3",
|
||||||
|
Endpoint: "http://memory-profile/v1",
|
||||||
|
Model: "memory-profile-model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected engine construction to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
ProfileID: "mistral-small-3",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||||
|
"glossary": scriptorium.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected prepare to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.Model != "memory-profile-model" {
|
||||||
|
t.Fatalf("expected in-memory profile to have highest precedence, got %q", prepared.EffectiveModelParams.Model)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWithProfilesRejectsDuplicateIDs(t *testing.T) {
|
||||||
|
_, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: "./examples/prompts"},
|
||||||
|
scriptorium.WithProfiles(
|
||||||
|
scriptorium.Profile{ID: "duplicate", Endpoint: "http://one/v1", Model: "one"},
|
||||||
|
scriptorium.Profile{ID: "duplicate", Endpoint: "http://two/v1", Model: "two"},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if !errors.Is(err, scriptorium.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("expected ErrInvalidConfig, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) {
|
||||||
|
fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}
|
||||||
|
prof := scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||||
|
ID: "template-profile",
|
||||||
|
Endpoint: "http://template/v1",
|
||||||
|
Model: "template-model",
|
||||||
|
APIKeyRequired: true,
|
||||||
|
ExtraParams: map[string]any{
|
||||||
|
"provider": "template",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||||
|
PromptDir: "./examples/prompts",
|
||||||
|
SchemaDir: "./examples/schemas",
|
||||||
|
}, scriptorium.WithProfiles(prof), scriptorium.WithLLMClient(fake))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected engine construction to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = engine.Run(context.Background(), scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
ProfileID: "template-profile",
|
||||||
|
APIKey: "template-key",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||||
|
"glossary": scriptorium.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected run to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
if len(fake.requests) != 1 {
|
||||||
|
t.Fatalf("expected one request, got %d", len(fake.requests))
|
||||||
|
}
|
||||||
|
if fake.requests[0].Target.Model != "template-model" || fake.requests[0].APIKey != "template-key" {
|
||||||
|
t.Fatalf("unexpected generated request: %+v", fake.requests[0])
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, map[string]any{"provider": "template"}) {
|
||||||
|
t.Fatalf("unexpected extra params: %#v", fake.requests[0].Target.ExtraParams)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInMemoryProfileAPIKeyRequiredBehavior(t *testing.T) {
|
||||||
|
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||||
|
PromptDir: "./examples/prompts",
|
||||||
|
SchemaDir: "./examples/schemas",
|
||||||
|
}, scriptorium.WithProfiles(scriptorium.Profile{
|
||||||
|
ID: "requires-key",
|
||||||
|
Endpoint: "http://requires-key/v1",
|
||||||
|
Model: "requires-key-model",
|
||||||
|
APIKeyRequired: true,
|
||||||
|
}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected engine construction to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req := scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
ProfileID: "requires-key",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||||
|
"glossary": scriptorium.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
_, err = engine.Prepare(context.Background(), req)
|
||||||
|
if !errors.Is(err, scriptorium.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("expected ErrInvalidRequest without API key, got %v", err)
|
||||||
|
}
|
||||||
|
req.APIKey = "direct-required-key"
|
||||||
|
if _, err := engine.Prepare(context.Background(), req); err != nil {
|
||||||
|
t.Fatalf("expected direct API key to satisfy APIKeyRequired, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInMemoryProfileWithoutAPIKeyRequiredWorksWithoutKey(t *testing.T) {
|
||||||
|
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||||
|
PromptDir: "./examples/prompts",
|
||||||
|
SchemaDir: "./examples/schemas",
|
||||||
|
}, scriptorium.WithProfiles(scriptorium.Profile{
|
||||||
|
ID: "no-key-required",
|
||||||
|
Endpoint: "http://no-key/v1",
|
||||||
|
Model: "no-key-model",
|
||||||
|
}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected engine construction to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
ProfileID: "no-key-required",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||||
|
"glossary": scriptorium.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected prepare without API key to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInMemoryProfileExtraParamsAreCopiedAcrossPublicBoundary(t *testing.T) {
|
||||||
|
fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}
|
||||||
|
labels := map[string]string{"route": "primary"}
|
||||||
|
ids := []int{1, 2, 3}
|
||||||
|
extraParams := map[string]any{
|
||||||
|
"labels": labels,
|
||||||
|
"ids": ids,
|
||||||
|
}
|
||||||
|
|
||||||
|
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||||
|
PromptDir: "./examples/prompts",
|
||||||
|
SchemaDir: "./examples/schemas",
|
||||||
|
},
|
||||||
|
scriptorium.WithProfiles(scriptorium.Profile{
|
||||||
|
ID: "copy-profile",
|
||||||
|
Endpoint: "http://copy/v1",
|
||||||
|
Model: "copy-model",
|
||||||
|
ExtraParams: extraParams,
|
||||||
|
}),
|
||||||
|
scriptorium.WithLLMClient(fake),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected engine construction to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
labels["route"] = "mutated-before-run"
|
||||||
|
ids[0] = 99
|
||||||
|
extraParams["added"] = "mutated"
|
||||||
|
|
||||||
|
_, err = engine.Run(context.Background(), scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
ProfileID: "copy-profile",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||||
|
"glossary": scriptorium.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected run to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
want := map[string]any{
|
||||||
|
"labels": map[string]string{"route": "primary"},
|
||||||
|
"ids": []int{1, 2, 3},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, want) {
|
||||||
|
t.Fatalf("captured extra params changed after mutation:\ngot=%#v\nwant=%#v", fake.requests[0].Target.ExtraParams, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunStructuredOutputWorksWithSchemaFS(t *testing.T) {
|
func TestRunStructuredOutputWorksWithSchemaFS(t *testing.T) {
|
||||||
fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: `{"events":[]}`}}
|
fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: `{"events":[]}`}}
|
||||||
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||||
|
|||||||
@@ -171,6 +171,7 @@ type ExecutionProfile struct {
|
|||||||
ServiceTier string `yaml:"service_tier"`
|
ServiceTier string `yaml:"service_tier"`
|
||||||
ReasoningEffort string `yaml:"reasoning_effort"`
|
ReasoningEffort string `yaml:"reasoning_effort"`
|
||||||
APIKeyEnv string `yaml:"api_key_env"`
|
APIKeyEnv string `yaml:"api_key_env"`
|
||||||
|
APIKeyRequired bool `yaml:"-" json:"-"`
|
||||||
ExtraParams map[string]any `yaml:"extra_params"`
|
ExtraParams map[string]any `yaml:"extra_params"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -209,6 +210,7 @@ type ExecutionTarget struct {
|
|||||||
ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"`
|
ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"`
|
||||||
APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"`
|
APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"`
|
||||||
APIKey string `yaml:"-" json:"-"`
|
APIKey string `yaml:"-" json:"-"`
|
||||||
|
APIKeyRequired bool `yaml:"-" json:"-"`
|
||||||
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
|
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ var (
|
|||||||
ErrInvalidRequest = errors.New("invalid run request")
|
ErrInvalidRequest = errors.New("invalid run request")
|
||||||
ErrProfileRequired = errors.New("profile selection is required")
|
ErrProfileRequired = errors.New("profile selection is required")
|
||||||
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
||||||
|
ErrAPIKeyRequired = errors.New("api key is required")
|
||||||
ErrProfileLoad = errors.New("failed to load prompt definition")
|
ErrProfileLoad = errors.New("failed to load prompt definition")
|
||||||
ErrArtifactLoad = errors.New("failed to load artifact")
|
ErrArtifactLoad = errors.New("failed to load artifact")
|
||||||
ErrPromptRender = errors.New("failed to render prompt")
|
ErrPromptRender = errors.New("failed to render prompt")
|
||||||
@@ -202,7 +203,7 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
|||||||
if strings.TrimSpace(effectiveModel.Model) == "" {
|
if strings.TrimSpace(effectiveModel.Model) == "" {
|
||||||
return nil, fmt.Errorf("%w: execution model is required", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: execution model is required", ErrInvalidRequest)
|
||||||
}
|
}
|
||||||
if err := validateAPIKey(effectiveModel.APIKeyEnv, effectiveModel.APIKey); err != nil {
|
if err := validateAPIKey(effectiveModel.APIKeyEnv, effectiveModel.APIKey, effectiveModel.APIKeyRequired); err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -363,6 +364,9 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution
|
|||||||
if strings.TrimSpace(override.APIKeyEnv) != "" {
|
if strings.TrimSpace(override.APIKeyEnv) != "" {
|
||||||
out.APIKeyEnv = override.APIKeyEnv
|
out.APIKeyEnv = override.APIKeyEnv
|
||||||
}
|
}
|
||||||
|
if override.APIKeyRequired {
|
||||||
|
out.APIKeyRequired = true
|
||||||
|
}
|
||||||
if len(override.ExtraParams) > 0 {
|
if len(override.ExtraParams) > 0 {
|
||||||
out.ExtraParams = copyExtraParams(override.ExtraParams)
|
out.ExtraParams = copyExtraParams(override.ExtraParams)
|
||||||
}
|
}
|
||||||
@@ -435,12 +439,15 @@ func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *dom
|
|||||||
return out, presence, nil
|
return out, presence, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateAPIKey(apiKeyEnv string, apiKey string) error {
|
func validateAPIKey(apiKeyEnv string, apiKey string, apiKeyRequired bool) error {
|
||||||
if strings.TrimSpace(apiKey) != "" {
|
if strings.TrimSpace(apiKey) != "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
envName := strings.TrimSpace(apiKeyEnv)
|
envName := strings.TrimSpace(apiKeyEnv)
|
||||||
if envName == "" {
|
if envName == "" {
|
||||||
|
if apiKeyRequired {
|
||||||
|
return ErrAPIKeyRequired
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(os.Getenv(envName)) == "" {
|
if strings.TrimSpace(os.Getenv(envName)) == "" {
|
||||||
@@ -463,6 +470,7 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
|
|||||||
ServiceTier: p.ServiceTier,
|
ServiceTier: p.ServiceTier,
|
||||||
ReasoningEffort: p.ReasoningEffort,
|
ReasoningEffort: p.ReasoningEffort,
|
||||||
APIKeyEnv: p.APIKeyEnv,
|
APIKeyEnv: p.APIKeyEnv,
|
||||||
|
APIKeyRequired: p.APIKeyRequired,
|
||||||
ExtraParams: copyExtraParams(p.ExtraParams),
|
ExtraParams: copyExtraParams(p.ExtraParams),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1205,6 +1205,49 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerPrepareAPIKeyRequiredFailsWithoutDirectKey(t *testing.T) {
|
||||||
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||||
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
|
||||||
|
}}
|
||||||
|
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||||
|
|
||||||
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrAPIKeyRequired) {
|
||||||
|
t.Fatalf("expected ErrAPIKeyRequired, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunAPIKeyRequiredSucceedsWithDirectKey(t *testing.T) {
|
||||||
|
const directKey = "direct-required-key"
|
||||||
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||||
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
|
||||||
|
}}
|
||||||
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
|
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||||
|
|
||||||
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
APIKey: directKey,
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if llmClient.lastReq.Target.APIKey != directKey {
|
||||||
|
t.Fatalf("expected direct API key to reach LLM request")
|
||||||
|
}
|
||||||
|
if !llmClient.lastReq.Target.APIKeyRequired {
|
||||||
|
t.Fatalf("expected APIKeyRequired to be carried to target")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
|
func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
|
||||||
const envName = "SCRIPTORIUM_RUNTIME_API_KEY"
|
const envName = "SCRIPTORIUM_RUNTIME_API_KEY"
|
||||||
t.Setenv(envName, "runtime-secret")
|
t.Setenv(envName, "runtime-secret")
|
||||||
@@ -1494,6 +1537,7 @@ func TestExecutionProfileToTargetPopulatesAllFieldsAndCopiesExtraParams(t *testi
|
|||||||
ServiceTier: "priority",
|
ServiceTier: "priority",
|
||||||
ReasoningEffort: "medium",
|
ReasoningEffort: "medium",
|
||||||
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
||||||
|
APIKeyRequired: true,
|
||||||
ExtraParams: map[string]any{
|
ExtraParams: map[string]any{
|
||||||
"provider_option": "on",
|
"provider_option": "on",
|
||||||
},
|
},
|
||||||
@@ -1508,7 +1552,8 @@ func TestExecutionProfileToTargetPopulatesAllFieldsAndCopiesExtraParams(t *testi
|
|||||||
target.TimeoutSeconds != src.TimeoutSeconds ||
|
target.TimeoutSeconds != src.TimeoutSeconds ||
|
||||||
target.ServiceTier != src.ServiceTier ||
|
target.ServiceTier != src.ServiceTier ||
|
||||||
target.ReasoningEffort != src.ReasoningEffort ||
|
target.ReasoningEffort != src.ReasoningEffort ||
|
||||||
target.APIKeyEnv != src.APIKeyEnv {
|
target.APIKeyEnv != src.APIKeyEnv ||
|
||||||
|
target.APIKeyRequired != src.APIKeyRequired {
|
||||||
t.Fatalf("expected all profile fields to populate target, got %+v", target)
|
t.Fatalf("expected all profile fields to populate target, got %+v", target)
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(target.ExtraParams, src.ExtraParams) {
|
if !reflect.DeepEqual(target.ExtraParams, src.ExtraParams) {
|
||||||
@@ -1533,6 +1578,7 @@ func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testin
|
|||||||
ServiceTier: "priority",
|
ServiceTier: "priority",
|
||||||
ReasoningEffort: "low",
|
ReasoningEffort: "low",
|
||||||
APIKeyEnv: "PROFILE_KEY",
|
APIKeyEnv: "PROFILE_KEY",
|
||||||
|
APIKeyRequired: true,
|
||||||
ExtraParams: map[string]any{
|
ExtraParams: map[string]any{
|
||||||
"profile_option": "enabled",
|
"profile_option": "enabled",
|
||||||
},
|
},
|
||||||
@@ -1553,7 +1599,8 @@ func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testin
|
|||||||
target.TimeoutSeconds != profileValue.TimeoutSeconds ||
|
target.TimeoutSeconds != profileValue.TimeoutSeconds ||
|
||||||
target.ServiceTier != profileValue.ServiceTier ||
|
target.ServiceTier != profileValue.ServiceTier ||
|
||||||
target.ReasoningEffort != profileValue.ReasoningEffort ||
|
target.ReasoningEffort != profileValue.ReasoningEffort ||
|
||||||
target.APIKeyEnv != profileValue.APIKeyEnv {
|
target.APIKeyEnv != profileValue.APIKeyEnv ||
|
||||||
|
target.APIKeyRequired != profileValue.APIKeyRequired {
|
||||||
t.Fatalf("expected profile values to populate target, got %+v", target)
|
t.Fatalf("expected profile values to populate target, got %+v", target)
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(target.ExtraParams, profileValue.ExtraParams) {
|
if !reflect.DeepEqual(target.ExtraParams, profileValue.ExtraParams) {
|
||||||
|
|||||||
105
profiles.go
Normal file
105
profiles.go
Normal file
@@ -0,0 +1,105 @@
|
|||||||
|
package scriptorium
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||||
|
)
|
||||||
|
|
||||||
|
// OpenAICompatibleProfile returns an in-memory profile for an OpenAI-compatible
|
||||||
|
// chat-completions endpoint.
|
||||||
|
func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile {
|
||||||
|
return Profile{
|
||||||
|
ID: cfg.ID,
|
||||||
|
Endpoint: cfg.Endpoint,
|
||||||
|
Model: cfg.Model,
|
||||||
|
Temperature: cfg.Temperature,
|
||||||
|
MaxTokens: cfg.MaxTokens,
|
||||||
|
TopP: cfg.TopP,
|
||||||
|
TimeoutSeconds: cfg.TimeoutSeconds,
|
||||||
|
ServiceTier: cfg.ServiceTier,
|
||||||
|
ReasoningEffort: cfg.ReasoningEffort,
|
||||||
|
APIKeyRequired: cfg.APIKeyRequired,
|
||||||
|
ExtraParams: copyAnyMap(cfg.ExtraParams),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type memoryProfileRepository struct {
|
||||||
|
profiles map[string]domain.ExecutionProfile
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMemoryProfileRepository(profiles []Profile) (*memoryProfileRepository, error) {
|
||||||
|
repo := &memoryProfileRepository{profiles: make(map[string]domain.ExecutionProfile, len(profiles))}
|
||||||
|
for _, publicProfile := range profiles {
|
||||||
|
prof, err := toDomainProfile(publicProfile)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if _, exists := repo.profiles[prof.ID]; exists {
|
||||||
|
return nil, fmt.Errorf("duplicate profile id %q", prof.ID)
|
||||||
|
}
|
||||||
|
repo.profiles[prof.ID] = prof
|
||||||
|
}
|
||||||
|
return repo, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *memoryProfileRepository) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
if r == nil {
|
||||||
|
return nil, profile.ErrProfileNotFound
|
||||||
|
}
|
||||||
|
prof, ok := r.profiles[id]
|
||||||
|
if !ok {
|
||||||
|
return nil, profile.ErrProfileNotFound
|
||||||
|
}
|
||||||
|
prof.ExtraParams = copyAnyMap(prof.ExtraParams)
|
||||||
|
return &prof, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func toDomainProfile(publicProfile Profile) (domain.ExecutionProfile, error) {
|
||||||
|
prof := domain.ExecutionProfile{
|
||||||
|
ID: strings.TrimSpace(publicProfile.ID),
|
||||||
|
Endpoint: publicProfile.Endpoint,
|
||||||
|
Model: publicProfile.Model,
|
||||||
|
Temperature: publicProfile.Temperature,
|
||||||
|
MaxTokens: publicProfile.MaxTokens,
|
||||||
|
TopP: publicProfile.TopP,
|
||||||
|
TimeoutSeconds: publicProfile.TimeoutSeconds,
|
||||||
|
ServiceTier: publicProfile.ServiceTier,
|
||||||
|
ReasoningEffort: publicProfile.ReasoningEffort,
|
||||||
|
APIKeyRequired: publicProfile.APIKeyRequired,
|
||||||
|
ExtraParams: copyAnyMap(publicProfile.ExtraParams),
|
||||||
|
}
|
||||||
|
if err := validatePublicProfile(prof); err != nil {
|
||||||
|
return domain.ExecutionProfile{}, err
|
||||||
|
}
|
||||||
|
return prof, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validatePublicProfile(prof domain.ExecutionProfile) error {
|
||||||
|
if strings.TrimSpace(prof.ID) == "" {
|
||||||
|
return errors.New("id is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(prof.Endpoint) == "" {
|
||||||
|
return errors.New("endpoint is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(prof.Model) == "" {
|
||||||
|
return errors.New("model is required")
|
||||||
|
}
|
||||||
|
if prof.Temperature < 0 || prof.Temperature > 2 {
|
||||||
|
return errors.New("temperature must be between 0 and 2")
|
||||||
|
}
|
||||||
|
if prof.MaxTokens < 0 {
|
||||||
|
return errors.New("max_tokens must be greater than or equal to 0")
|
||||||
|
}
|
||||||
|
if prof.TopP < 0 || prof.TopP > 1 {
|
||||||
|
return errors.New("top_p must be between 0 and 1")
|
||||||
|
}
|
||||||
|
if prof.TimeoutSeconds < 0 {
|
||||||
|
return errors.New("timeout_seconds must be greater than or equal to 0")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
30
types.go
30
types.go
@@ -154,6 +154,36 @@ type ExecutionTargetOverride struct {
|
|||||||
ExtraParams map[string]any
|
ExtraParams map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Profile is an in-memory execution profile for library consumers.
|
||||||
|
type Profile struct {
|
||||||
|
ID string
|
||||||
|
Endpoint string
|
||||||
|
Model string
|
||||||
|
Temperature float64
|
||||||
|
MaxTokens int
|
||||||
|
TopP float64
|
||||||
|
TimeoutSeconds int
|
||||||
|
ServiceTier string
|
||||||
|
ReasoningEffort string
|
||||||
|
APIKeyRequired bool
|
||||||
|
ExtraParams map[string]any
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenAICompatibleProfileConfig configures an OpenAI-compatible in-memory profile.
|
||||||
|
type OpenAICompatibleProfileConfig struct {
|
||||||
|
ID string
|
||||||
|
Endpoint string
|
||||||
|
Model string
|
||||||
|
APIKeyRequired bool
|
||||||
|
Temperature float64
|
||||||
|
MaxTokens int
|
||||||
|
TopP float64
|
||||||
|
TimeoutSeconds int
|
||||||
|
ServiceTier string
|
||||||
|
ReasoningEffort string
|
||||||
|
ExtraParams map[string]any
|
||||||
|
}
|
||||||
|
|
||||||
// ExecutionTargetPresence tracks which numeric runtime settings were explicit
|
// ExecutionTargetPresence tracks which numeric runtime settings were explicit
|
||||||
// request overrides.
|
// request overrides.
|
||||||
type ExecutionTargetPresence struct {
|
type ExecutionTargetPresence struct {
|
||||||
|
|||||||
Reference in New Issue
Block a user