28 Commits

Author SHA1 Message Date
e1e5351c5d Clean up completed roadmap docs 2026-07-04 17:24:02 -05:00
d60ef66f53 Implement a library profile API and built-in profile docs 2026-07-04 17:23:34 -05:00
4669b73d38 Update OpenAI-compatible auth docs 2026-07-04 17:04:25 +00:00
6f91603168 Add public asset source options 2026-07-04 17:02:09 +00:00
3ad247039b Add request API key support 2026-07-04 16:55:01 +00:00
32e2433628 Add built-in profile repository wiring 2026-07-04 16:48:18 +00:00
712c6b92b8 Add profile repository foundations 2026-07-04 16:41:28 +00:00
89cafcefec Add a roadmap, implementation plan, and built-in profile defaults for a production-ready public library package 2026-07-04 11:38:14 -05:00
1d7fac0a47 Implement fixes to the initial library facade 2026-07-04 11:32:47 -05:00
03d4f27d2b Document public library usage 2026-07-04 14:24:06 +00:00
4ac2038331 Add public run API with injectable LLM 2026-07-04 14:21:37 +00:00
14a7e7e04c Add public prepare engine API 2026-07-04 14:16:19 +00:00
5e522bad8b Add a roadmap and implementation plan for an initial public library package 2026-07-04 09:09:29 -05:00
23872dd742 Implement runtime parameter completion fixes
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-07-04 09:00:19 -05:00
7ffbf5f6ca Update woodpecker config to prepare only linux binaries 2026-07-04 08:53:30 -05:00
d0dc30fcc9 Document runtime provider parameters 2026-07-04 13:29:38 +00:00
b38f7b4dc3 Serialize runtime extra parameters outbound 2026-07-04 13:26:53 +00:00
0512995931 Allow JSON-compatible extra params 2026-07-04 13:24:27 +00:00
049a5feadb Make request execution overrides presence-aware 2026-07-04 13:20:39 +00:00
1798e9c575 Add a roadmap and implementation plan to support reasoning_effort and extra_params in outbound requests 2026-07-04 08:13:10 -05:00
5d4bc8c2b9 Remove completed feature roadmap docs 2026-07-02 20:12:10 -05:00
63fb8fc132 Implement support for OpenRouter sticky routing via a session_id variable 2026-07-02 20:08:44 -05:00
4d4bb7a121 Document prompt cache control behavior 2026-07-02 23:10:24 +00:00
5dcb3cd4fc Expose cache usage in adapters 2026-07-02 23:07:57 +00:00
efe346893c Serialize cache-controlled chat messages 2026-07-02 23:05:56 +00:00
c95d6fcfec Preserve cache control in rendered prompts 2026-07-02 23:03:50 +00:00
0badb4364d Add prompt cache control loading 2026-07-02 23:00:43 +00:00
1f63f8afbb Add a feature roadmap and implementation plan for cache_control values 2026-07-02 17:55:56 -05:00
75 changed files with 6801 additions and 289 deletions

1
.gitignore vendored
View File

@@ -1,6 +1,5 @@
# ---> Codex # ---> Codex
.codex .codex
AGENTS.md
# ---> Go # ---> Go
# If you prefer the allow list template instead of the deny list, see community template: # If you prefer the allow list template instead of the deny list, see community template:

View File

@@ -28,10 +28,6 @@ steps:
build_binary linux amd64 "" build_binary linux amd64 ""
build_binary linux arm64 "" build_binary linux arm64 ""
build_binary darwin amd64 ""
build_binary darwin arm64 ""
build_binary windows amd64 ".exe"
build_binary windows arm64 ".exe"
- name: publish-release - name: publish-release
image: woodpeckerci/plugin-release image: woodpeckerci/plugin-release

4
AGENTS.md Normal file
View File

@@ -0,0 +1,4 @@
Please carefully review the relevant documents in `docs/policy` before making any changes to this repository.
- `development.md` defines the contributor workflow for this application.
- `architecture.md` provides the canonical high-level architecture policy for this repository, and should be reviewed before writing or changing any code.
- `documentation.md` provides the canonical documentation policy for this repository, and should be reviewed before writing or changing any documentation.

View File

@@ -25,6 +25,7 @@ This command renders the prepared prompt and effective runtime settings without
- [Configuration reference](docs/config.md) - [Configuration reference](docs/config.md)
- [Operations guide](docs/operations.md) - [Operations guide](docs/operations.md)
- [Troubleshooting](docs/troubleshooting.md) - [Troubleshooting](docs/troubleshooting.md)
- [Go library package](docs/consumers/pkg-scriptorium.md)
- [HTTP API integration](docs/integrations/http-api.md) - [HTTP API integration](docs/integrations/http-api.md)
- [OpenAI-compatible chat integration](docs/integrations/openai-compatible-chat.md) - [OpenAI-compatible chat integration](docs/integrations/openai-compatible-chat.md)
- [Narratio subprocess integration](docs/integrations/narratio.md) - [Narratio subprocess integration](docs/integrations/narratio.md)
@@ -34,3 +35,4 @@ This command renders the prepared prompt and effective runtime settings without
- `examples/render-markdown-summary.sh` - `examples/render-markdown-summary.sh`
- `examples/http-run.json` - `examples/http-run.json`
- `examples/go-library/prepare`

398
convert.go Normal file
View File

@@ -0,0 +1,398 @@
package scriptorium
import (
"reflect"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
)
func toDomainRunRequest(req RunRequest) domain.RunRequest {
return domain.RunRequest{
PromptID: req.PromptID,
PromptVersion: req.PromptVersion,
ProfileID: req.ProfileID,
APIKey: req.APIKey,
Inputs: toDomainArtifactRefMap(req.Inputs),
Vars: copyStringMap(req.Vars),
Execution: toDomainExecutionTargetOverride(req.Execution),
Validation: toDomainOutputContractPtr(req.Validation),
Metadata: copyStringMap(req.Metadata),
}
}
func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
if prepared == nil {
return nil
}
return &PreparedRun{
PromptID: prepared.PromptID,
PromptVersion: prepared.PromptVersion,
PromptHash: prepared.PromptHash,
SelectedProfileID: prepared.SelectedProfileID,
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
OutputContract: fromDomainOutputContract(prepared.OutputContract),
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
InputHashes: copyStringMap(prepared.InputHashes),
SessionID: prepared.SessionID,
RenderedPromptHash: prepared.RenderedPromptHash,
Messages: fromDomainRenderedMessages(prepared.Messages),
StartTime: prepared.StartTime,
EndTime: prepared.EndTime,
DurationMS: prepared.DurationMS,
}
}
func fromDomainRunResult(result *domain.RunResult) *RunResult {
if result == nil {
return nil
}
return &RunResult{
RunID: result.RunID,
Artifact: fromDomainArtifact(result.Artifact),
RawOutput: result.RawOutput,
Validation: fromDomainValidationResult(result.Validation),
PromptID: result.PromptID,
PromptVersion: result.PromptVersion,
PromptHash: result.PromptHash,
RenderedPromptHash: result.RenderedPromptHash,
SelectedProfileID: result.SelectedProfileID,
ModelName: result.ModelName,
Endpoint: result.Endpoint,
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
InputHashes: copyStringMap(result.InputHashes),
Usage: fromDomainTokenUsage(result.Usage),
StartTime: result.StartTime,
EndTime: result.EndTime,
Duration: result.Duration,
}
}
func fromDomainGenerateRequest(req domain.GenerateRequest) GenerateRequest {
return GenerateRequest{
Prompt: fromDomainRenderedPrompt(req.Prompt),
Target: fromDomainExecutionTarget(req.Target),
TargetPresence: fromDomainExecutionTargetPresence(req.TargetPresence),
StructuredOutput: fromDomainStructuredOutputSpec(req.StructuredOutput),
APIKey: req.Target.APIKey,
}
}
func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse {
if resp == nil {
return nil
}
return &domain.GenerateResponse{
Content: resp.Content,
Usage: toDomainTokenUsage(resp.Usage),
}
}
func fromDomainRenderedPrompt(prompt domain.RenderedPrompt) RenderedPrompt {
return RenderedPrompt{
SessionID: prompt.SessionID,
Messages: fromDomainRenderedMessages(prompt.Messages),
}
}
func toDomainArtifactRefMap(src map[string]ArtifactRef) map[string]domain.ArtifactRef {
if src == nil {
return nil
}
out := make(map[string]domain.ArtifactRef, len(src))
for k, v := range src {
out[k] = toDomainArtifactRef(v)
}
return out
}
func toDomainArtifactRef(ref ArtifactRef) domain.ArtifactRef {
return domain.ArtifactRef{
Type: domain.ArtifactRefType(ref.Type),
URI: ref.URI,
Body: ref.Body,
}
}
func fromDomainArtifact(artifact domain.Artifact) Artifact {
return Artifact{
Name: artifact.Name,
ContentType: artifact.ContentType,
Body: copyBytes(artifact.Body),
URI: artifact.URI,
Size: artifact.Size,
Hash: artifact.Hash,
}
}
func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) *domain.ExecutionTargetOverride {
if override == nil {
return nil
}
return &domain.ExecutionTargetOverride{
Endpoint: override.Endpoint,
Model: override.Model,
Temperature: copyFloat64Ptr(override.Temperature),
MaxTokens: copyIntPtr(override.MaxTokens),
TopP: copyFloat64Ptr(override.TopP),
TimeoutSeconds: copyIntPtr(override.TimeoutSeconds),
ServiceTier: override.ServiceTier,
ReasoningEffort: override.ReasoningEffort,
APIKeyEnv: override.APIKeyEnv,
ExtraParams: copyAnyMap(override.ExtraParams),
}
}
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
return ExecutionTarget{
Endpoint: target.Endpoint,
Model: target.Model,
Temperature: target.Temperature,
MaxTokens: target.MaxTokens,
TopP: target.TopP,
TimeoutSeconds: target.TimeoutSeconds,
ServiceTier: target.ServiceTier,
ReasoningEffort: target.ReasoningEffort,
APIKeyEnv: target.APIKeyEnv,
ExtraParams: copyAnyMap(target.ExtraParams),
}
}
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
return ExecutionTargetPresence{
Temperature: presence.Temperature,
MaxTokens: presence.MaxTokens,
TopP: presence.TopP,
TimeoutSeconds: presence.TimeoutSeconds,
}
}
func toDomainOutputContractPtr(contract *OutputContract) *domain.OutputContract {
if contract == nil {
return nil
}
out := toDomainOutputContract(*contract)
return &out
}
func toDomainOutputContract(contract OutputContract) domain.OutputContract {
return domain.OutputContract{
Format: domain.OutputFormat(contract.Format),
ValidationMode: domain.ValidationMode(contract.ValidationMode),
SchemaPath: contract.SchemaPath,
RepairAttempts: contract.RepairAttempts,
}
}
func fromDomainOutputContract(contract domain.OutputContract) OutputContract {
return OutputContract{
Format: OutputFormat(contract.Format),
ValidationMode: ValidationMode(contract.ValidationMode),
SchemaPath: contract.SchemaPath,
RepairAttempts: contract.RepairAttempts,
}
}
func fromDomainValidationResult(result domain.ValidationResult) ValidationResult {
return ValidationResult{
Status: ValidationStatus(result.Status),
Mode: ValidationMode(result.Mode),
Errors: copyStringSlice(result.Errors),
SchemaPath: result.SchemaPath,
RepairAttempts: result.RepairAttempts,
IsValid: result.IsValid,
}
}
func fromDomainTokenUsage(usage domain.TokenUsage) TokenUsage {
return TokenUsage{
PromptTokens: usage.PromptTokens,
CompletionTokens: usage.CompletionTokens,
TotalTokens: usage.TotalTokens,
CachedTokens: usage.CachedTokens,
CacheWriteTokens: usage.CacheWriteTokens,
}
}
func toDomainTokenUsage(usage TokenUsage) domain.TokenUsage {
return domain.TokenUsage{
PromptTokens: usage.PromptTokens,
CompletionTokens: usage.CompletionTokens,
TotalTokens: usage.TotalTokens,
CachedTokens: usage.CachedTokens,
CacheWriteTokens: usage.CacheWriteTokens,
}
}
func fromDomainRenderedMessages(messages []domain.RenderedMessage) []RenderedMessage {
if messages == nil {
return nil
}
out := make([]RenderedMessage, len(messages))
for i, msg := range messages {
out[i] = RenderedMessage{
Role: msg.Role,
Content: msg.Content,
CacheControl: fromDomainCacheControl(msg.CacheControl),
}
}
return out
}
func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl {
if cacheControl == nil {
return nil
}
return &CacheControl{
Type: CacheControlType(cacheControl.Type),
TTL: cacheControl.TTL,
}
}
func fromDomainStructuredOutputSpec(spec *domain.StructuredOutputSpec) *StructuredOutputSpec {
if spec == nil {
return nil
}
out := &StructuredOutputSpec{
Type: StructuredOutputType(spec.Type),
}
if spec.JSONSchema != nil {
out.JSONSchema = &StructuredOutputJSONSpec{
Name: spec.JSONSchema.Name,
Strict: spec.JSONSchema.Strict,
Schema: copyAny(spec.JSONSchema.Schema),
}
}
return out
}
func copyStringMap(src map[string]string) map[string]string {
if src == nil {
return nil
}
out := make(map[string]string, len(src))
for k, v := range src {
out[k] = v
}
return out
}
func copyAnyMap(src map[string]any) map[string]any {
if src == nil {
return nil
}
out := make(map[string]any, len(src))
for k, v := range src {
out[k] = copyAny(v)
}
return out
}
func copyAny(value any) any {
if value == nil {
return nil
}
switch v := value.(type) {
case map[string]any:
return copyAnyMap(v)
case []any:
out := make([]any, len(v))
for i, item := range v {
out[i] = copyAny(item)
}
return out
case []string:
return copyStringSlice(v)
case []byte:
return copyBytes(v)
default:
return copyReflectValue(reflect.ValueOf(value)).Interface()
}
}
func copyReflectValue(value reflect.Value) reflect.Value {
if !value.IsValid() {
return value
}
switch value.Kind() {
case reflect.Interface:
if value.IsNil() {
return reflect.Zero(value.Type())
}
copied := copyReflectValue(value.Elem())
if copied.IsValid() && copied.Type().AssignableTo(value.Type()) {
return copied
}
out := reflect.New(value.Type()).Elem()
out.Set(copied)
return out
case reflect.Pointer:
if value.IsNil() {
return reflect.Zero(value.Type())
}
out := reflect.New(value.Type().Elem())
out.Elem().Set(copyReflectValue(value.Elem()))
return out
case reflect.Map:
if value.IsNil() {
return reflect.Zero(value.Type())
}
out := reflect.MakeMapWithSize(value.Type(), value.Len())
iter := value.MapRange()
for iter.Next() {
out.SetMapIndex(copyReflectValue(iter.Key()), copyReflectValue(iter.Value()))
}
return out
case reflect.Slice:
if value.IsNil() {
return reflect.Zero(value.Type())
}
out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap())
for i := 0; i < value.Len(); i++ {
out.Index(i).Set(copyReflectValue(value.Index(i)))
}
return out
case reflect.Array:
out := reflect.New(value.Type()).Elem()
for i := 0; i < value.Len(); i++ {
out.Index(i).Set(copyReflectValue(value.Index(i)))
}
return out
default:
return value
}
}
func copyStringSlice(src []string) []string {
if src == nil {
return nil
}
out := make([]string, len(src))
copy(out, src)
return out
}
func copyBytes(src []byte) []byte {
if src == nil {
return nil
}
out := make([]byte, len(src))
copy(out, src)
return out
}
func copyFloat64Ptr(src *float64) *float64 {
if src == nil {
return nil
}
v := *src
return &v
}
func copyIntPtr(src *int) *int {
if src == nil {
return nil
}
v := *src
return &v
}

View File

@@ -29,9 +29,13 @@ Integration references:
- `run` and `render` require: - `run` and `render` require:
- `--prompt` - `--prompt`
- at least one `--input` - at least one `--input`
- an effective `prompt_dir` and `profile_dir` (from flags or config) - an effective `prompt_dir` from flags or config
- `serve` requires an effective `prompt_dir` and `profile_dir` (from flags or config). - `serve` requires an effective `prompt_dir` from flags or config.
- `profile_dir` is optional. If omitted, only built-in profiles are available; if provided, custom profiles override built-ins with the same ID.
- Built-in profile IDs are listed in the [configuration reference](config.md#profile-definition-files).
- Positional arguments are rejected. - Positional arguments are rejected.
- Prompt cache control is configured in prompt YAML (`messages[].cache_control`), not with CLI flags.
- Provider-specific `reasoning_effort` and `extra_params` are configured in profile YAML or HTTP model overrides, not with CLI flags.
## Flag Reference ## Flag Reference
@@ -39,7 +43,7 @@ Integration references:
- `--config <path>`: app config file path. - `--config <path>`: app config file path.
- `--prompt-dir <dir>`: prompt definition directory. - `--prompt-dir <dir>`: prompt definition directory.
- `--profile-dir <dir>`: profile definition directory. - `--profile-dir <dir>`: custom profile definition directory.
- `--schema-dir <dir>`: schema base directory for `json_schema` validation. - `--schema-dir <dir>`: schema base directory for `json_schema` validation.
- `--prompt <id>`: prompt ID to execute. Required. - `--prompt <id>`: prompt ID to execute. Required.
- `--prompt-id <id>`: deprecated alias for `--prompt`. - `--prompt-id <id>`: deprecated alias for `--prompt`.
@@ -56,6 +60,11 @@ Integration references:
- `--top-p <float>`: runtime top-p override. - `--top-p <float>`: runtime top-p override.
- `--timeout <duration>`: runtime timeout override (Go duration syntax, for example `30s`, `2m`). - `--timeout <duration>`: runtime timeout override (Go duration syntax, for example `30s`, `2m`).
Numeric runtime override flags are presence-aware:
- omitted numeric flags preserve the selected profile/default value
- explicit zero values override the selected profile/default value (`--temperature 0`, `--max-tokens 0`, `--top-p 0`, `--timeout 0s`)
### `scriptorium render` ### `scriptorium render`
- Supports the same flags as `run`, except: - Supports the same flags as `run`, except:
@@ -72,7 +81,7 @@ Notes:
- `--config <path>`: app config file path. - `--config <path>`: app config file path.
- `--addr <listen-address>`: HTTP listen address. - `--addr <listen-address>`: HTTP listen address.
- `--prompt-dir <dir>`: prompt definition directory. - `--prompt-dir <dir>`: prompt definition directory.
- `--profile-dir <dir>`: profile definition directory. - `--profile-dir <dir>`: custom profile definition directory.
- `--schema-dir <dir>`: schema base directory for `json_schema` validation. - `--schema-dir <dir>`: schema base directory for `json_schema` validation.
Notes: Notes:
@@ -82,6 +91,7 @@ Notes:
- `--input name=path` maps prompt input names to local file paths. - `--input name=path` maps prompt input names to local file paths.
- `--var name=value` maps template variable names to values. - `--var name=value` maps template variable names to values.
- If a prompt defines `session_id: "{{ .session_id }}"`, pass the OpenRouter sticky-routing value with `--var session_id=<value>`.
- Both flags can be repeated. - Both flags can be repeated.
- Both flags also support comma-separated batches, for example: - Both flags also support comma-separated batches, for example:
- `--input transcript=./t.md,glossary=./g.yml` - `--input transcript=./t.md,glossary=./g.yml`
@@ -93,6 +103,7 @@ Notes:
- Writes generated artifact content to stdout by default. - Writes generated artifact content to stdout by default.
- Writes generated artifact content to `--out` when provided. - Writes generated artifact content to `--out` when provided.
- Prints run summary metadata to stderr on success. - Prints run summary metadata to stderr on success.
- Appends `cached_tokens=<n> cache_write_tokens=<n>` to the summary only when the provider reports non-zero cache usage.
- Prints errors to stderr on failure. - Prints errors to stderr on failure.
`render`: `render`:

View File

@@ -21,10 +21,9 @@ When `--config <path>` is provided, that file is required.
```yaml ```yaml
prompt_dir: ./examples/prompts prompt_dir: ./examples/prompts
profile_dir: ./examples/profiles
``` ```
This is enough to use `run` and `render` when prompt/profile files are valid. This is enough to use `run` and `render` when prompts select built-in profiles.
## Production-Oriented App Config ## Production-Oriented App Config
@@ -45,7 +44,7 @@ defaults:
Top-level fields: Top-level fields:
- `prompt_dir` (optional): default prompt definition directory. - `prompt_dir` (optional): default prompt definition directory.
- `profile_dir` (optional): default profile definition directory. - `profile_dir` (optional): default custom profile definition directory.
- `schema_dir` (optional): base directory for schema files used by `json_schema` validation. - `schema_dir` (optional): base directory for schema files used by `json_schema` validation.
- `server.addr` (optional): default listen address for `serve`. - `server.addr` (optional): default listen address for `serve`.
- `defaults.render_format` (optional): default `render` output format (`text` or `json`). - `defaults.render_format` (optional): default `render` output format (`text` or `json`).
@@ -104,6 +103,7 @@ Field reference:
- `version` (required): prompt version. - `version` (required): prompt version.
- `default_profile` (optional): profile ID used when request does not provide `profile_id`. - `default_profile` (optional): profile ID used when request does not provide `profile_id`.
- `description` (optional): prompt description. - `description` (optional): prompt description.
- `session_id` (optional): Go-template string for OpenRouter sticky-routing `session_id`; rendered from request vars.
- `inputs` (optional list): expected named inputs. - `inputs` (optional list): expected named inputs.
- `messages` (required list): prompt message templates. - `messages` (required list): prompt message templates.
- `output` (required object): output contract. - `output` (required object): output contract.
@@ -119,6 +119,7 @@ Field reference:
- `role` (required) - `role` (required)
- `content` or `content_file` (exactly one is required) - `content` or `content_file` (exactly one is required)
- `cache_control` (optional object): provider prompt-cache metadata for this message
Message rules: Message rules:
@@ -128,6 +129,35 @@ Message rules:
- Prompt decoding is strict; unknown YAML fields are rejected. - Prompt decoding is strict; unknown YAML fields are rejected.
- Duplicate prompt IDs are invalid. If multiple files declare the requested prompt ID, Scriptorium fails instead of choosing one. - Duplicate prompt IDs are invalid. If multiple files declare the requested prompt ID, Scriptorium fails instead of choosing one.
`messages[].cache_control` fields:
- `type` (required when `cache_control` is present): currently only `ephemeral`.
- `ttl` (optional): currently only `1h`; omitted from outbound requests when unset.
Example cache-controlled message:
```yaml
messages:
- role: system
content_file: ./stable_context.md
cache_control:
type: ephemeral
ttl: 1h
- role: user
content: |
{{input "transcript"}}
```
Use cache control on stable reusable prompt content. Dynamic per-run inputs before the cache-controlled message change the provider cache key.
Example prompt-level session ID:
```yaml
session_id: "{{ .session_id }}"
```
When configured, `session_id` is rendered with the same variable context as messages. The rendered value is trimmed, omitted when empty, and rejected if longer than 256 characters. CLI callers pass the value through `--var session_id=<value>`; HTTP callers pass it through `"vars": {"session_id": "<value>"}`.
`output` fields: `output` fields:
- `format` (required): `text`, `markdown`, or `json`. - `format` (required): `text`, `markdown`, or `json`.
@@ -142,7 +172,7 @@ Repair behavior boundary:
## Profile Definition Files ## Profile Definition Files
Execution profiles are YAML files anywhere under `profile_dir`, including nested subdirectories. Scriptorium includes built-in execution profiles. Custom execution profiles are YAML files anywhere under `profile_dir`, including nested subdirectories.
Subdirectories are organizational only. Callers still select profiles by the YAML `id`, not by file path. For example, `profiles/local/local-quality.yaml` may still declare `id: local-quality`, and callers use `--profile local-quality`. Subdirectories are organizational only. Callers still select profiles by the YAML `id`, not by file path. For example, `profiles/local/local-quality.yaml` may still declare `id: local-quality`, and callers use `--profile local-quality`.
@@ -158,6 +188,11 @@ top_p: 1.0
timeout_seconds: 90 timeout_seconds: 90
api_key_env: SCRIPTORIUM_API_KEY api_key_env: SCRIPTORIUM_API_KEY
service_tier: priority service_tier: priority
reasoning_effort: medium
extra_params:
provider_route: primary
provider_options:
retry_budget: 2
``` ```
Field reference: Field reference:
@@ -170,21 +205,55 @@ Field reference:
- `top_p` (optional): range `0..1` - `top_p` (optional): range `0..1`
- `timeout_seconds` (optional): `>= 0` - `timeout_seconds` (optional): `>= 0`
- `service_tier` (optional): provider-specific request tier such as OpenRouter `flex` or `priority` - `service_tier` (optional): provider-specific request tier such as OpenRouter `flex` or `priority`
- `reasoning_effort` (optional) - `reasoning_effort` (optional): serialized as top-level `reasoning_effort` in outbound chat-completions requests
- `api_key_env` (optional) - `api_key_env` (optional)
- `extra_params` (optional map of strings) - `extra_params` (optional map): JSON-compatible provider-specific parameters. Values may be strings, numbers, booleans, objects, or arrays.
Profile rules: Profile rules:
- `profile_dir` is optional. If omitted, only built-in profiles are available.
- If `profile_dir` is set, custom profiles from that directory override built-in profiles with the same `id`.
- Duplicate IDs within the custom profile directory are invalid. Matching IDs across custom and built-in profiles are valid override behavior.
- Profile decoding is strict; unknown YAML fields are rejected. - Profile decoding is strict; unknown YAML fields are rejected.
- Raw `api_key` is rejected; use `api_key_env`. - Raw `api_key` is rejected; use `api_key_env`.
- If `api_key_env` is set, that environment variable must be set when preparing/running. - If `api_key_env` is set, that environment variable must be set when preparing/running.
- Duplicate profile IDs are invalid. If multiple files declare the requested profile ID, Scriptorium fails instead of choosing one. - Duplicate profile IDs are invalid. If multiple files declare the requested profile ID, Scriptorium fails instead of choosing one.
- `extra_params` keys must not be empty and must not collide with reserved outbound request fields: `model`, `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, or `response_format`.
Built-in profile IDs:
| Provider | ID | Model | API key env |
| --- | --- | --- | --- |
| aion-labs | `aion-2` | `aion-labs/aion-2.0` | `OPENROUTER_API_KEY` |
| anthropic | `claude-fable-latest` | `~anthropic/claude-fable-latest` | `OPENROUTER_API_KEY` |
| anthropic | `claude-haiku-latest` | `~anthropic/claude-haiku-latest` | `OPENROUTER_API_KEY` |
| anthropic | `claude-opus-latest` | `~anthropic/claude-opus-latest` | `OPENROUTER_API_KEY` |
| anthropic | `claude-sonnet-latest` | `~anthropic/claude-sonnet-latest` | `OPENROUTER_API_KEY` |
| deepseek | `deepseek-3-2` | `deepseek/deepseek-v3.2` | `OPENROUTER_API_KEY` |
| deepseek | `deepseek-4-pro` | `deepseek/deepseek-v4-pro` | `OPENROUTER_API_KEY` |
| google | `gemini-2-flash` | `google/gemini-2.5-flash` | `OPENROUTER_API_KEY` |
| google | `gemini-2-flash-lite` | `google/gemini-2.5-flash-lite` | `OPENROUTER_API_KEY` |
| google | `gemini-2-pro` | `google/gemini-2.5-pro` | `OPENROUTER_API_KEY` |
| google | `gemini-3-flash-lite` | `google/gemini-3.1-flash-lite` | `OPENROUTER_API_KEY` |
| google | `gemini-flash-latest` | `~google/gemini-flash-latest` | `OPENROUTER_API_KEY` |
| google | `gemini-pro-latest` | `~google/gemini-pro-latest` | `OPENROUTER_API_KEY` |
| google | `gemma-4-31b` | `google/gemma-4-31b-it:exacto` | `OPENROUTER_API_KEY` |
| minimax | `minimax-m2` | `minimax/minimax-m2.5` | `OPENROUTER_API_KEY` |
| minimax | `minimax-m3` | `minimax/minimax-m3` | `OPENROUTER_API_KEY` |
| mistral | `mistral-large-2512` | `mistralai/mistral-large-2512` | `OPENROUTER_API_KEY` |
| mistral | `mistral-medium-3-5` | `mistralai/mistral-medium-3-5` | `OPENROUTER_API_KEY` |
| mistral | `mistral-small-3` | `mistralai/mistral-small-3.2-24b-instruct` | `OPENROUTER_API_KEY` |
| mistral | `mistral-small-4` | `mistralai/mistral-small-2603` | `OPENROUTER_API_KEY` |
| nvidia | `nemotron-3-ultra` | `nvidia/nemotron-3-ultra-550b-a55b` | `OPENROUTER_API_KEY` |
| openai | `gpt-5-mini` | `openai/gpt-5.4-mini` | `OPENROUTER_API_KEY` |
| openai | `gpt-5-nano` | `openai/gpt-5.4-nano` | `OPENROUTER_API_KEY` |
Current outbound request behavior: Current outbound request behavior:
- The OpenAI-compatible client currently serializes: `model`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, and optional `response_format` for `json_schema` prompts. - The OpenAI-compatible client currently serializes: `model`, optional `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, optional `response_format` for `json_schema` prompts, and `extra_params`.
- `reasoning_effort` and `extra_params` are parsed and carried in effective settings, but are not currently serialized into outbound chat-completions requests. - `extra_params` are flattened into provider-specific top-level JSON request fields. They are not wrapped in an `extra_params` object on the outbound provider request.
- Messages without `cache_control` serialize with string `content`.
- Messages with `cache_control` serialize as a single text content-block array containing `cache_control`.
## Schema Behavior ## Schema Behavior
@@ -211,7 +280,7 @@ Supported artifact reference types for request inputs are `file` and `inline`.
- App config: `examples/config.yml` - App config: `examples/config.yml`
- Prompt examples: `examples/prompts/` - Prompt examples: `examples/prompts/`
- Profile examples: `examples/profiles/` - Custom profile examples: `examples/profiles/`
- Schema examples: `examples/schemas/` - Schema examples: `examples/schemas/`
- Input fixtures: `examples/fixtures/` - Input fixtures: `examples/fixtures/`
- Render example script: `examples/render-markdown-summary.sh` - Render example script: `examples/render-markdown-summary.sh`

13
docs/consumers/api.md Normal file
View File

@@ -0,0 +1,13 @@
# Consumer API Overview
Scriptorium can be used by consumers through three implemented surfaces:
- CLI commands, documented in [CLI reference](../cli.md).
- HTTP `POST /v1/runs`, documented in [HTTP API integration](../integrations/http-api.md).
- Go package `gitea.maximumdirect.net/eric/scriptorium`, documented in [pkg-scriptorium](pkg-scriptorium.md).
The Go package is the typed in-process API. It prepares prompts, runs prompts, accepts file or inline artifacts, supports per-request execution overrides, and exposes stable public errors for `errors.Is`.
Use the Go package when the caller is a Go program that wants typed requests/results, context cancellation, repeated calls without subprocess overhead, or fake LLM injection for tests. Use the CLI or HTTP surfaces when process isolation, language neutrality, or an HTTP boundary is preferred.
Raw API key values are not accepted in public payloads and are not returned in prepared or run results. Execution profiles may reference an environment variable name through `api_key_env`.

View File

@@ -0,0 +1,161 @@
# Package scriptorium
Import path:
```go
import "gitea.maximumdirect.net/eric/scriptorium"
```
The root package is a public facade over Scriptorium's prompt execution use case. It keeps `internal/*` packages private while exposing typed construction, preparation, execution, inputs, results, and errors.
## Construct An Engine
```go
engine, err := scriptorium.NewEngine(scriptorium.Config{
PromptDir: "./examples/prompts",
ProfileDir: "./examples/profiles",
SchemaDir: "./examples/schemas",
})
if err != nil {
return err
}
```
`PromptDir` is required unless an explicit prompt source option is supplied. `ProfileDir` is optional; omit it to use built-in profiles only, or set it to overlay custom profiles above built-ins. `SchemaDir` defaults to the built-in schema directory. `Timeout` and `HTTPClient` configure the default OpenAI-compatible client used by `Run` when no custom LLM client is supplied.
## Asset Sources
Directory fields on `Config` remain the compatibility path. Explicit source options override the matching directory field:
- `WithPromptFS(fsys, root)` and `WithPromptFile(path)`
- `WithProfileFS(fsys, root)` and `WithProfileFile(path)`
- `WithSchemaFS(fsys, root)` and `WithSchemaFile(path)`
Prompt and profile sources load standard Scriptorium YAML with the same strict validation as directory sources. Prompt `content_file` paths resolve relative to the prompt file in the same source. Profile options overlay custom profiles above built-ins. Schema `fs.FS` sources preserve prompt `schema_path` semantics; schema file options expose the file by its base name.
## In-Memory Profiles
Use `WithProfiles` when the consuming application already has profile settings in typed Go configuration:
```go
profile := scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
ID: "app.default",
Endpoint: "https://openrouter.ai/api/v1",
Model: "mistralai/mistral-small-3.2-24b-instruct",
APIKeyRequired: true,
})
engine, err := scriptorium.NewEngine(cfg, scriptorium.WithProfiles(profile))
```
In-memory profiles have highest precedence, followed by configured profile file/FS/directory sources, then built-in profiles. Duplicate IDs in one `WithProfiles` call return `ErrInvalidConfig`.
`Profile` and `OpenAICompatibleProfileConfig` include endpoint, model, numeric defaults, service tier, reasoning effort, `APIKeyRequired`, and JSON-compatible `ExtraParams`. They do not accept raw API-key fields. When `APIKeyRequired` is true, pass the secret with `RunRequest.APIKey`.
## Prepare A Prompt
`Prepare` resolves the prompt definition, profile, inputs, variables, output contract, structured-output metadata, and rendered messages without calling an LLM.
```go
prepared, err := engine.Prepare(ctx, scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
},
})
if err != nil {
return err
}
_ = prepared.Messages
```
Input helpers:
- `scriptorium.File(path)` loads an input artifact from a file.
- `scriptorium.Inline(body)` passes inline input content.
- `scriptorium.InlineWithURI(uri, body)` passes inline content with URI metadata.
## Run A Prompt
`Run` prepares the prompt, calls the configured LLM client, builds the output artifact, and validates the output.
```go
result, err := engine.Run(ctx, scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
APIKey: apiKey,
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
},
})
if err != nil {
return err
}
_ = result.Artifact
```
`RunResult` includes the run ID, output artifact, raw output, validation result, prompt/profile/model metadata, effective model parameters, input hashes, token/cache usage, and timing fields. Validation content failures return a successful `RunResult` with failed validation status. Runtime validation errors return `ErrValidation`.
For the public Go API, pass provider credentials with `RunRequest.APIKey`. The value is request-scoped, uses `json:"-"`, is preferred over profile `api_key_env` by the default OpenAI-compatible client, and is not included in `PreparedRun` or `RunResult` JSON. Do not store raw keys in config, prompt files, or profile YAML.
## Inject An LLM Client
Use `WithLLMClient` for tests or custom model integrations:
```go
type fakeLLM struct{}
func (fakeLLM) Generate(ctx context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
return &scriptorium.GenerateResponse{
Content: "generated text",
Usage: scriptorium.TokenUsage{TotalTokens: 12},
}, nil
}
engine, err := scriptorium.NewEngine(cfg, scriptorium.WithLLMClient(fakeLLM{}))
```
The injected client receives the rendered prompt, effective execution target, target presence metadata for explicit numeric overrides, structured-output spec, and request API key when provided. `GenerateRequest.APIKey` also uses `json:"-"`; custom and fake clients should avoid logging or serializing it. `WithLLMClient(nil)` returns `ErrInvalidConfig`.
## Request Overrides
`RunRequest.Execution` accepts per-request overrides. Numeric override fields are pointers so explicit zero values are preserved:
```go
zero := 0
req.Execution = &scriptorium.ExecutionTargetOverride{
MaxTokens: &zero,
}
```
## Errors
Public methods wrap context while preserving stable sentinel checks with `errors.Is`:
- `ErrInvalidConfig`
- `ErrInvalidRequest`
- `ErrPromptNotFound`
- `ErrProfileNotFound`
- `ErrPromptLoad`
- `ErrProfileLoad`
- `ErrArtifactLoad`
- `ErrPromptRender`
- `ErrLLMGenerate`
- `ErrValidation`
Example:
```go
if errors.Is(err, scriptorium.ErrPromptNotFound) {
return err
}
```
## Examples
Run the prepare-only example from the repository root:
```bash
go run ./examples/go-library/prepare
```

View File

@@ -50,7 +50,10 @@ Copyable request example file:
"reasoning_effort": "medium", "reasoning_effort": "medium",
"api_key_env": "SCRIPTORIUM_API_KEY", "api_key_env": "SCRIPTORIUM_API_KEY",
"extra_params": { "extra_params": {
"route": "primary" "route": "primary",
"provider_options": {
"retry_budget": 2
}
} }
}, },
"include_raw_output": false "include_raw_output": false
@@ -67,6 +70,14 @@ Input reference types currently supported by runtime artifact loading:
- `file` - `file`
- `inline` - `inline`
Model override notes:
- Numeric model override fields distinguish omitted values from explicit zero values. For example, omitting `temperature` preserves the selected profile/default value, while `"temperature": 0` explicitly sets the effective temperature to zero.
- `extra_params` accepts JSON-compatible values: strings, numbers, booleans, objects, and arrays.
- `extra_params` are passed through effective model metadata and flattened into top-level provider request fields by the OpenAI-compatible client.
- `extra_params` keys must not be empty and must not collide with reserved outbound fields: `model`, `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, or `response_format`.
- Raw API-key values are not accepted. Use `api_key_env` to name an environment variable.
## Strict JSON Rules ## Strict JSON Rules
Request decoding uses strict JSON field checks: Request decoding uses strict JSON field checks:
@@ -119,7 +130,10 @@ Response shape:
"reasoning_effort": "medium", "reasoning_effort": "medium",
"api_key_env": "SCRIPTORIUM_API_KEY", "api_key_env": "SCRIPTORIUM_API_KEY",
"extra_params": { "extra_params": {
"route": "primary" "route": "primary",
"provider_options": {
"retry_budget": 2
}
} }
}, },
"input_hashes": { "input_hashes": {
@@ -128,7 +142,9 @@ Response shape:
"usage": { "usage": {
"prompt_tokens": 11, "prompt_tokens": 11,
"completion_tokens": 22, "completion_tokens": 22,
"total_tokens": 33 "total_tokens": 33,
"cached_tokens": 0,
"cache_write_tokens": 0
}, },
"start_time": "2026-05-04T12:00:00Z", "start_time": "2026-05-04T12:00:00Z",
"end_time": "2026-05-04T12:00:01Z", "end_time": "2026-05-04T12:00:01Z",
@@ -142,6 +158,8 @@ Response shape:
`raw_model_output` is omitted by default. `raw_model_output` is omitted by default.
`metadata.usage.cached_tokens` and `metadata.usage.cache_write_tokens` are always present as numbers. They are `0` when the provider omits compatible cache usage fields or reports no cache activity.
To include it, send: To include it, send:
- `"include_raw_output": true` - `"include_raw_output": true`

View File

@@ -26,15 +26,85 @@ Example:
Serialized JSON fields: Serialized JSON fields:
- `model` (required after fallback resolution) - `model` (required after fallback resolution)
- `messages` (role/content pairs from rendered prompt) - `session_id` (only when the rendered prompt includes a non-empty session ID)
- `temperature` (only when non-zero) - `messages` (rendered prompt messages)
- `max_tokens` (only when non-zero) - `temperature` (when non-zero, or when explicitly overridden to zero)
- `top_p` (only when non-zero) - `max_tokens` (when non-zero, or when explicitly overridden to zero)
- `top_p` (when non-zero, or when explicitly overridden to zero)
- `service_tier` (only when non-empty) - `service_tier` (only when non-empty)
- `reasoning_effort` (only when non-empty)
- `response_format` (only when structured output is provided) - `response_format` (only when structured output is provided)
- profile/request `extra_params` as additional provider-specific top-level fields
`service_tier` is provider-specific. OpenRouter currently documents request values such as `flex` and `priority`; Scriptorium forwards any non-empty configured value and lets the backend validate support. `service_tier` is provider-specific. OpenRouter currently documents request values such as `flex` and `priority`; Scriptorium forwards any non-empty configured value and lets the backend validate support.
`reasoning_effort` is provider-specific. Scriptorium forwards any non-empty configured value as top-level `reasoning_effort` and lets the backend validate support.
`extra_params` are flattened into the outbound JSON object. They are not wrapped in an `extra_params` object:
```json
{
"model": "gpt-4o-mini",
"messages": [
{
"role": "user",
"content": "rendered text"
}
],
"provider_route": "primary",
"provider_options": {
"retry_budget": 2
}
}
```
`extra_params` values must be JSON-compatible. Supported value shapes include strings, numbers, booleans, objects, and arrays.
Reserved `extra_params` keys are rejected before the HTTP request is made:
- `model`
- `session_id`
- `messages`
- `temperature`
- `max_tokens`
- `top_p`
- `service_tier`
- `reasoning_effort`
- `response_format`
Empty `extra_params` keys and values that cannot be encoded as JSON are also rejected before the HTTP request is made.
`session_id` is rendered from prompt YAML using request variables and serialized as a top-level JSON request field. Scriptorium does not send an `x-session-id` header. Empty rendered session IDs are omitted, and values longer than 256 characters are rejected before the HTTP request.
Messages without prompt cache control serialize with string `content`:
```json
{
"role": "system",
"content": "rendered text"
}
```
Messages with prompt cache control serialize as a single text content-block array:
```json
{
"role": "system",
"content": [
{
"type": "text",
"text": "rendered text",
"cache_control": {
"type": "ephemeral",
"ttl": "1h"
}
}
]
}
```
When cache-control `ttl` is unset in the prompt definition, `ttl` is omitted from the outbound payload.
Structured output is currently `json_schema` only, serialized as: Structured output is currently `json_schema` only, serialized as:
```json ```json
@@ -52,7 +122,12 @@ Structured output is currently `json_schema` only, serialized as:
## Authentication Header ## Authentication Header
If `Target.APIKeyEnv` is set: If `Target.APIKey` is set:
- set `Authorization: Bearer <value>`
- do not read `Target.APIKeyEnv`
If `Target.APIKey` is empty and `Target.APIKeyEnv` is set:
- resolve environment variable value at request time - resolve environment variable value at request time
- set `Authorization: Bearer <value>` - set `Authorization: Bearer <value>`
@@ -61,7 +136,7 @@ If the environment variable is unset/empty:
- request fails before HTTP call (`ErrInvalidRequest`) - request fails before HTTP call (`ErrInvalidRequest`)
If `Target.APIKeyEnv` is empty: If both `Target.APIKey` and `Target.APIKeyEnv` are empty:
- no `Authorization` header is sent - no `Authorization` header is sent
@@ -72,6 +147,7 @@ Base timeout comes from client configuration.
Per-request override: Per-request override:
- if `Target.TimeoutSeconds > 0`, use that value for request timeout - if `Target.TimeoutSeconds > 0`, use that value for request timeout
- if `Target.TimeoutSeconds == 0` and the value came from an explicit request override, disable the HTTP client timeout
- if `Target.TimeoutSeconds < 0`, request is rejected (`ErrInvalidRequest`) - if `Target.TimeoutSeconds < 0`, request is rejected (`ErrInvalidRequest`)
## Response Expectations ## Response Expectations
@@ -82,6 +158,13 @@ Expected successful response shape (subset used):
- `usage.prompt_tokens` - `usage.prompt_tokens`
- `usage.completion_tokens` - `usage.completion_tokens`
- `usage.total_tokens` - `usage.total_tokens`
- `usage.prompt_tokens_details.cached_tokens` (optional)
- `usage.cache_write_tokens` (optional)
Absent cache usage fields are treated as zero. Parsed cache usage is exposed through run results and adapter response surfaces as:
- `cached_tokens`
- `cache_write_tokens`
Malformed response conditions include: Malformed response conditions include:
@@ -99,10 +182,7 @@ Malformed responses return `ErrMalformedResponse`.
## Unsupported Or Non-Serialized Fields ## Unsupported Or Non-Serialized Fields
The following fields may exist in profile/effective settings but are not currently serialized into outbound chat-completions payloads: The client does not serialize top-level `cache_control`.
- `reasoning_effort`
- `extra_params`
No built-in retries, tool-calls, or multi-request payload modes are implemented in this client. No built-in retries, tool-calls, or multi-request payload modes are implemented in this client.

View File

@@ -8,12 +8,14 @@ This document describes implemented adapter/repository boundaries and their curr
- `internal/adapter/cli`: CLI command parsing, app wiring, stdout/stderr handling, exit codes. - `internal/adapter/cli`: CLI command parsing, app wiring, stdout/stderr handling, exit codes.
- `internal/adapter/http`: HTTP request/response mapping for `POST /v1/runs`. - `internal/adapter/http`: HTTP request/response mapping for `POST /v1/runs`.
- `internal/promptdef`: filesystem prompt-definition repository. - root package `scriptorium`: public Go library facade for preparing and running prompt requests.
- `internal/profile`: filesystem execution-profile repository. - `internal/promptdef`: filesystem and `fs.FS` prompt-definition repositories.
- `internal/profile`: filesystem, `fs.FS`, and overlay execution-profile repositories.
- `internal/profile/builtin`: embedded built-in execution-profile repository.
- `internal/artifact`: input artifact reader. - `internal/artifact`: input artifact reader.
- `internal/prompt`: Go-template renderer. - `internal/prompt`: Go-template renderer.
- `internal/llm`: OpenAI-compatible LLM client implementation. - `internal/llm`: OpenAI-compatible LLM client implementation.
- `internal/validate`: output validator. - `internal/validate`: filesystem and `fs.FS` output validators.
- `internal/format`: prepared-run formatters for `render` output. - `internal/format`: prepared-run formatters for `render` output.
## Inputs And Outputs ## Inputs And Outputs
@@ -22,16 +24,35 @@ CLI adapter:
- Input: process args, filesystem config/assets, environment. - Input: process args, filesystem config/assets, environment.
- Output: exit code, stdout artifact/prepared output, stderr summaries/errors. - Output: exit code, stdout artifact/prepared output, stderr summaries/errors.
- `run` summaries include cache usage counters only when either parsed cache counter is non-zero.
HTTP adapter: HTTP adapter:
- Input: JSON request body (`runRequestDTO`). - Input: JSON request body (`runRequestDTO`).
- Output: JSON success/error body with mapped status codes. - Output: JSON success/error body with mapped status codes.
- Success metadata includes token usage plus cache usage counters.
Filesystem repositories: Public library facade:
- Input: prompt/profile YAML files under configured directories. - Input: typed `scriptorium.RunRequest` values.
- Output: typed `PreparedRun` and `RunResult` values plus public sentinel errors.
- Custom LLM behavior is injected with `WithLLMClient`; otherwise the default OpenAI-compatible client is used.
- `RunRequest.APIKey` is a request-scoped Go value only; it is converted into internal execution state for LLM generation and stripped from public result types.
- Prompt, profile, and schema source options can use directories, single files, or `fs.FS` roots. Explicit source options override the matching `Config` directory field.
- Public types are facade types converted at the package boundary; internal domain types remain internal.
Prompt/profile repositories:
- Input: prompt/profile YAML files under configured directories or `fs.FS` roots.
- Output: normalized domain definitions/profiles or typed errors. - Output: normalized domain definitions/profiles or typed errors.
- Single-file public sources are represented as `fs.FS` roots containing one YAML file; lookup still uses YAML `id` values.
Profile repository composition:
- Built-in profiles are embedded and loaded through the same profile validation rules as filesystem profiles.
- When no custom profile directory is configured, the runner receives the built-in profile repository.
- When a custom profile directory/file/`fs.FS` source is configured, the runner receives an overlay repository with custom profiles as primary and built-ins as fallback.
- Overlay lookup falls back only after custom profile-not-found errors; custom load/validation/raw-key errors are returned directly.
Artifact reader: Artifact reader:
@@ -42,11 +63,13 @@ LLM adapter:
- Input: `domain.GenerateRequest`. - Input: `domain.GenerateRequest`.
- Output: `domain.GenerateResponse`. - Output: `domain.GenerateResponse`.
- Direct API-key values are preferred when present; otherwise `api_key_env` is resolved from the process environment.
Validator: Validator:
- Input: artifact body + output contract. - Input: artifact body + output contract.
- Output: validation result or runtime validation error. - Output: validation result or runtime validation error.
- Schema documents may be loaded from a directory, single file, or `fs.FS` root in the public package. CLI and HTTP continue to use directory-backed schema loading.
## Boundaries ## Boundaries
@@ -59,7 +82,7 @@ Validator:
Primary app settings consumed by adapters: Primary app settings consumed by adapters:
- `prompt_dir` - `prompt_dir`
- `profile_dir` - `profile_dir` (optional custom profile source)
- `schema_dir` - `schema_dir`
- `server.addr` - `server.addr`
- `defaults.render_format` - `defaults.render_format`
@@ -67,6 +90,8 @@ Primary app settings consumed by adapters:
Execution profile/request settings used through runner: Execution profile/request settings used through runner:
- `endpoint`, `model`, `temperature`, `max_tokens`, `top_p`, `timeout_seconds`, `service_tier`, `api_key_env`, `reasoning_effort`, `extra_params` - `endpoint`, `model`, `temperature`, `max_tokens`, `top_p`, `timeout_seconds`, `service_tier`, `api_key_env`, `reasoning_effort`, `extra_params`
- CLI and HTTP request adapters preserve caller intent for numeric runtime overrides. Omitted values remain absent; explicit zero values are mapped as explicit overrides.
- HTTP `extra_params` accepts JSON-compatible values and maps them to domain request overrides without provider-specific adapter logic.
## External Dependencies ## External Dependencies
@@ -81,7 +106,9 @@ Strict decoding and input checks:
- config/prompt/profile loaders reject unknown YAML fields. - config/prompt/profile loaders reject unknown YAML fields.
- prompt/profile repositories scan nested subdirectories recursively. - prompt/profile repositories scan nested subdirectories recursively.
- prompt/profile lookup uses YAML `id` values; subdirectory paths are organizational only. - prompt/profile lookup uses YAML `id` values; subdirectory paths are organizational only.
- prompt `content_file` paths resolve relative to the prompt YAML file within the same source.
- duplicate prompt/profile IDs are invalid and fail instead of using first-match behavior. - duplicate prompt/profile IDs are invalid and fail instead of using first-match behavior.
- duplicate profile IDs across custom and built-in sources are allowed; the custom source overrides the built-in profile.
- HTTP DTO decoder rejects unknown JSON fields. - HTTP DTO decoder rejects unknown JSON fields.
- raw API key payload fields are rejected by strict decoding in profile/http paths. - raw API key payload fields are rejected by strict decoding in profile/http paths.
@@ -93,8 +120,16 @@ Artifact refs:
LLM adapter: LLM adapter:
- endpoint appends `/chat/completions`. - endpoint appends `/chat/completions`.
- rendered messages without cache control serialize with string `content`.
- rendered messages with cache control serialize as one text content block with `cache_control`.
- non-empty `reasoning_effort` serializes as a top-level provider request field.
- `extra_params` flatten into provider-specific top-level JSON request fields.
- reserved `extra_params` keys are rejected before the provider call: `model`, `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, and `response_format`.
- empty `extra_params` keys and values that cannot be JSON-encoded are rejected before the provider call.
- compatible cache usage response fields are parsed into domain token usage.
- non-2xx responses map to request failure errors. - non-2xx responses map to request failure errors.
- malformed responses (including missing/empty first choice content) are errors. - malformed responses (including missing/empty first choice content) are errors.
- direct API-key values are never serialized in provider request bodies.
Validator: Validator:
@@ -141,4 +176,5 @@ Behavior highlights:
- Adapter packages do not own runner decision logic. - Adapter packages do not own runner decision logic.
- External request/response strictness is part of contract stability. - External request/response strictness is part of contract stability.
- Prepared-render output never includes resolved API key values. - Prepared-render output never includes resolved API key values.
- Outbound OpenAI-compatible request includes only currently serialized fields (`model`, `messages`, optional `temperature`, `max_tokens`, `top_p`, optional `service_tier`, optional `response_format`). - Outbound OpenAI-compatible request includes currently serialized first-class fields (`model`, optional `session_id`, `messages`, optional `temperature`, `max_tokens`, `top_p`, optional `service_tier`, optional `reasoning_effort`, optional `response_format`) plus validated `extra_params` flattened as provider-specific top-level fields.
- Outbound cache control is message-level only; no top-level cache-control field is serialized.

View File

@@ -103,16 +103,29 @@ Validation content failures are not run errors:
- built-in execution defaults - built-in execution defaults
- selected profile values - selected profile values
- request overrides - request overrides
6. verify required `api_key_env` environment variable: - request numeric overrides are presence-aware, so omitted values preserve the current effective value and explicit zero values override it
- missing/empty env value returns an invalid request with `ErrAPIKeyEnvMissing` 6. verify credentials when the effective target names `api_key_env`:
- only the environment-variable name is retained; secret value is never returned - a request-scoped direct API key satisfies the credential requirement
- otherwise a missing/empty env value returns an invalid request with `ErrAPIKeyEnvMissing`
- only the environment-variable name is returned in public output; secret values are never returned
7. resolve output contract and structured-output schema payload when `json_schema` mode is active. 7. resolve output contract and structured-output schema payload when `json_schema` mode is active.
8. read input artifacts. 8. read input artifacts.
9. render prompt messages. 9. render prompt messages, including any normalized message cache-control metadata.
10. compute prompt/input/render hashes and return `PreparedRun`. 10. compute prompt/input/render hashes and return `PreparedRun`.
`rendered_prompt_hash` includes cache-control metadata when present because it affects the outbound provider request. Prompts without cache control keep the role/content hash behavior.
`Prepare` does not call the LLM. `Prepare` does not call the LLM.
Runtime target notes:
- Profile `extra_params` and request `extra_params` carry JSON-compatible values through prepared output, run metadata, and `domain.GenerateRequest.Target`.
- The OpenAI-compatible client serializes non-empty `reasoning_effort` as a top-level provider request field.
- The OpenAI-compatible client flattens `extra_params` into provider-specific top-level JSON request fields.
- Empty `extra_params` keys, reserved outbound field names, and values that cannot be JSON-encoded fail before the provider request.
- Resolved API-key values are never serialized in prepared/run output, public results, logs, or HTTP responses.
- Public direct API-key values are carried only far enough to call the configured LLM client and are excluded from JSON/YAML serialization.
## Run Flow ## Run Flow
`Run` performs: `Run` performs:
@@ -123,7 +136,7 @@ Validation content failures are not run errors:
4. build output artifact content type from output format. 4. build output artifact content type from output format.
5. validate output. 5. validate output.
6. optionally attempt bounded repair when repairer is injected and contract allows it. 6. optionally attempt bounded repair when repairer is injected and contract allows it.
7. return `RunResult` with artifact, raw output, validation, hashes, profile/model metadata, usage, and timestamps. 7. return `RunResult` with artifact, raw output, validation, hashes, profile/model metadata, token/cache usage, and timestamps.
## Repair Hook Boundary ## Repair Hook Boundary

View File

@@ -2,12 +2,13 @@
## Purpose ## Purpose
Project documentation must help four audiences: Project documentation must help five audiences:
1. users who need to run the application; 1. users who need to run the application;
2. administrators/operators who need to configure and operate it; 2. administrators/operators who need to configure and operate it;
3. developers who need to understand and change it safely; 3. developers who need to understand and change it safely;
4. LLM coding agents that need clear scope, boundaries, and invariants. 4. LLM coding agents that need clear scope, boundaries, and invariants;
5. developers and LLM coding agents integrating this project from another codebase.
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition. Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
@@ -42,11 +43,14 @@ Canonical homes:
- project purpose and quickstart: `README.md` - project purpose and quickstart: `README.md`
- development principles: `docs/policy/architecture.md` - development principles: `docs/policy/architecture.md`
- public HTTP API reference: `docs/api.md`
- configuration reference: `docs/config.md` - configuration reference: `docs/config.md`
- CLI reference: `docs/cli.md` - CLI reference: `docs/cli.md`
- operations and recovery: `docs/operations.md` - operations and recovery: `docs/operations.md`
- troubleshooting: `docs/troubleshooting.md` - troubleshooting: `docs/troubleshooting.md`
- public API/package consumer guidance: `docs/consumers/`
- implemented internals: `docs/internal/` - implemented internals: `docs/internal/`
- external protocol, service, and file-format contracts: `docs/integrations/`
- future work: `docs/roadmap/` - future work: `docs/roadmap/`
- contributor workflow: `docs/policy/development.md` - contributor workflow: `docs/policy/development.md`
- copyable examples: `examples/` - copyable examples: `examples/`
@@ -106,7 +110,7 @@ Recommended:
- `examples/` - `examples/`
- `docs/policy/development.md` - `docs/policy/development.md`
### Modular, staged, service-oriented, or orchestration application ### Modular, service-oriented, or orchestration application
Required: Required:
- `docs/cli.md`, if CLI-based - `docs/cli.md`, if CLI-based
@@ -119,6 +123,31 @@ Recommended:
- `docs/troubleshooting.md` - `docs/troubleshooting.md`
- validated examples under `examples/` - validated examples under `examples/`
### Public HTTP API service
Required:
- `docs/api.md`
- `docs/cli.md`, if CLI-based
- `docs/config.md`, if config-driven
- `docs/operations.md`
- `docs/internal/`
- `docs/policy/development.md`
Recommended:
- `docs/troubleshooting.md`
- `docs/consumers/`, for task-oriented client integration guides
- `docs/integrations/`, for upstream/downstream service contracts
- validated examples under `examples/`
### Project with public packages or consumer APIs
Required:
- `docs/consumers/api.md`
- one `docs/consumers/pkg-<name>.md` file per public package, if public packages exist
Recommended:
- copyable consumer examples under `examples/`, if practical
## Required Documents ## Required Documents
### README.md ### README.md
@@ -159,7 +188,35 @@ It should include:
- architectural invariants; - architectural invariants;
- explicit non-goals, if useful. - explicit non-goals, if useful.
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light. Notably, this file should prescribe a core development *policy* that should remain unchanged as the application evolves. It is not a place for details (e.g., CLI flags) that could change over time.
The contents of `architecture.md` should be trim and concise. LLMs may be directed to review it routinely via AGENTS.md, CLAUDE.md, or similar.
### docs/api.md
**Audience:** external HTTP API consumers, developers, LLM coding agents integrating by HTTP
Required for projects whose primary public interface is HTTP.
`docs/api.md` is the canonical public HTTP API contract. It should be normative for external consumers and should not be duplicated by README, operations docs, consumer guides, or integration docs.
It should include:
1. base URL conventions;
2. authentication and authorization behavior, if implemented;
3. response envelope;
4. supported media types and content negotiation behavior;
5. shared query parameters;
6. endpoint reference grouped by route family;
7. request parameters and validation rules;
8. response fields, units, nullability, and optionality;
9. error response shape and status codes;
10. pagination, caching, rate-limit, idempotency, and retry behavior, if implemented;
11. compact request and response examples.
It must document only implemented endpoints and behavior. Planned endpoints, proposed fields, future filters, and experimental response shapes belong only under `docs/roadmap/`.
For HTTP API projects, `docs/consumers/` may provide task-oriented client integration guides, but those guides should link to `docs/api.md` for the authoritative endpoint contract.
### docs/policy/development.md ### docs/policy/development.md
@@ -175,7 +232,7 @@ It should include:
- dependency policy; - dependency policy;
- how to add config fields; - how to add config fields;
- how to add CLI flags; - how to add CLI flags;
- how to add stages/modules/adapters, if applicable; - how to add modules or adapters, if applicable;
- how to update examples; - how to update examples;
- documentation update expectations. - documentation update expectations.
@@ -216,7 +273,7 @@ Explain when commands are useful, not just their syntax.
**Audience:** administrators, operators **Audience:** administrators, operators
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures. Required for applications that maintain state, support resume behavior, run multi-step workflows, write durable artifacts, use remote storage, or require recovery procedures.
It should cover: It should cover:
@@ -244,11 +301,40 @@ Each entry should include:
- safe fix; - safe fix;
- relevant links. - relevant links.
### docs/consumers/
**Audience:** developers and LLM coding agents integrating this project from another codebase
Required for projects with public packages, SDKs, client APIs, plugin APIs, or other application-facing integration surfaces.
This directory describes how an external codebase should consume the project's public API. It should be task-oriented and copyable where useful. It is not the place for internal implementation details or operator procedures.
For projects whose public API is HTTP, `docs/consumers/` is not required, and it should not duplicate the endpoint reference in `docs/api.md`. If present, it may provide practical integration workflows, client-specific examples, or migration notes that link back to `docs/api.md`.
`docs/consumers/api.md` should provide the consumer-facing overview and primary implementation workflow. It should include:
1. intended consumer audience and use cases;
2. required inputs supplied by operators or deployment configuration;
3. recommended public package or API workflow;
4. minimal copyable example;
5. consumer responsibilities and boundaries;
6. retry, idempotency, or status behavior, if applicable;
7. links to package-specific docs and canonical integration contracts.
Package-specific docs should be named `pkg-<name>.md` and should include:
1. import path;
2. intended use cases;
3. primary types and functions needed by consumers;
4. minimal examples;
5. validation, error, retry, and boundary behavior;
6. links to canonical file-format or wire-protocol contracts.
### docs/internal/ ### docs/internal/
**Audience:** developers, LLM coding agents **Audience:** developers, LLM coding agents
Required for modular, staged, service-oriented, or orchestration projects. Required for modular, service-oriented, or orchestration projects.
This directory describes implemented internal components. It is not the roadmap. This directory describes implemented internal components. It is not the roadmap.
@@ -289,7 +375,9 @@ Roadmap docs should not be confused with current behavior.
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain. Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses. This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses or exposes.
For public HTTP API services, `docs/integrations/` should document upstream, downstream, storage, protocol, or runtime contracts that the service depends on or bridges. It should not become a second copy of the public HTTP endpoint reference; that belongs in `docs/api.md`.
Use one file per integration where useful. Use one file per integration where useful.
@@ -346,8 +434,10 @@ Before merging documentation changes, verify:
- README is concise and orientation-focused. - README is concise and orientation-focused.
- `docs/policy/architecture.md` describes development principles. - `docs/policy/architecture.md` describes development principles.
- `docs/api.md` is the canonical HTTP contract for HTTP API services.
- Future work appears only under `docs/roadmap/`. - Future work appears only under `docs/roadmap/`.
- User-facing docs avoid unnecessary internals. - User-facing docs avoid unnecessary internals.
- Consumer-facing docs explain public APIs without duplicating HTTP endpoint or integration contracts.
- Developer-facing docs preserve boundaries and invariants. - Developer-facing docs preserve boundaries and invariants.
- Config examples match the schema. - Config examples match the schema.
- CLI examples match real commands and flags. - CLI examples match real commands and flags.

View File

@@ -33,23 +33,23 @@ Relevant links:
- [Configuration reference](config.md) - [Configuration reference](config.md)
- [CLI reference](cli.md) - [CLI reference](cli.md)
## Missing Prompt/Profile Directory Settings ## Missing Prompt Directory Settings
Symptom: Symptom:
- CLI parse errors saying prompt directory or profile directory is required. - CLI parse errors saying prompt directory is required.
Likely cause: Likely cause:
- Neither CLI flags nor config provide effective `prompt_dir` / `profile_dir`. - Neither CLI flags nor config provide an effective `prompt_dir`.
Diagnostic step: Diagnostic step:
- Run the failing command with explicit `--prompt-dir` and `--profile-dir` once to verify. - Run the failing command with explicit `--prompt-dir` once to verify.
Safe fix: Safe fix:
- Set `prompt_dir` and `profile_dir` in config, or always pass both flags. - Set `prompt_dir` in config, or always pass `--prompt-dir`.
Relevant links: Relevant links:
@@ -252,6 +252,39 @@ Relevant links:
- [Configuration reference](config.md) - [Configuration reference](config.md)
- [Operations guide](operations.md) - [Operations guide](operations.md)
## Prompt Cache Misses Or No Cache Usage
Symptom:
- CLI run summary omits `cached_tokens` / `cache_write_tokens`.
- HTTP `metadata.usage.cached_tokens` and `metadata.usage.cache_write_tokens` are both `0`.
- Provider cost or latency does not improve after repeated similar runs.
Likely cause:
- The selected prompt has no `messages[].cache_control`.
- Dynamic per-run input appears before the cache-controlled message and changes the provider cache key.
- The provider does not support the serialized cache-control shape for the selected model.
- The provider imposes minimum token thresholds or cache-breakpoint limits.
Diagnostic step:
- Run `render --format json` and verify the intended rendered message includes `cache_control`.
- Confirm stable reusable context appears before the cache-controlled message, with dynamic input after it.
- Check provider docs/logs for model support, minimum token thresholds, and breakpoint limits.
Safe fix:
- Move stable reusable context before the cache-controlled message.
- Move highly dynamic input after the cache breakpoint.
- Keep `cache_control.type: ephemeral` and, when using `ttl`, set `ttl: 1h`.
- Use CLI cache counters or HTTP cache usage fields to verify cache reads/writes after rerunning.
Relevant links:
- [Configuration reference](config.md)
- [OpenAI-compatible chat integration](integrations/openai-compatible-chat.md)
## Validation Status Failed (`run` Exit 2 Or HTTP 200 With Failed Status) ## Validation Status Failed (`run` Exit 2 Or HTTP 200 With Failed Status)
Symptom: Symptom:

278
engine.go Normal file
View File

@@ -0,0 +1,278 @@
package scriptorium
import (
"context"
"errors"
"fmt"
"io/fs"
"net/http"
"os"
"path/filepath"
"strings"
"time"
artifactadapter "gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile/builtin"
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
)
// ErrInvalidConfig indicates invalid public engine configuration.
var ErrInvalidConfig = errors.New("invalid engine configuration")
var (
ErrInvalidRequest = errors.New("invalid run request")
ErrPromptNotFound = errors.New("prompt not found")
ErrProfileNotFound = errors.New("profile not found")
ErrPromptLoad = errors.New("failed to load prompt definition")
ErrProfileLoad = errors.New("failed to load execution profile")
ErrArtifactLoad = errors.New("failed to load artifact")
ErrPromptRender = errors.New("failed to render prompt")
ErrLLMGenerate = errors.New("failed to generate output")
ErrValidation = errors.New("failed to validate output")
)
// Engine prepares and runs Scriptorium prompt requests.
type Engine struct {
runner *usecase.Runner
}
// Config configures a public Scriptorium engine.
type Config struct {
PromptDir string
ProfileDir string
SchemaDir string
Timeout time.Duration
HTTPClient *http.Client
}
// Option customizes engine construction.
type Option func(*engineOptions) error
type engineOptions struct {
llmClient llm.Client
promptDefs promptdef.Repository
profiles profile.Repository
memoryProfiles profile.Repository
validator validate.Validator
promptSource bool
profileSource bool
memorySource bool
validatorSource bool
}
// WithLLMClient injects a custom LLM client for execution.
func WithLLMClient(client LLMClient) Option {
return func(options *engineOptions) error {
if client == nil {
return ErrInvalidConfig
}
options.llmClient = publicLLMClientAdapter{client: client}
return nil
}
}
func WithPromptFS(fsys fs.FS, root string) Option {
return func(options *engineOptions) error {
if fsys == nil {
return ErrInvalidConfig
}
if strings.TrimSpace(root) == "" {
return ErrInvalidConfig
}
options.promptDefs = promptdef.NewFSRepository(fsys, root)
options.promptSource = true
return nil
}
}
func WithPromptFile(path string) Option {
return func(options *engineOptions) error {
fsys, root, err := fileSource(path)
if err != nil {
return err
}
options.promptDefs = promptdef.NewFSRepository(fsys, root)
options.promptSource = true
return nil
}
}
func WithProfileFS(fsys fs.FS, root string) Option {
return func(options *engineOptions) error {
if fsys == nil {
return ErrInvalidConfig
}
if strings.TrimSpace(root) == "" {
return ErrInvalidConfig
}
options.profiles = profile.NewFSRepository(fsys, root)
options.profileSource = true
return nil
}
}
func WithProfileFile(path string) Option {
return func(options *engineOptions) error {
fsys, root, err := fileSource(path)
if err != nil {
return err
}
options.profiles = profile.NewFSRepository(fsys, root)
options.profileSource = true
return nil
}
}
// WithProfiles configures in-memory profiles that take precedence over
// configured profile files and built-in profiles.
func WithProfiles(profiles ...Profile) Option {
return func(options *engineOptions) error {
repo, err := newMemoryProfileRepository(profiles)
if err != nil {
return err
}
options.memoryProfiles = repo
options.memorySource = true
return nil
}
}
func WithSchemaFS(fsys fs.FS, root string) Option {
return func(options *engineOptions) error {
if fsys == nil {
return ErrInvalidConfig
}
if strings.TrimSpace(root) == "" {
return ErrInvalidConfig
}
options.validator = validate.NewFSValidator(fsys, root)
options.validatorSource = true
return nil
}
}
func WithSchemaFile(path string) Option {
return func(options *engineOptions) error {
fsys, root, err := fileSource(path)
if err != nil {
return err
}
options.validator = validate.NewFSValidator(fsys, root)
options.validatorSource = true
return nil
}
}
// NewEngine constructs an Engine using the same default internal components as
// the CLI and HTTP adapters.
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
var options engineOptions
for _, opt := range opts {
if opt == nil {
continue
}
if err := opt(&options); err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
}
}
promptDefs := options.promptDefs
if !options.promptSource {
if strings.TrimSpace(cfg.PromptDir) == "" {
return nil, fmt.Errorf("%w: prompt directory is required", ErrInvalidConfig)
}
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
}
profiles := builtin.NewRepositoryWithDirectory(cfg.ProfileDir)
if options.profileSource {
profiles = builtin.NewRepositoryWithPrimary(options.profiles)
}
if options.memorySource {
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
}
validator := options.validator
if !options.validatorSource {
schemaDir := cfg.SchemaDir
if strings.TrimSpace(schemaDir) == "" {
schemaDir = defaults.SchemaDirDefault
}
validator = validate.NewStandardValidator(schemaDir)
}
llmClient := options.llmClient
if llmClient == nil {
var err error
llmClient, err = llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
Timeout: cfg.Timeout,
HTTPClient: cfg.HTTPClient,
})
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
}
}
return &Engine{
runner: usecase.NewRunner(
promptDefs,
profiles,
artifactadapter.NewCompositeReader(),
prompt.NewGoRenderer(),
llmClient,
validator,
),
}, nil
}
func fileSource(name string) (fs.FS, string, error) {
cleanName := strings.TrimSpace(name)
if cleanName == "" {
return nil, "", ErrInvalidConfig
}
dir := filepath.Dir(cleanName)
base := filepath.Base(cleanName)
if base == "." || base == string(filepath.Separator) || strings.TrimSpace(base) == "" {
return nil, "", ErrInvalidConfig
}
info, err := os.Stat(cleanName)
if err != nil {
return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, cleanName, err)
}
if info.IsDir() {
return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, cleanName)
}
return os.DirFS(dir), filepath.ToSlash(base), nil
}
// Prepare resolves a prompt request without calling an LLM.
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
prepared, err := e.runner.Prepare(ctx, toDomainRunRequest(req))
if err != nil {
return nil, mapPublicError(err)
}
return fromDomainPreparedRun(prepared), nil
}
// Run executes a prompt request and returns the generated artifact and metadata.
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
result, err := e.runner.Run(ctx, toDomainRunRequest(req))
if err != nil {
return nil, mapPublicError(err)
}
return fromDomainRunResult(result), nil
}

1395
engine_test.go Normal file

File diff suppressed because it is too large Load Diff

77
errors.go Normal file
View File

@@ -0,0 +1,77 @@
package scriptorium
import (
"errors"
"fmt"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
)
func mapPublicError(err error) error {
if err == nil {
return nil
}
if hasPublicError(err) {
return err
}
publicErr := publicErrorFor(err)
if publicErr == nil {
return err
}
return fmt.Errorf("%w: %w", publicErr, err)
}
func hasPublicError(err error) bool {
for _, publicErr := range []error{
ErrInvalidConfig,
ErrInvalidRequest,
ErrPromptNotFound,
ErrProfileNotFound,
ErrPromptLoad,
ErrProfileLoad,
ErrArtifactLoad,
ErrPromptRender,
ErrLLMGenerate,
ErrValidation,
} {
if errors.Is(err, publicErr) {
return true
}
}
return false
}
func publicErrorFor(err error) error {
switch {
case errors.Is(err, promptdef.ErrPromptDefinitionNotFound):
return ErrPromptNotFound
case errors.Is(err, profile.ErrProfileNotFound):
return ErrProfileNotFound
case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition):
return ErrPromptLoad
case isProfileLoadCause(err):
return ErrProfileLoad
case errors.Is(err, usecase.ErrArtifactLoad):
return ErrArtifactLoad
case errors.Is(err, usecase.ErrPromptRender):
return ErrPromptRender
case errors.Is(err, usecase.ErrLLMGenerate):
return ErrLLMGenerate
case errors.Is(err, usecase.ErrValidation):
return ErrValidation
case errors.Is(err, usecase.ErrInvalidRequest):
return ErrInvalidRequest
case errors.Is(err, usecase.ErrProfileLoad):
return ErrPromptLoad
default:
return nil
}
}
func isProfileLoadCause(err error) bool {
return errors.Is(err, profile.ErrInvalidYAML) ||
errors.Is(err, profile.ErrInvalidProfile) ||
errors.Is(err, profile.ErrRawAPIKeyNotAllowed)
}

View File

@@ -0,0 +1,50 @@
package main
import (
"context"
"encoding/json"
"log"
"os"
"gitea.maximumdirect.net/eric/scriptorium"
)
func main() {
engine, err := scriptorium.NewEngine(scriptorium.Config{
PromptDir: "./examples/prompts",
ProfileDir: "./examples/profiles",
SchemaDir: "./examples/schemas",
})
if err != nil {
log.Fatal(err)
}
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
},
})
if err != nil {
log.Fatal(err)
}
summary := struct {
PromptID string `json:"prompt_id"`
SelectedProfileID string `json:"selected_profile_id"`
Model string `json:"model"`
MessageCount int `json:"message_count"`
InputHashes map[string]string `json:"input_hashes"`
}{
PromptID: prepared.PromptID,
SelectedProfileID: prepared.SelectedProfileID,
Model: prepared.EffectiveModelParams.Model,
MessageCount: len(prepared.Messages),
InputHashes: prepared.InputHashes,
}
if err := json.NewEncoder(os.Stdout).Encode(summary); err != nil {
log.Fatal(err)
}
}

View File

@@ -19,7 +19,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format" renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
"gitea.maximumdirect.net/eric/scriptorium/internal/llm" "gitea.maximumdirect.net/eric/scriptorium/internal/llm"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile" "gitea.maximumdirect.net/eric/scriptorium/internal/profile/builtin"
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt" "gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef" "gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase" "gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
@@ -33,8 +33,7 @@ const (
) )
const ( const (
errPromptDirRequired = "prompt directory is required; provide --prompt-dir or config.yml prompt_dir" errPromptDirRequired = "prompt directory is required; provide --prompt-dir or config.yml prompt_dir"
errProfileDirRequired = "profile directory is required; provide --profile-dir or config.yml profile_dir"
) )
type runConfig struct { type runConfig struct {
@@ -305,12 +304,14 @@ func parseServeArgs(args []string) (*serveConfig, error) {
cfg.schemaDir = settings.schemaDir cfg.schemaDir = settings.schemaDir
cfg.addr = settings.serverAddr cfg.addr = settings.serverAddr
if err := validateRequiredLibraryDirs(cfg.promptDir, cfg.profileDir); err != nil { if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
return nil, err return nil, err
} }
cfg.promptDir = filepath.Clean(cfg.promptDir) cfg.promptDir = filepath.Clean(cfg.promptDir)
cfg.profileDir = filepath.Clean(cfg.profileDir) if strings.TrimSpace(cfg.profileDir) != "" {
cfg.profileDir = filepath.Clean(cfg.profileDir)
}
cfg.schemaDir = filepath.Clean(cfg.schemaDir) cfg.schemaDir = filepath.Clean(cfg.schemaDir)
return cfg, nil return cfg, nil
} }
@@ -353,7 +354,7 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
cfg.schemaDir = settings.schemaDir cfg.schemaDir = settings.schemaDir
cfg.defaultRenderFormat = settings.defaultRenderFormat cfg.defaultRenderFormat = settings.defaultRenderFormat
if err := validateRequiredLibraryDirs(cfg.promptDir, cfg.profileDir); err != nil { if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
return err return err
} }
if strings.TrimSpace(cfg.promptID) == "" { if strings.TrimSpace(cfg.promptID) == "" {
@@ -363,7 +364,9 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
return errors.New("at least one --input is required") return errors.New("at least one --input is required")
} }
cfg.promptDir = filepath.Clean(cfg.promptDir) cfg.promptDir = filepath.Clean(cfg.promptDir)
cfg.profileDir = filepath.Clean(cfg.profileDir) if strings.TrimSpace(cfg.profileDir) != "" {
cfg.profileDir = filepath.Clean(cfg.profileDir)
}
if cfg.outputPath != "" { if cfg.outputPath != "" {
cfg.outputPath = filepath.Clean(cfg.outputPath) cfg.outputPath = filepath.Clean(cfg.outputPath)
} }
@@ -467,20 +470,17 @@ func resolveCommonSettings(fs *flag.FlagSet, configPath string, overrides appcon
}, nil }, nil
} }
func validateRequiredLibraryDirs(promptDir, profileDir string) error { func validateRequiredLibraryDirs(promptDir string) error {
if strings.TrimSpace(promptDir) == "" { if strings.TrimSpace(promptDir) == "" {
return errors.New(errPromptDirRequired) return errors.New(errPromptDirRequired)
} }
if strings.TrimSpace(profileDir) == "" {
return errors.New(errProfileDirRequired)
}
return nil return nil
} }
func newRunner(promptDir, profileDir, schemaDir string, llmClient llm.Client) *usecase.Runner { func newRunner(promptDir, profileDir, schemaDir string, llmClient llm.Client) *usecase.Runner {
return usecase.NewRunner( return usecase.NewRunner(
promptdef.NewFilesystemRepository(promptDir), promptdef.NewFilesystemRepository(promptDir),
profile.NewFilesystemRepository(profileDir), builtin.NewRepositoryWithDirectory(profileDir),
artifactadapter.NewCompositeReader(), artifactadapter.NewCompositeReader(),
prompt.NewGoRenderer(), prompt.NewGoRenderer(),
llmClient, llmClient,
@@ -513,18 +513,25 @@ func buildRunRequestFromConfig(cfg *runConfig) (domain.RunRequest, error) {
inputs[name] = domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path} inputs[name] = domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path}
} }
var modelOverride *domain.ExecutionTarget var modelOverride *domain.ExecutionTargetOverride
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.apiKeyEnvSet || cfg.timeoutSet { if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.apiKeyEnvSet || cfg.timeoutSet {
modelOverride = &domain.ExecutionTarget{ modelOverride = &domain.ExecutionTargetOverride{
Endpoint: cfg.llmBaseURL, Endpoint: cfg.llmBaseURL,
Model: cfg.model, Model: cfg.model,
Temperature: cfg.temperature, APIKeyEnv: cfg.apiKeyEnv,
MaxTokens: cfg.maxTokens, }
TopP: cfg.topP, if cfg.temperatureSet {
APIKeyEnv: cfg.apiKeyEnv, modelOverride.Temperature = &cfg.temperature
}
if cfg.maxTokensSet {
modelOverride.MaxTokens = &cfg.maxTokens
}
if cfg.topPSet {
modelOverride.TopP = &cfg.topP
} }
if cfg.timeoutSet { if cfg.timeoutSet {
modelOverride.TimeoutSeconds = int(cfg.timeout.Seconds()) timeoutSeconds := int(cfg.timeout.Seconds())
modelOverride.TimeoutSeconds = &timeoutSeconds
} }
} }
@@ -606,7 +613,7 @@ func printSummary(stderr io.Writer, res *domain.RunResult) {
if res == nil { if res == nil {
return return
} }
fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d\n", fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d",
res.PromptID, res.PromptID,
res.PromptVersion, res.PromptVersion,
res.SelectedProfileID, res.SelectedProfileID,
@@ -620,6 +627,10 @@ func printSummary(stderr io.Writer, res *domain.RunResult) {
res.Usage.CompletionTokens, res.Usage.CompletionTokens,
res.Usage.TotalTokens, res.Usage.TotalTokens,
) )
if res.Usage.CachedTokens != 0 || res.Usage.CacheWriteTokens != 0 {
fmt.Fprintf(stderr, " cached_tokens=%d cache_write_tokens=%d", res.Usage.CachedTokens, res.Usage.CacheWriteTokens)
}
fmt.Fprintln(stderr)
} }
func printUsage(w io.Writer) { func printUsage(w io.Writer) {

View File

@@ -74,12 +74,12 @@ func TestParseRunArgsRequiredFlags(t *testing.T) {
t.Fatalf("expected clear prompt-dir guidance, got %v", err) t.Fatalf("expected clear prompt-dir guidance, got %v", err)
} }
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p", "--input", "a=b"}) cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p", "--input", "a=b"})
if err == nil { if err != nil {
t.Fatal("expected missing --profile-dir error") t.Fatalf("expected missing --profile-dir to be accepted, got %v", err)
} }
if !strings.Contains(err.Error(), "profile directory is required") { if cfg.profileDir != "" {
t.Fatalf("expected clear profile-dir guidance, got %v", err) t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
} }
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--input", "a=b"}) _, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--input", "a=b"})
@@ -161,17 +161,12 @@ func TestParseServeArgsRequiredFlags(t *testing.T) {
t.Fatalf("expected clear prompt-dir guidance, got %v", err) t.Fatalf("expected clear prompt-dir guidance, got %v", err)
} }
_, err = parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts"}) cfg, err := parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts"})
if err == nil {
t.Fatal("expected missing --profile-dir error")
}
if !strings.Contains(err.Error(), "profile directory is required") {
t.Fatalf("expected clear profile-dir guidance, got %v", err)
}
cfg, err := parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles"})
if err != nil { if err != nil {
t.Fatalf("expected valid serve args, got %v", err) t.Fatalf("expected missing --profile-dir to be accepted, got %v", err)
}
if cfg.profileDir != "" {
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
} }
if cfg.addr != defaults.HTTPAddrDefault { if cfg.addr != defaults.HTTPAddrDefault {
t.Fatalf("expected default addr %s, got %q", defaults.HTTPAddrDefault, cfg.addr) t.Fatalf("expected default addr %s, got %q", defaults.HTTPAddrDefault, cfg.addr)
@@ -565,21 +560,21 @@ profile_dir: ./profiles
} }
} }
func TestParseRunArgsFailsClearlyWhenNoEffectiveProfileDir(t *testing.T) { func TestParseRunArgsAcceptsMissingEffectiveProfileDir(t *testing.T) {
configPath := writeAppConfigFile(t, ` configPath := writeAppConfigFile(t, `
prompt_dir: ./prompts prompt_dir: ./prompts
`) `)
_, err := parseRunArgs([]string{ cfg, err := parseRunArgs([]string{
"--config", configPath, "--config", configPath,
"--prompt", "p", "--prompt", "p",
"--input", "a=b", "--input", "a=b",
}) })
if err == nil { if err != nil {
t.Fatal("expected missing profile_dir error") t.Fatalf("expected missing profile_dir to be accepted, got %v", err)
} }
if !strings.Contains(err.Error(), "profile directory is required") || !strings.Contains(err.Error(), "config.yml profile_dir") { if cfg.profileDir != "" {
t.Fatalf("expected clear profile_dir guidance, got %v", err) t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
} }
} }
@@ -601,21 +596,21 @@ profile_dir: ./profiles
} }
} }
func TestParseRenderArgsFailsClearlyWhenNoEffectiveProfileDir(t *testing.T) { func TestParseRenderArgsAcceptsMissingEffectiveProfileDir(t *testing.T) {
configPath := writeAppConfigFile(t, ` configPath := writeAppConfigFile(t, `
prompt_dir: ./prompts prompt_dir: ./prompts
`) `)
_, err := parseRenderArgs([]string{ cfg, err := parseRenderArgs([]string{
"--config", configPath, "--config", configPath,
"--prompt", "p", "--prompt", "p",
"--input", "a=b", "--input", "a=b",
}) })
if err == nil { if err != nil {
t.Fatal("expected missing profile_dir error") t.Fatalf("expected missing profile_dir to be accepted, got %v", err)
} }
if !strings.Contains(err.Error(), "profile directory is required") || !strings.Contains(err.Error(), "config.yml profile_dir") { if cfg.profileDir != "" {
t.Fatalf("expected clear profile_dir guidance, got %v", err) t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
} }
} }
@@ -747,6 +742,35 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te
} }
} }
func TestRenderCommandExplicitZeroTemperatureReachesEffectiveSettings(t *testing.T) {
lib := newCLITestLibrary(t)
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
profile := `id: local-default
endpoint: http://127.0.0.1:1/v1
model: profile-model
temperature: 0.7
`
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-default.yaml"), []byte(profile), 0o644); err != nil {
t.Fatalf("failed to write profile fixture: %v", err)
}
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--prompt-dir", lib.promptDir,
"--profile-dir", lib.profileDir,
"--prompt", "prompt.render",
"--input", "transcript=" + inputPath,
"--temperature", "0",
})
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
if !strings.Contains(stdout, "\n temperature: 0\n") {
t.Fatalf("expected explicit zero temperature in effective settings, got:\n%s", stdout)
}
}
func TestRenderCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) { func TestRenderCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
lib := newCLITestLibrary(t) lib := newCLITestLibrary(t)
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript") inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
@@ -900,6 +924,29 @@ func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
} }
} }
func TestRenderCommandUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
t.Setenv("OPENROUTER_API_KEY", "test-key")
lib := newCLITestLibrary(t)
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
writePromptFile(t, lib.promptDir, "prompt.builtin", "mistral-small-3")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--prompt-dir", lib.promptDir,
"--prompt", "prompt.builtin",
"--input", "transcript=" + inputPath,
})
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
if !strings.Contains(stdout, "selected_profile_id: mistral-small-3") {
t.Fatalf("expected built-in selected profile, got %q", stdout)
}
if !strings.Contains(stdout, "model: mistralai/mistral-small-3.2-24b-instruct") {
t.Fatalf("expected built-in model, got %q", stdout)
}
}
func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) { func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
lib := newCLITestLibrary(t) lib := newCLITestLibrary(t)
inputPath := lib.writeInputFile(t, "transcript.md", "hello") inputPath := lib.writeInputFile(t, "transcript.md", "hello")
@@ -1064,6 +1111,38 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
if !strings.Contains(stderr.String(), "prompt=p@1") { if !strings.Contains(stderr.String(), "prompt=p@1") {
t.Fatalf("expected summary on stderr, got %q", stderr.String()) t.Fatalf("expected summary on stderr, got %q", stderr.String())
} }
if strings.Contains(stderr.String(), "cached_tokens=") || strings.Contains(stderr.String(), "cache_write_tokens=") {
t.Fatalf("expected zero cache usage to be omitted from summary, got %q", stderr.String())
}
}
func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) {
var stderr bytes.Buffer
printSummary(&stderr, &domain.RunResult{
PromptID: "p",
PromptVersion: "1",
SelectedProfileID: "exec",
ModelName: "m",
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic},
RenderedPromptHash: "h",
InputHashes: map[string]string{"in": "x"},
Usage: domain.TokenUsage{
PromptTokens: 10,
CompletionTokens: 5,
TotalTokens: 15,
CachedTokens: 0,
CacheWriteTokens: 3,
},
})
summary := stderr.String()
if !strings.Contains(summary, "usage=10/5/15") {
t.Fatalf("expected base usage summary, got %q", summary)
}
if !strings.Contains(summary, "cached_tokens=0 cache_write_tokens=3") {
t.Fatalf("expected cache usage in summary, got %q", summary)
}
} }
type cliTestLibrary struct { type cliTestLibrary struct {

View File

@@ -21,16 +21,16 @@ type inputRefDTO struct {
} }
type modelOverrideRequestDTO struct { type modelOverrideRequestDTO struct {
Endpoint string `json:"endpoint,omitempty"` Endpoint string `json:"endpoint,omitempty"`
Model string `json:"model,omitempty"` Model string `json:"model,omitempty"`
Temperature float64 `json:"temperature,omitempty"` Temperature *float64 `json:"temperature,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"` MaxTokens *int `json:"max_tokens,omitempty"`
TopP float64 `json:"top_p,omitempty"` TopP *float64 `json:"top_p,omitempty"`
TimeoutSeconds int `json:"timeout_seconds,omitempty"` TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
ServiceTier string `json:"service_tier,omitempty"` ServiceTier string `json:"service_tier,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"` ReasoningEffort string `json:"reasoning_effort,omitempty"`
APIKeyEnv string `json:"api_key_env,omitempty"` APIKeyEnv string `json:"api_key_env,omitempty"`
ExtraParams map[string]string `json:"extra_params,omitempty"` ExtraParams map[string]any `json:"extra_params,omitempty"`
} }
type runResponseDTO struct { type runResponseDTO struct {
@@ -70,22 +70,24 @@ type metadataDTO struct {
} }
type modelParamsDTO struct { type modelParamsDTO struct {
Endpoint string `json:"endpoint"` Endpoint string `json:"endpoint"`
Model string `json:"model"` Model string `json:"model"`
Temperature float64 `json:"temperature"` Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"` MaxTokens int `json:"max_tokens"`
TopP float64 `json:"top_p"` TopP float64 `json:"top_p"`
TimeoutSeconds int `json:"timeout_seconds"` TimeoutSeconds int `json:"timeout_seconds"`
ServiceTier string `json:"service_tier,omitempty"` ServiceTier string `json:"service_tier,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"` ReasoningEffort string `json:"reasoning_effort,omitempty"`
APIKeyEnv string `json:"api_key_env,omitempty"` APIKeyEnv string `json:"api_key_env,omitempty"`
ExtraParams map[string]string `json:"extra_params,omitempty"` ExtraParams map[string]any `json:"extra_params,omitempty"`
} }
type tokenUsageDTO struct { type tokenUsageDTO struct {
PromptTokens int `json:"prompt_tokens"` PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"` CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"` TotalTokens int `json:"total_tokens"`
CachedTokens int `json:"cached_tokens"`
CacheWriteTokens int `json:"cache_write_tokens"`
} }
type validationDTO struct { type validationDTO struct {

View File

@@ -61,9 +61,9 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
} }
var model *domain.ExecutionTarget var model *domain.ExecutionTargetOverride
if req.Model != nil { if req.Model != nil {
model = executionTargetFromModelOverrideDTO(req.Model) model = executionTargetOverrideFromModelOverrideDTO(req.Model)
} }
res, err := h.runner.Run(r.Context(), domain.RunRequest{ res, err := h.runner.Run(r.Context(), domain.RunRequest{
@@ -105,6 +105,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
PromptTokens: res.Usage.PromptTokens, PromptTokens: res.Usage.PromptTokens,
CompletionTokens: res.Usage.CompletionTokens, CompletionTokens: res.Usage.CompletionTokens,
TotalTokens: res.Usage.TotalTokens, TotalTokens: res.Usage.TotalTokens,
CachedTokens: res.Usage.CachedTokens,
CacheWriteTokens: res.Usage.CacheWriteTokens,
}, },
StartTime: res.StartTime, StartTime: res.StartTime,
EndTime: res.EndTime, EndTime: res.EndTime,
@@ -121,11 +123,11 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, resp) writeJSON(w, http.StatusOK, resp)
} }
func executionTargetFromModelOverrideDTO(dto *modelOverrideRequestDTO) *domain.ExecutionTarget { func executionTargetOverrideFromModelOverrideDTO(dto *modelOverrideRequestDTO) *domain.ExecutionTargetOverride {
if dto == nil { if dto == nil {
return nil return nil
} }
return &domain.ExecutionTarget{ return &domain.ExecutionTargetOverride{
Endpoint: dto.Endpoint, Endpoint: dto.Endpoint,
Model: dto.Model, Model: dto.Model,
Temperature: dto.Temperature, Temperature: dto.Temperature,

View File

@@ -13,6 +13,7 @@ import (
"time" "time"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile" "gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef" "gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase" "gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
@@ -32,6 +33,34 @@ func (f *fakeRunner) Run(ctx context.Context, req domain.RunRequest) (*domain.Ru
return f.result, nil return f.result, nil
} }
type handlerPromptRepo struct {
def *domain.PromptDefinition
}
func (r handlerPromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
return r.def, nil
}
type handlerProfileRepo struct {
profile *domain.ExecutionProfile
}
func (r handlerProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
return r.profile, nil
}
type handlerArtifactReader struct{}
func (handlerArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
return &domain.Artifact{Name: "input", Body: []byte("input"), Hash: "hash"}, nil
}
type handlerRenderer struct{}
func (handlerRenderer) Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
return &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, nil
}
func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) { func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
start := time.Now().UTC() start := time.Now().UTC()
end := start.Add(2 * time.Second) end := start.Add(2 * time.Second)
@@ -66,11 +95,17 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
APIKeyEnv: envName, APIKeyEnv: envName,
}, },
InputHashes: map[string]string{"transcript": "h1"}, InputHashes: map[string]string{"transcript": "h1"},
Usage: domain.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3}, Usage: domain.TokenUsage{
StartTime: start, PromptTokens: 1,
EndTime: end, CompletionTokens: 2,
Duration: 2 * time.Second, TotalTokens: 3,
RawOutput: "hello", CachedTokens: 4,
CacheWriteTokens: 5,
},
StartTime: start,
EndTime: end,
Duration: 2 * time.Second,
RawOutput: "hello",
}} }}
h := NewHandler(r) h := NewHandler(r)
@@ -111,6 +146,13 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" { if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" {
t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"]) t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"])
} }
usage := metadata["usage"].(map[string]any)
if usage["prompt_tokens"] != float64(1) || usage["completion_tokens"] != float64(2) || usage["total_tokens"] != float64(3) {
t.Fatalf("unexpected base usage metadata: %#v", usage)
}
if usage["cached_tokens"] != float64(4) || usage["cache_write_tokens"] != float64(5) {
t.Fatalf("unexpected cache usage metadata: %#v", usage)
}
modelParams := metadata["model_params"].(map[string]any) modelParams := metadata["model_params"].(map[string]any)
if modelParams["api_key_env"] != envName { if modelParams["api_key_env"] != envName {
t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"]) t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"])
@@ -134,7 +176,7 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
if r.last.Execution == nil || r.last.Execution.Model != "gpt-x" { if r.last.Execution == nil || r.last.Execution.Model != "gpt-x" {
t.Fatalf("expected model override, got %#v", r.last.Execution) t.Fatalf("expected model override, got %#v", r.last.Execution)
} }
if r.last.Execution.TimeoutSeconds != 120 { if r.last.Execution.TimeoutSeconds == nil || *r.last.Execution.TimeoutSeconds != 120 {
t.Fatalf("expected timeout_seconds override 120, got %#v", r.last.Execution) t.Fatalf("expected timeout_seconds override 120, got %#v", r.last.Execution)
} }
if r.last.Execution.ServiceTier != "flex" { if r.last.Execution.ServiceTier != "flex" {
@@ -171,6 +213,10 @@ func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
if metadata["selected_profile_id"] != "prompt-default" { if metadata["selected_profile_id"] != "prompt-default" {
t.Fatalf("expected selected_profile_id from result, got %#v", metadata["selected_profile_id"]) t.Fatalf("expected selected_profile_id from result, got %#v", metadata["selected_profile_id"])
} }
usage := metadata["usage"].(map[string]any)
if usage["cached_tokens"] != float64(0) || usage["cache_write_tokens"] != float64(0) {
t.Fatalf("expected zero cache usage fields to be included, got %#v", usage)
}
} }
func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) { func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
@@ -211,20 +257,136 @@ func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
got := r.last.Execution got := r.last.Execution
if got.Endpoint != "http://override/v1" || if got.Endpoint != "http://override/v1" ||
got.Model != "override-model" || got.Model != "override-model" ||
got.Temperature != 0.6 ||
got.MaxTokens != 250 ||
got.TopP != 0.85 ||
got.TimeoutSeconds != 33 ||
got.ServiceTier != "flex" || got.ServiceTier != "flex" ||
got.ReasoningEffort != "medium" || got.ReasoningEffort != "medium" ||
got.APIKeyEnv != "SCRIPTORIUM_API_KEY" { got.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
t.Fatalf("unexpected mapped execution target: %+v", got) t.Fatalf("unexpected mapped execution target: %+v", got)
} }
if !reflect.DeepEqual(got.ExtraParams, map[string]string{"provider_option": "on"}) { if got.Temperature == nil || *got.Temperature != 0.6 {
t.Fatalf("unexpected mapped temperature: %#v", got.Temperature)
}
if got.MaxTokens == nil || *got.MaxTokens != 250 {
t.Fatalf("unexpected mapped max_tokens: %#v", got.MaxTokens)
}
if got.TopP == nil || *got.TopP != 0.85 {
t.Fatalf("unexpected mapped top_p: %#v", got.TopP)
}
if got.TimeoutSeconds == nil || *got.TimeoutSeconds != 33 {
t.Fatalf("unexpected mapped timeout_seconds: %#v", got.TimeoutSeconds)
}
if !reflect.DeepEqual(got.ExtraParams, map[string]any{"provider_option": "on"}) {
t.Fatalf("unexpected mapped extra_params: %#v", got.ExtraParams) t.Fatalf("unexpected mapped extra_params: %#v", got.ExtraParams)
} }
} }
func TestHandlerModelOverrideAcceptsJSONCompatibleExtraParams(t *testing.T) {
r := &fakeRunner{result: &domain.RunResult{
Artifact: domain.Artifact{Body: []byte("ok")},
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
}}
h := NewHandler(r)
reqBody := `{
"prompt_id": "prompt-1",
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
"model": {
"extra_params": {
"string_value": "enabled",
"number_value": 42,
"boolean_value": true,
"object_value": {"nested": "value", "count": 2},
"array_value": ["first", 3, false]
}
}
}`
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(reqBody))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
}
if r.last.Execution == nil {
t.Fatal("expected execution override in run request")
}
want := map[string]any{
"string_value": "enabled",
"number_value": float64(42),
"boolean_value": true,
"object_value": map[string]any{"nested": "value", "count": float64(2)},
"array_value": []any{"first", float64(3), false},
}
if !reflect.DeepEqual(r.last.Execution.ExtraParams, want) {
t.Fatalf("unexpected mapped extra_params:\ngot=%#v\nwant=%#v", r.last.Execution.ExtraParams, want)
}
}
func TestHandlerModelOverrideExplicitZeroTemperatureMapsAsPresent(t *testing.T) {
r := &fakeRunner{result: &domain.RunResult{
Artifact: domain.Artifact{Body: []byte("ok")},
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0},
}}
h := NewHandler(r)
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
"prompt_id": "prompt-1",
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
"model": {"temperature": 0}
}`))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
}
if r.last.Execution == nil || r.last.Execution.Temperature == nil {
t.Fatalf("expected temperature override to be present, got %#v", r.last.Execution)
}
if *r.last.Execution.Temperature != 0 {
t.Fatalf("expected zero temperature override, got %v", *r.last.Execution.Temperature)
}
}
func TestHandlerModelOverrideOmittedTemperatureMapsAsAbsent(t *testing.T) {
r := &fakeRunner{result: &domain.RunResult{
Artifact: domain.Artifact{Body: []byte("ok")},
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0.7},
}}
h := NewHandler(r)
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
"prompt_id": "prompt-1",
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
"model": {"model": "override-model"}
}`))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
}
if r.last.Execution == nil {
t.Fatal("expected model override")
}
if r.last.Execution.Temperature != nil {
t.Fatalf("expected omitted temperature to remain absent, got %#v", r.last.Execution.Temperature)
}
var resp map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("invalid JSON response: %v", err)
}
metadata := resp["metadata"].(map[string]any)
params := metadata["model_params"].(map[string]any)
if params["temperature"] != 0.7 {
t.Fatalf("expected effective profile/default temperature in response, got %#v", params["temperature"])
}
}
func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing.T) { func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing.T) {
r := &fakeRunner{result: &domain.RunResult{ r := &fakeRunner{result: &domain.RunResult{
Artifact: domain.Artifact{ Artifact: domain.Artifact{
@@ -245,8 +407,10 @@ func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing
ServiceTier: "priority", ServiceTier: "priority",
ReasoningEffort: "high", ReasoningEffort: "high",
APIKeyEnv: "SCRIPTORIUM_API_KEY", APIKeyEnv: "SCRIPTORIUM_API_KEY",
ExtraParams: map[string]string{ ExtraParams: map[string]any{
"provider_option": "on", "provider_option": "on",
"number_value": 42,
"object_value": map[string]any{"nested": "value"},
}, },
}, },
}} }}
@@ -301,6 +465,13 @@ func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing
if extraParams["provider_option"] != "on" { if extraParams["provider_option"] != "on" {
t.Fatalf("unexpected extra_params.provider_option: %#v", extraParams["provider_option"]) t.Fatalf("unexpected extra_params.provider_option: %#v", extraParams["provider_option"])
} }
if extraParams["number_value"] != float64(42) {
t.Fatalf("unexpected extra_params.number_value: %#v", extraParams["number_value"])
}
objectValue, ok := extraParams["object_value"].(map[string]any)
if !ok || objectValue["nested"] != "value" {
t.Fatalf("unexpected extra_params.object_value: %#v", extraParams["object_value"])
}
} }
func TestHandlerInvalidJSON(t *testing.T) { func TestHandlerInvalidJSON(t *testing.T) {
@@ -335,6 +506,54 @@ func TestHandlerMissingPromptID(t *testing.T) {
} }
} }
func TestHandlerReservedExtraParamsThroughRunnerMapsToInvalidRequest(t *testing.T) {
llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{})
if err != nil {
t.Fatal(err)
}
runner := usecase.NewRunner(
handlerPromptRepo{def: &domain.PromptDefinition{
ID: "p",
Version: "1",
DefaultProfile: "exec",
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "hi"}},
OutputFormat: domain.FormatText,
Validation: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone},
}},
handlerProfileRepo{profile: &domain.ExecutionProfile{
ID: "exec",
Endpoint: "http://example.invalid/v1",
Model: "model",
}},
handlerArtifactReader{},
handlerRenderer{},
llmClient,
nil,
)
h := NewHandler(runner)
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
"prompt_id":"p",
"inputs":{"x":{"type":"file","uri":"a"}},
"model":{"extra_params":{"model":"collision"}}
}`))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusBadRequest {
t.Fatalf("expected 400, got %d body=%s", w.Code, w.Body.String())
}
var resp map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("invalid JSON response: %v", err)
}
errBody := resp["error"].(map[string]any)
if errBody["code"] != "invalid_request" {
t.Fatalf("expected invalid_request code, got %#v", errBody["code"])
}
}
func TestHandlerUsecaseErrorMapping(t *testing.T) { func TestHandlerUsecaseErrorMapping(t *testing.T) {
tests := []struct { tests := []struct {
name string name string

View File

@@ -40,14 +40,33 @@ const (
ValidationSkipped ValidationStatus = "skipped" ValidationSkipped ValidationStatus = "skipped"
) )
// CacheControlType defines provider cache behavior for prompt content.
type CacheControlType string
const (
CacheControlEphemeral CacheControlType = "ephemeral"
)
const (
// SessionIDMaxLength is OpenRouter's documented maximum session_id length.
SessionIDMaxLength = 256
)
// CacheControl describes provider cache metadata attached to prompt content.
type CacheControl struct {
Type CacheControlType `yaml:"type" json:"type"`
TTL string `yaml:"ttl,omitempty" json:"ttl,omitempty"`
}
// RunRequest represents a request to generate a single artifact. // RunRequest represents a request to generate a single artifact.
type RunRequest struct { type RunRequest struct {
PromptID string PromptID string
PromptVersion string PromptVersion string
ProfileID string ProfileID string
APIKey string `json:"-" yaml:"-"`
Inputs map[string]ArtifactRef Inputs map[string]ArtifactRef
Vars map[string]string Vars map[string]string
Execution *ExecutionTarget Execution *ExecutionTargetOverride
Validation *OutputContract Validation *OutputContract
Metadata map[string]string Metadata map[string]string
} }
@@ -77,19 +96,21 @@ type RunResult struct {
// PreparedRun contains pre-LLM execution state from the prepare/render phase. // PreparedRun contains pre-LLM execution state from the prepare/render phase.
// It must never include resolved API key values, model output, or validation data. // It must never include resolved API key values, model output, or validation data.
type PreparedRun struct { type PreparedRun struct {
PromptID string `json:"prompt_id"` PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version,omitempty"` PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"` PromptHash string `json:"prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id"` SelectedProfileID string `json:"selected_profile_id"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"` EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
OutputContract OutputContract `json:"output_contract"` TargetPresence ExecutionTargetPresence `json:"-"`
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"` OutputContract OutputContract `json:"output_contract"`
InputHashes map[string]string `json:"input_hashes,omitempty"` StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
RenderedPromptHash string `json:"rendered_prompt_hash"` InputHashes map[string]string `json:"input_hashes,omitempty"`
Messages []RenderedMessage `json:"messages"` SessionID string `json:"session_id,omitempty"`
StartTime time.Time `json:"start_time,omitempty"` RenderedPromptHash string `json:"rendered_prompt_hash"`
EndTime time.Time `json:"end_time,omitempty"` Messages []RenderedMessage `json:"messages"`
DurationMS int64 `json:"duration_ms,omitempty"` StartTime time.Time `json:"start_time,omitempty"`
EndTime time.Time `json:"end_time,omitempty"`
DurationMS int64 `json:"duration_ms,omitempty"`
} }
// ArtifactRef represents a reference to an input artifact. // ArtifactRef represents a reference to an input artifact.
@@ -115,6 +136,7 @@ type PromptDefinition struct {
Version string `yaml:"version"` Version string `yaml:"version"`
DefaultProfile string `yaml:"default_profile"` DefaultProfile string `yaml:"default_profile"`
Description string `yaml:"description"` Description string `yaml:"description"`
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
Inputs []PromptInput `yaml:"inputs"` Inputs []PromptInput `yaml:"inputs"`
Templates []PromptMessageTemplate `yaml:"templates"` Templates []PromptMessageTemplate `yaml:"templates"`
OutputFormat OutputFormat `yaml:"output_format"` OutputFormat OutputFormat `yaml:"output_format"`
@@ -131,38 +153,65 @@ type PromptInput struct {
// PromptMessageTemplate defines a template for a chat message. // PromptMessageTemplate defines a template for a chat message.
type PromptMessageTemplate struct { type PromptMessageTemplate struct {
Role string `yaml:"role"` Role string `yaml:"role"`
Content string `yaml:"content"` Content string `yaml:"content"`
ContentFile string `yaml:"content_file"` ContentFile string `yaml:"content_file"`
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
} }
// ExecutionProfile describes how and where to execute a model. // ExecutionProfile describes how and where to execute a model.
type ExecutionProfile struct { type ExecutionProfile struct {
ID string `yaml:"id"` ID string `yaml:"id"`
Endpoint string `yaml:"endpoint"` Endpoint string `yaml:"endpoint"`
Model string `yaml:"model"` Model string `yaml:"model"`
Temperature float64 `yaml:"temperature"` Temperature float64 `yaml:"temperature"`
MaxTokens int `yaml:"max_tokens"` MaxTokens int `yaml:"max_tokens"`
TopP float64 `yaml:"top_p"` TopP float64 `yaml:"top_p"`
TimeoutSeconds int `yaml:"timeout_seconds"` TimeoutSeconds int `yaml:"timeout_seconds"`
ServiceTier string `yaml:"service_tier"` ServiceTier string `yaml:"service_tier"`
ReasoningEffort string `yaml:"reasoning_effort"` ReasoningEffort string `yaml:"reasoning_effort"`
APIKeyEnv string `yaml:"api_key_env"` APIKeyEnv string `yaml:"api_key_env"`
ExtraParams map[string]string `yaml:"extra_params"` APIKeyRequired bool `yaml:"-" json:"-"`
ExtraParams map[string]any `yaml:"extra_params"`
}
// ExecutionTargetOverride represents per-request runtime setting overrides.
type ExecutionTargetOverride struct {
Endpoint string `json:"endpoint,omitempty"`
Model string `json:"model,omitempty"`
Temperature *float64 `json:"temperature,omitempty"`
MaxTokens *int `json:"max_tokens,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
ServiceTier string `json:"service_tier,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"`
APIKeyEnv string `json:"api_key_env,omitempty"`
ExtraParams map[string]any `json:"extra_params,omitempty"`
}
// ExecutionTargetPresence tracks which effective runtime fields came from an
// explicit request override even when the resolved value is a zero value.
type ExecutionTargetPresence struct {
Temperature bool
MaxTokens bool
TopP bool
TimeoutSeconds bool
} }
// ExecutionTarget represents effective model runtime settings for a run. // ExecutionTarget represents effective model runtime settings for a run.
type ExecutionTarget struct { type ExecutionTarget struct {
Endpoint string `yaml:"endpoint" json:"endpoint"` Endpoint string `yaml:"endpoint" json:"endpoint"`
Model string `yaml:"model" json:"model"` Model string `yaml:"model" json:"model"`
Temperature float64 `yaml:"temperature" json:"temperature"` Temperature float64 `yaml:"temperature" json:"temperature"`
MaxTokens int `yaml:"max_tokens" json:"max_tokens"` MaxTokens int `yaml:"max_tokens" json:"max_tokens"`
TopP float64 `yaml:"top_p" json:"top_p"` TopP float64 `yaml:"top_p" json:"top_p"`
TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"` TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"`
ServiceTier string `yaml:"service_tier" json:"service_tier"` ServiceTier string `yaml:"service_tier" json:"service_tier"`
ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"` ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"`
APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"` APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"`
ExtraParams map[string]string `yaml:"extra_params" json:"extra_params"` APIKey string `yaml:"-" json:"-"`
APIKeyRequired bool `yaml:"-" json:"-"`
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
} }
// OutputContract defines the requirements for the output artifact. // OutputContract defines the requirements for the output artifact.
@@ -175,19 +224,22 @@ type OutputContract struct {
// RenderedPrompt represents the prompt after template application. // RenderedPrompt represents the prompt after template application.
type RenderedPrompt struct { type RenderedPrompt struct {
Messages []RenderedMessage `json:"messages"` SessionID string `json:"session_id,omitempty"`
Messages []RenderedMessage `json:"messages"`
} }
// RenderedMessage is a single message in a rendered prompt. // RenderedMessage is a single message in a rendered prompt.
type RenderedMessage struct { type RenderedMessage struct {
Role string `json:"role"` Role string `json:"role"`
Content string `json:"content"` Content string `json:"content"`
CacheControl *CacheControl `json:"cache_control,omitempty"`
} }
// GenerateRequest is the internal request passed to the LLM client. // GenerateRequest is the internal request passed to the LLM client.
type GenerateRequest struct { type GenerateRequest struct {
Prompt RenderedPrompt Prompt RenderedPrompt
Target ExecutionTarget Target ExecutionTarget
TargetPresence ExecutionTargetPresence
StructuredOutput *StructuredOutputSpec StructuredOutput *StructuredOutputSpec
} }
@@ -222,6 +274,8 @@ type TokenUsage struct {
PromptTokens int PromptTokens int
CompletionTokens int CompletionTokens int
TotalTokens int TotalTokens int
CachedTokens int
CacheWriteTokens int
} }
// ValidationResult represents the outcome of an output validation. // ValidationResult represents the outcome of an output validation.

View File

@@ -20,6 +20,7 @@ func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
Endpoint: "http://llm/v1", Endpoint: "http://llm/v1",
Model: "gpt-test", Model: "gpt-test",
APIKeyEnv: envName, APIKeyEnv: envName,
APIKey: secret,
}, },
InputHashes: map[string]string{"transcript": "hash-1"}, InputHashes: map[string]string{"transcript": "hash-1"},
RenderedPromptHash: "rendered-hash", RenderedPromptHash: "rendered-hash",
@@ -53,3 +54,88 @@ func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
} }
} }
} }
func TestPreparedRunJSONIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
prepared := PreparedRun{
PromptID: "prompt.id",
SelectedProfileID: "local-fast",
EffectiveModelParams: ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "gpt-test",
},
RenderedPromptHash: "rendered-hash",
Messages: []RenderedMessage{
{
Role: "system",
Content: "You are helpful.",
CacheControl: &CacheControl{
Type: CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "Summarize this."},
},
}
b, err := json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal failed: %v", err)
}
var decoded struct {
Messages []map[string]any `json:"messages"`
}
if err := json.Unmarshal(b, &decoded); err != nil {
t.Fatalf("unmarshal failed: %v", err)
}
if len(decoded.Messages) != 2 {
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
}
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
if !ok {
t.Fatalf("expected cache_control on first message, got %#v", decoded.Messages[0])
}
if cacheControl["type"] != string(CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
}
if _, ok := decoded.Messages[1]["cache_control"]; ok {
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
}
}
func TestPreparedRunJSONIncludesSessionIDOnlyWhenPresent(t *testing.T) {
prepared := PreparedRun{
PromptID: "prompt.id",
SelectedProfileID: "local-fast",
EffectiveModelParams: ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "gpt-test",
},
SessionID: "session-123",
RenderedPromptHash: "rendered-hash",
Messages: []RenderedMessage{{Role: "user", Content: "Summarize this."}},
}
b, err := json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal failed: %v", err)
}
var decoded map[string]any
if err := json.Unmarshal(b, &decoded); err != nil {
t.Fatalf("unmarshal failed: %v", err)
}
if decoded["session_id"] != "session-123" {
t.Fatalf("expected session_id in prepared run JSON, got %#v", decoded["session_id"])
}
prepared.SessionID = ""
b, err = json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal failed: %v", err)
}
if strings.Contains(string(b), "session_id") {
t.Fatalf("expected empty session_id to be omitted, got %s", b)
}
}

View File

@@ -96,6 +96,9 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
if prepared.PromptHash != "" { if prepared.PromptHash != "" {
fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash) fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash)
} }
if prepared.SessionID != "" {
fmt.Fprintf(&b, "session_id: %s\n", prepared.SessionID)
}
fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash) fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash)
target := prepared.EffectiveModelParams target := prepared.EffectiveModelParams
@@ -123,7 +126,11 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
} }
sort.Strings(keys) sort.Strings(keys)
for _, k := range keys { for _, k := range keys {
fmt.Fprintf(&b, " %s: %s\n", k, target.ExtraParams[k]) renderedValue, err := formatExtraParamTextValue(target.ExtraParams[k])
if err != nil {
return nil, fmt.Errorf("failed to format extra_params.%s: %w", k, err)
}
fmt.Fprintf(&b, " %s: %s\n", k, renderedValue)
} }
} }
@@ -151,6 +158,13 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
messages := byRole[role] messages := byRole[role]
for i, msg := range messages { for i, msg := range messages {
fmt.Fprintf(&b, " - message: %d\n", i+1) fmt.Fprintf(&b, " - message: %d\n", i+1)
if msg.CacheControl != nil {
fmt.Fprintf(&b, " cache_control: %s", msg.CacheControl.Type)
if msg.CacheControl.TTL != "" {
fmt.Fprintf(&b, " ttl=%s", msg.CacheControl.TTL)
}
fmt.Fprintln(&b)
}
fmt.Fprintln(&b, " content: |") fmt.Fprintln(&b, " content: |")
content := msg.Content content := msg.Content
if content == "" { if content == "" {
@@ -165,3 +179,15 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
return b.Bytes(), nil return b.Bytes(), nil
} }
func formatExtraParamTextValue(value any) (string, error) {
if s, ok := value.(string); ok {
return s, nil
}
b, err := json.Marshal(value)
if err != nil {
return "", err
}
return string(b), nil
}

View File

@@ -49,6 +49,36 @@ func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) {
} }
} }
func TestTextFormatterRendersExtraParamsDeterministically(t *testing.T) {
prepared := samplePreparedRun()
prepared.EffectiveModelParams.ExtraParams = map[string]any{
"z_string": "enabled",
"b_number": 42,
"a_object": map[string]any{
"nested": "value",
"count": 2,
},
"c_array": []any{"first", 3, false},
}
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
s := string(out)
want := strings.Join([]string{
" extra_params:",
" a_object: {\"count\":2,\"nested\":\"value\"}",
" b_number: 42",
" c_array: [\"first\",3,false]",
" z_string: enabled",
}, "\n")
if !strings.Contains(s, want) {
t.Fatalf("expected deterministic extra_params block %q, got:\n%s", want, s)
}
}
func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) { func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
const secret = "super-secret-api-key" const secret = "super-secret-api-key"
t.Setenv("SCRIPTORIUM_API_KEY", secret) t.Setenv("SCRIPTORIUM_API_KEY", secret)
@@ -62,8 +92,94 @@ func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
} }
} }
func TestTextFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) {
const directKey = "direct-format-key"
prepared := samplePreparedRun()
prepared.EffectiveModelParams.APIKey = directKey
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if strings.Contains(string(out), directKey) {
t.Fatalf("text output should not include direct api key value: %s", out)
}
}
func TestTextFormatterIncludesMessageCacheControlBeforeContent(t *testing.T) {
prepared := samplePreparedRun()
prepared.Messages = []domain.RenderedMessage{
{
Role: "system",
Content: "System guidance.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "Summarize the transcript."},
}
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
s := string(out)
if !strings.Contains(s, " system:\n - message: 1\n cache_control: ephemeral ttl=1h\n content: |") {
t.Fatalf("expected system message cache control before content, got:\n%s", s)
}
if strings.Count(s, "cache_control:") != 1 {
t.Fatalf("expected exactly one cache_control line, got:\n%s", s)
}
}
func TestTextFormatterIncludesSessionIDWhenPresent(t *testing.T) {
prepared := samplePreparedRun()
prepared.SessionID = "session-123"
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if !strings.Contains(string(out), "session_id: session-123\n") {
t.Fatalf("expected session_id in text output, got:\n%s", out)
}
}
func TestTextFormatterOmitsEmptyCacheControlTTL(t *testing.T) {
prepared := samplePreparedRun()
prepared.Messages = []domain.RenderedMessage{
{
Role: "system",
Content: "System guidance.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
},
},
}
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
s := string(out)
if !strings.Contains(s, " cache_control: ephemeral\n") {
t.Fatalf("expected cache_control line without ttl, got:\n%s", s)
}
if strings.Contains(s, "ttl=") {
t.Fatalf("expected empty ttl to be omitted, got:\n%s", s)
}
}
func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) { func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
prepared := samplePreparedRun() prepared := samplePreparedRun()
prepared.SessionID = "session-123"
prepared.EffectiveModelParams.ExtraParams = map[string]any{
"number": 42,
"nested": map[string]any{
"enabled": true,
},
}
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON) out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
if err != nil { if err != nil {
@@ -87,9 +203,24 @@ func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
if decoded["rendered_prompt_hash"] != "rendered-hash" { if decoded["rendered_prompt_hash"] != "rendered-hash" {
t.Fatalf("expected rendered_prompt_hash in json output, got %#v", decoded["rendered_prompt_hash"]) t.Fatalf("expected rendered_prompt_hash in json output, got %#v", decoded["rendered_prompt_hash"])
} }
if _, ok := decoded["effective_model_params"]; !ok { if decoded["session_id"] != "session-123" {
t.Fatalf("expected session_id in json output, got %#v", decoded["session_id"])
}
modelParams, ok := decoded["effective_model_params"].(map[string]any)
if !ok {
t.Fatalf("expected effective_model_params in json output, got %#v", decoded) t.Fatalf("expected effective_model_params in json output, got %#v", decoded)
} }
extraParams, ok := modelParams["extra_params"].(map[string]any)
if !ok {
t.Fatalf("expected extra_params in json output, got %#v", modelParams["extra_params"])
}
if extraParams["number"] != float64(42) {
t.Fatalf("unexpected numeric extra param in json output: %#v", extraParams["number"])
}
nested, ok := extraParams["nested"].(map[string]any)
if !ok || nested["enabled"] != true {
t.Fatalf("unexpected nested extra param in json output: %#v", extraParams["nested"])
}
if _, ok := decoded["input_hashes"]; !ok { if _, ok := decoded["input_hashes"]; !ok {
t.Fatalf("expected input_hashes in json output, got %#v", decoded) t.Fatalf("expected input_hashes in json output, got %#v", decoded)
} }
@@ -98,6 +229,47 @@ func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
} }
} }
func TestJSONFormatterIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
prepared := samplePreparedRun()
prepared.Messages = []domain.RenderedMessage{
{
Role: "system",
Content: "System guidance.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "Summarize the transcript."},
}
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
var decoded struct {
Messages []map[string]any `json:"messages"`
}
if err := json.Unmarshal(out, &decoded); err != nil {
t.Fatalf("expected valid json output, got %v", err)
}
if len(decoded.Messages) != 2 {
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
}
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
if !ok {
t.Fatalf("expected first message cache_control, got %#v", decoded.Messages[0])
}
if cacheControl["type"] != string(domain.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
}
if _, ok := decoded.Messages[1]["cache_control"]; ok {
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
}
}
func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) { func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
const secret = "super-secret-api-key" const secret = "super-secret-api-key"
t.Setenv("SCRIPTORIUM_API_KEY", secret) t.Setenv("SCRIPTORIUM_API_KEY", secret)
@@ -111,6 +283,20 @@ func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
} }
} }
func TestJSONFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) {
const directKey = "direct-format-key"
prepared := samplePreparedRun()
prepared.EffectiveModelParams.APIKey = directKey
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if strings.Contains(string(out), directKey) {
t.Fatalf("json output should not include direct api key value: %s", out)
}
}
func TestParsePreparedRunOutputFormatRecognizesSupportedNames(t *testing.T) { func TestParsePreparedRunOutputFormatRecognizesSupportedNames(t *testing.T) {
tests := []struct { tests := []struct {
name string name string

View File

@@ -12,6 +12,7 @@ import (
"os" "os"
"strings" "strings"
"time" "time"
"unicode/utf8"
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults" "gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
@@ -89,7 +90,12 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
} }
payload, err := json.Marshal(wireReq) wirePayload, err := openAIChatRequestPayload(wireReq)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
}
payload, err := json.Marshal(wirePayload)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err) return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
} }
@@ -99,7 +105,9 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err) return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
} }
httpReq.Header.Set("Content-Type", "application/json") httpReq.Header.Set("Content-Type", "application/json")
if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" { if apiKey := strings.TrimSpace(req.Target.APIKey); apiKey != "" {
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
} else if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" {
apiKey := strings.TrimSpace(os.Getenv(envName)) apiKey := strings.TrimSpace(os.Getenv(envName))
if apiKey == "" { if apiKey == "" {
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName) return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
@@ -110,6 +118,8 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
effectiveTimeout := c.timeout effectiveTimeout := c.timeout
if req.Target.TimeoutSeconds > 0 { if req.Target.TimeoutSeconds > 0 {
effectiveTimeout = time.Duration(req.Target.TimeoutSeconds) * time.Second effectiveTimeout = time.Duration(req.Target.TimeoutSeconds) * time.Second
} else if req.TargetPresence.TimeoutSeconds {
effectiveTimeout = 0
} }
httpClient := c.httpClient httpClient := c.httpClient
@@ -151,6 +161,8 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
PromptTokens: wireResp.Usage.PromptTokens, PromptTokens: wireResp.Usage.PromptTokens,
CompletionTokens: wireResp.Usage.CompletionTokens, CompletionTokens: wireResp.Usage.CompletionTokens,
TotalTokens: wireResp.Usage.TotalTokens, TotalTokens: wireResp.Usage.TotalTokens,
CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens,
CacheWriteTokens: wireResp.Usage.CacheWriteTokens,
}, },
}, nil }, nil
} }
@@ -167,27 +179,36 @@ func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultMod
wireReq := openAIChatRequest{ wireReq := openAIChatRequest{
Model: model, Model: model,
} }
if sessionID := strings.TrimSpace(req.Prompt.SessionID); sessionID != "" {
wireReq.Messages = make([]openAIChatMessage, 0, len(req.Prompt.Messages)) if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
for _, msg := range req.Prompt.Messages { return openAIChatRequest{}, fmt.Errorf("session_id length %d exceeds maximum %d", n, domain.SessionIDMaxLength)
wireReq.Messages = append(wireReq.Messages, openAIChatMessage{ }
Role: msg.Role, wireReq.SessionID = sessionID
Content: msg.Content,
})
} }
if req.Target.Temperature != 0 { wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
for _, msg := range req.Prompt.Messages {
wireReq.Messages = append(wireReq.Messages, openAIChatRequestMessageFromRenderedMessage(msg))
}
if req.Target.Temperature != 0 || req.TargetPresence.Temperature {
wireReq.Temperature = &req.Target.Temperature wireReq.Temperature = &req.Target.Temperature
} }
if req.Target.MaxTokens != 0 { if req.Target.MaxTokens != 0 || req.TargetPresence.MaxTokens {
wireReq.MaxTokens = &req.Target.MaxTokens wireReq.MaxTokens = &req.Target.MaxTokens
} }
if req.Target.TopP != 0 { if req.Target.TopP != 0 || req.TargetPresence.TopP {
wireReq.TopP = &req.Target.TopP wireReq.TopP = &req.Target.TopP
} }
if strings.TrimSpace(req.Target.ServiceTier) != "" { if strings.TrimSpace(req.Target.ServiceTier) != "" {
wireReq.ServiceTier = req.Target.ServiceTier wireReq.ServiceTier = req.Target.ServiceTier
} }
if strings.TrimSpace(req.Target.ReasoningEffort) != "" {
wireReq.ReasoningEffort = req.Target.ReasoningEffort
}
if len(req.Target.ExtraParams) > 0 {
wireReq.ExtraParams = req.Target.ExtraParams
}
if req.StructuredOutput != nil { if req.StructuredOutput != nil {
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput) responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
if err != nil { if err != nil {
@@ -200,28 +221,106 @@ func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultMod
} }
type openAIChatRequest struct { type openAIChatRequest struct {
Model string `json:"model"` Model string `json:"model"`
Messages []openAIChatMessage `json:"messages"` SessionID string `json:"session_id,omitempty"`
Temperature *float64 `json:"temperature,omitempty"` Messages []openAIChatRequestMessage `json:"messages"`
MaxTokens *int `json:"max_tokens,omitempty"` Temperature *float64 `json:"temperature,omitempty"`
TopP *float64 `json:"top_p,omitempty"` MaxTokens *int `json:"max_tokens,omitempty"`
ServiceTier string `json:"service_tier,omitempty"` TopP *float64 `json:"top_p,omitempty"`
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"` ServiceTier string `json:"service_tier,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"`
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"`
ExtraParams map[string]any `json:"-"`
} }
type openAIChatMessage struct { func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
out := map[string]any{
"model": req.Model,
"messages": req.Messages,
}
if req.SessionID != "" {
out["session_id"] = req.SessionID
}
if req.Temperature != nil {
out["temperature"] = *req.Temperature
}
if req.MaxTokens != nil {
out["max_tokens"] = *req.MaxTokens
}
if req.TopP != nil {
out["top_p"] = *req.TopP
}
if req.ServiceTier != "" {
out["service_tier"] = req.ServiceTier
}
if req.ReasoningEffort != "" {
out["reasoning_effort"] = req.ReasoningEffort
}
if req.ResponseFormat != nil {
out["response_format"] = req.ResponseFormat
}
for key, value := range req.ExtraParams {
if key == "" {
return nil, errors.New("extra_params key must not be empty")
}
if _, reserved := reservedOpenAIChatRequestFields[key]; reserved {
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
}
if _, err := json.Marshal(value); err != nil {
return nil, fmt.Errorf("extra_params.%s must be JSON-serializable: %w", key, err)
}
out[key] = value
}
return out, nil
}
var reservedOpenAIChatRequestFields = map[string]struct{}{
"model": {},
"session_id": {},
"messages": {},
"temperature": {},
"max_tokens": {},
"top_p": {},
"service_tier": {},
"reasoning_effort": {},
"response_format": {},
}
type openAIChatRequestMessage struct {
Role string `json:"role"`
Content any `json:"content"`
}
type openAIChatTextContentBlock struct {
Type string `json:"type"`
Text string `json:"text"`
CacheControl *openAICacheControl `json:"cache_control,omitempty"`
}
type openAICacheControl struct {
Type string `json:"type"`
TTL string `json:"ttl,omitempty"`
}
type openAIChatResponseMessage struct {
Role string `json:"role"` Role string `json:"role"`
Content string `json:"content"` Content string `json:"content"`
} }
type openAIChatResponse struct { type openAIChatResponse struct {
Choices []struct { Choices []struct {
Message openAIChatMessage `json:"message"` Message openAIChatResponseMessage `json:"message"`
} `json:"choices"` } `json:"choices"`
Usage struct { Usage struct {
PromptTokens int `json:"prompt_tokens"` PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"` CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"` TotalTokens int `json:"total_tokens"`
PromptTokensDetails struct {
CachedTokens int `json:"cached_tokens"`
} `json:"prompt_tokens_details"`
CacheWriteTokens int `json:"cache_write_tokens"`
} `json:"usage"` } `json:"usage"`
} }
@@ -236,6 +335,28 @@ type openAIJSONSchemaEnvelope struct {
Schema any `json:"schema"` Schema any `json:"schema"`
} }
func openAIChatRequestMessageFromRenderedMessage(msg domain.RenderedMessage) openAIChatRequestMessage {
wireMsg := openAIChatRequestMessage{
Role: msg.Role,
Content: msg.Content,
}
if msg.CacheControl == nil {
return wireMsg
}
wireMsg.Content = []openAIChatTextContentBlock{
{
Type: "text",
Text: msg.Content,
CacheControl: &openAICacheControl{
Type: string(msg.CacheControl.Type),
TTL: msg.CacheControl.TTL,
},
},
}
return wireMsg
}
func toOpenAIResponseFormat(spec *domain.StructuredOutputSpec) (*openAIResponseFormat, error) { func toOpenAIResponseFormat(spec *domain.StructuredOutputSpec) (*openAIResponseFormat, error) {
if spec == nil { if spec == nil {
return nil, nil return nil, nil

View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"math"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"strings" "strings"
@@ -89,6 +90,9 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
if resp.Usage.PromptTokens != 11 || resp.Usage.CompletionTokens != 22 || resp.Usage.TotalTokens != 33 { if resp.Usage.PromptTokens != 11 || resp.Usage.CompletionTokens != 22 || resp.Usage.TotalTokens != 33 {
t.Fatalf("unexpected usage: %+v", resp.Usage) t.Fatalf("unexpected usage: %+v", resp.Usage)
} }
if resp.Usage.CachedTokens != 0 || resp.Usage.CacheWriteTokens != 0 {
t.Fatalf("expected absent cache usage fields to remain zero, got %+v", resp.Usage)
}
if obs.Authorization != "Bearer secret-key" { if obs.Authorization != "Bearer secret-key" {
t.Fatalf("unexpected Authorization header: %q", obs.Authorization) t.Fatalf("unexpected Authorization header: %q", obs.Authorization)
@@ -144,6 +148,277 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
} }
} }
func TestOpenAICompatibleClientDirectAPIKeyPreferredOverEnv(t *testing.T) {
const directKey = "direct-llm-key"
t.Setenv("SCRIPTORIUM_TEST_API_KEY", "env-key")
var gotAuth string
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotAuth = r.Header.Get("Authorization")
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{
Model: "model",
APIKeyEnv: "SCRIPTORIUM_TEST_API_KEY",
APIKey: directKey,
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if gotAuth != "Bearer "+directKey {
t.Fatalf("unexpected Authorization header: %q", gotAuth)
}
}
func TestOpenAICompatibleClientSerializesCacheControlledMessageAsContentBlock(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "Stable instructions.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "Dynamic request."},
}},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
for _, forbidden := range []string{"cache_control", "extra_params"} {
if _, exists := observedBody[forbidden]; exists {
t.Fatalf("expected top-level %s to be omitted, got %#v", forbidden, observedBody[forbidden])
}
}
msgs, ok := observedBody["messages"].([]any)
if !ok || len(msgs) != 2 {
t.Fatalf("unexpected messages payload: %#v", observedBody["messages"])
}
msg0 := msgs[0].(map[string]any)
if msg0["role"] != "system" {
t.Fatalf("unexpected first message role: %#v", msg0["role"])
}
contentBlocks, ok := msg0["content"].([]any)
if !ok || len(contentBlocks) != 1 {
t.Fatalf("expected first message content block array, got %#v", msg0["content"])
}
block := contentBlocks[0].(map[string]any)
if block["type"] != "text" || block["text"] != "Stable instructions." {
t.Fatalf("unexpected text content block: %#v", block)
}
cacheControl, ok := block["cache_control"].(map[string]any)
if !ok {
t.Fatalf("expected cache_control on content block, got %#v", block)
}
if cacheControl["type"] != string(domain.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
}
msg1 := msgs[1].(map[string]any)
if msg1["role"] != "user" || msg1["content"] != "Dynamic request." {
t.Fatalf("expected uncached message to keep string content, got %#v", msg1)
}
}
func TestOpenAICompatibleClientOmitsEmptyCacheControlTTL(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "Stable instructions.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
},
},
}},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
msgs := observedBody["messages"].([]any)
msg0 := msgs[0].(map[string]any)
contentBlocks := msg0["content"].([]any)
block := contentBlocks[0].(map[string]any)
cacheControl := block["cache_control"].(map[string]any)
if cacheControl["type"] != string(domain.CacheControlEphemeral) {
t.Fatalf("unexpected cache_control type: %#v", cacheControl)
}
if _, exists := cacheControl["ttl"]; exists {
t.Fatalf("expected empty ttl to be omitted, got %#v", cacheControl)
}
}
func TestOpenAICompatibleClientSerializesSessionID(t *testing.T) {
var observedBody map[string]any
var observedSessionHeader string
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
observedSessionHeader = r.Header.Get("x-session-id")
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{
SessionID: " session-123 ",
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if observedBody["session_id"] != "session-123" {
t.Fatalf("expected top-level session_id, got %#v", observedBody["session_id"])
}
if observedSessionHeader != "" {
t.Fatalf("did not expect x-session-id header, got %q", observedSessionHeader)
}
}
func TestOpenAICompatibleClientOmitsEmptySessionID(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{
SessionID: " ",
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if _, exists := observedBody["session_id"]; exists {
t.Fatalf("expected empty session_id to be omitted, got %#v", observedBody["session_id"])
}
}
func TestOpenAICompatibleClientRejectsTooLongSessionID(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "http://example.com/v1",
Model: "model",
})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{
SessionID: strings.Repeat("x", domain.SessionIDMaxLength+1),
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
},
})
if err == nil {
t.Fatal("expected invalid request error")
}
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
}
func TestOpenAICompatibleClientParsesCacheUsage(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{
"choices": [{"message": {"role": "assistant", "content": "ok"}}],
"usage": {
"prompt_tokens": 100,
"completion_tokens": 20,
"total_tokens": 120,
"prompt_tokens_details": {"cached_tokens": 80},
"cache_write_tokens": 60
}
}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "model"})
if err != nil {
t.Fatal(err)
}
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if resp.Usage.PromptTokens != 100 || resp.Usage.CompletionTokens != 20 || resp.Usage.TotalTokens != 120 {
t.Fatalf("unexpected base usage fields: %+v", resp.Usage)
}
if resp.Usage.CachedTokens != 80 || resp.Usage.CacheWriteTokens != 60 {
t.Fatalf("unexpected cache usage fields: %+v", resp.Usage)
}
}
func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *testing.T) { func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *testing.T) {
var observedBody map[string]any var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -175,7 +450,7 @@ func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *test
} }
} }
func TestOpenAICompatibleClientOmitsReasoningEffortAndExtraParams(t *testing.T) { func TestOpenAICompatibleClientSerializesReasoningEffortAndExtraParams(t *testing.T) {
var observedBody map[string]any var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close() defer r.Body.Close()
@@ -196,19 +471,257 @@ func TestOpenAICompatibleClientOmitsReasoningEffortAndExtraParams(t *testing.T)
Target: domain.ExecutionTarget{ Target: domain.ExecutionTarget{
Model: "model", Model: "model",
ReasoningEffort: "high", ReasoningEffort: "high",
ExtraParams: map[string]string{ ExtraParams: map[string]any{
"provider_option": "on", "string_value": "on",
"number_value": 42,
"boolean_value": true,
"object_value": map[string]any{"nested": "value", "count": 2},
"array_value": []any{"first", 3, false},
}, },
}, },
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
} }
if observedBody["reasoning_effort"] != "high" {
t.Fatalf("expected reasoning_effort high, got %#v", observedBody["reasoning_effort"])
}
if observedBody["string_value"] != "on" {
t.Fatalf("unexpected string extra param: %#v", observedBody["string_value"])
}
if observedBody["number_value"] != float64(42) {
t.Fatalf("unexpected number extra param: %#v", observedBody["number_value"])
}
if observedBody["boolean_value"] != true {
t.Fatalf("unexpected boolean extra param: %#v", observedBody["boolean_value"])
}
objectValue, ok := observedBody["object_value"].(map[string]any)
if !ok || objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
t.Fatalf("unexpected object extra param: %#v", observedBody["object_value"])
}
if _, exists := observedBody["extra_params"]; exists {
t.Fatalf("expected extra_params wrapper omitted, got %#v", observedBody["extra_params"])
}
arrayValue, ok := observedBody["array_value"].([]any)
if !ok || len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
t.Fatalf("unexpected array extra param: %#v", observedBody["array_value"])
}
}
func TestOpenAICompatibleClientOmitsReasoningEffortWhenUnset(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if _, exists := observedBody["reasoning_effort"]; exists { if _, exists := observedBody["reasoning_effort"]; exists {
t.Fatalf("expected reasoning_effort omitted, got %#v", observedBody["reasoning_effort"]) t.Fatalf("expected reasoning_effort omitted, got %#v", observedBody["reasoning_effort"])
} }
if _, exists := observedBody["extra_params"]; exists { if _, exists := observedBody["extra_params"]; exists {
t.Fatalf("expected extra_params omitted, got %#v", observedBody["extra_params"]) t.Fatalf("expected extra_params wrapper omitted, got %#v", observedBody["extra_params"])
}
}
func TestOpenAICompatibleClientSerializesExplicitZeroNumericOverrides(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model"},
TargetPresence: domain.ExecutionTargetPresence{
Temperature: true,
MaxTokens: true,
TopP: true,
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if observedBody["temperature"] != float64(0) {
t.Fatalf("expected explicit zero temperature, got %#v", observedBody["temperature"])
}
if observedBody["max_tokens"] != float64(0) {
t.Fatalf("expected explicit zero max_tokens, got %#v", observedBody["max_tokens"])
}
if observedBody["top_p"] != float64(0) {
t.Fatalf("expected explicit zero top_p, got %#v", observedBody["top_p"])
}
}
func TestOpenAICompatibleClientOmitsImplicitZeroNumericFields(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
for _, field := range []string{"temperature", "max_tokens", "top_p"} {
if _, exists := observedBody[field]; exists {
t.Fatalf("expected implicit zero field %q to be omitted, got body %#v", field, observedBody)
}
}
}
func TestOpenAICompatibleClientExplicitZeroTimeoutDisablesClientTimeout(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(20 * time.Millisecond)
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: ts.URL + "/v1",
Timeout: time.Nanosecond,
})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model", TimeoutSeconds: 0},
TargetPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true},
})
if err != nil {
t.Fatalf("expected explicit zero timeout to disable client timeout, got %v", err)
}
}
func TestOpenAICompatibleClientOmittedTimeoutUsesClientTimeout(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(20 * time.Millisecond)
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: ts.URL + "/v1",
Timeout: time.Nanosecond,
})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model", TimeoutSeconds: 0},
})
if err == nil {
t.Fatal("expected omitted timeout to use client timeout")
}
if !errors.Is(err, ErrRequestFailed) {
t.Fatalf("expected ErrRequestFailed, got %v", err)
}
}
func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *testing.T) {
tests := []struct {
name string
extraParams map[string]any
want string
}{
{name: "empty key", extraParams: map[string]any{"": "empty"}, want: "key must not be empty"},
{name: "unserializable value", extraParams: map[string]any{"bad": math.Inf(1)}, want: "JSON-serializable"},
}
for _, key := range []string{
"model",
"session_id",
"messages",
"temperature",
"max_tokens",
"top_p",
"service_tier",
"reasoning_effort",
"response_format",
} {
tests = append(tests, struct {
name string
extraParams map[string]any
want string
}{
name: "reserved key " + key,
extraParams: map[string]any{key: "collision"},
want: "reserved request field",
})
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
called := false
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
called = true
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model", ExtraParams: tc.extraParams},
})
if err == nil {
t.Fatal("expected invalid request error")
}
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
if !strings.Contains(err.Error(), tc.want) {
t.Fatalf("expected error to contain %q, got %v", tc.want, err)
}
if called {
t.Fatal("provider should not be called for invalid extra_params")
}
})
} }
} }

View File

@@ -0,0 +1,9 @@
id: aion-2
endpoint: https://openrouter.ai/api/v1
model: aion-labs/aion-2.0
temperature: 0.72
reasoning_effort: high
top_p: 0.95
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,7 @@
id: claude-fable-latest
endpoint: https://openrouter.ai/api/v1
model: "~anthropic/claude-fable-latest"
reasoning_effort: high
timeout_seconds: 600
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,7 @@
id: claude-haiku-latest
endpoint: https://openrouter.ai/api/v1
model: "~anthropic/claude-haiku-latest"
reasoning_effort: medium
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,7 @@
id: claude-opus-latest
endpoint: https://openrouter.ai/api/v1
model: "~anthropic/claude-opus-latest"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,7 @@
id: claude-sonnet-latest
endpoint: https://openrouter.ai/api/v1
model: "~anthropic/claude-sonnet-latest"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,7 @@
id: deepseek-3-2
endpoint: https://openrouter.ai/api/v1
model: deepseek/deepseek-v3.2
reasoning_effort: high
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,7 @@
id: deepseek-4-pro
endpoint: https://openrouter.ai/api/v1
model: deepseek/deepseek-v4-pro
reasoning_effort: high
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,9 @@
id: gemini-2-flash-lite
endpoint: https://openrouter.ai/api/v1
model: "google/gemini-2.5-flash-lite"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,9 @@
id: gemini-2-flash
endpoint: https://openrouter.ai/api/v1
model: "google/gemini-2.5-flash"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,9 @@
id: gemini-2-pro
endpoint: https://openrouter.ai/api/v1
model: "google/gemini-2.5-pro"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,9 @@
id: gemini-3-flash-lite
endpoint: https://openrouter.ai/api/v1
model: "google/gemini-3.1-flash-lite"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,9 @@
id: gemini-flash-latest
endpoint: https://openrouter.ai/api/v1
model: "~google/gemini-flash-latest"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,9 @@
id: gemini-pro-latest
endpoint: https://openrouter.ai/api/v1
model: "~google/gemini-pro-latest"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,9 @@
id: gemma-4-31b
endpoint: https://openrouter.ai/api/v1
model: google/gemma-4-31b-it:exacto
temperature: 0.15
reasoning_effort: high
top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,9 @@
id: minimax-m2
endpoint: https://openrouter.ai/api/v1
model: minimax/minimax-m2.5
temperature: 0.5
reasoning_effort: high
top_p: 0.95
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,9 @@
id: minimax-m3
endpoint: https://openrouter.ai/api/v1
model: minimax/minimax-m3
#temperature: 0.5
reasoning_effort: high
#top_p: 0.95
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,7 @@
id: mistral-large-2512
endpoint: https://openrouter.ai/api/v1
model: mistralai/mistral-large-2512
temperature: 0.15
top_p: 0.98
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -0,0 +1,8 @@
id: mistral-medium-3-5
endpoint: https://openrouter.ai/api/v1
model: mistralai/mistral-medium-3-5
temperature: 0.15
reasoning_effort: high
top_p: 0.98
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -0,0 +1,7 @@
id: mistral-small-3
endpoint: https://openrouter.ai/api/v1
model: mistralai/mistral-small-3.2-24b-instruct
temperature: 0.05
top_p: 1.0
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -0,0 +1,8 @@
id: mistral-small-4
endpoint: https://openrouter.ai/api/v1
model: mistralai/mistral-small-2603
temperature: 0.1
reasoning_effort: high
top_p: 0.98
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -0,0 +1,7 @@
id: nemotron-3-ultra
endpoint: https://openrouter.ai/api/v1
model: nvidia/nemotron-3-ultra-550b-a55b
reasoning_effort: high
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,7 @@
id: gpt-5-mini
endpoint: https://openrouter.ai/api/v1
model: "openai/gpt-5.4-mini"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,7 @@
id: gpt-5-nano
endpoint: https://openrouter.ai/api/v1
model: "openai/gpt-5.4-nano"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,31 @@
package builtin
import (
"embed"
"strings"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
)
const assetRoot = "assets"
//go:embed assets/**/*.yml
var assets embed.FS
func NewRepository() profile.Repository {
return profile.NewFSRepository(assets, assetRoot)
}
func NewRepositoryWithPrimary(primary profile.Repository) profile.Repository {
if primary == nil {
return NewRepository()
}
return profile.NewOverlayRepository(primary, NewRepository())
}
func NewRepositoryWithDirectory(dir string) profile.Repository {
if strings.TrimSpace(dir) == "" {
return NewRepository()
}
return NewRepositoryWithPrimary(profile.NewFilesystemRepository(dir))
}

View File

@@ -0,0 +1,127 @@
package builtin
import (
"context"
"errors"
"io/fs"
"strings"
"testing"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gopkg.in/yaml.v3"
)
func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
repo := NewRepository()
ids := loadBuiltInProfileIDs(t)
if len(ids) == 0 {
t.Fatal("expected built-in profiles")
}
for id := range ids {
t.Run(id, func(t *testing.T) {
p, err := repo.GetProfile(context.Background(), id)
if err != nil {
t.Fatalf("expected built-in profile %q to load, got %v", id, err)
}
if p.ID != id {
t.Fatalf("expected profile id %q, got %q", id, p.ID)
}
})
}
}
func TestBuiltInProfilesDoNotContainDuplicateIDsOrRawAPIKeys(t *testing.T) {
loadBuiltInProfileIDs(t)
}
func loadBuiltInProfileIDs(t *testing.T) map[string]string {
t.Helper()
ids := map[string]string{}
err := fs.WalkDir(assets, assetRoot, func(name string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() || !strings.HasSuffix(name, ".yml") {
return nil
}
data, err := assets.ReadFile(name)
if err != nil {
t.Fatalf("failed to read built-in profile %s: %v", name, err)
}
var raw map[string]any
if err := yaml.Unmarshal(data, &raw); err != nil {
t.Fatalf("failed to decode built-in profile %s: %v", name, err)
}
if _, ok := raw["api_key"]; ok {
t.Fatalf("built-in profile %s contains raw api_key", name)
}
id, ok := raw["id"].(string)
if !ok || strings.TrimSpace(id) == "" {
t.Fatalf("built-in profile %s has missing id", name)
}
if previous, ok := ids[id]; ok {
t.Fatalf("duplicate built-in profile id %q in %s and %s", id, previous, name)
}
ids[id] = name
return nil
})
if err != nil {
t.Fatalf("failed to walk built-in profiles: %v", err)
}
return ids
}
func TestRepositoryWithPrimaryUsesPrimaryBeforeBuiltIns(t *testing.T) {
repo := NewRepositoryWithPrimary(staticProfileRepo{
profiles: map[string]string{"mistral-small-3": "custom-model"},
})
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
if err != nil {
t.Fatalf("expected profile to load, got %v", err)
}
if p.Model != "custom-model" {
t.Fatalf("expected primary profile to override built-in, got %+v", p)
}
}
func TestRepositoryWithPrimaryFallsBackToBuiltIns(t *testing.T) {
repo := NewRepositoryWithPrimary(staticProfileRepo{})
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
if err != nil {
t.Fatalf("expected built-in profile to load, got %v", err)
}
if p.ID != "mistral-small-3" {
t.Fatalf("unexpected profile: %+v", p)
}
}
func TestRepositoryWithPrimaryDoesNotFallBackAfterPrimaryError(t *testing.T) {
repo := NewRepositoryWithPrimary(staticProfileRepo{err: profile.ErrInvalidProfile})
_, err := repo.GetProfile(context.Background(), "mistral-small-3")
if !errors.Is(err, profile.ErrInvalidProfile) {
t.Fatalf("expected primary error, got %v", err)
}
}
type staticProfileRepo struct {
profiles map[string]string
err error
}
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
if r.err != nil {
return nil, r.err
}
if model, ok := r.profiles[id]; ok {
return &domain.ExecutionProfile{ID: id, Endpoint: "http://primary/v1", Model: model}, nil
}
return nil, profile.ErrProfileNotFound
}

View File

@@ -5,12 +5,12 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"io/fs"
"os" "os"
"path/filepath" "path"
"strings" "strings"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/filecatalog"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
@@ -30,11 +30,56 @@ func NewFilesystemRepository(dir string) Repository {
} }
func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
return loadProfile(ctx, os.DirFS(r.dir), ".", id)
}
type fsRepository struct {
fsys fs.FS
root string
}
func NewFSRepository(fsys fs.FS, root string) Repository {
return &fsRepository{fsys: fsys, root: root}
}
func (r *fsRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
return loadProfile(ctx, r.fsys, r.root, id)
}
type overlayRepository struct {
primary Repository
fallback Repository
}
func NewOverlayRepository(primary, fallback Repository) Repository {
return &overlayRepository{primary: primary, fallback: fallback}
}
func (r *overlayRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
if r.primary != nil {
prof, err := r.primary.GetProfile(ctx, id)
if err == nil {
return prof, nil
}
if !errors.Is(err, ErrProfileNotFound) {
return nil, err
}
}
if r.fallback == nil {
return nil, ErrProfileNotFound
}
return r.fallback.GetProfile(ctx, id)
}
func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*domain.ExecutionProfile, error) {
if strings.TrimSpace(id) == "" { if strings.TrimSpace(id) == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile) return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
} }
if fsys == nil {
return nil, fmt.Errorf("failed to read profile directory: filesystem is nil")
}
files, err := filecatalog.FindYAMLFiles(ctx, r.dir) files, err := findProfileYAMLFiles(ctx, fsys, root)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to read profile directory: %w", err) return nil, fmt.Errorf("failed to read profile directory: %w", err)
} }
@@ -47,9 +92,9 @@ func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*doma
default: default:
} }
relPath := filecatalog.RelativePath(r.dir, fullPath) relPath := displayPath(root, fullPath)
fileMatch := filecatalog.Stem(filepath.Base(fullPath)) == id fileMatch := profileFileStem(path.Base(fullPath)) == id
data, err := os.ReadFile(fullPath) data, err := fs.ReadFile(fsys, fullPath)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err) return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
} }
@@ -102,6 +147,61 @@ func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*doma
return nil, ErrProfileNotFound return nil, ErrProfileNotFound
} }
func findProfileYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, error) {
cleanRoot := cleanFSRoot(root)
var files []string
err := fs.WalkDir(fsys, cleanRoot, func(name string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if d.IsDir() {
return nil
}
if !isProfileYAMLFile(d.Name()) {
return nil
}
files = append(files, name)
return nil
})
return files, err
}
func cleanFSRoot(root string) string {
root = strings.TrimSpace(root)
if root == "" || root == "." {
return "."
}
return path.Clean(root)
}
func displayPath(root string, name string) string {
cleanRoot := cleanFSRoot(root)
cleanName := path.Clean(name)
if cleanRoot == "." {
return cleanName
}
prefix := strings.TrimSuffix(cleanRoot, "/") + "/"
if strings.HasPrefix(cleanName, prefix) {
return strings.TrimPrefix(cleanName, prefix)
}
return cleanName
}
func profileFileStem(name string) string {
name = strings.TrimSuffix(name, ".yaml")
name = strings.TrimSuffix(name, ".yml")
return name
}
func isProfileYAMLFile(name string) bool {
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
}
type profileMatch struct { type profileMatch struct {
profile *domain.ExecutionProfile profile *domain.ExecutionProfile
path string path string

View File

@@ -2,11 +2,15 @@ package profile
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"testing" "testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
) )
func TestFilesystemRepository_GetProfile(t *testing.T) { func TestFilesystemRepository_GetProfile(t *testing.T) {
@@ -85,6 +89,63 @@ temperature: 0.1
} }
}) })
t.Run("valid profile with JSON-compatible extra params", func(t *testing.T) {
writeProfileTestFile(t, filepath.Join(tmpDir, "json-extra-params.yaml"), `
id: json-extra-params
endpoint: http://localhost:8000/v1
model: nested-model
extra_params:
string_value: enabled
number_value: 42
boolean_value: true
object_value:
nested: value
count: 2
array_value:
- first
- 3
- false
`)
p, err := repo.GetProfile(ctx, "json-extra-params")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
var got map[string]any
encoded, err := json.Marshal(p.ExtraParams)
if err != nil {
t.Fatalf("expected extra_params to marshal as JSON, got %v", err)
}
if err := json.Unmarshal(encoded, &got); err != nil {
t.Fatalf("expected extra_params JSON to decode, got %v", err)
}
if got["string_value"] != "enabled" {
t.Fatalf("unexpected string extra param: %#v", got["string_value"])
}
if got["number_value"] != float64(42) {
t.Fatalf("unexpected number extra param: %#v", got["number_value"])
}
if got["boolean_value"] != true {
t.Fatalf("unexpected boolean extra param: %#v", got["boolean_value"])
}
objectValue, ok := got["object_value"].(map[string]any)
if !ok {
t.Fatalf("expected object extra param, got %#v", got["object_value"])
}
if objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
t.Fatalf("unexpected object extra param: %#v", objectValue)
}
arrayValue, ok := got["array_value"].([]any)
if !ok {
t.Fatalf("expected array extra param, got %#v", got["array_value"])
}
if len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
t.Fatalf("unexpected array extra param: %#v", arrayValue)
}
})
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) { t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), ` writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
id: duplicate-profile id: duplicate-profile
@@ -203,3 +264,216 @@ func writeProfileTestFile(t *testing.T, path string, content string) {
t.Fatalf("failed to write profile test file %q: %v", path, err) t.Fatalf("failed to write profile test file %q: %v", path, err)
} }
} }
func TestFSRepository(t *testing.T) {
ctx := context.Background()
t.Run("loads valid profiles from nested directories", func(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"profiles/provider/nested.yaml": profileMapFile(`
id: nested-profile
endpoint: http://localhost:8000/v1
model: nested-model
temperature: 0.1
`),
}, "profiles")
p, err := repo.GetProfile(ctx, "nested-profile")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.ID != "nested-profile" || p.Model != "nested-model" {
t.Fatalf("unexpected profile: %+v", p)
}
})
t.Run("rejects unknown YAML fields", func(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"profiles/unknown.yaml": profileMapFile(`
id: unknown-profile
endpoint: http://localhost:8000/v1
model: model
unknown: value
`),
}, "profiles")
_, err := repo.GetProfile(ctx, "unknown-profile")
if !errors.Is(err, ErrInvalidYAML) {
t.Fatalf("expected ErrInvalidYAML, got %v", err)
}
})
t.Run("rejects raw api_key in selected profile", func(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"profiles/raw.yaml": profileMapFile(`
id: raw-profile
endpoint: http://localhost:8000/v1
model: model
api_key: secret
`),
}, "profiles")
_, err := repo.GetProfile(ctx, "raw-profile")
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
}
})
t.Run("ignores raw api_key in non-selected profiles", func(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"profiles/raw.yaml": profileMapFile(`
id: raw-profile
endpoint: http://localhost:8000/v1
model: model
api_key: secret
`),
"profiles/valid.yaml": profileMapFile(`
id: valid-profile
endpoint: http://localhost:8000/v1
model: model
`),
}, "profiles")
p, err := repo.GetProfile(ctx, "valid-profile")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.ID != "valid-profile" {
t.Fatalf("unexpected profile: %+v", p)
}
})
t.Run("rejects duplicate IDs within one source", func(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"profiles/a.yaml": profileMapFile(`
id: duplicate-profile
endpoint: http://localhost:8000/v1
model: first
`),
"profiles/nested/b.yaml": profileMapFile(`
id: duplicate-profile
endpoint: http://localhost:8000/v1
model: second
`),
}, "profiles")
_, err := repo.GetProfile(ctx, "duplicate-profile")
if !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("expected ErrInvalidProfile, got %v", err)
}
for _, want := range []string{"duplicate execution profile id", "a.yaml", "nested/b.yaml"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("expected error to contain %q, got %v", want, err)
}
}
})
}
func TestOverlayRepository(t *testing.T) {
ctx := context.Background()
primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"}
fallbackProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://fallback", Model: "fallback"}
t.Run("returns primary matches before fallback matches", func(t *testing.T) {
repo := NewOverlayRepository(
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": primaryProfile}},
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
)
p, err := repo.GetProfile(ctx, "shared")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.Model != "primary" {
t.Fatalf("expected primary profile, got %+v", p)
}
})
t.Run("falls back on primary not found", func(t *testing.T) {
repo := NewOverlayRepository(
staticProfileRepo{},
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
)
p, err := repo.GetProfile(ctx, "shared")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.Model != "fallback" {
t.Fatalf("expected fallback profile, got %+v", p)
}
})
t.Run("does not fall back after primary load errors", func(t *testing.T) {
for _, tc := range []struct {
name string
err error
}{
{name: "invalid yaml", err: ErrInvalidYAML},
{name: "invalid profile", err: ErrInvalidProfile},
{name: "raw api key", err: ErrRawAPIKeyNotAllowed},
} {
t.Run(tc.name, func(t *testing.T) {
repo := NewOverlayRepository(
staticProfileRepo{err: tc.err},
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
)
_, err := repo.GetProfile(ctx, "shared")
if !errors.Is(err, tc.err) {
t.Fatalf("expected %v, got %v", tc.err, err)
}
})
}
})
t.Run("returns not found when both sources miss", func(t *testing.T) {
repo := NewOverlayRepository(staticProfileRepo{}, staticProfileRepo{})
_, err := repo.GetProfile(ctx, "missing")
if !errors.Is(err, ErrProfileNotFound) {
t.Fatalf("expected ErrProfileNotFound, got %v", err)
}
})
t.Run("nil primary uses fallback", func(t *testing.T) {
repo := NewOverlayRepository(nil, staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}})
p, err := repo.GetProfile(ctx, "shared")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.Model != "fallback" {
t.Fatalf("expected fallback profile, got %+v", p)
}
})
t.Run("nil fallback returns not found after primary miss", func(t *testing.T) {
repo := NewOverlayRepository(staticProfileRepo{}, nil)
_, err := repo.GetProfile(ctx, "missing")
if !errors.Is(err, ErrProfileNotFound) {
t.Fatalf("expected ErrProfileNotFound, got %v", err)
}
})
}
func profileMapFile(content string) *fstest.MapFile {
return &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))}
}
type staticProfileRepo struct {
profiles map[string]*domain.ExecutionProfile
err error
}
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
if r.err != nil {
return nil, r.err
}
if p, ok := r.profiles[id]; ok {
cp := *p
return &cp, nil
}
return nil, ErrProfileNotFound
}

View File

@@ -6,7 +6,9 @@ import (
"errors" "errors"
"fmt" "fmt"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"strings"
"text/template" "text/template"
"unicode/utf8"
) )
var ( var (
@@ -50,6 +52,11 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
}, },
} }
sessionID, err := renderSessionID(definition.SessionID, funcs, vars)
if err != nil {
return nil, err
}
var renderedMessages []domain.RenderedMessage var renderedMessages []domain.RenderedMessage
for i, tmplMsg := range definition.Templates { for i, tmplMsg := range definition.Templates {
@@ -75,12 +82,44 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
} }
renderedMessages = append(renderedMessages, domain.RenderedMessage{ renderedMessages = append(renderedMessages, domain.RenderedMessage{
Role: tmplMsg.Role, Role: tmplMsg.Role,
Content: buf.String(), Content: buf.String(),
CacheControl: cloneCacheControl(tmplMsg.CacheControl),
}) })
} }
return &domain.RenderedPrompt{ return &domain.RenderedPrompt{
Messages: renderedMessages, SessionID: sessionID,
Messages: renderedMessages,
}, nil }, nil
} }
func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
if strings.TrimSpace(raw) == "" {
return "", nil
}
tmpl, err := template.New("session_id").Funcs(funcs).Option("missingkey=error").Parse(raw)
if err != nil {
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, err)
}
var buf bytes.Buffer
if err := tmpl.Execute(&buf, vars); err != nil {
return "", fmt.Errorf("%w: session_id: %w", ErrRenderFailure, err)
}
sessionID := strings.TrimSpace(buf.String())
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
return "", fmt.Errorf("%w: session_id length %d exceeds maximum %d", ErrRenderFailure, n, domain.SessionIDMaxLength)
}
return sessionID, nil
}
func cloneCacheControl(in *domain.CacheControl) *domain.CacheControl {
if in == nil {
return nil
}
out := *in
return &out
}

View File

@@ -3,6 +3,7 @@ package prompt
import ( import (
"context" "context"
"errors" "errors"
"strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
@@ -78,6 +79,66 @@ func TestGoRenderer_Render(t *testing.T) {
} }
}) })
t.Run("copying cache control to rendered messages", func(t *testing.T) {
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{
Role: "system",
Content: "You are concise.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "Analyze this: {{input \"transcript\"}}"},
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(res.Messages) != 2 {
t.Fatalf("expected 2 messages, got %d", len(res.Messages))
}
if res.Messages[0].CacheControl == nil {
t.Fatal("expected rendered cache control")
}
if res.Messages[0].CacheControl.Type != domain.CacheControlEphemeral {
t.Fatalf("unexpected cache control type: %q", res.Messages[0].CacheControl.Type)
}
if res.Messages[0].CacheControl.TTL != "1h" {
t.Fatalf("unexpected cache control ttl: %q", res.Messages[0].CacheControl.TTL)
}
if res.Messages[1].CacheControl != nil {
t.Fatalf("expected no cache control on second message, got %#v", res.Messages[1].CacheControl)
}
})
t.Run("rendered cache control does not alias source template", func(t *testing.T) {
source := &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "You are concise.", CacheControl: source},
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if res.Messages[0].CacheControl == source {
t.Fatal("expected rendered cache control to be cloned")
}
res.Messages[0].CacheControl.TTL = ""
if source.TTL != "1h" {
t.Fatalf("source cache control was mutated, ttl=%q", source.TTL)
}
})
t.Run("accessing vars", func(t *testing.T) { t.Run("accessing vars", func(t *testing.T) {
def := &domain.PromptDefinition{ def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
@@ -95,6 +156,78 @@ func TestGoRenderer_Render(t *testing.T) {
} }
}) })
t.Run("rendering session id from vars", func(t *testing.T) {
def := &domain.PromptDefinition{
SessionID: " {{ .session_id }} ",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "Speak in a {{.tone}} tone."},
},
}
res, err := renderer.Render(ctx, def, inputs, map[string]string{
"tone": "concise",
"session_id": "agent-session-123",
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if res.SessionID != "agent-session-123" {
t.Fatalf("unexpected session id: %q", res.SessionID)
}
})
t.Run("empty rendered session id is omitted", func(t *testing.T) {
def := &domain.PromptDefinition{
SessionID: " ",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "Speak in a {{.tone}} tone."},
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if res.SessionID != "" {
t.Fatalf("expected empty session id, got %q", res.SessionID)
}
})
t.Run("missing session id var fails rendering", func(t *testing.T) {
def := &domain.PromptDefinition{
SessionID: "{{ .session_id }}",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "Speak in a {{.tone}} tone."},
},
}
_, err := renderer.Render(ctx, def, inputs, vars)
if !errors.Is(err, ErrRenderFailure) {
t.Fatalf("expected ErrRenderFailure, got %v", err)
}
})
t.Run("too long rendered session id fails rendering", func(t *testing.T) {
def := &domain.PromptDefinition{
SessionID: "{{ .session_id }}",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "Speak in a {{.tone}} tone."},
},
}
_, err := renderer.Render(ctx, def, inputs, map[string]string{
"tone": "concise",
"session_id": strings.Repeat("x", domain.SessionIDMaxLength+1),
})
if !errors.Is(err, ErrRenderFailure) {
t.Fatalf("expected ErrRenderFailure, got %v", err)
}
})
t.Run("inserting required input artifact", func(t *testing.T) { t.Run("inserting required input artifact", func(t *testing.T) {
def := &domain.PromptDefinition{ def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},

View File

@@ -5,7 +5,9 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"io/fs"
"os" "os"
"path"
"path/filepath" "path/filepath"
"strings" "strings"
@@ -24,11 +26,17 @@ type filesystemRepository struct {
dir string dir string
} }
type fsRepository struct {
fsys fs.FS
root string
}
type promptDefinitionFile struct { type promptDefinitionFile struct {
ID string `yaml:"id"` ID string `yaml:"id"`
Version string `yaml:"version"` Version string `yaml:"version"`
DefaultProfile *string `yaml:"default_profile"` DefaultProfile *string `yaml:"default_profile"`
Description string `yaml:"description"` Description string `yaml:"description"`
SessionID string `yaml:"session_id"`
Inputs []promptInputFile `yaml:"inputs"` Inputs []promptInputFile `yaml:"inputs"`
Messages []promptMessageFile `yaml:"messages"` Messages []promptMessageFile `yaml:"messages"`
Output promptOutputContractFile `yaml:"output"` Output promptOutputContractFile `yaml:"output"`
@@ -42,9 +50,15 @@ type promptInputFile struct {
} }
type promptMessageFile struct { type promptMessageFile struct {
Role string `yaml:"role"` Role string `yaml:"role"`
Content string `yaml:"content"` Content string `yaml:"content"`
ContentFile string `yaml:"content_file"` ContentFile string `yaml:"content_file"`
CacheControl *cacheControlFile `yaml:"cache_control"`
}
type cacheControlFile struct {
Type string `yaml:"type"`
TTL string `yaml:"ttl"`
} }
type promptOutputContractFile struct { type promptOutputContractFile struct {
@@ -58,6 +72,10 @@ func NewFilesystemRepository(dir string) Repository {
return &filesystemRepository{dir: dir} return &filesystemRepository{dir: dir}
} }
func NewFSRepository(fsys fs.FS, root string) Repository {
return &fsRepository{fsys: fsys, root: root}
}
func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) { func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
if strings.TrimSpace(id) == "" { if strings.TrimSpace(id) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition) return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
@@ -125,6 +143,10 @@ func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id strin
return nil, ErrPromptDefinitionNotFound return nil, ErrPromptDefinitionNotFound
} }
func (r *fsRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
return loadPromptDefinition(ctx, r.fsys, r.root, id, version)
}
type promptDefinitionMatch struct { type promptDefinitionMatch struct {
def *domain.PromptDefinition def *domain.PromptDefinition
path string path string
@@ -159,7 +181,187 @@ func promptDefinitionFileHasID(path string, id string) bool {
return strings.TrimSpace(raw.ID) == id return strings.TrimSpace(raw.ID) == id
} }
func loadPromptDefinition(ctx context.Context, fsys fs.FS, root string, id string, version string) (*domain.PromptDefinition, error) {
if strings.TrimSpace(id) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
}
if fsys == nil {
return nil, fmt.Errorf("failed to read prompt definition directory: filesystem is nil")
}
files, err := findPromptDefinitionYAMLFiles(ctx, fsys, root)
if err != nil {
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
}
var matches []promptDefinitionMatch
for _, fullPath := range files {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
relPath := displayPath(root, fullPath)
fileMatch := filecatalog.Stem(path.Base(fullPath)) == id
data, err := fs.ReadFile(fsys, fullPath)
if err != nil {
if fileMatch {
return nil, fmt.Errorf("%w: %s: failed to read prompt definition file: %v", ErrInvalidYAML, relPath, err)
}
continue
}
raw, err := decodePromptDefinition(data)
if err != nil {
if fileMatch || promptDefinitionDataHasID(data, id) {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
}
continue
}
def, err := normalizePromptDefinitionFromFS(raw, fsys, fullPath)
if err != nil {
if fileMatch || strings.TrimSpace(raw.ID) == id {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, relPath, err)
}
continue
}
if def.ID != id {
continue
}
if version != "" && def.Version != version {
continue
}
matches = append(matches, promptDefinitionMatch{
def: def,
path: relPath,
})
}
if len(matches) > 1 {
paths := make([]string, 0, len(matches))
for _, match := range matches {
paths = append(paths, match.path)
}
if version != "" {
return nil, fmt.Errorf("%w: duplicate prompt definition id %q version %q found in: %s", ErrInvalidPromptDefinition, id, version, strings.Join(paths, ", "))
}
return nil, fmt.Errorf("%w: duplicate prompt definition id %q found in: %s", ErrInvalidPromptDefinition, id, strings.Join(paths, ", "))
}
if len(matches) == 1 {
return matches[0].def, nil
}
return nil, ErrPromptDefinitionNotFound
}
func findPromptDefinitionYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, error) {
cleanRoot := cleanFSRoot(root)
var files []string
err := fs.WalkDir(fsys, cleanRoot, func(name string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if d.IsDir() {
return nil
}
if !isPromptDefinitionYAMLFile(d.Name()) {
return nil
}
files = append(files, name)
return nil
})
return files, err
}
func cleanFSRoot(root string) string {
root = strings.TrimSpace(root)
if root == "" || root == "." {
return "."
}
return path.Clean(root)
}
func displayPath(root string, name string) string {
cleanRoot := cleanFSRoot(root)
cleanName := path.Clean(name)
if cleanRoot == "." {
return cleanName
}
prefix := strings.TrimSuffix(cleanRoot, "/") + "/"
if strings.HasPrefix(cleanName, prefix) {
return strings.TrimPrefix(cleanName, prefix)
}
return cleanName
}
func isPromptDefinitionYAMLFile(name string) bool {
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
}
func decodePromptDefinition(data []byte) (*promptDefinitionFile, error) {
var raw promptDefinitionFile
decoder := yaml.NewDecoder(bytes.NewReader(data))
decoder.KnownFields(true)
if err := decoder.Decode(&raw); err != nil {
return nil, err
}
return &raw, nil
}
func promptDefinitionDataHasID(data []byte, id string) bool {
var raw struct {
ID string `yaml:"id"`
}
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&raw); err != nil {
return false
}
return strings.TrimSpace(raw.ID) == id
}
func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*domain.PromptDefinition, error) { func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*domain.PromptDefinition, error) {
promptDir := filepath.Dir(sourcePath)
return normalizePromptDefinitionWithContent(raw, func(contentFile string) (string, string, error) {
resolvedPath := strings.TrimSpace(contentFile)
if !filepath.IsAbs(resolvedPath) {
resolvedPath = filepath.Join(promptDir, resolvedPath)
}
resolvedPath = filepath.Clean(resolvedPath)
body, err := os.ReadFile(resolvedPath)
if err != nil {
return "", "", err
}
return string(body), resolvedPath, nil
})
}
func normalizePromptDefinitionFromFS(raw *promptDefinitionFile, fsys fs.FS, sourcePath string) (*domain.PromptDefinition, error) {
promptDir := path.Dir(sourcePath)
return normalizePromptDefinitionWithContent(raw, func(contentFile string) (string, string, error) {
resolvedPath := strings.TrimSpace(contentFile)
if !path.IsAbs(resolvedPath) {
resolvedPath = path.Join(promptDir, resolvedPath)
}
resolvedPath = strings.TrimPrefix(path.Clean(resolvedPath), "/")
body, err := fs.ReadFile(fsys, resolvedPath)
if err != nil {
return "", "", err
}
return string(body), resolvedPath, nil
})
}
func normalizePromptDefinitionWithContent(raw *promptDefinitionFile, readContentFile func(string) (string, string, error)) (*domain.PromptDefinition, error) {
if raw == nil { if raw == nil {
return nil, errors.New("prompt definition is nil") return nil, errors.New("prompt definition is nil")
} }
@@ -199,7 +401,6 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
} }
templates := make([]domain.PromptMessageTemplate, 0, len(raw.Messages)) templates := make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
promptDir := filepath.Dir(sourcePath)
for i, msg := range raw.Messages { for i, msg := range raw.Messages {
role := strings.TrimSpace(msg.Role) role := strings.TrimSpace(msg.Role)
if role == "" { if role == "" {
@@ -212,27 +413,27 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
return nil, fmt.Errorf("message %d (%s) must set exactly one of content or content_file", i, role) return nil, fmt.Errorf("message %d (%s) must set exactly one of content or content_file", i, role)
} }
cacheControl, err := normalizeCacheControl(msg.CacheControl)
if err != nil {
return nil, fmt.Errorf("message %d (%s) cache_control: %w", i, role, err)
}
templateContent := msg.Content templateContent := msg.Content
resolvedContentFile := "" resolvedContentFile := ""
if hasContentFile { if hasContentFile {
resolvedPath := strings.TrimSpace(msg.ContentFile) body, resolvedPath, err := readContentFile(msg.ContentFile)
if !filepath.IsAbs(resolvedPath) {
resolvedPath = filepath.Join(promptDir, resolvedPath)
}
resolvedPath = filepath.Clean(resolvedPath)
body, err := os.ReadFile(resolvedPath)
if err != nil { if err != nil {
return nil, fmt.Errorf("prompt %q message %d (%s): failed to read content_file %q: %w", id, i, role, msg.ContentFile, err) return nil, fmt.Errorf("prompt %q message %d (%s): failed to read content_file %q: %w", id, i, role, msg.ContentFile, err)
} }
templateContent = string(body) templateContent = body
resolvedContentFile = resolvedPath resolvedContentFile = resolvedPath
} }
templates = append(templates, domain.PromptMessageTemplate{ templates = append(templates, domain.PromptMessageTemplate{
Role: role, Role: role,
Content: templateContent, Content: templateContent,
ContentFile: resolvedContentFile, ContentFile: resolvedContentFile,
CacheControl: cacheControl,
}) })
} }
@@ -262,6 +463,7 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
Version: version, Version: version,
DefaultProfile: defaultProfile, DefaultProfile: defaultProfile,
Description: strings.TrimSpace(raw.Description), Description: strings.TrimSpace(raw.Description),
SessionID: strings.TrimSpace(raw.SessionID),
Inputs: inputs, Inputs: inputs,
Templates: templates, Templates: templates,
OutputFormat: raw.Output.Format, OutputFormat: raw.Output.Format,
@@ -274,6 +476,30 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
}, nil }, nil
} }
func normalizeCacheControl(raw *cacheControlFile) (*domain.CacheControl, error) {
if raw == nil {
return nil, nil
}
cacheType := strings.TrimSpace(raw.Type)
if cacheType == "" {
return nil, errors.New("type is required")
}
if domain.CacheControlType(cacheType) != domain.CacheControlEphemeral {
return nil, fmt.Errorf("unsupported type %q", cacheType)
}
ttl := strings.TrimSpace(raw.TTL)
if ttl != "" && ttl != "1h" {
return nil, fmt.Errorf("unsupported ttl %q", ttl)
}
return &domain.CacheControl{
Type: domain.CacheControlType(cacheType),
TTL: ttl,
}, nil
}
func isValidOutputFormat(f domain.OutputFormat) bool { func isValidOutputFormat(f domain.OutputFormat) bool {
switch f { switch f {
case domain.FormatText, domain.FormatMarkdown, domain.FormatJSON: case domain.FormatText, domain.FormatMarkdown, domain.FormatJSON:

View File

@@ -8,6 +8,7 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"testing" "testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
) )
@@ -68,6 +69,44 @@ func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
} }
}) })
t.Run("valid cache control with ttl", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "valid-cache-control-ttl", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(p.Templates) != 2 {
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
}
assertCacheControl(t, p.Templates[0].CacheControl, domain.CacheControlEphemeral, "1h")
if p.Templates[1].CacheControl != nil {
t.Fatalf("expected second message cache control to be nil, got %#v", p.Templates[1].CacheControl)
}
})
t.Run("valid cache control without ttl", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "valid-cache-control-without-ttl", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(p.Templates) != 2 {
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
}
assertCacheControl(t, p.Templates[0].CacheControl, domain.CacheControlEphemeral, "")
if p.Templates[1].CacheControl != nil {
t.Fatalf("expected second message cache control to be nil, got %#v", p.Templates[1].CacheControl)
}
})
t.Run("valid session id template", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "valid-session-id", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.SessionID != "{{ .session_id }}" {
t.Fatalf("expected trimmed session_id template, got %q", p.SessionID)
}
})
t.Run("valid nested file-backed prompt resolves content file relative to nested YAML", func(t *testing.T) { t.Run("valid nested file-backed prompt resolves content file relative to nested YAML", func(t *testing.T) {
nestedDir := filepath.Join(tmpDir, "dnd", "recap") nestedDir := filepath.Join(tmpDir, "dnd", "recap")
if err := os.MkdirAll(nestedDir, 0o755); err != nil { if err := os.MkdirAll(nestedDir, 0o755); err != nil {
@@ -258,6 +297,10 @@ output:
{name: "invalid validation mode", id: "invalid_validation_mode", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"invalid validation mode"}}, {name: "invalid validation mode", id: "invalid_validation_mode", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"invalid validation mode"}},
{name: "json_schema without schema_path", id: "json_schema_without_schema_path", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"schema_path"}}, {name: "json_schema without schema_path", id: "json_schema_without_schema_path", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"schema_path"}},
{name: "unknown input field", id: "unknown_input_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unknown_input_setting not found"}}, {name: "unknown input field", id: "unknown_input_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unknown_input_setting not found"}},
{name: "empty cache control type", id: "empty_cache_control_type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "type is required"}},
{name: "unsupported cache control type", id: "unsupported_cache_control_type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported type"}},
{name: "unsupported cache control ttl", id: "unsupported_cache_control_ttl", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported ttl"}},
{name: "unknown cache control field", id: "unknown_cache_control_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unexpected not found"}},
} }
for _, tc := range cases { for _, tc := range cases {
@@ -282,6 +325,110 @@ output:
}) })
} }
func TestFSRepositoryGetPromptDefinition(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"prompts/nested/prompt.yaml": &fstest.MapFile{Data: []byte(`
id: fs-prompt
version: "1.0.0"
inputs:
- name: transcript
required: true
messages:
- role: user
content_file: ./messages/user.tmpl
output:
format: markdown
validation_mode: basic
repair_attempts: 0
`)},
"prompts/nested/messages/user.tmpl": &fstest.MapFile{Data: []byte(`Summarize {{input "transcript"}}.`)},
}, "prompts")
got, err := repo.GetPromptDefinition(context.Background(), "fs-prompt", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if got.ID != "fs-prompt" {
t.Fatalf("unexpected prompt id: %q", got.ID)
}
if len(got.Templates) != 1 || !strings.Contains(got.Templates[0].Content, `{{input "transcript"}}`) {
t.Fatalf("expected content_file body to be loaded, got %+v", got.Templates)
}
if got.Templates[0].ContentFile != "prompts/nested/messages/user.tmpl" {
t.Fatalf("unexpected content file path: %q", got.Templates[0].ContentFile)
}
}
func TestFSRepositoryRejectsDuplicatePromptIDs(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"one.yaml": &fstest.MapFile{Data: []byte(`
id: duplicate-fs-prompt
version: "1.0.0"
messages:
- role: user
content: First.
output:
format: text
validation_mode: none
repair_attempts: 0
`)},
"nested/two.yaml": &fstest.MapFile{Data: []byte(`
id: duplicate-fs-prompt
version: "1.0.0"
messages:
- role: user
content: Second.
output:
format: text
validation_mode: none
repair_attempts: 0
`)},
}, ".")
_, err := repo.GetPromptDefinition(context.Background(), "duplicate-fs-prompt", "")
if !errors.Is(err, ErrInvalidPromptDefinition) {
t.Fatalf("expected ErrInvalidPromptDefinition, got %v", err)
}
if !strings.Contains(err.Error(), "one.yaml") || !strings.Contains(err.Error(), "nested/two.yaml") {
t.Fatalf("expected duplicate paths in error, got %v", err)
}
}
func TestFSRepositoryRejectsUnknownYAMLFields(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"not_named_like_id.yaml": &fstest.MapFile{Data: []byte(`
id: strict-fs-prompt
version: "1.0.0"
unknown: true
messages:
- role: user
content: Invalid.
output:
format: text
validation_mode: none
repair_attempts: 0
`)},
}, ".")
_, err := repo.GetPromptDefinition(context.Background(), "strict-fs-prompt", "")
if !errors.Is(err, ErrInvalidYAML) {
t.Fatalf("expected ErrInvalidYAML, got %v", err)
}
}
func assertCacheControl(t *testing.T, got *domain.CacheControl, wantType domain.CacheControlType, wantTTL string) {
t.Helper()
if got == nil {
t.Fatal("expected cache control, got nil")
}
if got.Type != wantType {
t.Fatalf("unexpected cache control type: got %q want %q", got.Type, wantType)
}
if got.TTL != wantTTL {
t.Fatalf("unexpected cache control ttl: got %q want %q", got.TTL, wantTTL)
}
}
func writePromptTestFile(t *testing.T, path string, content string) { func writePromptTestFile(t *testing.T, path string, content string) {
t.Helper() t.Helper()
if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil { if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil {

View File

@@ -0,0 +1,10 @@
id: empty-cache-control-type
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control: {}
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -0,0 +1,12 @@
id: unknown-cache-control-field
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: ephemeral
unexpected: true
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -0,0 +1,12 @@
id: unsupported-cache-control-ttl
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: ephemeral
ttl: 5m
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -0,0 +1,11 @@
id: unsupported-cache-control-type
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: persistent
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -0,0 +1,14 @@
id: valid-cache-control-ttl
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: ephemeral
ttl: 1h
- role: user
content: "Summarize the input."
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -0,0 +1,13 @@
id: valid-cache-control-without-ttl
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: ephemeral
- role: user
content: "Summarize the input."
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -0,0 +1,10 @@
id: valid-session-id
version: "1.0.0"
session_id: " {{ .session_id }} "
messages:
- role: user
content: Hello.
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -27,6 +27,7 @@ var (
ErrInvalidRequest = errors.New("invalid run request") ErrInvalidRequest = errors.New("invalid run request")
ErrProfileRequired = errors.New("profile selection is required") ErrProfileRequired = errors.New("profile selection is required")
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable") ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
ErrAPIKeyRequired = errors.New("api key is required")
ErrProfileLoad = errors.New("failed to load prompt definition") ErrProfileLoad = errors.New("failed to load prompt definition")
ErrArtifactLoad = errors.New("failed to load artifact") ErrArtifactLoad = errors.New("failed to load artifact")
ErrPromptRender = errors.New("failed to render prompt") ErrPromptRender = errors.New("failed to render prompt")
@@ -90,11 +91,15 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
} }
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{ genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: prepared.Messages}, Prompt: domain.RenderedPrompt{SessionID: prepared.SessionID, Messages: prepared.Messages},
Target: prepared.EffectiveModelParams, Target: prepared.EffectiveModelParams,
TargetPresence: prepared.TargetPresence,
StructuredOutput: prepared.StructuredOutput, StructuredOutput: prepared.StructuredOutput,
}) })
if err != nil { if err != nil {
if errors.Is(err, llm.ErrInvalidRequest) {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err) return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err)
} }
@@ -187,14 +192,18 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err) return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
} }
effectiveModel := resolveExecutionTarget(execProfile, req.Execution) effectiveModel, targetPresence, err := resolveExecutionTarget(execProfile, req.Execution)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
effectiveModel.APIKey = req.APIKey
if strings.TrimSpace(effectiveModel.Endpoint) == "" { if strings.TrimSpace(effectiveModel.Endpoint) == "" {
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest) return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
} }
if strings.TrimSpace(effectiveModel.Model) == "" { if strings.TrimSpace(effectiveModel.Model) == "" {
return nil, fmt.Errorf("%w: execution model is required", ErrInvalidRequest) return nil, fmt.Errorf("%w: execution model is required", ErrInvalidRequest)
} }
if err := validateAPIKeyEnv(effectiveModel.APIKeyEnv); err != nil { if err := validateAPIKey(effectiveModel.APIKeyEnv, effectiveModel.APIKey, effectiveModel.APIKeyRequired); err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
} }
@@ -230,9 +239,11 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
PromptHash: promptDefinitionHash, PromptHash: promptDefinitionHash,
SelectedProfileID: selectedProfileID, SelectedProfileID: selectedProfileID,
EffectiveModelParams: effectiveModel, EffectiveModelParams: effectiveModel,
TargetPresence: targetPresence,
OutputContract: effectiveContract, OutputContract: effectiveContract,
StructuredOutput: structuredOutput, StructuredOutput: structuredOutput,
InputHashes: inputHashes, InputHashes: inputHashes,
SessionID: renderedPrompt.SessionID,
RenderedPromptHash: hashRenderedPrompt(*renderedPrompt), RenderedPromptHash: hashRenderedPrompt(*renderedPrompt),
Messages: renderedPrompt.Messages, Messages: renderedPrompt.Messages,
StartTime: start, StartTime: start,
@@ -353,28 +364,90 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution
if strings.TrimSpace(override.APIKeyEnv) != "" { if strings.TrimSpace(override.APIKeyEnv) != "" {
out.APIKeyEnv = override.APIKeyEnv out.APIKeyEnv = override.APIKeyEnv
} }
if override.APIKeyRequired {
out.APIKeyRequired = true
}
if len(override.ExtraParams) > 0 { if len(override.ExtraParams) > 0 {
cp := make(map[string]string, len(override.ExtraParams)) out.ExtraParams = copyExtraParams(override.ExtraParams)
for k, v := range override.ExtraParams {
cp[k] = v
}
out.ExtraParams = cp
} }
return out return out
} }
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTarget) domain.ExecutionTarget { func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
out := base
var presence domain.ExecutionTargetPresence
if override.Endpoint != "" {
out.Endpoint = override.Endpoint
}
if override.Model != "" {
out.Model = override.Model
}
if override.Temperature != nil {
if *override.Temperature < 0 || *override.Temperature > 2 {
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("temperature must be between 0 and 2")
}
out.Temperature = *override.Temperature
presence.Temperature = true
}
if override.MaxTokens != nil {
if *override.MaxTokens < 0 {
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("max_tokens must be greater than or equal to 0")
}
out.MaxTokens = *override.MaxTokens
presence.MaxTokens = true
}
if override.TopP != nil {
if *override.TopP < 0 || *override.TopP > 1 {
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("top_p must be between 0 and 1")
}
out.TopP = *override.TopP
presence.TopP = true
}
if override.TimeoutSeconds != nil {
if *override.TimeoutSeconds < 0 {
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("timeout_seconds must be greater than or equal to 0")
}
out.TimeoutSeconds = *override.TimeoutSeconds
presence.TimeoutSeconds = true
}
if strings.TrimSpace(override.ServiceTier) != "" {
out.ServiceTier = override.ServiceTier
}
if strings.TrimSpace(override.ReasoningEffort) != "" {
out.ReasoningEffort = override.ReasoningEffort
}
if strings.TrimSpace(override.APIKeyEnv) != "" {
out.APIKeyEnv = override.APIKeyEnv
}
if len(override.ExtraParams) > 0 {
out.ExtraParams = copyExtraParams(override.ExtraParams)
}
return out, presence, nil
}
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
out := defaults.ExecutionTargetDefault() out := defaults.ExecutionTargetDefault()
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue)) out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
var presence domain.ExecutionTargetPresence
if override != nil { if override != nil {
out = mergeExecutionTarget(out, *override) var err error
out, presence, err = mergeExecutionTargetOverride(out, *override)
if err != nil {
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, err
}
} }
return out return out, presence, nil
} }
func validateAPIKeyEnv(apiKeyEnv string) error { func validateAPIKey(apiKeyEnv string, apiKey string, apiKeyRequired bool) error {
if strings.TrimSpace(apiKey) != "" {
return nil
}
envName := strings.TrimSpace(apiKeyEnv) envName := strings.TrimSpace(apiKeyEnv)
if envName == "" { if envName == "" {
if apiKeyRequired {
return ErrAPIKeyRequired
}
return nil return nil
} }
if strings.TrimSpace(os.Getenv(envName)) == "" { if strings.TrimSpace(os.Getenv(envName)) == "" {
@@ -387,13 +460,6 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
if p == nil { if p == nil {
return domain.ExecutionTarget{} return domain.ExecutionTarget{}
} }
cp := map[string]string(nil)
if len(p.ExtraParams) > 0 {
cp = make(map[string]string, len(p.ExtraParams))
for k, v := range p.ExtraParams {
cp[k] = v
}
}
return domain.ExecutionTarget{ return domain.ExecutionTarget{
Endpoint: p.Endpoint, Endpoint: p.Endpoint,
Model: p.Model, Model: p.Model,
@@ -404,10 +470,22 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
ServiceTier: p.ServiceTier, ServiceTier: p.ServiceTier,
ReasoningEffort: p.ReasoningEffort, ReasoningEffort: p.ReasoningEffort,
APIKeyEnv: p.APIKeyEnv, APIKeyEnv: p.APIKeyEnv,
ExtraParams: cp, APIKeyRequired: p.APIKeyRequired,
ExtraParams: copyExtraParams(p.ExtraParams),
} }
} }
func copyExtraParams(src map[string]any) map[string]any {
if len(src) == 0 {
return nil
}
cp := make(map[string]any, len(src))
for k, v := range src {
cp[k] = v
}
return cp
}
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract { func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract {
contract := def.Validation contract := def.Validation
if contract.Format == "" { if contract.Format == "" {
@@ -424,10 +502,23 @@ func resolveOutputContract(def *domain.PromptDefinition, override *domain.Output
func hashRenderedPrompt(p domain.RenderedPrompt) string { func hashRenderedPrompt(p domain.RenderedPrompt) string {
var b strings.Builder var b strings.Builder
if p.SessionID != "" {
b.WriteString("session_id=")
b.WriteString(p.SessionID)
b.WriteString("\n---\n")
}
for _, msg := range p.Messages { for _, msg := range p.Messages {
b.WriteString(msg.Role) b.WriteString(msg.Role)
b.WriteByte('\n') b.WriteByte('\n')
b.WriteString(msg.Content) b.WriteString(msg.Content)
if msg.CacheControl != nil {
b.WriteString("\ncache_control.type=")
b.WriteString(string(msg.CacheControl.Type))
if msg.CacheControl.TTL != "" {
b.WriteString("\ncache_control.ttl=")
b.WriteString(msg.CacheControl.TTL)
}
}
b.WriteString("\n---\n") b.WriteString("\n---\n")
} }
h := sha256.Sum256([]byte(b.String())) h := sha256.Sum256([]byte(b.String()))

View File

@@ -14,6 +14,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults" "gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile" "gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt" "gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef" "gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
@@ -160,7 +161,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")}, "a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")}, "a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
}} }}
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{forbid: true} llmClient := &fakeLLM{forbid: true}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
@@ -172,7 +173,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"}, "glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
}, },
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90}, Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)},
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
@@ -198,6 +199,9 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
if len(prepared.Messages) != 2 { if len(prepared.Messages) != 2 {
t.Fatalf("expected two messages, got %d", len(prepared.Messages)) t.Fatalf("expected two messages, got %d", len(prepared.Messages))
} }
if prepared.SessionID != "session-123" {
t.Fatalf("expected prepared session id, got %q", prepared.SessionID)
}
if llmClient.calls != 0 { if llmClient.calls != 0 {
t.Fatalf("prepare should not call llm, calls=%d", llmClient.calls) t.Fatalf("prepare should not call llm, calls=%d", llmClient.calls)
} }
@@ -269,11 +273,11 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
Inputs: singleInputRef(), Inputs: singleInputRef(),
Execution: &domain.ExecutionTarget{ Execution: &domain.ExecutionTargetOverride{
Endpoint: "http://override/v1", Endpoint: "http://override/v1",
Model: "override-model", Model: "override-model",
Temperature: 0.7, Temperature: float64Ptr(0.7),
TimeoutSeconds: 30, TimeoutSeconds: intPtr(30),
ServiceTier: "flex", ServiceTier: "flex",
}, },
}) })
@@ -291,6 +295,143 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
} }
} }
func TestRunnerPrepareRequestNumericOverridePresence(t *testing.T) {
tests := []struct {
name string
override *domain.ExecutionTargetOverride
wantTemperature float64
wantMaxTokens int
wantTopP float64
wantTimeoutSecs int
wantPresence domain.ExecutionTargetPresence
}{
{
name: "omitted preserves profile values",
override: &domain.ExecutionTargetOverride{},
wantTemperature: 0.7,
wantMaxTokens: 321,
wantTopP: 0.8,
wantTimeoutSecs: 45,
},
{
name: "explicit zero temperature",
override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(0)},
wantTemperature: 0,
wantMaxTokens: 321,
wantTopP: 0.8,
wantTimeoutSecs: 45,
wantPresence: domain.ExecutionTargetPresence{Temperature: true},
},
{
name: "explicit zero max tokens",
override: &domain.ExecutionTargetOverride{MaxTokens: intPtr(0)},
wantTemperature: 0.7,
wantMaxTokens: 0,
wantTopP: 0.8,
wantTimeoutSecs: 45,
wantPresence: domain.ExecutionTargetPresence{MaxTokens: true},
},
{
name: "explicit zero top p",
override: &domain.ExecutionTargetOverride{TopP: float64Ptr(0)},
wantTemperature: 0.7,
wantMaxTokens: 321,
wantTopP: 0,
wantTimeoutSecs: 45,
wantPresence: domain.ExecutionTargetPresence{TopP: true},
},
{
name: "explicit zero timeout",
override: &domain.ExecutionTargetOverride{TimeoutSeconds: intPtr(0)},
wantTemperature: 0.7,
wantMaxTokens: 321,
wantTopP: 0.8,
wantTimeoutSecs: 0,
wantPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.7,
MaxTokens: 321,
TopP: 0.8,
TimeoutSeconds: 45,
},
}},
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
nil,
)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
Execution: tc.override,
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
got := prepared.EffectiveModelParams
if got.Temperature != tc.wantTemperature ||
got.MaxTokens != tc.wantMaxTokens ||
got.TopP != tc.wantTopP ||
got.TimeoutSeconds != tc.wantTimeoutSecs {
t.Fatalf("unexpected effective numeric settings: %+v", got)
}
if prepared.TargetPresence != tc.wantPresence {
t.Fatalf("unexpected target presence: got %+v want %+v", prepared.TargetPresence, tc.wantPresence)
}
})
}
}
func TestRunnerPrepareInvalidRequestNumericOverridesFail(t *testing.T) {
tests := []struct {
name string
override *domain.ExecutionTargetOverride
}{
{name: "temperature below range", override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(-0.1)}},
{name: "temperature above range", override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(2.1)}},
{name: "max tokens below range", override: &domain.ExecutionTargetOverride{MaxTokens: intPtr(-1)}},
{name: "top p below range", override: &domain.ExecutionTargetOverride{TopP: float64Ptr(-0.1)}},
{name: "top p above range", override: &domain.ExecutionTargetOverride{TopP: float64Ptr(1.1)}},
{name: "timeout below range", override: &domain.ExecutionTargetOverride{TimeoutSeconds: intPtr(-1)}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
nil,
)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
Execution: tc.override,
})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
})
}
}
func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) { func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
@@ -577,6 +718,100 @@ func TestDeriveStructuredSchemaName(t *testing.T) {
} }
} }
func TestHashRenderedPromptIncludesCacheControlWhenPresent(t *testing.T) {
uncached := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{Role: "system", Content: "sys"},
{Role: "user", Content: "usr"},
}}
wantLegacyHash := hashString("system\nsys\n---\nuser\nusr\n---\n")
if got := hashRenderedPrompt(uncached); got != wantLegacyHash {
t.Fatalf("expected no-cache hash to preserve legacy input, got %q want %q", got, wantLegacyHash)
}
withCache := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "sys",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "usr"},
}}
alsoWithCache := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "sys",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "usr"},
}}
withoutTTL := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "sys",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
},
},
{Role: "user", Content: "usr"},
}}
cachedHash := hashRenderedPrompt(withCache)
if cachedHash == hashRenderedPrompt(uncached) {
t.Fatal("expected cache control to change rendered prompt hash")
}
if cachedHash != hashRenderedPrompt(alsoWithCache) {
t.Fatal("expected identical cache control metadata to produce stable hash")
}
if cachedHash == hashRenderedPrompt(withoutTTL) {
t.Fatal("expected ttl changes to affect rendered prompt hash")
}
}
func TestHashRenderedPromptIncludesSessionIDWhenPresent(t *testing.T) {
withoutSession := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{Role: "system", Content: "sys"},
{Role: "user", Content: "usr"},
}}
withSession := domain.RenderedPrompt{
SessionID: "session-123",
Messages: []domain.RenderedMessage{
{Role: "system", Content: "sys"},
{Role: "user", Content: "usr"},
},
}
alsoWithSession := domain.RenderedPrompt{
SessionID: "session-123",
Messages: []domain.RenderedMessage{
{Role: "system", Content: "sys"},
{Role: "user", Content: "usr"},
},
}
otherSession := domain.RenderedPrompt{
SessionID: "session-456",
Messages: []domain.RenderedMessage{
{Role: "system", Content: "sys"},
{Role: "user", Content: "usr"},
},
}
sessionHash := hashRenderedPrompt(withSession)
if sessionHash == hashRenderedPrompt(withoutSession) {
t.Fatal("expected session_id to change rendered prompt hash")
}
if sessionHash != hashRenderedPrompt(alsoWithSession) {
t.Fatal("expected identical session_id to produce stable hash")
}
if sessionHash == hashRenderedPrompt(otherSession) {
t.Fatal("expected session_id value changes to affect rendered prompt hash")
}
}
func TestRunnerRunSuccessful(t *testing.T) { func TestRunnerRunSuccessful(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)} promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}
@@ -584,7 +819,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")}, "a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")}, "a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
}} }}
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
@@ -596,7 +831,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"}, "glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
}, },
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90}, Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)},
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
@@ -631,6 +866,48 @@ func TestRunnerRunSuccessful(t *testing.T) {
if llmClient.lastReq.Target.TimeoutSeconds != 90 { if llmClient.lastReq.Target.TimeoutSeconds != 90 {
t.Fatalf("expected timeout propagation, got %d", llmClient.lastReq.Target.TimeoutSeconds) t.Fatalf("expected timeout propagation, got %d", llmClient.lastReq.Target.TimeoutSeconds)
} }
if !llmClient.lastReq.TargetPresence.Temperature || !llmClient.lastReq.TargetPresence.TimeoutSeconds {
t.Fatalf("expected numeric override presence to be sent to llm, got %+v", llmClient.lastReq.TargetPresence)
}
if llmClient.lastReq.Prompt.SessionID != "session-123" {
t.Fatalf("expected session id to be sent to llm, got %q", llmClient.lastReq.Prompt.SessionID)
}
}
func TestRunnerRunPassesExtraParamsToGenerateRequestTarget(t *testing.T) {
extraParams := map[string]any{
"string_value": "enabled",
"number_value": 42,
"boolean_value": true,
"object_value": map[string]any{"nested": "value"},
"array_value": []any{"first", 3, false},
}
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
ExtraParams: extraParams,
},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if !reflect.DeepEqual(res.EffectiveModelParams.ExtraParams, extraParams) {
t.Fatalf("expected run result extra_params to match profile values, got %#v", res.EffectiveModelParams.ExtraParams)
}
if !reflect.DeepEqual(llmClient.lastReq.Target.ExtraParams, extraParams) {
t.Fatalf("expected generate request extra_params to match profile values, got %#v", llmClient.lastReq.Target.ExtraParams)
}
} }
func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T) { func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T) {
@@ -649,7 +926,7 @@ func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T)
Inputs: map[string]domain.ArtifactRef{ Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
}, },
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90}, Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)},
} }
prepared, err := runner.Prepare(context.Background(), req) prepared, err := runner.Prepare(context.Background(), req)
@@ -777,11 +1054,11 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T)
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
Inputs: singleInputRef(), Inputs: singleInputRef(),
Execution: &domain.ExecutionTarget{ Execution: &domain.ExecutionTargetOverride{
Endpoint: "http://override/v1", Endpoint: "http://override/v1",
Model: "override-model", Model: "override-model",
Temperature: 0.7, Temperature: float64Ptr(0.7),
TimeoutSeconds: 30, TimeoutSeconds: intPtr(30),
ServiceTier: "flex", ServiceTier: "flex",
}, },
}) })
@@ -902,6 +1179,75 @@ func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) {
} }
} }
func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
const directKey = "direct-runner-key"
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "SCRIPTORIUM_MISSING_KEY"},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
APIKey: directKey,
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if llmClient.lastReq.Target.APIKey != directKey {
t.Fatalf("expected direct API key to reach LLM request")
}
if llmClient.lastReq.Target.APIKeyEnv != "SCRIPTORIUM_MISSING_KEY" {
t.Fatalf("expected api_key_env name to remain on target, got %q", llmClient.lastReq.Target.APIKeyEnv)
}
}
func TestRunnerPrepareAPIKeyRequiredFailsWithoutDirectKey(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if !errors.Is(err, ErrAPIKeyRequired) {
t.Fatalf("expected ErrAPIKeyRequired, got %v", err)
}
}
func TestRunnerRunAPIKeyRequiredSucceedsWithDirectKey(t *testing.T) {
const directKey = "direct-required-key"
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
APIKey: directKey,
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if llmClient.lastReq.Target.APIKey != directKey {
t.Fatalf("expected direct API key to reach LLM request")
}
if !llmClient.lastReq.Target.APIKeyRequired {
t.Fatalf("expected APIKeyRequired to be carried to target")
}
}
func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) { func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
const envName = "SCRIPTORIUM_RUNTIME_API_KEY" const envName = "SCRIPTORIUM_RUNTIME_API_KEY"
t.Setenv(envName, "runtime-secret") t.Setenv(envName, "runtime-secret")
@@ -916,7 +1262,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
Inputs: singleInputRef(), Inputs: singleInputRef(),
Execution: &domain.ExecutionTarget{APIKeyEnv: envName}, Execution: &domain.ExecutionTargetOverride{APIKeyEnv: envName},
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
@@ -941,7 +1287,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) {
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
Inputs: singleInputRef(), Inputs: singleInputRef(),
Execution: &domain.ExecutionTarget{APIKeyEnv: runtimeEnv}, Execution: &domain.ExecutionTargetOverride{APIKeyEnv: runtimeEnv},
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
@@ -1040,6 +1386,28 @@ func TestRunnerRunLLMFailure(t *testing.T) {
} }
} }
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{err: llm.ErrInvalidRequest},
nil,
)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
if errors.Is(err, ErrLLMGenerate) {
t.Fatalf("did not expect ErrLLMGenerate, got %v", err)
}
}
func TestRunnerRunValidationStillWorks(t *testing.T) { func TestRunnerRunValidationStillWorks(t *testing.T) {
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}} validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
runner := NewRunner( runner := NewRunner(
@@ -1082,7 +1450,7 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
Inputs: singleInputRef(), Inputs: singleInputRef(),
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "override-model", TimeoutSeconds: 22}, Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "override-model", TimeoutSeconds: intPtr(22)},
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
@@ -1169,7 +1537,8 @@ func TestExecutionProfileToTargetPopulatesAllFieldsAndCopiesExtraParams(t *testi
ServiceTier: "priority", ServiceTier: "priority",
ReasoningEffort: "medium", ReasoningEffort: "medium",
APIKeyEnv: "SCRIPTORIUM_API_KEY", APIKeyEnv: "SCRIPTORIUM_API_KEY",
ExtraParams: map[string]string{ APIKeyRequired: true,
ExtraParams: map[string]any{
"provider_option": "on", "provider_option": "on",
}, },
} }
@@ -1183,7 +1552,8 @@ func TestExecutionProfileToTargetPopulatesAllFieldsAndCopiesExtraParams(t *testi
target.TimeoutSeconds != src.TimeoutSeconds || target.TimeoutSeconds != src.TimeoutSeconds ||
target.ServiceTier != src.ServiceTier || target.ServiceTier != src.ServiceTier ||
target.ReasoningEffort != src.ReasoningEffort || target.ReasoningEffort != src.ReasoningEffort ||
target.APIKeyEnv != src.APIKeyEnv { target.APIKeyEnv != src.APIKeyEnv ||
target.APIKeyRequired != src.APIKeyRequired {
t.Fatalf("expected all profile fields to populate target, got %+v", target) t.Fatalf("expected all profile fields to populate target, got %+v", target)
} }
if !reflect.DeepEqual(target.ExtraParams, src.ExtraParams) { if !reflect.DeepEqual(target.ExtraParams, src.ExtraParams) {
@@ -1208,12 +1578,19 @@ func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testin
ServiceTier: "priority", ServiceTier: "priority",
ReasoningEffort: "low", ReasoningEffort: "low",
APIKeyEnv: "PROFILE_KEY", APIKeyEnv: "PROFILE_KEY",
ExtraParams: map[string]string{ APIKeyRequired: true,
ExtraParams: map[string]any{
"profile_option": "enabled", "profile_option": "enabled",
}, },
} }
target := resolveExecutionTarget(profileValue, nil) target, presence, err := resolveExecutionTarget(profileValue, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if presence != (domain.ExecutionTargetPresence{}) {
t.Fatalf("expected no request override presence, got %+v", presence)
}
if target.Endpoint != profileValue.Endpoint || if target.Endpoint != profileValue.Endpoint ||
target.Model != profileValue.Model || target.Model != profileValue.Model ||
target.Temperature != profileValue.Temperature || target.Temperature != profileValue.Temperature ||
@@ -1222,7 +1599,8 @@ func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testin
target.TimeoutSeconds != profileValue.TimeoutSeconds || target.TimeoutSeconds != profileValue.TimeoutSeconds ||
target.ServiceTier != profileValue.ServiceTier || target.ServiceTier != profileValue.ServiceTier ||
target.ReasoningEffort != profileValue.ReasoningEffort || target.ReasoningEffort != profileValue.ReasoningEffort ||
target.APIKeyEnv != profileValue.APIKeyEnv { target.APIKeyEnv != profileValue.APIKeyEnv ||
target.APIKeyRequired != profileValue.APIKeyRequired {
t.Fatalf("expected profile values to populate target, got %+v", target) t.Fatalf("expected profile values to populate target, got %+v", target)
} }
if !reflect.DeepEqual(target.ExtraParams, profileValue.ExtraParams) { if !reflect.DeepEqual(target.ExtraParams, profileValue.ExtraParams) {
@@ -1242,32 +1620,38 @@ func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFiel
ServiceTier: "priority", ServiceTier: "priority",
ReasoningEffort: "medium", ReasoningEffort: "medium",
APIKeyEnv: "PROFILE_KEY", APIKeyEnv: "PROFILE_KEY",
ExtraParams: map[string]string{ ExtraParams: map[string]any{
"profile_only": "yes", "profile_only": "yes",
}, },
} }
override := &domain.ExecutionTarget{ override := &domain.ExecutionTargetOverride{
Endpoint: "http://override/v1", Endpoint: "http://override/v1",
Model: "override-model", Model: "override-model",
Temperature: 0.9, Temperature: float64Ptr(0.9),
MaxTokens: 111, MaxTokens: intPtr(111),
TopP: 0.5, TopP: float64Ptr(0.5),
TimeoutSeconds: 30, TimeoutSeconds: intPtr(30),
ServiceTier: "flex", ServiceTier: "flex",
ReasoningEffort: "high", ReasoningEffort: "high",
APIKeyEnv: "RUNTIME_KEY", APIKeyEnv: "RUNTIME_KEY",
ExtraParams: map[string]string{ ExtraParams: map[string]any{
"runtime_only": "yes", "runtime_only": "yes",
}, },
} }
target := resolveExecutionTarget(profileValue, override) target, presence, err := resolveExecutionTarget(profileValue, override)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if presence != (domain.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}) {
t.Fatalf("unexpected override presence: %+v", presence)
}
if target.Endpoint != override.Endpoint || if target.Endpoint != override.Endpoint ||
target.Model != override.Model || target.Model != override.Model ||
target.Temperature != override.Temperature || target.Temperature != *override.Temperature ||
target.MaxTokens != override.MaxTokens || target.MaxTokens != *override.MaxTokens ||
target.TopP != override.TopP || target.TopP != *override.TopP ||
target.TimeoutSeconds != override.TimeoutSeconds || target.TimeoutSeconds != *override.TimeoutSeconds ||
target.ServiceTier != override.ServiceTier || target.ServiceTier != override.ServiceTier ||
target.ReasoningEffort != override.ReasoningEffort || target.ReasoningEffort != override.ReasoningEffort ||
target.APIKeyEnv != override.APIKeyEnv { target.APIKeyEnv != override.APIKeyEnv {
@@ -1311,12 +1695,12 @@ func TestMergeExecutionTargetEmptyStringOverridesDoNotErase(t *testing.T) {
func TestMergeExecutionTargetEmptyExtraParamsDoesNotErase(t *testing.T) { func TestMergeExecutionTargetEmptyExtraParamsDoesNotErase(t *testing.T) {
base := domain.ExecutionTarget{ base := domain.ExecutionTarget{
ExtraParams: map[string]string{ ExtraParams: map[string]any{
"keep": "value", "keep": "value",
}, },
} }
override := domain.ExecutionTarget{ override := domain.ExecutionTarget{
ExtraParams: map[string]string{}, ExtraParams: map[string]any{},
} }
merged := mergeExecutionTarget(base, override) merged := mergeExecutionTarget(base, override)
@@ -1392,6 +1776,14 @@ func singleInputRef() map[string]domain.ArtifactRef {
return map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}} return map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}}
} }
func float64Ptr(v float64) *float64 {
return &v
}
func intPtr(v int) *int {
return &v
}
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner { func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
return NewRunner( return NewRunner(
promptRepo, promptRepo,

View File

@@ -5,7 +5,9 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io/fs"
"os" "os"
"path"
"path/filepath" "path/filepath"
"strings" "strings"
@@ -18,10 +20,19 @@ type StandardValidator struct {
schemaBaseDir string schemaBaseDir string
} }
type FSValidator struct {
fsys fs.FS
root string
}
func NewStandardValidator(schemaBaseDir string) Validator { func NewStandardValidator(schemaBaseDir string) Validator {
return &StandardValidator{schemaBaseDir: schemaBaseDir} return &StandardValidator{schemaBaseDir: schemaBaseDir}
} }
func NewFSValidator(fsys fs.FS, root string) Validator {
return &FSValidator{fsys: fsys, root: root}
}
func (v *StandardValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) { func (v *StandardValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
select { select {
case <-ctx.Done(): case <-ctx.Done():
@@ -100,6 +111,88 @@ func (v *StandardValidator) Validate(ctx context.Context, artifact *domain.Artif
} }
} }
func (v *FSValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
select {
case <-ctx.Done():
return domain.ValidationResult{}, ctx.Err()
default:
}
res := domain.ValidationResult{
Mode: contract.ValidationMode,
SchemaPath: contract.SchemaPath,
RepairAttempts: contract.RepairAttempts,
}
if artifact == nil {
return domain.ValidationResult{}, errors.New("artifact is required for validation")
}
switch contract.ValidationMode {
case domain.ValidationNone:
res.Status = domain.ValidationSkipped
res.IsValid = true
return res, nil
case domain.ValidationBasic:
if strings.TrimSpace(string(artifact.Body)) == "" {
res.Status = domain.ValidationFailed
res.IsValid = false
res.Errors = []string{"output is empty"}
return res, nil
}
res.Status = domain.ValidationPassed
res.IsValid = true
return res, nil
case domain.ValidationJSON:
_, jsonErr := parseJSON(artifact.Body)
if jsonErr != nil {
res.Status = domain.ValidationFailed
res.IsValid = false
res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)}
return res, nil
}
res.Status = domain.ValidationPassed
res.IsValid = true
return res, nil
case domain.ValidationJSONSchema:
instance, jsonErr := parseJSON(artifact.Body)
if jsonErr != nil {
res.Status = domain.ValidationFailed
res.IsValid = false
res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)}
return res, nil
}
schemaName, schemaDoc, err := v.loadSchemaDocument(contract.SchemaPath)
if err != nil {
return domain.ValidationResult{}, err
}
resourceURL := fsSchemaResourceURL(schemaName)
compiler := jsonschema.NewCompiler()
if err := compiler.AddResource(resourceURL, schemaDoc); err != nil {
return domain.ValidationResult{}, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
}
schema, err := compiler.Compile(resourceURL)
if err != nil {
return domain.ValidationResult{}, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
}
if err := schema.Validate(instance); err != nil {
res.Status = domain.ValidationFailed
res.IsValid = false
res.Errors = []string{fmt.Sprintf("json schema validation failed: %v", err)}
return res, nil
}
res.Status = domain.ValidationPassed
res.IsValid = true
return res, nil
default:
return domain.ValidationResult{}, fmt.Errorf("unsupported validation mode: %q", contract.ValidationMode)
}
}
func parseJSON(body []byte) (any, error) { func parseJSON(body []byte) (any, error) {
var v any var v any
if err := json.Unmarshal(body, &v); err != nil { if err := json.Unmarshal(body, &v); err != nil {
@@ -132,6 +225,20 @@ func (v *StandardValidator) LoadSchemaDocument(ctx context.Context, schemaPath s
return doc, nil return doc, nil
} }
func (v *FSValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
_, doc, err := v.loadSchemaDocument(schemaPath)
if err != nil {
return nil, err
}
return doc, nil
}
func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error) { func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error) {
if strings.TrimSpace(schemaPath) == "" { if strings.TrimSpace(schemaPath) == "" {
return "", errors.New("schema path is required for json_schema validation") return "", errors.New("schema path is required for json_schema validation")
@@ -149,3 +256,70 @@ func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error)
return resolved, nil return resolved, nil
} }
func (v *FSValidator) loadSchemaDocument(schemaPath string) (string, any, error) {
resolved, err := v.resolveSchemaPath(schemaPath)
if err != nil {
return "", nil, err
}
raw, err := fs.ReadFile(v.fsys, resolved)
if err != nil {
return "", nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err)
}
var doc any
if err := json.Unmarshal(raw, &doc); err != nil {
return "", nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
}
return resolved, doc, nil
}
func (v *FSValidator) resolveSchemaPath(schemaPath string) (string, error) {
if strings.TrimSpace(schemaPath) == "" {
return "", errors.New("schema path is required for json_schema validation")
}
if v.fsys == nil {
return "", errors.New("schema filesystem is nil")
}
cleanRoot := cleanFSRoot(v.root)
rootInfo, err := fs.Stat(v.fsys, cleanRoot)
if err != nil {
return "", fmt.Errorf("failed to access schema source %q: %w", cleanRoot, err)
}
cleanSchemaPath := cleanSchemaFSPath(schemaPath)
var resolved string
if rootInfo.IsDir() {
resolved = path.Join(cleanRoot, cleanSchemaPath)
} else {
if cleanSchemaPath != path.Base(cleanRoot) {
return "", fmt.Errorf("schema path %q does not match schema file %q", cleanSchemaPath, path.Base(cleanRoot))
}
resolved = cleanRoot
}
if _, err := fs.Stat(v.fsys, resolved); err != nil {
return "", fmt.Errorf("failed to access schema file %q: %w", resolved, err)
}
return resolved, nil
}
func cleanSchemaFSPath(schemaPath string) string {
cleaned := strings.TrimSpace(schemaPath)
cleaned = strings.TrimPrefix(path.Clean(cleaned), "/")
return cleaned
}
func cleanFSRoot(root string) string {
root = strings.TrimSpace(root)
if root == "" || root == "." {
return "."
}
return strings.TrimPrefix(path.Clean(root), "/")
}
func fsSchemaResourceURL(schemaName string) string {
return "scriptorium-schema:///" + strings.TrimPrefix(path.Clean(schemaName), "/")
}

View File

@@ -5,6 +5,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
) )
@@ -250,3 +251,77 @@ func TestStandardValidatorLoadSchemaDocumentInvalidJSON(t *testing.T) {
t.Fatal("expected decode error") t.Fatal("expected decode error")
} }
} }
func TestFSValidatorJSONSchemaSuccess(t *testing.T) {
v := NewFSValidator(fstest.MapFS{
"schemas/events.schema.json": &fstest.MapFile{Data: []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"required": ["events"],
"properties": {
"events": {"type": "array"}
}
}`)},
}, "schemas")
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "events.schema.json",
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.Status != domain.ValidationPassed || !res.IsValid {
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
}
}
func TestFSValidatorSingleSchemaFileUsesBaseName(t *testing.T) {
v := NewFSValidator(fstest.MapFS{
"events.schema.json": &fstest.MapFile{Data: []byte(`{
"type": "object",
"required": ["events"],
"properties": {
"events": {"type": "array"}
}
}`)},
}, "events.schema.json")
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "events.schema.json",
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.Status != domain.ValidationPassed || !res.IsValid {
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
}
_, err = v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "other.schema.json",
})
if err == nil {
t.Fatal("expected schema path mismatch error")
}
}
func TestFSValidatorLoadSchemaDocument(t *testing.T) {
v := NewFSValidator(fstest.MapFS{
"schemas/schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
}, "schemas")
loader, ok := v.(SchemaDocumentLoader)
if !ok {
t.Fatal("fs validator must implement SchemaDocumentLoader")
}
doc, err := loader.LoadSchemaDocument(context.Background(), "schema.json")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
obj, ok := doc.(map[string]any)
if !ok || obj["type"] != "object" {
t.Fatalf("unexpected schema document: %#v", doc)
}
}

23
llm_adapter.go Normal file
View File

@@ -0,0 +1,23 @@
package scriptorium
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
)
type publicLLMClientAdapter struct {
client LLMClient
}
func (a publicLLMClientAdapter) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
resp, err := a.client.Generate(ctx, fromDomainGenerateRequest(req))
if err != nil {
return nil, err
}
if resp == nil {
return nil, fmt.Errorf("%w: llm client returned nil response", ErrLLMGenerate)
}
return toDomainGenerateResponse(resp), nil
}

105
profiles.go Normal file
View File

@@ -0,0 +1,105 @@
package scriptorium
import (
"context"
"errors"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
)
// OpenAICompatibleProfile returns an in-memory profile for an OpenAI-compatible
// chat-completions endpoint.
func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile {
return Profile{
ID: cfg.ID,
Endpoint: cfg.Endpoint,
Model: cfg.Model,
Temperature: cfg.Temperature,
MaxTokens: cfg.MaxTokens,
TopP: cfg.TopP,
TimeoutSeconds: cfg.TimeoutSeconds,
ServiceTier: cfg.ServiceTier,
ReasoningEffort: cfg.ReasoningEffort,
APIKeyRequired: cfg.APIKeyRequired,
ExtraParams: copyAnyMap(cfg.ExtraParams),
}
}
type memoryProfileRepository struct {
profiles map[string]domain.ExecutionProfile
}
func newMemoryProfileRepository(profiles []Profile) (*memoryProfileRepository, error) {
repo := &memoryProfileRepository{profiles: make(map[string]domain.ExecutionProfile, len(profiles))}
for _, publicProfile := range profiles {
prof, err := toDomainProfile(publicProfile)
if err != nil {
return nil, err
}
if _, exists := repo.profiles[prof.ID]; exists {
return nil, fmt.Errorf("duplicate profile id %q", prof.ID)
}
repo.profiles[prof.ID] = prof
}
return repo, nil
}
func (r *memoryProfileRepository) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
if r == nil {
return nil, profile.ErrProfileNotFound
}
prof, ok := r.profiles[id]
if !ok {
return nil, profile.ErrProfileNotFound
}
prof.ExtraParams = copyAnyMap(prof.ExtraParams)
return &prof, nil
}
func toDomainProfile(publicProfile Profile) (domain.ExecutionProfile, error) {
prof := domain.ExecutionProfile{
ID: strings.TrimSpace(publicProfile.ID),
Endpoint: publicProfile.Endpoint,
Model: publicProfile.Model,
Temperature: publicProfile.Temperature,
MaxTokens: publicProfile.MaxTokens,
TopP: publicProfile.TopP,
TimeoutSeconds: publicProfile.TimeoutSeconds,
ServiceTier: publicProfile.ServiceTier,
ReasoningEffort: publicProfile.ReasoningEffort,
APIKeyRequired: publicProfile.APIKeyRequired,
ExtraParams: copyAnyMap(publicProfile.ExtraParams),
}
if err := validatePublicProfile(prof); err != nil {
return domain.ExecutionProfile{}, err
}
return prof, nil
}
func validatePublicProfile(prof domain.ExecutionProfile) error {
if strings.TrimSpace(prof.ID) == "" {
return errors.New("id is required")
}
if strings.TrimSpace(prof.Endpoint) == "" {
return errors.New("endpoint is required")
}
if strings.TrimSpace(prof.Model) == "" {
return errors.New("model is required")
}
if prof.Temperature < 0 || prof.Temperature > 2 {
return errors.New("temperature must be between 0 and 2")
}
if prof.MaxTokens < 0 {
return errors.New("max_tokens must be greater than or equal to 0")
}
if prof.TopP < 0 || prof.TopP > 1 {
return errors.New("top_p must be between 0 and 1")
}
if prof.TimeoutSeconds < 0 {
return errors.New("timeout_seconds must be greater than or equal to 0")
}
return nil
}

288
types.go Normal file
View File

@@ -0,0 +1,288 @@
package scriptorium
import (
"context"
"time"
)
// ArtifactRefType defines how an artifact is referenced.
type ArtifactRefType string
const (
ArtifactRefInline ArtifactRefType = "inline"
ArtifactRefFile ArtifactRefType = "file"
)
// OutputFormat defines the desired output format.
type OutputFormat string
const (
FormatText OutputFormat = "text"
FormatMarkdown OutputFormat = "markdown"
FormatJSON OutputFormat = "json"
)
// ValidationMode defines the output validation strategy.
type ValidationMode string
const (
ValidationNone ValidationMode = "none"
ValidationBasic ValidationMode = "basic"
ValidationJSON ValidationMode = "json"
ValidationJSONSchema ValidationMode = "json_schema"
)
// ValidationStatus defines the result of a validation check.
type ValidationStatus string
const (
ValidationPassed ValidationStatus = "passed"
ValidationFailed ValidationStatus = "failed"
ValidationSkipped ValidationStatus = "skipped"
)
// CacheControlType defines provider cache behavior for prompt content.
type CacheControlType string
const (
CacheControlEphemeral CacheControlType = "ephemeral"
)
// StructuredOutputType identifies provider-level structured output modes.
type StructuredOutputType string
const (
StructuredOutputJSONSchema StructuredOutputType = "json_schema"
)
// RunRequest represents a request to prepare or run a single prompt.
type RunRequest struct {
PromptID string
PromptVersion string
ProfileID string
APIKey string `json:"-"`
Inputs map[string]ArtifactRef
Vars map[string]string
Execution *ExecutionTargetOverride
Validation *OutputContract
Metadata map[string]string
}
// PreparedRun contains prepared prompt execution state. It does not include
// resolved API key values, model output, validation results, or internal target
// presence metadata.
type PreparedRun struct {
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
OutputContract OutputContract `json:"output_contract"`
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
InputHashes map[string]string `json:"input_hashes,omitempty"`
SessionID string `json:"session_id,omitempty"`
RenderedPromptHash string `json:"rendered_prompt_hash"`
Messages []RenderedMessage `json:"messages"`
StartTime time.Time `json:"start_time,omitempty"`
EndTime time.Time `json:"end_time,omitempty"`
DurationMS int64 `json:"duration_ms,omitempty"`
}
// RunResult contains generated output, validation state, and run metadata.
type RunResult struct {
RunID string `json:"run_id"`
Artifact Artifact `json:"artifact"`
RawOutput string `json:"raw_output"`
Validation ValidationResult `json:"validation"`
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"`
RenderedPromptHash string `json:"rendered_prompt_hash"`
SelectedProfileID string `json:"selected_profile_id"`
ModelName string `json:"model_name"`
Endpoint string `json:"endpoint"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
InputHashes map[string]string `json:"input_hashes,omitempty"`
Usage TokenUsage `json:"usage"`
StartTime time.Time `json:"start_time,omitempty"`
EndTime time.Time `json:"end_time,omitempty"`
Duration time.Duration `json:"duration,omitempty"`
}
// ArtifactRef represents a reference to prompt input content.
type ArtifactRef struct {
Type ArtifactRefType
URI string
Body string
}
// Artifact represents loaded artifact content.
type Artifact struct {
Name string
ContentType string
Body []byte
URI string
Size int64
Hash string
}
// ExecutionTarget represents effective model runtime settings.
type ExecutionTarget struct {
Endpoint string `json:"endpoint"`
Model string `json:"model"`
Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"`
TopP float64 `json:"top_p"`
TimeoutSeconds int `json:"timeout_seconds"`
ServiceTier string `json:"service_tier"`
ReasoningEffort string `json:"reasoning_effort"`
APIKeyEnv string `json:"api_key_env"`
ExtraParams map[string]any `json:"extra_params"`
}
// ExecutionTargetOverride represents per-request runtime setting overrides.
type ExecutionTargetOverride struct {
Endpoint string
Model string
Temperature *float64
MaxTokens *int
TopP *float64
TimeoutSeconds *int
ServiceTier string
ReasoningEffort string
APIKeyEnv string
ExtraParams map[string]any
}
// Profile is an in-memory execution profile for library consumers.
type Profile struct {
ID string
Endpoint string
Model string
Temperature float64
MaxTokens int
TopP float64
TimeoutSeconds int
ServiceTier string
ReasoningEffort string
APIKeyRequired bool
ExtraParams map[string]any
}
// OpenAICompatibleProfileConfig configures an OpenAI-compatible in-memory profile.
type OpenAICompatibleProfileConfig struct {
ID string
Endpoint string
Model string
APIKeyRequired bool
Temperature float64
MaxTokens int
TopP float64
TimeoutSeconds int
ServiceTier string
ReasoningEffort string
ExtraParams map[string]any
}
// ExecutionTargetPresence tracks which numeric runtime settings were explicit
// request overrides.
type ExecutionTargetPresence struct {
Temperature bool
MaxTokens bool
TopP bool
TimeoutSeconds bool
}
// OutputContract defines output and validation requirements.
type OutputContract struct {
Format OutputFormat `json:"format"`
ValidationMode ValidationMode `json:"validation_mode"`
SchemaPath string `json:"schema_path"`
RepairAttempts int `json:"repair_attempts"`
}
// ValidationResult represents output validation state.
type ValidationResult struct {
Status ValidationStatus `json:"status"`
Mode ValidationMode `json:"mode"`
Errors []string `json:"errors,omitempty"`
SchemaPath string `json:"schema_path,omitempty"`
RepairAttempts int `json:"repair_attempts"`
IsValid bool `json:"is_valid"`
}
// TokenUsage tracks token consumption.
type TokenUsage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
CachedTokens int `json:"cached_tokens"`
CacheWriteTokens int `json:"cache_write_tokens"`
}
// RenderedPrompt is the fully rendered prompt passed to an LLM client.
type RenderedPrompt struct {
SessionID string `json:"session_id,omitempty"`
Messages []RenderedMessage `json:"messages"`
}
// RenderedMessage is a rendered chat message.
type RenderedMessage struct {
Role string `json:"role"`
Content string `json:"content"`
CacheControl *CacheControl `json:"cache_control,omitempty"`
}
// CacheControl describes provider cache metadata attached to prompt content.
type CacheControl struct {
Type CacheControlType `json:"type"`
TTL string `json:"ttl,omitempty"`
}
// StructuredOutputSpec describes provider-level structured output.
type StructuredOutputSpec struct {
Type StructuredOutputType `json:"type"`
JSONSchema *StructuredOutputJSONSpec `json:"json_schema,omitempty"`
}
// StructuredOutputJSONSpec contains JSON Schema output constraints.
type StructuredOutputJSONSpec struct {
Name string `json:"name"`
Strict bool `json:"strict"`
Schema any `json:"schema"`
}
// LLMClient executes rendered prompts for Engine.Run.
type LLMClient interface {
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
}
// GenerateRequest is passed to an injected LLM client.
type GenerateRequest struct {
Prompt RenderedPrompt `json:"prompt"`
Target ExecutionTarget `json:"target"`
TargetPresence ExecutionTargetPresence `json:"target_presence"`
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
APIKey string `json:"-"`
}
// GenerateResponse is returned by an injected LLM client.
type GenerateResponse struct {
Content string `json:"content"`
Usage TokenUsage `json:"usage"`
}
// File returns a file-backed artifact reference.
func File(path string) ArtifactRef {
return ArtifactRef{Type: ArtifactRefFile, URI: path}
}
// Inline returns an inline artifact reference.
func Inline(body string) ArtifactRef {
return ArtifactRef{Type: ArtifactRefInline, Body: body}
}
// InlineWithURI returns an inline artifact reference with URI metadata.
func InlineWithURI(uri string, body string) ArtifactRef {
return ArtifactRef{Type: ArtifactRefInline, URI: uri, Body: body}
}