30 Commits

Author SHA1 Message Date
23872dd742 Implement runtime parameter completion fixes
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-07-04 09:00:19 -05:00
7ffbf5f6ca Update woodpecker config to prepare only linux binaries 2026-07-04 08:53:30 -05:00
d0dc30fcc9 Document runtime provider parameters 2026-07-04 13:29:38 +00:00
b38f7b4dc3 Serialize runtime extra parameters outbound 2026-07-04 13:26:53 +00:00
0512995931 Allow JSON-compatible extra params 2026-07-04 13:24:27 +00:00
049a5feadb Make request execution overrides presence-aware 2026-07-04 13:20:39 +00:00
1798e9c575 Add a roadmap and implementation plan to support reasoning_effort and extra_params in outbound requests 2026-07-04 08:13:10 -05:00
5d4bc8c2b9 Remove completed feature roadmap docs 2026-07-02 20:12:10 -05:00
63fb8fc132 Implement support for OpenRouter sticky routing via a session_id variable 2026-07-02 20:08:44 -05:00
4d4bb7a121 Document prompt cache control behavior 2026-07-02 23:10:24 +00:00
5dcb3cd4fc Expose cache usage in adapters 2026-07-02 23:07:57 +00:00
efe346893c Serialize cache-controlled chat messages 2026-07-02 23:05:56 +00:00
c95d6fcfec Preserve cache control in rendered prompts 2026-07-02 23:03:50 +00:00
0badb4364d Add prompt cache control loading 2026-07-02 23:00:43 +00:00
1f63f8afbb Add a feature roadmap and implementation plan for cache_control values 2026-07-02 17:55:56 -05:00
bc099a31ad Finish cleanup roadmap follow-through
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-26 11:05:08 -05:00
4ff55221a3 Update internal docs for stable runner error reasons and serialized model fields 2026-05-26 15:00:06 +00:00
8d8024099f Complete final cleanup verification and align unsupported artifact test naming 2026-05-26 13:23:45 +00:00
18792fd8d1 Remove unsupported S3 artifact reference placeholder 2026-05-26 13:21:52 +00:00
8860aa033c Reduce CLI test fixture duplication with local setup helpers 2026-05-26 13:20:07 +00:00
3ca14d8b6e Add regression test for JSON schema load failure during prepare 2026-05-26 13:16:26 +00:00
099e9c4a3e Use stable usecase sentinels for HTTP invalid-request mapping 2026-05-26 13:14:57 +00:00
cfe6b9408a Refactor CLI command wiring with shared settings and runner helpers 2026-05-26 13:12:34 +00:00
6ececc749f Centralize YAML catalog scanning for prompt and profile repositories 2026-05-26 13:10:09 +00:00
79901fbb86 Refine execution target mapping helpers and coverage across usecase, HTTP, and LLM 2026-05-26 13:07:35 +00:00
75fa0a030a Added a roadmap to address the issues identifed by the audit 2026-05-26 08:01:33 -05:00
ef64966897 Audit code quality and deduplication opportunities 2026-05-26 07:52:48 -05:00
c6c5e3cb69 Added support for the service_tier key in profiles 2026-05-26 07:44:49 -05:00
3f4fd230b9 Implemented support for loading configuration from nested subdirectories 2026-05-26 07:36:10 -05:00
2091b58066 Completed the documentation update and removed the completed roadmap 2026-05-26 07:15:36 -05:00
60 changed files with 4178 additions and 985 deletions

View File

@@ -28,10 +28,6 @@ steps:
build_binary linux amd64 "" build_binary linux amd64 ""
build_binary linux arm64 "" build_binary linux arm64 ""
build_binary darwin amd64 ""
build_binary darwin arm64 ""
build_binary windows amd64 ".exe"
build_binary windows arm64 ".exe"
- name: publish-release - name: publish-release
image: woodpeckerci/plugin-release image: woodpeckerci/plugin-release

View File

@@ -29,7 +29,6 @@ This command renders the prepared prompt and effective runtime settings without
- [OpenAI-compatible chat integration](docs/integrations/openai-compatible-chat.md) - [OpenAI-compatible chat integration](docs/integrations/openai-compatible-chat.md)
- [Narratio subprocess integration](docs/integrations/narratio.md) - [Narratio subprocess integration](docs/integrations/narratio.md)
- [Architecture policy](docs/policy/architecture.md) - [Architecture policy](docs/policy/architecture.md)
- [Documentation roadmap](docs/roadmap/documentation.md)
## Examples ## Examples

View File

@@ -1,14 +0,0 @@
# Architecture (Moved)
The canonical architecture policy is:
- `docs/policy/architecture.md`
Implemented internal component documentation is:
- `docs/internal/runner.md`
- `docs/internal/adapters.md`
Roadmap-only planning content is:
- `docs/roadmap/documentation.md`

View File

@@ -20,8 +20,8 @@ go run ./cmd/scriptorium render \
Integration references: Integration references:
- HTTP contract: `docs/integrations/http-api.md` - [HTTP contract](integrations/http-api.md)
- Narratio subprocess contract: `docs/integrations/narratio.md` - [Narratio subprocess contract](integrations/narratio.md)
## Common Argument Rules ## Common Argument Rules
@@ -32,6 +32,8 @@ Integration references:
- an effective `prompt_dir` and `profile_dir` (from flags or config) - an effective `prompt_dir` and `profile_dir` (from flags or config)
- `serve` requires an effective `prompt_dir` and `profile_dir` (from flags or config). - `serve` requires an effective `prompt_dir` and `profile_dir` (from flags or config).
- Positional arguments are rejected. - Positional arguments are rejected.
- Prompt cache control is configured in prompt YAML (`messages[].cache_control`), not with CLI flags.
- Provider-specific `reasoning_effort` and `extra_params` are configured in profile YAML or HTTP model overrides, not with CLI flags.
## Flag Reference ## Flag Reference
@@ -56,6 +58,11 @@ Integration references:
- `--top-p <float>`: runtime top-p override. - `--top-p <float>`: runtime top-p override.
- `--timeout <duration>`: runtime timeout override (Go duration syntax, for example `30s`, `2m`). - `--timeout <duration>`: runtime timeout override (Go duration syntax, for example `30s`, `2m`).
Numeric runtime override flags are presence-aware:
- omitted numeric flags preserve the selected profile/default value
- explicit zero values override the selected profile/default value (`--temperature 0`, `--max-tokens 0`, `--top-p 0`, `--timeout 0s`)
### `scriptorium render` ### `scriptorium render`
- Supports the same flags as `run`, except: - Supports the same flags as `run`, except:
@@ -82,6 +89,7 @@ Notes:
- `--input name=path` maps prompt input names to local file paths. - `--input name=path` maps prompt input names to local file paths.
- `--var name=value` maps template variable names to values. - `--var name=value` maps template variable names to values.
- If a prompt defines `session_id: "{{ .session_id }}"`, pass the OpenRouter sticky-routing value with `--var session_id=<value>`.
- Both flags can be repeated. - Both flags can be repeated.
- Both flags also support comma-separated batches, for example: - Both flags also support comma-separated batches, for example:
- `--input transcript=./t.md,glossary=./g.yml` - `--input transcript=./t.md,glossary=./g.yml`
@@ -93,6 +101,7 @@ Notes:
- Writes generated artifact content to stdout by default. - Writes generated artifact content to stdout by default.
- Writes generated artifact content to `--out` when provided. - Writes generated artifact content to `--out` when provided.
- Prints run summary metadata to stderr on success. - Prints run summary metadata to stderr on success.
- Appends `cached_tokens=<n> cache_write_tokens=<n>` to the summary only when the provider reports non-zero cache usage.
- Prints errors to stderr on failure. - Prints errors to stderr on failure.
`render`: `render`:
@@ -134,7 +143,7 @@ go run ./cmd/scriptorium run \
--profile local-fast \ --profile local-fast \
--input transcript=./examples/fixtures/transcript.md \ --input transcript=./examples/fixtures/transcript.md \
--input glossary=./examples/fixtures/glossary.yml \ --input glossary=./examples/fixtures/glossary.yml \
--out ./out/summary.md --out ./summary.md
``` ```
Start the HTTP server with explicit config: Start the HTTP server with explicit config:

View File

@@ -20,8 +20,8 @@ When `--config <path>` is provided, that file is required.
## Minimal App Config ## Minimal App Config
```yaml ```yaml
prompt_dir: ./prompts prompt_dir: ./examples/prompts
profile_dir: ./profiles profile_dir: ./examples/profiles
``` ```
This is enough to use `run` and `render` when prompt/profile files are valid. This is enough to use `run` and `render` when prompt/profile files are valid.
@@ -63,7 +63,9 @@ Validation behavior:
## Prompt Definition Files ## Prompt Definition Files
Prompt definitions are YAML files in `prompt_dir`. Prompt definitions are YAML files anywhere under `prompt_dir`, including nested subdirectories.
Subdirectories are organizational only. Callers still select prompts by the YAML `id`, not by file path. For example, `prompts/dnd/recap.yaml` may still declare `id: dnd.recap`, and callers use `--prompt dnd.recap`.
Example: Example:
@@ -102,6 +104,7 @@ Field reference:
- `version` (required): prompt version. - `version` (required): prompt version.
- `default_profile` (optional): profile ID used when request does not provide `profile_id`. - `default_profile` (optional): profile ID used when request does not provide `profile_id`.
- `description` (optional): prompt description. - `description` (optional): prompt description.
- `session_id` (optional): Go-template string for OpenRouter sticky-routing `session_id`; rendered from request vars.
- `inputs` (optional list): expected named inputs. - `inputs` (optional list): expected named inputs.
- `messages` (required list): prompt message templates. - `messages` (required list): prompt message templates.
- `output` (required object): output contract. - `output` (required object): output contract.
@@ -117,12 +120,44 @@ Field reference:
- `role` (required) - `role` (required)
- `content` or `content_file` (exactly one is required) - `content` or `content_file` (exactly one is required)
- `cache_control` (optional object): provider prompt-cache metadata for this message
Message rules: Message rules:
- Repeated roles are allowed. - Repeated roles are allowed.
- `content_file` is resolved relative to the prompt YAML file location. - `content_file` is resolved relative to the prompt YAML file location.
- Nested prompt files keep the same relative `content_file` behavior; `./recap.user.md` next to `dnd/recap.yaml` resolves from `dnd/`.
- Prompt decoding is strict; unknown YAML fields are rejected. - Prompt decoding is strict; unknown YAML fields are rejected.
- Duplicate prompt IDs are invalid. If multiple files declare the requested prompt ID, Scriptorium fails instead of choosing one.
`messages[].cache_control` fields:
- `type` (required when `cache_control` is present): currently only `ephemeral`.
- `ttl` (optional): currently only `1h`; omitted from outbound requests when unset.
Example cache-controlled message:
```yaml
messages:
- role: system
content_file: ./stable_context.md
cache_control:
type: ephemeral
ttl: 1h
- role: user
content: |
{{input "transcript"}}
```
Use cache control on stable reusable prompt content. Dynamic per-run inputs before the cache-controlled message change the provider cache key.
Example prompt-level session ID:
```yaml
session_id: "{{ .session_id }}"
```
When configured, `session_id` is rendered with the same variable context as messages. The rendered value is trimmed, omitted when empty, and rejected if longer than 256 characters. CLI callers pass the value through `--var session_id=<value>`; HTTP callers pass it through `"vars": {"session_id": "<value>"}`.
`output` fields: `output` fields:
@@ -138,7 +173,9 @@ Repair behavior boundary:
## Profile Definition Files ## Profile Definition Files
Execution profiles are YAML files in `profile_dir`. Execution profiles are YAML files anywhere under `profile_dir`, including nested subdirectories.
Subdirectories are organizational only. Callers still select profiles by the YAML `id`, not by file path. For example, `profiles/local/local-quality.yaml` may still declare `id: local-quality`, and callers use `--profile local-quality`.
Example: Example:
@@ -151,6 +188,12 @@ max_tokens: 500
top_p: 1.0 top_p: 1.0
timeout_seconds: 90 timeout_seconds: 90
api_key_env: SCRIPTORIUM_API_KEY api_key_env: SCRIPTORIUM_API_KEY
service_tier: priority
reasoning_effort: medium
extra_params:
provider_route: primary
provider_options:
retry_budget: 2
``` ```
Field reference: Field reference:
@@ -162,20 +205,25 @@ Field reference:
- `max_tokens` (optional): `>= 0` - `max_tokens` (optional): `>= 0`
- `top_p` (optional): range `0..1` - `top_p` (optional): range `0..1`
- `timeout_seconds` (optional): `>= 0` - `timeout_seconds` (optional): `>= 0`
- `reasoning_effort` (optional) - `service_tier` (optional): provider-specific request tier such as OpenRouter `flex` or `priority`
- `reasoning_effort` (optional): serialized as top-level `reasoning_effort` in outbound chat-completions requests
- `api_key_env` (optional) - `api_key_env` (optional)
- `extra_params` (optional map of strings) - `extra_params` (optional map): JSON-compatible provider-specific parameters. Values may be strings, numbers, booleans, objects, or arrays.
Profile rules: Profile rules:
- Profile decoding is strict; unknown YAML fields are rejected. - Profile decoding is strict; unknown YAML fields are rejected.
- Raw `api_key` is rejected; use `api_key_env`. - Raw `api_key` is rejected; use `api_key_env`.
- If `api_key_env` is set, that environment variable must be set when preparing/running. - If `api_key_env` is set, that environment variable must be set when preparing/running.
- Duplicate profile IDs are invalid. If multiple files declare the requested profile ID, Scriptorium fails instead of choosing one.
- `extra_params` keys must not be empty and must not collide with reserved outbound request fields: `model`, `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, or `response_format`.
Current outbound request behavior: Current outbound request behavior:
- The OpenAI-compatible client currently serializes: `model`, `messages`, `temperature`, `max_tokens`, `top_p`, and optional `response_format` for `json_schema` prompts. - The OpenAI-compatible client currently serializes: `model`, optional `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, optional `response_format` for `json_schema` prompts, and `extra_params`.
- `reasoning_effort` and `extra_params` are parsed and carried in effective settings, but are not currently serialized into outbound chat-completions requests. - `extra_params` are flattened into provider-specific top-level JSON request fields. They are not wrapped in an `extra_params` object on the outbound provider request.
- Messages without `cache_control` serialize with string `content`.
- Messages with `cache_control` serialize as a single text content-block array containing `cache_control`.
## Schema Behavior ## Schema Behavior
@@ -184,8 +232,9 @@ Schemas are JSON files, typically in `schema_dir`.
Rules: Rules:
- `output.validation_mode: json_schema` requires `output.schema_path`. - `output.validation_mode: json_schema` requires `output.schema_path`.
- Relative `schema_path` values resolve from `schema_dir`. - Relative `schema_path` values resolve from `schema_dir`, including explicit nested paths such as `dnd/structured_events.schema.json`.
- Absolute `schema_path` values are used directly. - Absolute `schema_path` values are used directly.
- Scriptorium does not recursively search schemas by basename; nested schemas must be referenced by their relative path.
- Missing or invalid schema documents cause runtime validation errors. - Missing or invalid schema documents cause runtime validation errors.
- Invalid generated JSON causes validation status `failed` (not a runtime error). - Invalid generated JSON causes validation status `failed` (not a runtime error).
@@ -200,14 +249,22 @@ Supported artifact reference types for request inputs are `file` and `inline`.
## Maintained Examples ## Maintained Examples
- App config: `examples/config.yml` - App config: `examples/config.yml`
- Prompt examples: `prompts/` - Prompt examples: `examples/prompts/`
- Profile examples: `profiles/` - Profile examples: `examples/profiles/`
- Schema examples: `schemas/` - Schema examples: `examples/schemas/`
- Input fixtures: `examples/fixtures/` - Input fixtures: `examples/fixtures/`
- Render example script: `examples/render-markdown-summary.sh` - Render example script: `examples/render-markdown-summary.sh`
- HTTP request example: `examples/http-run.json` - HTTP request example: `examples/http-run.json`
Example organizational layout:
```text
examples/prompts/dnd/recap.yaml
examples/profiles/local/local-quality.yaml
examples/schemas/dnd/structured_events.schema.json
```
## Integration References ## Integration References
- Inbound HTTP contract: `docs/integrations/http-api.md` - [Inbound HTTP contract](integrations/http-api.md)
- Outbound OpenAI-compatible contract: `docs/integrations/openai-compatible-chat.md` - [Outbound OpenAI-compatible contract](integrations/openai-compatible-chat.md)

View File

@@ -8,7 +8,7 @@ Current scope is only:
- `POST /v1/runs` - `POST /v1/runs`
For CLI behavior, see `docs/cli.md`. For CLI behavior, see the [CLI reference](../cli.md).
## Endpoint ## Endpoint
@@ -46,10 +46,14 @@ Copyable request example file:
"max_tokens": 800, "max_tokens": 800,
"top_p": 1.0, "top_p": 1.0,
"timeout_seconds": 120, "timeout_seconds": 120,
"service_tier": "priority",
"reasoning_effort": "medium", "reasoning_effort": "medium",
"api_key_env": "SCRIPTORIUM_API_KEY", "api_key_env": "SCRIPTORIUM_API_KEY",
"extra_params": { "extra_params": {
"route": "primary" "route": "primary",
"provider_options": {
"retry_budget": 2
}
} }
}, },
"include_raw_output": false "include_raw_output": false
@@ -66,6 +70,14 @@ Input reference types currently supported by runtime artifact loading:
- `file` - `file`
- `inline` - `inline`
Model override notes:
- Numeric model override fields distinguish omitted values from explicit zero values. For example, omitting `temperature` preserves the selected profile/default value, while `"temperature": 0` explicitly sets the effective temperature to zero.
- `extra_params` accepts JSON-compatible values: strings, numbers, booleans, objects, and arrays.
- `extra_params` are passed through effective model metadata and flattened into top-level provider request fields by the OpenAI-compatible client.
- `extra_params` keys must not be empty and must not collide with reserved outbound fields: `model`, `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, or `response_format`.
- Raw API-key values are not accepted. Use `api_key_env` to name an environment variable.
## Strict JSON Rules ## Strict JSON Rules
Request decoding uses strict JSON field checks: Request decoding uses strict JSON field checks:
@@ -114,10 +126,14 @@ Response shape:
"max_tokens": 800, "max_tokens": 800,
"top_p": 1, "top_p": 1,
"timeout_seconds": 120, "timeout_seconds": 120,
"service_tier": "priority",
"reasoning_effort": "medium", "reasoning_effort": "medium",
"api_key_env": "SCRIPTORIUM_API_KEY", "api_key_env": "SCRIPTORIUM_API_KEY",
"extra_params": { "extra_params": {
"route": "primary" "route": "primary",
"provider_options": {
"retry_budget": 2
}
} }
}, },
"input_hashes": { "input_hashes": {
@@ -126,7 +142,9 @@ Response shape:
"usage": { "usage": {
"prompt_tokens": 11, "prompt_tokens": 11,
"completion_tokens": 22, "completion_tokens": 22,
"total_tokens": 33 "total_tokens": 33,
"cached_tokens": 0,
"cache_write_tokens": 0
}, },
"start_time": "2026-05-04T12:00:00Z", "start_time": "2026-05-04T12:00:00Z",
"end_time": "2026-05-04T12:00:01Z", "end_time": "2026-05-04T12:00:01Z",
@@ -140,6 +158,8 @@ Response shape:
`raw_model_output` is omitted by default. `raw_model_output` is omitted by default.
`metadata.usage.cached_tokens` and `metadata.usage.cache_write_tokens` are always present as numbers. They are `0` when the provider omits compatible cache usage fields or reports no cache activity.
To include it, send: To include it, send:
- `"include_raw_output": true` - `"include_raw_output": true`

View File

@@ -109,6 +109,6 @@ A `run` exit code `2` can still produce output (stdout or `--out`).
## Canonical References ## Canonical References
- CLI behavior: `docs/cli.md` - CLI behavior: [CLI reference](../cli.md)
- Config behavior: `docs/config.md` - Config behavior: [Configuration reference](../config.md)
- Operations and failure handling: `docs/operations.md`, `docs/troubleshooting.md` - Operations and failure handling: [Operations guide](../operations.md), [Troubleshooting](../troubleshooting.md)

View File

@@ -26,11 +26,84 @@ Example:
Serialized JSON fields: Serialized JSON fields:
- `model` (required after fallback resolution) - `model` (required after fallback resolution)
- `messages` (role/content pairs from rendered prompt) - `session_id` (only when the rendered prompt includes a non-empty session ID)
- `temperature` (only when non-zero) - `messages` (rendered prompt messages)
- `max_tokens` (only when non-zero) - `temperature` (when non-zero, or when explicitly overridden to zero)
- `top_p` (only when non-zero) - `max_tokens` (when non-zero, or when explicitly overridden to zero)
- `top_p` (when non-zero, or when explicitly overridden to zero)
- `service_tier` (only when non-empty)
- `reasoning_effort` (only when non-empty)
- `response_format` (only when structured output is provided) - `response_format` (only when structured output is provided)
- profile/request `extra_params` as additional provider-specific top-level fields
`service_tier` is provider-specific. OpenRouter currently documents request values such as `flex` and `priority`; Scriptorium forwards any non-empty configured value and lets the backend validate support.
`reasoning_effort` is provider-specific. Scriptorium forwards any non-empty configured value as top-level `reasoning_effort` and lets the backend validate support.
`extra_params` are flattened into the outbound JSON object. They are not wrapped in an `extra_params` object:
```json
{
"model": "gpt-4o-mini",
"messages": [
{
"role": "user",
"content": "rendered text"
}
],
"provider_route": "primary",
"provider_options": {
"retry_budget": 2
}
}
```
`extra_params` values must be JSON-compatible. Supported value shapes include strings, numbers, booleans, objects, and arrays.
Reserved `extra_params` keys are rejected before the HTTP request is made:
- `model`
- `session_id`
- `messages`
- `temperature`
- `max_tokens`
- `top_p`
- `service_tier`
- `reasoning_effort`
- `response_format`
Empty `extra_params` keys and values that cannot be encoded as JSON are also rejected before the HTTP request is made.
`session_id` is rendered from prompt YAML using request variables and serialized as a top-level JSON request field. Scriptorium does not send an `x-session-id` header. Empty rendered session IDs are omitted, and values longer than 256 characters are rejected before the HTTP request.
Messages without prompt cache control serialize with string `content`:
```json
{
"role": "system",
"content": "rendered text"
}
```
Messages with prompt cache control serialize as a single text content-block array:
```json
{
"role": "system",
"content": [
{
"type": "text",
"text": "rendered text",
"cache_control": {
"type": "ephemeral",
"ttl": "1h"
}
}
]
}
```
When cache-control `ttl` is unset in the prompt definition, `ttl` is omitted from the outbound payload.
Structured output is currently `json_schema` only, serialized as: Structured output is currently `json_schema` only, serialized as:
@@ -69,6 +142,7 @@ Base timeout comes from client configuration.
Per-request override: Per-request override:
- if `Target.TimeoutSeconds > 0`, use that value for request timeout - if `Target.TimeoutSeconds > 0`, use that value for request timeout
- if `Target.TimeoutSeconds == 0` and the value came from an explicit request override, disable the HTTP client timeout
- if `Target.TimeoutSeconds < 0`, request is rejected (`ErrInvalidRequest`) - if `Target.TimeoutSeconds < 0`, request is rejected (`ErrInvalidRequest`)
## Response Expectations ## Response Expectations
@@ -79,6 +153,13 @@ Expected successful response shape (subset used):
- `usage.prompt_tokens` - `usage.prompt_tokens`
- `usage.completion_tokens` - `usage.completion_tokens`
- `usage.total_tokens` - `usage.total_tokens`
- `usage.prompt_tokens_details.cached_tokens` (optional)
- `usage.cache_write_tokens` (optional)
Absent cache usage fields are treated as zero. Parsed cache usage is exposed through run results and adapter response surfaces as:
- `cached_tokens`
- `cache_write_tokens`
Malformed response conditions include: Malformed response conditions include:
@@ -96,10 +177,7 @@ Malformed responses return `ErrMalformedResponse`.
## Unsupported Or Non-Serialized Fields ## Unsupported Or Non-Serialized Fields
The following fields may exist in profile/effective settings but are not currently serialized into outbound chat-completions payloads: The client does not serialize top-level `cache_control`.
- `reasoning_effort`
- `extra_params`
No built-in retries, tool-calls, or multi-request payload modes are implemented in this client. No built-in retries, tool-calls, or multi-request payload modes are implemented in this client.

View File

@@ -22,15 +22,17 @@ CLI adapter:
- Input: process args, filesystem config/assets, environment. - Input: process args, filesystem config/assets, environment.
- Output: exit code, stdout artifact/prepared output, stderr summaries/errors. - Output: exit code, stdout artifact/prepared output, stderr summaries/errors.
- `run` summaries include cache usage counters only when either parsed cache counter is non-zero.
HTTP adapter: HTTP adapter:
- Input: JSON request body (`runRequestDTO`). - Input: JSON request body (`runRequestDTO`).
- Output: JSON success/error body with mapped status codes. - Output: JSON success/error body with mapped status codes.
- Success metadata includes token usage plus cache usage counters.
Filesystem repositories: Filesystem repositories:
- Input: prompt/profile YAML files. - Input: prompt/profile YAML files under configured directories.
- Output: normalized domain definitions/profiles or typed errors. - Output: normalized domain definitions/profiles or typed errors.
Artifact reader: Artifact reader:
@@ -66,7 +68,9 @@ Primary app settings consumed by adapters:
Execution profile/request settings used through runner: Execution profile/request settings used through runner:
- `endpoint`, `model`, `temperature`, `max_tokens`, `top_p`, `timeout_seconds`, `api_key_env`, `reasoning_effort`, `extra_params` - `endpoint`, `model`, `temperature`, `max_tokens`, `top_p`, `timeout_seconds`, `service_tier`, `api_key_env`, `reasoning_effort`, `extra_params`
- CLI and HTTP request adapters preserve caller intent for numeric runtime overrides. Omitted values remain absent; explicit zero values are mapped as explicit overrides.
- HTTP `extra_params` accepts JSON-compatible values and maps them to domain request overrides without provider-specific adapter logic.
## External Dependencies ## External Dependencies
@@ -79,6 +83,9 @@ Execution profile/request settings used through runner:
Strict decoding and input checks: Strict decoding and input checks:
- config/prompt/profile loaders reject unknown YAML fields. - config/prompt/profile loaders reject unknown YAML fields.
- prompt/profile repositories scan nested subdirectories recursively.
- prompt/profile lookup uses YAML `id` values; subdirectory paths are organizational only.
- duplicate prompt/profile IDs are invalid and fail instead of using first-match behavior.
- HTTP DTO decoder rejects unknown JSON fields. - HTTP DTO decoder rejects unknown JSON fields.
- raw API key payload fields are rejected by strict decoding in profile/http paths. - raw API key payload fields are rejected by strict decoding in profile/http paths.
@@ -90,6 +97,13 @@ Artifact refs:
LLM adapter: LLM adapter:
- endpoint appends `/chat/completions`. - endpoint appends `/chat/completions`.
- rendered messages without cache control serialize with string `content`.
- rendered messages with cache control serialize as one text content block with `cache_control`.
- non-empty `reasoning_effort` serializes as a top-level provider request field.
- `extra_params` flatten into provider-specific top-level JSON request fields.
- reserved `extra_params` keys are rejected before the provider call: `model`, `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, and `response_format`.
- empty `extra_params` keys and values that cannot be JSON-encoded are rejected before the provider call.
- compatible cache usage response fields are parsed into domain token usage.
- non-2xx responses map to request failure errors. - non-2xx responses map to request failure errors.
- malformed responses (including missing/empty first choice content) are errors. - malformed responses (including missing/empty first choice content) are errors.
@@ -97,10 +111,12 @@ Validator:
- `basic`, `json`, `json_schema` content failures return `ValidationFailed` results. - `basic`, `json`, `json_schema` content failures return `ValidationFailed` results.
- schema load/compile/path failures are runtime errors. - schema load/compile/path failures are runtime errors.
- schema lookup uses explicit `schema_path` values relative to `schema_dir`; it does not recursively search by basename.
HTTP error mapping: HTTP error mapping:
- maps domain/use-case errors to stable HTTP code + error code/message. - maps domain/use-case errors to stable HTTP code + error code/message.
- distinguishes missing profile selection and missing `api_key_env` variable using stable use-case sentinel errors.
- avoids returning internal wrapped-cause details in response payload. - avoids returning internal wrapped-cause details in response payload.
## CLI Adapter Semantics ## CLI Adapter Semantics
@@ -136,4 +152,5 @@ Behavior highlights:
- Adapter packages do not own runner decision logic. - Adapter packages do not own runner decision logic.
- External request/response strictness is part of contract stability. - External request/response strictness is part of contract stability.
- Prepared-render output never includes resolved API key values. - Prepared-render output never includes resolved API key values.
- Outbound OpenAI-compatible request includes only currently serialized fields (`model`, `messages`, optional `temperature`, `max_tokens`, `top_p`, optional `response_format`). - Outbound OpenAI-compatible request includes currently serialized first-class fields (`model`, optional `session_id`, `messages`, optional `temperature`, `max_tokens`, `top_p`, optional `service_tier`, optional `reasoning_effort`, optional `response_format`) plus validated `extra_params` flattened as provider-specific top-level fields.
- Outbound cache control is message-level only; no top-level cache-control field is serialized.

View File

@@ -66,14 +66,21 @@ It receives fully constructed repositories/readers/validators from adapters. Eff
## Failure Behavior ## Failure Behavior
Key error classes surfaced from `Runner`: Primary runner error classes:
- `ErrInvalidRequest`: invalid prompt/profile/request/runtime/API-key-env prerequisites. - `ErrInvalidRequest`: invalid run request envelope.
- `ErrProfileLoad`: prompt or profile load failures. - `ErrProfileRequired`: specific invalid-request reason when neither request `profile_id` nor prompt `default_profile` is available.
- `ErrAPIKeyEnvMissing`: specific invalid-request reason when `api_key_env` is set but the named environment variable is unset/empty.
- `ErrProfileLoad`: prompt/profile repository load failures.
- `ErrArtifactLoad`: artifact read failures. - `ErrArtifactLoad`: artifact read failures.
- `ErrPromptRender`: template render failures. - `ErrPromptRender`: template render failures.
- `ErrLLMGenerate`: model request failures. - `ErrLLMGenerate`: outbound model request failures.
- `ErrValidation`: validation runtime failures (including schema load/compile failures). - `ErrValidation`: validation runtime failures (including structured-output schema load/compile failures).
Reason sentinel behavior:
- `ErrProfileRequired` and `ErrAPIKeyEnvMissing` are wrapped with `ErrInvalidRequest`.
- Adapters can use `errors.Is` for stable reason mapping without matching runner prose.
Validation content failures are not run errors: Validation content failures are not run errors:
@@ -90,20 +97,33 @@ Validation content failures are not run errors:
3. select profile ID: 3. select profile ID:
- explicit request profile ID - explicit request profile ID
- prompt `default_profile` - prompt `default_profile`
- otherwise request error - otherwise return an invalid request with `ErrProfileRequired`
4. load execution profile. 4. load execution profile.
5. merge effective runtime target: 5. merge effective runtime target:
- built-in execution defaults - built-in execution defaults
- selected profile values - selected profile values
- request overrides - request overrides
6. verify required `api_key_env` environment variable (name only; value is not returned). - request numeric overrides are presence-aware, so omitted values preserve the current effective value and explicit zero values override it
6. verify required `api_key_env` environment variable:
- missing/empty env value returns an invalid request with `ErrAPIKeyEnvMissing`
- only the environment-variable name is retained; secret value is never returned
7. resolve output contract and structured-output schema payload when `json_schema` mode is active. 7. resolve output contract and structured-output schema payload when `json_schema` mode is active.
8. read input artifacts. 8. read input artifacts.
9. render prompt messages. 9. render prompt messages, including any normalized message cache-control metadata.
10. compute prompt/input/render hashes and return `PreparedRun`. 10. compute prompt/input/render hashes and return `PreparedRun`.
`rendered_prompt_hash` includes cache-control metadata when present because it affects the outbound provider request. Prompts without cache control keep the role/content hash behavior.
`Prepare` does not call the LLM. `Prepare` does not call the LLM.
Runtime target notes:
- Profile `extra_params` and request `extra_params` carry JSON-compatible values through prepared output, run metadata, and `domain.GenerateRequest.Target`.
- The OpenAI-compatible client serializes non-empty `reasoning_effort` as a top-level provider request field.
- The OpenAI-compatible client flattens `extra_params` into provider-specific top-level JSON request fields.
- Empty `extra_params` keys, reserved outbound field names, and values that cannot be JSON-encoded fail before the provider request.
- Resolved API-key values are never stored in `PreparedRun`, `RunResult`, logs, or HTTP responses.
## Run Flow ## Run Flow
`Run` performs: `Run` performs:
@@ -114,7 +134,7 @@ Validation content failures are not run errors:
4. build output artifact content type from output format. 4. build output artifact content type from output format.
5. validate output. 5. validate output.
6. optionally attempt bounded repair when repairer is injected and contract allows it. 6. optionally attempt bounded repair when repairer is injected and contract allows it.
7. return `RunResult` with artifact, raw output, validation, hashes, profile/model metadata, usage, and timestamps. 7. return `RunResult` with artifact, raw output, validation, hashes, profile/model metadata, token/cache usage, and timestamps.
## Repair Hook Boundary ## Repair Hook Boundary

View File

@@ -48,8 +48,9 @@ Typical sequence:
1. Confirm prompt/profile directories resolve through config or flags. 1. Confirm prompt/profile directories resolve through config or flags.
2. Confirm required input files exist and map to prompt input names. 2. Confirm required input files exist and map to prompt input names.
3. Confirm required API-key environment variables are set. 3. Confirm required API-key environment variables are set.
4. Run `render` for preflight when changing prompt/profile/input wiring. 4. Confirm the selected profile's model endpoint is reachable from the process environment.
5. Run `run` for actual generation. 5. Run `render` for preflight when changing prompt/profile/input wiring.
6. Run `run` for actual generation.
## Secrets Handling ## Secrets Handling

View File

@@ -1,355 +0,0 @@
# Documentation Roadmap
## Purpose
This roadmap defines the work required to bring scriptorium's documentation into compliance with `docs/policy/documentation.md` and the current implementation. It is a planning document only; future agents should use it to update the canonical documentation without describing unimplemented behavior outside `docs/roadmap/`.
## Repository Documentation Inventory
- `README.md` - keep and rewrite. It is currently a full manual covering config, CLI, HTTP, prompt/profile authoring, examples, and build commands; policy says README should be a short orientation page with a quickstart and links.
- `architecture.md` - move/merge/delete after rewrite. It overlaps with `docs/policy/architecture.md`, contains "should" guidance and future extension notes outside `docs/roadmap/`, and references architecture that is partly stale or aspirational.
- `docs/policy/documentation.md` - keep and lightly update only if policy itself changes. It is the controlling documentation policy.
- `docs/policy/architecture.md` - keep and lightly update. It is the canonical development architecture policy, but some package-layout defaults do not exactly match this repository (`internal/adapter/...` versus policy examples such as `internal/adapters/...`).
- `docs/policy/development.md` - create new. Required by policy for projects maintained by humans and LLM coding agents.
- `docs/config/config-yml.md` - merge into `docs/config.md`. The content mostly matches code but lives in the wrong canonical home.
- `docs/config/prompt-definitions.md` - merge into `docs/config.md`. The field reference is mostly accurate, but it needs caveats about `repair_attempts`, repeated message roles, schema path resolution, and examples.
- `docs/config/profile-definitions.md` - merge into `docs/config.md`. It must stop implying that every profile field is sent to the provider; the current OpenAI-compatible client does not send `reasoning_effort` or `extra_params`.
- `docs/config/schema-definitions.md` - merge into `docs/config.md` or link from it. Schema behavior is implemented, but the canonical config reference should own this material.
- `docs/cli.md` - create new. Required by policy for the implemented CLI.
- `docs/operations.md` - create new. Required by policy for this CLI/service application; scope should cover normal operation, config paths, secret handling, stdout/stderr, exit codes, HTTP serving, and the fact that there is no durable run state/resume behavior.
- `docs/troubleshooting.md` - create new. Recommended by policy and justified by implemented failure modes in parser, config, prompt/profile loading, artifact reading, validation, LLM calls, and HTTP error mapping.
- `docs/internal/` - create new. Required by policy for this modular application.
- `docs/integrations/narratio.md` - keep and rewrite. It documents an actual CLI integration contract, but it includes future extension notes outside roadmap and illustrative prompt IDs that are not all present in the repository.
- `docs/integrations/http-api.md` - create new. The implemented `POST /v1/runs` API is an external integration contract and should not live in README.
- `docs/integrations/openai-compatible-chat.md` - create new. The outbound LLM contract is important and implemented in `internal/llm/openai_compatible_client.go`.
- `examples/config.yml` - keep and lightly update if paths move. It is a valid app config example for the current root `prompts/`, `profiles/`, and `schemas/` directories.
- `examples/fixtures/transcript.md` - keep. It is used by integration tests.
- `examples/fixtures/glossary.yml` - keep. It is used by integration tests.
- `prompts/` - keep as maintained sample prompt library for now; recommended to move or mirror under `examples/` only if tests and docs are updated together.
- `profiles/` - keep as maintained sample profile library for now; recommended to move or mirror under `examples/` only if tests and docs are updated together.
- `schemas/` - keep as maintained sample schema library for now; recommended to move or mirror under `examples/` only if tests and docs are updated together.
- `local-test/` - delete, move out of the repository, or explicitly exclude from maintained docs. It contains ad hoc local artifacts and provider profiles; it should not be linked from canonical docs unless promoted to maintained examples with tests and secret review.
## Policy Compliance Assessment
Required documents that are missing:
- `docs/cli.md`
- `docs/config.md`
- `docs/operations.md`
- `docs/internal/`
- `docs/policy/development.md`
Recommended documents that should be added:
- `docs/troubleshooting.md`
- Validated examples under `examples/` beyond the current config and fixtures, especially command examples that can be checked with `render`.
- Integration docs for the implemented HTTP API and outbound OpenAI-compatible chat API.
Documents that exist but are stale or in the wrong canonical home:
- `README.md` duplicates material that belongs in `docs/cli.md`, `docs/config.md`, `docs/integrations/`, and `docs/internal/`.
- `docs/config/*.md` should be merged into `docs/config.md`.
- `architecture.md` should be merged into `docs/policy/architecture.md`, `docs/internal/`, or `docs/roadmap/`, then removed.
- `docs/integrations/narratio.md` should remain under integrations but must be narrowed to implemented CLI behavior and actual integration guidance.
Content that appears to describe deprecated, historical, planned, or unimplemented behavior outside `docs/roadmap/`:
- `architecture.md` has future extension notes for S3 artifact references, additional LLM providers, streaming, batch execution, database-backed repositories, profile versioning, and HTTP render endpoints.
- `architecture.md` and `README.md` describe bounded repair as if it is generally active. The code has an injected repairer hook, but the CLI and HTTP server construct `Runner` without a repairer, so production commands do not currently perform repair attempts.
- `README.md` says "additional output formats can be added later"; this belongs in roadmap only.
- `README.md` references `go build -o scriptorium ./cmd/scriptorium`, which is valid, but the README should not be the build/test manual after `docs/policy/development.md` exists.
- `docs/integrations/narratio.md` has "Future Extension Notes" and examples for prompt IDs not present in the repository, such as `dnd.structured_events`, `dnd.glossary_suggestions`, and `dnd.player_summary`.
- Any docs implying S3 artifact support should be removed from current-behavior docs. `domain.ArtifactRefS3` exists, but `artifact.CompositeReader` supports only `inline` and `file`.
Examples that are missing, stale, invalid, or untested:
- `examples/config.yml` points at root `prompts/`, `profiles/`, and `schemas/`; it is valid for repository-root execution but should be tested or explicitly checked.
- There are no copyable CLI example scripts or expected-output files under `examples/`.
- The maintained prompt/profile/schema examples live outside `examples/`; this is usable, but policy prefers copyable examples under `examples/`.
- `local-test/` appears unmaintained and should not be treated as documentation.
Links that are likely stale or need verification:
- Existing links from README to docs should be rewritten after canonical files are created.
- Any references to `docs/config/config-yml.md`, `docs/config/prompt-definitions.md`, `docs/config/profile-definitions.md`, or `docs/config/schema-definitions.md` should be updated after those files are merged.
- References to default config paths must use the implemented search order: `/usr/local/etc/scriptorium/config.yml`, then `/etc/scriptorium/config.yml`.
## Target Documentation Set
### `README.md`
- Audience: users, administrators, operators.
- Purpose: concise orientation and shortest useful command.
- Canonical scope: project purpose, elevator pitch, quickstart, and links.
- Recommended outline: description; why scriptorium exists; shortest useful `scriptorium render` or `scriptorium run` example; documentation links.
- Source of truth: `cmd/scriptorium/main.go`, `internal/adapter/cli/run.go`, `examples/config.yml`, `prompts/generic.markdown_summary.yaml`.
- Acceptance criteria: no full flag reference; no HTTP schema; no prompt/profile field tables; no future work; all links point to existing target docs.
### `docs/cli.md`
- Audience: users, administrators, operators.
- Purpose: complete CLI reference and common workflows.
- Canonical scope: commands, flags, outputs, exit codes, and command examples.
- Recommended outline: shortest useful command; command overview; `run`; `render`; `serve`; flag reference; input and variable mapping syntax; output behavior; exit codes; common workflows.
- Source of truth: `internal/adapter/cli/run.go`, `internal/adapter/cli/run_test.go`, `internal/format/prepared_run.go`, `cmd/scriptorium/main.go`.
- Acceptance criteria: documents real flags only; notes deprecated aliases `--prompt-id` and `--profile-id`; documents that `render` does not currently accept `--schema-dir`; documents stdout/stderr split and exit code `2` for validation failure.
### `docs/config.md`
- Audience: administrators, operators, advanced users.
- Purpose: canonical reference for app config, prompt definitions, profiles, and schemas.
- Canonical scope: all implemented YAML/JSON file formats and precedence rules.
- Recommended outline: config file discovery and precedence; minimal app config; production-oriented app config; app config reference; prompt definition reference; profile definition reference; schema behavior; secrets handling; maintained examples.
- Source of truth: `internal/config/config.go`, `internal/defaults/defaults.go`, `internal/promptdef/filesystem_repository.go`, `internal/profile/filesystem_repository.go`, `internal/validate/standard_validator.go`, repository `prompts/`, `profiles/`, `schemas/`, and config/profile/prompt tests.
- Acceptance criteria: replaces split `docs/config/*.md`; documents strict YAML decoding; documents raw API key rejection; documents `schema_dir` default `.`; documents prompt `content_file` relative to prompt YAML; states `content_type` is metadata only; does not claim operational repair unless a repairer is configured.
### `docs/operations.md`
- Audience: administrators, operators.
- Purpose: operational use of CLI and HTTP service.
- Canonical scope: normal workflow, filesystem expectations, config deployment, secrets, logs/output, validation behavior, and recovery from failed runs.
- Recommended outline: normal run/render workflow; config and library directories; environment variables for API keys; serving HTTP; output and stderr summaries; validation failure handling; no durable state/resume/archive behavior; safe recovery steps.
- Source of truth: `internal/adapter/cli/run.go`, `internal/adapter/http/handler.go`, `internal/config/config.go`, `internal/llm/openai_compatible_client.go`, `internal/usecase/runner.go`.
- Acceptance criteria: makes clear scriptorium does not persist run state; does not invent cleanup/archive/resume; documents that HTTP has no built-in authentication and should be deployed behind trusted controls.
### `docs/troubleshooting.md`
- Audience: administrators, operators.
- Purpose: safe diagnosis and fixes for recurring implemented failure modes.
- Canonical scope: symptoms, likely causes, diagnostics, safe fixes, and links.
- Recommended outline: missing config; missing prompt/profile dirs; unknown flags; prompt/profile load failures; missing input files; template render failures; missing API-key environment values; LLM non-2xx/malformed responses; JSON/schema validation failures; HTTP error codes.
- Source of truth: `internal/adapter/cli/run_test.go`, `internal/adapter/http/handler_test.go`, `internal/config/config_test.go`, `internal/promptdef/repository_test.go`, `internal/profile/repository_test.go`, `internal/validate/standard_validator_test.go`, `internal/llm/openai_compatible_client_test.go`.
- Acceptance criteria: every entry includes symptom, likely cause, diagnostic step, safe fix, and links to canonical CLI/config/operations docs.
### `docs/policy/architecture.md`
- Audience: developers, LLM coding agents.
- Purpose: controlling development architecture and invariants.
- Canonical scope: development principles, boundaries, invariants, non-goals.
- Recommended outline: keep current policy shape; add scriptorium-specific package map or link to `docs/internal/`; clarify no orchestration creep; clarify current adapters.
- Source of truth: existing policy, `internal/` package layout, `architecture.md`.
- Acceptance criteria: remains policy-oriented; does not become user docs; future work stays in roadmap; no stale package names.
### `docs/policy/development.md`
- Audience: developers, LLM coding agents.
- Purpose: contributor workflow and change checklist.
- Canonical scope: repository layout, build/test commands, coding conventions, dependency policy, adding config/CLI/adapters, updating examples/docs.
- Recommended outline: repository layout; common commands; coding conventions; dependency policy; how to add config fields; how to add CLI flags; how to add adapters; how to update examples; documentation expectations.
- Source of truth: `go.mod`, `cmd/scriptorium/main.go`, `internal/adapter/cli/run.go`, `internal/config/config.go`, `docs/policy/architecture.md`, existing tests.
- Acceptance criteria: includes `go test ./...`; references `go build ./cmd/scriptorium`; tells contributors to update docs and tests with behavior changes.
### `docs/internal/runner.md`
- Audience: developers, LLM coding agents.
- Purpose: implemented core prepare/run behavior.
- Canonical scope: `Runner.Prepare`, `Runner.Run`, profile selection, runtime merge, artifact loading, rendering, structured output setup, validation, repair hook boundary.
- Recommended outline: purpose; inputs/outputs; prepare flow; run flow; boundary contracts; failure behavior; tests; invariants.
- Source of truth: `internal/usecase/runner.go`, `internal/usecase/repairer.go`, `internal/usecase/runner_test.go`, `internal/usecase/integration_test.go`.
- Acceptance criteria: states CLI/HTTP currently construct `Runner` without a repairer; documents validation content failures versus runtime validation errors; no provider-specific details except through ports.
### `docs/internal/adapters.md`
- Audience: developers, LLM coding agents.
- Purpose: implemented adapter boundaries.
- Canonical scope: CLI adapter, HTTP adapter, filesystem repositories, artifact reader, prompt renderer, OpenAI-compatible LLM client, validator, prepared-run formatter.
- Recommended outline: adapter map; inputs/outputs; config fields used; external dependencies; failure behavior; tests to inspect.
- Source of truth: `internal/adapter/cli`, `internal/adapter/http`, `internal/promptdef`, `internal/profile`, `internal/artifact`, `internal/prompt`, `internal/llm`, `internal/validate`, `internal/format`.
- Acceptance criteria: documents only implemented adapters; states `inline` and `file` artifact refs are supported and S3 is not; states OpenAI request fields actually sent.
### `docs/integrations/http-api.md`
- Audience: developers, LLM coding agents, API clients.
- Purpose: implemented inbound HTTP contract.
- Canonical scope: `POST /v1/runs`, request/response shape, raw output opt-in, error mapping, validation-failed status behavior.
- Recommended outline: scope; endpoint; request fields; response fields; error responses; validation behavior; security/auth note.
- Source of truth: `internal/adapter/http/dto.go`, `internal/adapter/http/handler.go`, `internal/adapter/http/handler_test.go`.
- Acceptance criteria: no unimplemented render endpoint; no built-in auth claim; unknown JSON fields rejected; raw API key fields rejected by strict JSON.
### `docs/integrations/openai-compatible-chat.md`
- Audience: developers, LLM operators, LLM adapter maintainers.
- Purpose: implemented outbound LLM API contract.
- Canonical scope: OpenAI-compatible chat completions request/response subset and provider-level structured output behavior.
- Recommended outline: endpoint construction; request fields sent; auth header from `api_key_env`; timeout behavior; response expectations; error handling; unsupported profile fields.
- Source of truth: `internal/llm/openai_compatible_client.go`, `internal/llm/openai_compatible_client_test.go`, `internal/usecase/runner.go`.
- Acceptance criteria: says endpoint appends `/chat/completions`; says empty first choice content is malformed; says `reasoning_effort` and `extra_params` are not currently serialized into the outbound request.
### `docs/integrations/narratio.md`
- Audience: developers, LLM coding agents maintaining Narratio integration.
- Purpose: CLI subprocess contract for Narratio.
- Canonical scope: how Narratio should call implemented `scriptorium run` and `scriptorium render`.
- Recommended outline: purpose; assumptions; command shapes; inputs/vars; profile selection; runtime overrides; config behavior; environment handling; output handling; exit statuses; security notes; non-goals.
- Source of truth: `internal/adapter/cli/run.go`, `internal/adapter/cli/run_test.go`, `docs/cli.md`, `docs/config.md`.
- Acceptance criteria: removes future extensions; labels any Narratio-specific prompt IDs as external examples only or removes them; links to canonical CLI/config docs.
## File-by-File Rewrite Guidance
- `README.md`: cover what scriptorium does and show one minimal command. Avoid field tables, complete flag lists, HTTP schema, internal package details, future extensions, and long examples. Link to `docs/cli.md`, `docs/config.md`, `docs/operations.md`, `docs/troubleshooting.md`, and `docs/integrations/`.
- `docs/cli.md`: cover `run`, `render`, `serve`, flags, output behavior, and exit codes. Avoid duplicating prompt/profile YAML field references; link to `docs/config.md`. Inspect CLI parser tests before writing examples. Do not carry forward README's claim that render supports `--schema-dir`.
- `docs/config.md`: cover app config, prompt YAML, profile YAML, and schema behavior. Avoid provider API details except where needed for profile fields; link to OpenAI integration doc. Do not carry forward claims that `repair_attempts` enables repair for normal CLI/HTTP runs unless code later wires a repairer.
- `docs/operations.md`: cover deployed operation and recovery boundaries. Avoid inventing durable state, resume behavior, cleanup, backups, or archives. State that rerunning a command is the recovery model.
- `docs/troubleshooting.md`: use tested errors and behavior. Avoid exposing internal wrapped error details that HTTP intentionally suppresses. Link to CLI/config/operations instead of repeating full references.
- `docs/policy/architecture.md`: preserve policy authority and update only scriptorium-specific facts. Avoid copying the long historical `architecture.md` wholesale. Move future extension ideas to roadmap docs only.
- `docs/policy/development.md`: cover contributor mechanics and how to update behavior safely. Avoid user-facing manuals. Include tests and docs update expectations.
- `docs/internal/runner.md`: explain implemented core flow and invariants. Avoid CLI flag tables and HTTP DTO detail; link to adapter docs.
- `docs/internal/adapters.md`: explain implemented adapter boundaries and tests. Avoid proposing new adapters. Do not imply `ArtifactRefS3` works.
- `docs/integrations/http-api.md`: document only `POST /v1/runs`. Avoid documenting a render/prepare HTTP endpoint.
- `docs/integrations/openai-compatible-chat.md`: document the outbound request subset. Avoid documenting unsupported OpenAI fields or provider-specific options unless code sends them.
- `docs/integrations/narratio.md`: keep it as a subprocess contract. Avoid future work, S3, HTTP-as-primary-path, and undeployed prompt IDs as current examples.
- `architecture.md`: after target docs exist, delete it or replace it with a short pointer to `docs/policy/architecture.md` and `docs/internal/`. Do not leave future notes in this root file.
- `docs/config/*.md`: after `docs/config.md` exists and links are updated, delete these split files or replace them with pointers only if backwards-compatible links are necessary.
## Examples Plan
Existing maintained examples:
- `examples/config.yml`: minimal app config pointing at root prompt/profile/schema libraries. Validity check: `go test ./internal/config ./internal/adapter/cli` and `go run ./cmd/scriptorium render --config ./examples/config.yml --prompt generic.markdown_summary --input transcript=./examples/fixtures/transcript.md --format json`. Link from README, `docs/config.md`, and `docs/cli.md`.
- `examples/fixtures/transcript.md`: sample transcript input. Validity check: used by `internal/usecase/integration_test.go` and render smoke command. Link from README and `docs/cli.md`.
- `examples/fixtures/glossary.yml`: sample optional glossary input. Validity check: used by `internal/usecase/integration_test.go`. Link from `docs/config.md` and examples section in `docs/cli.md`.
- `prompts/generic.markdown_summary.yaml`: sample markdown prompt. Validity check: render smoke command. Link from config docs until or unless it is moved under `examples/`.
- `prompts/generic.structured_events.yaml` plus `schemas/structured_events.schema.json`: sample JSON-schema prompt. Validity check: `go test ./internal/usecase`. Link from `docs/config.md`.
- `profiles/local-fast.yaml` and `profiles/local-quality.yaml`: sample profiles. Validity check: profile repository tests plus integration test. Link from `docs/config.md`, with a note that `local-quality` requires `SCRIPTORIUM_API_KEY` because it sets `api_key_env`.
Recommended example additions, all based on implemented behavior:
- `examples/render-markdown-summary.sh`: copyable render smoke command using `generic.markdown_summary`. Expected check: run script or equivalent `go run` command exits `0`. Link from README and `docs/cli.md`.
- `examples/http-run.json`: copyable `POST /v1/runs` request body using `inline` or `file` artifact refs. Expected check: parse as JSON and keep aligned with `internal/adapter/http/dto.go`. Link from `docs/integrations/http-api.md`.
- `examples/prompts/`, `examples/profiles/`, `examples/schemas/`: optional future move or mirror of maintained sample libraries. Expected check: update integration tests and `examples/config.yml` together. This is recommended for policy alignment but should be done as its own implementation stage to avoid breaking tests.
Do not document `local-test/` as maintained examples.
## Internal Documentation Plan
### Core runner
- Path: `docs/internal/runner.md`
- Purpose: explain implemented prepare/run lifecycle.
- Inputs and outputs: `domain.RunRequest`, `domain.PreparedRun`, `domain.RunResult`, `domain.GenerateRequest`.
- Boundaries: usecase owns profile selection, runtime merge, artifact resolution orchestration, rendering orchestration, structured output setup, validation, and run metadata; adapters own transport/config parsing.
- Config fields used: none directly; adapters pass resolved repositories, validators, and request values.
- Adapters used: promptdef repository, profile repository, artifact reader, prompt renderer, LLM client, validator, optional injected repairer.
- Failure behavior: invalid request, prompt/profile load, artifact load, render failure, LLM failure, validation runtime failure; validation content failures return a result.
- Tests to inspect before changing: `internal/usecase/runner_test.go`, `internal/usecase/integration_test.go`.
- Architectural invariants: `Run` reuses `Prepare`; no resolved API key values in prepared data; repair attempts bounded and only possible when a repairer is injected; no orchestration creep.
### Adapters and repositories
- Path: `docs/internal/adapters.md`
- Purpose: explain implemented external boundaries.
- Inputs and outputs: CLI args/stdout/stderr/exit codes; HTTP JSON DTOs; YAML prompt/profile/config files; file/inline artifacts; OpenAI-compatible HTTP requests; prepared-run text/JSON output.
- Boundaries: adapters translate external forms into domain requests/results and must not own domain decisions.
- Config fields used: `prompt_dir`, `profile_dir`, `schema_dir`, `server.addr`, `defaults.render_format`; profile `endpoint`, `model`, generation fields, timeout, `api_key_env`.
- Adapters used: CLI, HTTP, filesystem repositories, artifact reader, Go template renderer, OpenAI-compatible client, standard validator, prepared-run formatter.
- Failure behavior: strict YAML/JSON decoding, unknown fields rejected, unsupported artifact refs rejected, LLM non-2xx/malformed responses become errors.
- Tests to inspect before changing: adapter, repository, artifact, renderer, LLM, validator, and formatter tests under `internal/**`.
- Architectural invariants: no raw API keys; no S3 docs until reader exists; OpenAI client sends only implemented request fields.
### Validation and structured output
- Path: include in `docs/internal/runner.md` or create `docs/internal/validation.md` if the section grows.
- Purpose: explain `none`, `basic`, `json`, `json_schema`, schema loading, and provider-level JSON schema request setup.
- Inputs and outputs: `domain.Artifact`, `domain.OutputContract`, `domain.ValidationResult`, `domain.StructuredOutputSpec`.
- Boundaries: validator checks output; runner creates provider-level structured output spec for `json_schema`; OpenAI adapter serializes `response_format`.
- Config fields used: `schema_dir`; prompt `output.schema_path`, `output.validation_mode`, `output.format`.
- Adapters used: standard validator and OpenAI-compatible client.
- Failure behavior: invalid generated JSON is validation failure; missing/invalid schema file is runtime validation error before or during run preparation.
- Tests to inspect before changing: `internal/validate/standard_validator_test.go`, `internal/usecase/runner_test.go`, `internal/llm/openai_compatible_client_test.go`.
- Architectural invariants: schema docs must load before `json_schema` LLM request; schema paths resolve relative to `schema_dir`.
## Integration Documentation Plan
### HTTP API
- Path: `docs/integrations/http-api.md`
- External system or contract: inbound HTTP clients of scriptorium.
- Current usage in scriptorium: `scriptorium serve` exposes `POST /v1/runs`.
- Version or compatibility notes: route is `/v1/runs`; request decoding rejects unknown JSON fields.
- What should be documented: request fields, `file` and `inline` artifact refs, model overrides, raw output opt-in, response shape, error codes, validation-failed `200 OK`, no built-in auth.
- What should not be documented: unimplemented render endpoint, streaming, batch, authentication middleware, remote artifact storage.
### OpenAI-compatible chat completions
- Path: `docs/integrations/openai-compatible-chat.md`
- External system or contract: outbound OpenAI-compatible `/chat/completions` API.
- Current usage in scriptorium: `OpenAICompatibleClient.Generate` posts chat messages and optional JSON schema response format.
- Version or compatibility notes: compatibility is defined by the subset used in code, not by a pinned OpenAI API version.
- What should be documented: endpoint construction, request fields, auth header behavior, timeout behavior, expected response shape, error handling, structured output payload.
- What should not be documented: provider features not serialized by code, retries, streaming, tool calls, reasoning controls, or extra provider params.
### Narratio CLI subprocess
- Path: `docs/integrations/narratio.md`
- External system or contract: Narratio calling scriptorium as a subprocess.
- Current usage in scriptorium: public CLI commands `run` and `render`.
- Version or compatibility notes: contract should be tied to implemented CLI flags and exit codes.
- What should be documented: command construction, config use, input files, vars, profile overrides, timeout override, output paths, stdout/stderr handling, exit status semantics.
- What should not be documented: future S3 support, future HTTP primary integration, unimplemented prompt IDs as current examples, or Narratio stage state internals.
JSON Schema is important but does not need a separate integration doc in the first migration; keep schema behavior in `docs/config.md` and internal validation docs unless compatibility issues require a dedicated page later.
## Recommended Implementation Sequence
### Stage 1: Canonical README, CLI, and Config
- Goal: make user-facing docs accurate and move reference material to canonical homes.
- Files to create/update/delete/move: rewrite `README.md`; create `docs/cli.md`; create `docs/config.md`; leave old `docs/config/*.md` temporarily with pointers or delete them only after links are updated.
- Repository areas to inspect: `cmd/scriptorium/main.go`, `internal/adapter/cli/run.go`, `internal/adapter/cli/run_test.go`, `internal/config`, `internal/promptdef`, `internal/profile`, `internal/validate`, `examples/config.yml`, `prompts/`, `profiles/`, `schemas/`.
- Acceptance criteria: README is short; CLI flags match parser; config paths and precedence match code; no future work outside roadmap; no repair claims beyond implemented behavior.
- Suggested validation commands: `go test ./...`; `go run ./cmd/scriptorium render --config ./examples/config.yml --prompt generic.markdown_summary --input transcript=./examples/fixtures/transcript.md --format json`; `rg -n "future|planned|may be added|can be added later|S3|streaming|batch" README.md docs/cli.md docs/config.md`.
- One-prompt size: yes, if old split config files are deleted or replaced with pointers in the same change.
### Stage 2: Operations and Troubleshooting
- Goal: document operational behavior and known failure modes.
- Files to create/update/delete/move: create `docs/operations.md`; create `docs/troubleshooting.md`; update README links.
- Repository areas to inspect: CLI and HTTP adapters, config tests, LLM client tests, validator tests, prompt/profile repository tests.
- Acceptance criteria: no invented state/resume/backup behavior; troubleshooting entries are actionable and link to canonical docs; HTTP no-auth caveat is clear.
- Suggested validation commands: `go test ./internal/adapter/cli ./internal/adapter/http ./internal/config ./internal/llm ./internal/validate`; `rg -n "resume|archive|backup|cleanup|state" docs/operations.md docs/troubleshooting.md`.
- One-prompt size: yes.
### Stage 3: Development Policy and Internal Docs
- Goal: give developers and LLM agents accurate package boundaries and invariants.
- Files to create/update/delete/move: create `docs/policy/development.md`; update `docs/policy/architecture.md`; create `docs/internal/runner.md`; create `docs/internal/adapters.md`; optionally create `docs/internal/validation.md`.
- Repository areas to inspect: all `internal/` packages, `go.mod`, root `architecture.md`, tests.
- Acceptance criteria: package names match repository; current adapters only; future extension ideas absent except links to roadmap; repairer hook boundary is accurate.
- Suggested validation commands: `go test ./...`; `rg -n "should expose|may be added|future|S3|batch|streaming|database-backed|additional providers" docs/policy docs/internal`.
- One-prompt size: maybe split into two prompts if `docs/internal/` becomes too large.
### Stage 4: Integration Docs
- Goal: move external contracts out of README and make integration docs precise.
- Files to create/update/delete/move: create `docs/integrations/http-api.md`; create `docs/integrations/openai-compatible-chat.md`; rewrite `docs/integrations/narratio.md`; update README and CLI/config links.
- Repository areas to inspect: `internal/adapter/http`, `internal/llm`, `internal/usecase`, CLI tests, HTTP tests, LLM tests.
- Acceptance criteria: HTTP docs cover only `POST /v1/runs`; OpenAI docs cover only serialized fields; Narratio docs include only implemented CLI integration and no future notes.
- Suggested validation commands: `go test ./internal/adapter/http ./internal/llm ./internal/adapter/cli`; `rg -n "POST /v1/renders|S3|future|later|batch|streaming" docs/integrations`.
- One-prompt size: yes.
### Stage 5: Examples and Link Cleanup
- Goal: make examples policy-compliant and verify links after moves.
- Files to create/update/delete/move: optionally add `examples/render-markdown-summary.sh`; optionally add `examples/http-run.json`; decide whether to move or mirror `prompts/`, `profiles/`, `schemas/` under `examples/`; delete or exclude `local-test/`; remove or replace root `architecture.md`; delete obsolete `docs/config/*.md` if not already removed.
- Repository areas to inspect: `examples/`, `prompts/`, `profiles/`, `schemas/`, `internal/usecase/integration_test.go`, docs links.
- Acceptance criteria: examples are copyable, secret-free, and tested where practical; no links to deleted docs; no maintained docs link to `local-test/`.
- Suggested validation commands: `go test ./...`; `go run ./cmd/scriptorium render --config ./examples/config.yml --prompt generic.markdown_summary --input transcript=./examples/fixtures/transcript.md --format text`; `rg -n "docs/config/|architecture.md|local-test|dnd.structured_events|dnd.glossary_suggestions|dnd.player_summary" README.md docs examples`.
- One-prompt size: split if moving prompt/profile/schema assets because tests and paths must be updated carefully.
## Validation Plan
- Run `go test ./...` after documentation changes that touch examples, paths, command examples, or config references.
- Smoke-test the documented render quickstart with `go run ./cmd/scriptorium render --config ./examples/config.yml --prompt generic.markdown_summary --input transcript=./examples/fixtures/transcript.md --format json`.
- If documenting JSON-schema render/run examples, set `SCRIPTORIUM_API_KEY` or use a profile without `api_key_env`; `Runner.Prepare` validates the named environment variable.
- Validate CLI flags against `internal/adapter/cli/run.go` and parser tests, especially `render` lacking `--schema-dir` and `serve` rejecting runtime override flags.
- Validate app config examples against `internal/config/config.go` strict YAML decoding.
- Validate prompt/profile examples against `internal/promptdef/filesystem_repository.go` and `internal/profile/filesystem_repository.go`.
- Validate HTTP request examples against `internal/adapter/http/dto.go` strict JSON decoding.
- Run grep checks for stale or roadmap-only terms outside `docs/roadmap/`: `future`, `planned`, `may be added`, `can be added later`, `S3`, `streaming`, `batch`, `database-backed`, `render endpoint`, and prompt IDs not present in `prompts/`.
- Run grep checks for stale paths after file moves: `docs/config/`, `architecture.md`, and `local-test`.
- No automated documentation link checker is currently configured; perform manual link review or add a link checker in a separate roadmap item if desired.
## Open Questions
No open questions block the documentation migration. The recommended path is to document the current implementation conservatively, move future ideas into `docs/roadmap/`, and avoid claiming production behavior for hooks that are present in code but not wired into CLI or HTTP adapters.

View File

@@ -0,0 +1,262 @@
# Runtime Parameter Implementation Plan
This plan implements the target state in `docs/roadmap/params.md`.
Audience: LLM coding agents implementing the feature in order. Follow `docs/policy/architecture.md`, `docs/policy/development.md`, and `docs/policy/documentation.md` before changing code.
## Constraints
- Keep adapters thin. CLI and HTTP should capture caller intent and map it into domain request types; merge decisions belong in `internal/usecase`.
- Keep external decoding strict. Unknown YAML/JSON fields must continue to fail.
- Do not accept or emit raw API key values.
- Do not add dependencies unless there is a clear need. This feature should use the standard library plus existing dependencies.
- Do not expand the HTTP API surface beyond `POST /v1/runs`.
- Do not add provider-specific adapter packages.
- Keep each stage passing `go test ./...` before moving to the next stage.
## Stage 1: Presence-Aware Request Overrides
Goal: make per-request numeric execution overrides presence-aware while keeping resolved execution settings concrete.
### Domain Changes
1. In `internal/domain/domain.go`, add a request-only type:
```go
type ExecutionTargetOverride struct {
Endpoint string `json:"endpoint,omitempty"`
Model string `json:"model,omitempty"`
Temperature *float64 `json:"temperature,omitempty"`
MaxTokens *int `json:"max_tokens,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
ServiceTier string `json:"service_tier,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"`
APIKeyEnv string `json:"api_key_env,omitempty"`
ExtraParams map[string]any `json:"extra_params,omitempty"`
}
```
2. Change `domain.RunRequest.Execution` from `*ExecutionTarget` to `*ExecutionTargetOverride`.
3. Change `ExecutionProfile.ExtraParams` and `ExecutionTarget.ExtraParams` from `map[string]string` to `map[string]any`.
4. Keep `ExecutionTarget` concrete. It represents the resolved effective runtime target after defaults, profile, and request overrides are merged.
### Runner Changes
1. Update `internal/usecase/runner.go` so profile values still merge over built-in defaults and request overrides merge over that result.
2. Keep the existing concrete profile merge semantics for profile numeric fields.
3. Add a separate request override merge path that uses pointer presence:
- `nil` numeric pointer means omitted; preserve the current value.
- non-nil numeric pointer means explicit override, even when the value is `0`.
4. Validate request override numeric values before or during merge:
- `temperature`: `0 <= value <= 2`
- `max_tokens`: `value >= 0`
- `top_p`: `0 <= value <= 1`
- `timeout_seconds`: `value >= 0`
5. Preserve existing validation after merge:
- effective endpoint required
- effective model required
- `api_key_env`, when set, must name a non-empty environment variable
6. Preserve secret handling. The resolved API key value must never be stored in `PreparedRun`, `RunResult`, logs, or HTTP responses.
### CLI Changes
1. Update `internal/adapter/cli/run.go` request construction to build `domain.ExecutionTargetOverride`.
2. Use the existing `flagWasSet` booleans to populate numeric pointers only when the user provided the flag.
3. Required behavior:
- omitted `--temperature` preserves profile/default temperature;
- `--temperature 0` explicitly sets temperature to zero;
- omitted `--top-p` preserves profile/default top-p;
- `--top-p 0` explicitly sets top-p to zero;
- omitted `--max-tokens` preserves profile/default max tokens;
- `--max-tokens 0` explicitly sets max tokens to zero;
- omitted `--timeout` preserves profile/default timeout;
- `--timeout 0s` explicitly sets timeout seconds to zero.
4. Do not add new CLI flags in this stage.
### HTTP Changes
1. Update `internal/adapter/http/dto.go` so numeric model override fields are pointers:
- `Temperature *float64`
- `MaxTokens *int`
- `TopP *float64`
- `TimeoutSeconds *int`
2. Update DTO mapping in `internal/adapter/http/handler.go` to build `domain.ExecutionTargetOverride`.
3. Preserve strict JSON decoding and existing error mapping.
4. Required behavior:
- omitted numeric JSON fields preserve profile/default values;
- explicit numeric zero JSON fields override profile/default values.
### Tests
Add or update tests in:
- `internal/usecase/runner_test.go`
- `internal/adapter/cli/run_test.go`
- `internal/adapter/http/handler_test.go`
Required test coverage:
- Runner preserves profile value when request numeric override is omitted.
- Runner applies explicit zero request override for `temperature`.
- Runner applies explicit zero request override for `top_p`.
- Runner applies explicit zero request override for `max_tokens`.
- Runner applies explicit zero request override for `timeout_seconds`.
- Invalid request override ranges fail as invalid request errors.
- CLI `--temperature 0` reaches effective settings as zero.
- HTTP `"temperature": 0` reaches effective settings as zero.
- HTTP omitted `temperature` preserves profile/default value.
### Verification
Run:
```bash
go test ./...
```
## Stage 2: JSON-Compatible `extra_params`
Goal: allow provider-specific parameters to carry JSON-compatible values throughout profile, HTTP, prepared output, metadata, and LLM request construction.
### Domain And Loader Changes
1. Complete all compile fixes from changing `ExtraParams` to `map[string]any`.
2. Ensure `internal/profile/filesystem_repository.go` continues to decode profiles strictly while allowing nested JSON-compatible values under `extra_params`.
3. Add profile repository tests for `extra_params` containing:
- string
- number
- boolean
- nested object or array
4. Ensure formatter output remains deterministic:
- keep sorting `extra_params` keys in `internal/format/prepared_run.go`;
- render non-string values with stable JSON encoding in text output.
5. Preserve JSON formatter behavior through normal `encoding/json` output.
### HTTP Changes
1. Change HTTP model override `ExtraParams` to `map[string]any`.
2. Add handler tests proving HTTP accepts JSON-compatible `extra_params` values.
3. Preserve strict rejection of unknown fields and raw API-key payload fields.
### Verification
Run:
```bash
go test ./...
```
## Stage 3: Outbound Serialization
Goal: serialize `reasoning_effort` and `extra_params` to the OpenAI-compatible chat-completions request.
### LLM Adapter Changes
1. In `internal/llm/openai_compatible_client.go`, add first-class outbound support for `reasoning_effort`.
2. Add `extra_params` support by flattening `domain.ExecutionTarget.ExtraParams` into additional top-level JSON request fields.
3. Implement reserved-field collision checks before the HTTP request is made.
4. Reserved keys must include:
- `model`
- `session_id`
- `messages`
- `temperature`
- `max_tokens`
- `top_p`
- `service_tier`
- `reasoning_effort`
- `response_format`
5. Reject empty `extra_params` keys.
6. Ensure each `extra_params` value can be marshaled as JSON. If marshaling fails, return `ErrInvalidRequest` with context.
7. Keep existing request behavior unchanged when `reasoning_effort` and `extra_params` are unset.
### Recommended Implementation Shape
Use a custom marshal path for the outbound chat request rather than string manipulation.
One acceptable shape:
- Add `ReasoningEffort string` and `ExtraParams map[string]any` to the internal `openAIChatRequest`.
- Add a helper that converts `openAIChatRequest` into `map[string]any`, inserts first-class fields when set, then inserts `ExtraParams` after collision validation.
- Marshal that map with `encoding/json`.
Do not construct outbound JSON with manual string concatenation.
### Tests
Update `internal/llm/openai_compatible_client_test.go`.
Required test coverage:
- outbound JSON includes `reasoning_effort` when set;
- outbound JSON omits `reasoning_effort` when unset;
- outbound JSON includes string, number, boolean, object, and array `extra_params`;
- reserved `extra_params` keys fail before provider call;
- empty `extra_params` keys fail before provider call;
- existing message, cache-control, service-tier, response-format, and usage parsing tests continue to pass.
### Verification
Run:
```bash
go test ./...
```
## Stage 4: Documentation And Examples
Goal: move implemented behavior from roadmap to canonical docs after code is complete.
Update only after Stages 1 through 3 are implemented.
### Required Docs
Update:
- `docs/config.md`
- `docs/cli.md`
- `docs/integrations/http-api.md`
- `docs/integrations/openai-compatible-chat.md`
- `docs/internal/runner.md`
- `docs/internal/adapters.md`
Required documentation content:
- `reasoning_effort` is serialized outbound when set.
- `extra_params` serializes as provider-specific top-level outbound JSON fields.
- `extra_params` supports JSON-compatible values.
- reserved `extra_params` fields are rejected.
- per-request numeric overrides distinguish omitted values from explicit zero values.
- CLI explicit zero behavior for existing numeric flags.
- HTTP explicit zero behavior for model override numeric fields.
- no raw API-key values are accepted or emitted.
### Examples
Update examples only if needed to keep them accurate and runnable.
If adding an `extra_params` example, keep it secret-free and simple. Prefer a harmless provider-routing example over a vendor-specific feature that requires special credentials.
### Verification
Run:
```bash
go test ./...
go run ./cmd/scriptorium render \
--config ./examples/config.yml \
--prompt generic.markdown_summary \
--input transcript=./examples/fixtures/transcript.md \
--input glossary=./examples/fixtures/glossary.yml \
--format json
```
## Final Checks
Before considering the feature complete:
1. Confirm `git diff` contains only intended code, test, doc, and example changes.
2. Confirm all non-roadmap docs describe implemented behavior only.
3. Confirm no output path exposes raw API key values.
4. Confirm `go test ./...` passes.
5. Confirm the render smoke command passes.

96
docs/roadmap/params.md Normal file
View File

@@ -0,0 +1,96 @@
# Runtime Parameter Feature Roadmap
This roadmap defines the target behavior for runtime model parameters.
Current behavior has two limitations:
- `reasoning_effort` and `extra_params` are parsed into effective execution settings but are not serialized into outbound OpenAI-compatible chat-completions requests.
- Per-request numeric execution overrides use zero-value merge semantics, so callers cannot reliably override a profile value with an explicit zero such as `temperature: 0`.
The implementation plan for this feature lives in `docs/roadmap/implementation.md`.
## Target State
Scriptorium should preserve the existing separation between prompt definitions, execution profiles, and per-request execution overrides while making runtime parameter behavior explicit and predictable.
Expected end state:
- Effective execution settings remain visible in prepared-run output, run metadata, and HTTP metadata without exposing raw secret values.
- `reasoning_effort` is treated as a first-class effective execution setting and is serialized to the outbound OpenAI-compatible request when set.
- `extra_params` supports provider-specific OpenAI-compatible request fields.
- `extra_params` is serialized as additional top-level outbound JSON fields.
- `extra_params` values support JSON-compatible scalar, object, and array values.
- `extra_params` cannot override first-class outbound request fields.
- Per-request numeric overrides preserve caller intent, including explicit zero values.
- Omitted per-request numeric overrides continue to inherit the selected profile and built-in defaults.
- External decoding remains strict for config, prompt, profile, and HTTP request payloads.
## Policy Decisions
### `extra_params`
`extra_params` should serialize as additional top-level outbound JSON fields in the OpenAI-compatible chat-completions request.
Reasoning:
Most OpenAI-compatible providers expose vendor-specific chat-completions parameters as top-level fields. This keeps Scriptorium's adapter compatible with that ecosystem without adding first-class fields for every provider option.
`extra_params` must not silently override Scriptorium-owned fields. Reserved outbound fields include at least:
- `model`
- `session_id`
- `messages`
- `temperature`
- `max_tokens`
- `top_p`
- `service_tier`
- `reasoning_effort`
- `response_format`
If a caller supplies a reserved key through `extra_params`, Scriptorium should fail before making the outbound HTTP request.
`extra_params` should use JSON-compatible values rather than only strings.
Reasoning:
Provider-specific parameters commonly need booleans, numbers, objects, or arrays. String-only values would force awkward encoding and would likely require a later compatibility break.
### Presence-Aware Overrides
Per-request execution overrides should use a presence-aware type with pointer fields for optional numeric values.
Reasoning:
The resolved execution target should remain a concrete value used by prepared runs, generated requests, and metadata. Optionality matters at the request boundary, not after the runner has resolved the effective target.
This keeps adapter and merge logic precise while avoiding nil checks in formatter, metadata, and LLM serialization paths.
## Scope
In scope:
- Runtime merge behavior for per-request execution overrides.
- HTTP model override decoding for explicit zero numeric values.
- CLI execution override handling for explicit zero numeric flags.
- Outbound serialization of `reasoning_effort`.
- Outbound serialization of JSON-compatible `extra_params`.
- Tests and documentation for the changed implemented behavior.
Out of scope:
- Expanding the HTTP API beyond `POST /v1/runs`.
- Adding built-in HTTP authentication or authorization.
- Adding durable run state, run history, or multi-step orchestration.
- Adding broad provider-specific adapter packages.
- Adding new CLI flags for every provider-specific parameter.
## Acceptance Criteria
- A profile containing `reasoning_effort: medium` produces an outbound request with `reasoning_effort`.
- HTTP callers can pass `reasoning_effort` through the existing `model` override object and have it appear outbound.
- A profile or HTTP request containing JSON-compatible `extra_params` produces outbound top-level JSON fields according to the reserved-field policy.
- Reserved `extra_params` collisions fail before the outbound provider call.
- CLI callers can pass `--temperature 0` and observe `temperature: 0` in rendered/effective settings and outbound requests.
- HTTP callers can send `"temperature": 0` and observe the same behavior.
- Omitting `temperature` continues to preserve the selected profile/default value.
- Raw API key values remain unsupported in config, profiles, CLI flags, HTTP payloads, logs, and rendered output.

View File

@@ -252,6 +252,39 @@ Relevant links:
- [Configuration reference](config.md) - [Configuration reference](config.md)
- [Operations guide](operations.md) - [Operations guide](operations.md)
## Prompt Cache Misses Or No Cache Usage
Symptom:
- CLI run summary omits `cached_tokens` / `cache_write_tokens`.
- HTTP `metadata.usage.cached_tokens` and `metadata.usage.cache_write_tokens` are both `0`.
- Provider cost or latency does not improve after repeated similar runs.
Likely cause:
- The selected prompt has no `messages[].cache_control`.
- Dynamic per-run input appears before the cache-controlled message and changes the provider cache key.
- The provider does not support the serialized cache-control shape for the selected model.
- The provider imposes minimum token thresholds or cache-breakpoint limits.
Diagnostic step:
- Run `render --format json` and verify the intended rendered message includes `cache_control`.
- Confirm stable reusable context appears before the cache-controlled message, with dynamic input after it.
- Check provider docs/logs for model support, minimum token thresholds, and breakpoint limits.
Safe fix:
- Move stable reusable context before the cache-controlled message.
- Move highly dynamic input after the cache breakpoint.
- Keep `cache_control.type: ephemeral` and, when using `ttl`, set `ttl: 1h`.
- Use CLI cache counters or HTTP cache usage fields to verify cache reads/writes after rerunning.
Relevant links:
- [Configuration reference](config.md)
- [OpenAI-compatible chat integration](integrations/openai-compatible-chat.md)
## Validation Status Failed (`run` Exit 2 Or HTTP 200 With Failed Status) ## Validation Status Failed (`run` Exit 2 Or HTTP 200 With Failed Status)
Symptom: Symptom:

View File

@@ -1,6 +1,6 @@
prompt_dir: ./prompts prompt_dir: ./examples/prompts
profile_dir: ./profiles profile_dir: ./examples/profiles
schema_dir: ./schemas schema_dir: ./examples/schemas
server: server:
addr: :8080 addr: :8080

View File

@@ -81,6 +81,14 @@ type serveConfig struct {
schemaDir string schemaDir string
} }
type commonCommandSettings struct {
promptDir string
profileDir string
schemaDir string
serverAddr string
defaultRenderFormat renderformat.PreparedRunOutputFormat
}
type listFlag []string type listFlag []string
func (l *listFlag) String() string { func (l *listFlag) String() string {
@@ -125,22 +133,13 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
return ExitRuntimeError return ExitRuntimeError
} }
llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{ llmClient, err := newOpenAIClient()
Timeout: defaults.LLMRequestTimeoutDefault,
})
if err != nil { if err != nil {
fmt.Fprintf(stderr, "llm client error: %v\n", err) fmt.Fprintf(stderr, "llm client error: %v\n", err)
return ExitRuntimeError return ExitRuntimeError
} }
runner := usecase.NewRunner( runner := newRunner(cfg.promptDir, cfg.profileDir, cfg.schemaDir, llmClient)
promptdef.NewFilesystemRepository(cfg.promptDir),
profile.NewFilesystemRepository(cfg.profileDir),
artifactadapter.NewCompositeReader(),
prompt.NewGoRenderer(),
llmClient,
validate.NewStandardValidator(cfg.schemaDir),
)
res, runErr := runner.Run(context.Background(), req) res, runErr := runner.Run(context.Background(), req)
if runErr != nil { if runErr != nil {
@@ -170,14 +169,7 @@ func renderCommand(args []string, stdout, stderr io.Writer) int {
return ExitRuntimeError return ExitRuntimeError
} }
runner := usecase.NewRunner( runner := newRunner(cfg.promptDir, cfg.profileDir, cfg.schemaDir, nil)
promptdef.NewFilesystemRepository(cfg.promptDir),
profile.NewFilesystemRepository(cfg.profileDir),
artifactadapter.NewCompositeReader(),
prompt.NewGoRenderer(),
nil,
validate.NewStandardValidator(cfg.schemaDir),
)
prepared, prepErr := runner.Prepare(context.Background(), req) prepared, prepErr := runner.Prepare(context.Background(), req)
if prepErr != nil { if prepErr != nil {
@@ -205,22 +197,13 @@ func serveCommand(args []string, stderr io.Writer) int {
return ExitRuntimeError return ExitRuntimeError
} }
llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{ llmClient, err := newOpenAIClient()
Timeout: defaults.LLMRequestTimeoutDefault,
})
if err != nil { if err != nil {
fmt.Fprintf(stderr, "llm client error: %v\n", err) fmt.Fprintf(stderr, "llm client error: %v\n", err)
return ExitRuntimeError return ExitRuntimeError
} }
runner := usecase.NewRunner( runner := newRunner(cfg.promptDir, cfg.profileDir, cfg.schemaDir, llmClient)
promptdef.NewFilesystemRepository(cfg.promptDir),
profile.NewFilesystemRepository(cfg.profileDir),
artifactadapter.NewCompositeReader(),
prompt.NewGoRenderer(),
llmClient,
validate.NewStandardValidator(cfg.schemaDir),
)
h := httpadapter.NewHandler(runner) h := httpadapter.NewHandler(runner)
srv := &http.Server{ srv := &http.Server{
@@ -307,7 +290,7 @@ func parseServeArgs(args []string) (*serveConfig, error) {
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args()) return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
} }
settings, err := resolveAppSettings(fs, cfg.configPath, appconfig.CLIOverrides{ settings, err := resolveCommonSettings(fs, cfg.configPath, appconfig.CLIOverrides{
PromptDir: cfg.promptDirIfSet(fs), PromptDir: cfg.promptDirIfSet(fs),
ProfileDir: cfg.profileDirIfSet(fs), ProfileDir: cfg.profileDirIfSet(fs),
SchemaDir: cfg.schemaDirIfSet(fs), SchemaDir: cfg.schemaDirIfSet(fs),
@@ -317,16 +300,13 @@ func parseServeArgs(args []string) (*serveConfig, error) {
return nil, err return nil, err
} }
cfg.promptDir = settings.PromptDir cfg.promptDir = settings.promptDir
cfg.profileDir = settings.ProfileDir cfg.profileDir = settings.profileDir
cfg.schemaDir = settings.SchemaDir cfg.schemaDir = settings.schemaDir
cfg.addr = settings.ServerAddr cfg.addr = settings.serverAddr
if strings.TrimSpace(cfg.promptDir) == "" { if err := validateRequiredLibraryDirs(cfg.promptDir, cfg.profileDir); err != nil {
return nil, errors.New(errPromptDirRequired) return nil, err
}
if strings.TrimSpace(cfg.profileDir) == "" {
return nil, errors.New(errProfileDirRequired)
} }
cfg.promptDir = filepath.Clean(cfg.promptDir) cfg.promptDir = filepath.Clean(cfg.promptDir)
@@ -359,7 +339,7 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
return fmt.Errorf("unexpected positional args: %v", fs.Args()) return fmt.Errorf("unexpected positional args: %v", fs.Args())
} }
settings, err := resolveAppSettings(fs, cfg.configPath, appconfig.CLIOverrides{ settings, err := resolveCommonSettings(fs, cfg.configPath, appconfig.CLIOverrides{
PromptDir: cfg.promptDirIfSet(fs), PromptDir: cfg.promptDirIfSet(fs),
ProfileDir: cfg.profileDirIfSet(fs), ProfileDir: cfg.profileDirIfSet(fs),
SchemaDir: cfg.schemaDirIfSet(fs), SchemaDir: cfg.schemaDirIfSet(fs),
@@ -368,16 +348,13 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
return err return err
} }
cfg.promptDir = settings.PromptDir cfg.promptDir = settings.promptDir
cfg.profileDir = settings.ProfileDir cfg.profileDir = settings.profileDir
cfg.schemaDir = settings.SchemaDir cfg.schemaDir = settings.schemaDir
cfg.defaultRenderFormat = settings.DefaultRenderFormat cfg.defaultRenderFormat = settings.defaultRenderFormat
if strings.TrimSpace(cfg.promptDir) == "" { if err := validateRequiredLibraryDirs(cfg.promptDir, cfg.profileDir); err != nil {
return errors.New(errPromptDirRequired) return err
}
if strings.TrimSpace(cfg.profileDir) == "" {
return errors.New(errProfileDirRequired)
} }
if strings.TrimSpace(cfg.promptID) == "" { if strings.TrimSpace(cfg.promptID) == "" {
return errors.New("--prompt is required") return errors.New("--prompt is required")
@@ -476,6 +453,47 @@ func resolveAppSettings(fs *flag.FlagSet, configPath string, overrides appconfig
return merged, nil return merged, nil
} }
func resolveCommonSettings(fs *flag.FlagSet, configPath string, overrides appconfig.CLIOverrides) (commonCommandSettings, error) {
settings, err := resolveAppSettings(fs, configPath, overrides)
if err != nil {
return commonCommandSettings{}, err
}
return commonCommandSettings{
promptDir: settings.PromptDir,
profileDir: settings.ProfileDir,
schemaDir: settings.SchemaDir,
serverAddr: settings.ServerAddr,
defaultRenderFormat: settings.DefaultRenderFormat,
}, nil
}
func validateRequiredLibraryDirs(promptDir, profileDir string) error {
if strings.TrimSpace(promptDir) == "" {
return errors.New(errPromptDirRequired)
}
if strings.TrimSpace(profileDir) == "" {
return errors.New(errProfileDirRequired)
}
return nil
}
func newRunner(promptDir, profileDir, schemaDir string, llmClient llm.Client) *usecase.Runner {
return usecase.NewRunner(
promptdef.NewFilesystemRepository(promptDir),
profile.NewFilesystemRepository(profileDir),
artifactadapter.NewCompositeReader(),
prompt.NewGoRenderer(),
llmClient,
validate.NewStandardValidator(schemaDir),
)
}
func newOpenAIClient() (*llm.OpenAICompatibleClient, error) {
return llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
Timeout: defaults.LLMRequestTimeoutDefault,
})
}
func buildRunRequestFromConfig(cfg *runConfig) (domain.RunRequest, error) { func buildRunRequestFromConfig(cfg *runConfig) (domain.RunRequest, error) {
inputMappings, err := parseMappings(cfg.inputRaw, false) inputMappings, err := parseMappings(cfg.inputRaw, false)
if err != nil { if err != nil {
@@ -495,18 +513,25 @@ func buildRunRequestFromConfig(cfg *runConfig) (domain.RunRequest, error) {
inputs[name] = domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path} inputs[name] = domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path}
} }
var modelOverride *domain.ExecutionTarget var modelOverride *domain.ExecutionTargetOverride
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.apiKeyEnvSet || cfg.timeoutSet { if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.apiKeyEnvSet || cfg.timeoutSet {
modelOverride = &domain.ExecutionTarget{ modelOverride = &domain.ExecutionTargetOverride{
Endpoint: cfg.llmBaseURL, Endpoint: cfg.llmBaseURL,
Model: cfg.model, Model: cfg.model,
Temperature: cfg.temperature, APIKeyEnv: cfg.apiKeyEnv,
MaxTokens: cfg.maxTokens, }
TopP: cfg.topP, if cfg.temperatureSet {
APIKeyEnv: cfg.apiKeyEnv, modelOverride.Temperature = &cfg.temperature
}
if cfg.maxTokensSet {
modelOverride.MaxTokens = &cfg.maxTokens
}
if cfg.topPSet {
modelOverride.TopP = &cfg.topP
} }
if cfg.timeoutSet { if cfg.timeoutSet {
modelOverride.TimeoutSeconds = int(cfg.timeout.Seconds()) timeoutSeconds := int(cfg.timeout.Seconds())
modelOverride.TimeoutSeconds = &timeoutSeconds
} }
} }
@@ -588,7 +613,7 @@ func printSummary(stderr io.Writer, res *domain.RunResult) {
if res == nil { if res == nil {
return return
} }
fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d\n", fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d",
res.PromptID, res.PromptID,
res.PromptVersion, res.PromptVersion,
res.SelectedProfileID, res.SelectedProfileID,
@@ -602,6 +627,10 @@ func printSummary(stderr io.Writer, res *domain.RunResult) {
res.Usage.CompletionTokens, res.Usage.CompletionTokens,
res.Usage.TotalTokens, res.Usage.TotalTokens,
) )
if res.Usage.CachedTokens != 0 || res.Usage.CacheWriteTokens != 0 {
fmt.Fprintf(stderr, " cached_tokens=%d cache_write_tokens=%d", res.Usage.CachedTokens, res.Usage.CacheWriteTokens)
}
fmt.Fprintln(stderr)
} }
func printUsage(w io.Writer) { func printUsage(w io.Writer) {

View File

@@ -11,6 +11,7 @@ import (
"net/http/httptest" "net/http/httptest"
"os" "os"
"path/filepath" "path/filepath"
"reflect"
"strings" "strings"
"sync/atomic" "sync/atomic"
"testing" "testing"
@@ -410,6 +411,28 @@ defaults:
} }
} }
func TestParseRenderArgsExplicitFormatOverridesConfigDefaultFormat(t *testing.T) {
configPath := writeAppConfigFile(t, `
prompt_dir: ./from-config/prompts
profile_dir: ./from-config/profiles
defaults:
render_format: json
`)
cfg, err := parseRenderArgs([]string{
"--config", configPath,
"--prompt", "p",
"--input", "a=b",
"--format", "text",
})
if err != nil {
t.Fatalf("expected valid args, got %v", err)
}
if cfg.outputFormat != renderformat.PreparedRunFormatText {
t.Fatalf("expected explicit --format text to override config default, got %q", cfg.outputFormat)
}
}
func TestParseServeArgsWithExplicitConfigLoadsSettingsAndCLIAddrOverrides(t *testing.T) { func TestParseServeArgsWithExplicitConfigLoadsSettingsAndCLIAddrOverrides(t *testing.T) {
configPath := writeAppConfigFile(t, ` configPath := writeAppConfigFile(t, `
prompt_dir: ./from-config/prompts prompt_dir: ./from-config/prompts
@@ -471,6 +494,59 @@ server:
} }
} }
func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *testing.T) {
runCfg, err := parseRunArgs([]string{
"--prompt-dir", "./prompts",
"--profile-dir", "./profiles",
"--prompt", "prompt-1",
"--profile", "profile-1",
"--input", "transcript=./transcript.md",
"--var", "session_date=2026-05-01",
"--llm-base-url", "http://localhost:8000/v1",
"--model", "model-x",
"--temperature", "0.8",
"--max-tokens", "123",
"--top-p", "0.6",
"--timeout", "90s",
"--api-key-env", "SCRIPTORIUM_API_KEY",
})
if err != nil {
t.Fatalf("expected valid run args, got %v", err)
}
renderCfg, err := parseRenderArgs([]string{
"--prompt-dir", "./prompts",
"--profile-dir", "./profiles",
"--prompt", "prompt-1",
"--profile", "profile-1",
"--input", "transcript=./transcript.md",
"--var", "session_date=2026-05-01",
"--llm-base-url", "http://localhost:8000/v1",
"--model", "model-x",
"--temperature", "0.8",
"--max-tokens", "123",
"--top-p", "0.6",
"--timeout", "90s",
"--api-key-env", "SCRIPTORIUM_API_KEY",
})
if err != nil {
t.Fatalf("expected valid render args, got %v", err)
}
runReq, err := buildRunRequestFromConfig(runCfg)
if err != nil {
t.Fatalf("expected run request build success, got %v", err)
}
renderReq, err := buildRunRequestFromConfig(&renderCfg.runConfig)
if err != nil {
t.Fatalf("expected render request build success, got %v", err)
}
if !reflect.DeepEqual(runReq, renderReq) {
t.Fatalf("expected run/render shared flag requests to match.\nrun=%#v\nrender=%#v", runReq, renderReq)
}
}
func TestParseRunArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) { func TestParseRunArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) {
configPath := writeAppConfigFile(t, ` configPath := writeAppConfigFile(t, `
profile_dir: ./profiles profile_dir: ./profiles
@@ -586,42 +662,29 @@ func TestRunCommandVarsOptional(t *testing.T) {
} }
func TestRunCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) { func TestRunCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil {
t.Fatal(err)
}
ts := newTestLLMServer("from-config-dirs", nil) ts := newTestLLMServer("from-config-dirs", nil)
defer ts.Close() defer ts.Close()
writePromptFile(t, promptDir, "prompt.default", "local-default") writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
writeProfileFile(t, profileDir, "local-default", ts.URL+"/v1", "profile-model") writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model")
configPath := writeAppConfigFile(t, fmt.Sprintf(` configPath := writeAppConfigFile(t, fmt.Sprintf(`
prompt_dir: %s prompt_dir: %s
profile_dir: %s profile_dir: %s
`, promptDir, profileDir)) `, lib.promptDir, lib.profileDir))
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, runCommand, []string{
var stderr bytes.Buffer
code := runCommand([]string{
"--config", configPath, "--config", configPath,
"--prompt", "prompt.default", "--prompt", "prompt.default",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
if stdout.String() != "from-config-dirs" { if stdout != "from-config-dirs" {
t.Fatalf("unexpected stdout output: %q", stdout.String()) t.Fatalf("unexpected stdout output: %q", stdout)
} }
} }
@@ -630,28 +693,15 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te
const secret = "super-secret-render-key" const secret = "super-secret-render-key"
t.Setenv(envName, secret) t.Setenv(envName, secret)
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello transcript"), 0o644); err != nil {
t.Fatal(err)
}
writePromptFileWithTemplate(t, promptDir, "prompt.render", "local-default", "Date {{.session_date}} - Summarize: {{input \"transcript\"}}") writePromptFileWithTemplate(t, lib.promptDir, "prompt.render", "local-default", "Date {{.session_date}} - Summarize: {{input \"transcript\"}}")
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, renderCommand, []string{
var stderr bytes.Buffer "--prompt-dir", lib.promptDir,
code := renderCommand([]string{ "--profile-dir", lib.profileDir,
"--prompt-dir", promptDir,
"--profile-dir", profileDir,
"--prompt", "prompt.render", "--prompt", "prompt.render",
"--profile", "local-default", "--profile", "local-default",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
@@ -663,15 +713,15 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te
"--top-p", "0.2", "--top-p", "0.2",
"--timeout", "20s", "--timeout", "20s",
"--api-key-env", envName, "--api-key-env", envName,
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
if stderr.Len() != 0 { if stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", stderr.String()) t.Fatalf("expected empty stderr on success, got %q", stderr)
} }
out := stdout.String() out := stdout
for _, want := range []string{ for _, want := range []string{
"prompt: prompt.render", "prompt: prompt.render",
"selected_profile_id: local-default", "selected_profile_id: local-default",
@@ -697,112 +747,102 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te
} }
} }
func TestRenderCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) { func TestRenderCommandExplicitZeroTemperatureReachesEffectiveSettings(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil { writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
t.Fatal(err) profile := `id: local-default
} endpoint: http://127.0.0.1:1/v1
if err := os.MkdirAll(profileDir, 0o755); err != nil { model: profile-model
t.Fatal(err) temperature: 0.7
} `
inputPath := filepath.Join(tmp, "transcript.md") if err := os.WriteFile(filepath.Join(lib.profileDir, "local-default.yaml"), []byte(profile), 0o644); err != nil {
if err := os.WriteFile(inputPath, []byte("hello transcript"), 0o644); err != nil { t.Fatalf("failed to write profile fixture: %v", err)
t.Fatal(err)
} }
writePromptFile(t, promptDir, "prompt.render", "local-default") code, stdout, stderr := runCLICommand(t, renderCommand, []string{
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") "--prompt-dir", lib.promptDir,
"--profile-dir", lib.profileDir,
"--prompt", "prompt.render",
"--input", "transcript=" + inputPath,
"--temperature", "0",
})
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
if !strings.Contains(stdout, "\n temperature: 0\n") {
t.Fatalf("expected explicit zero temperature in effective settings, got:\n%s", stdout)
}
}
func TestRenderCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
lib := newCLITestLibrary(t)
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
configPath := writeAppConfigFile(t, fmt.Sprintf(` configPath := writeAppConfigFile(t, fmt.Sprintf(`
prompt_dir: %s prompt_dir: %s
profile_dir: %s profile_dir: %s
`, promptDir, profileDir)) `, lib.promptDir, lib.profileDir))
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, renderCommand, []string{
var stderr bytes.Buffer
code := renderCommand([]string{
"--config", configPath, "--config", configPath,
"--prompt", "prompt.render", "--prompt", "prompt.render",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
if !strings.Contains(stdout.String(), "prompt: prompt.render") { if !strings.Contains(stdout, "prompt: prompt.render") {
t.Fatalf("expected rendered output, got %q", stdout.String()) t.Fatalf("expected rendered output, got %q", stdout)
} }
} }
func TestRenderCommandExplicitTextFormatWorks(t *testing.T) { func TestRenderCommandExplicitTextFormatWorks(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello transcript"), 0o644); err != nil {
t.Fatal(err)
}
writePromptFile(t, promptDir, "prompt.render", "local-default") writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, renderCommand, []string{
var stderr bytes.Buffer "--prompt-dir", lib.promptDir,
code := renderCommand([]string{ "--profile-dir", lib.profileDir,
"--prompt-dir", promptDir,
"--profile-dir", profileDir,
"--prompt", "prompt.render", "--prompt", "prompt.render",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
"--format", "text", "--format", "text",
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
if !strings.Contains(stdout.String(), "prompt: prompt.render") { if !strings.Contains(stdout, "prompt: prompt.render") {
t.Fatalf("expected text output for explicit --format text, got %q", stdout.String()) t.Fatalf("expected text output for explicit --format text, got %q", stdout)
} }
} }
func TestRenderCommandExplicitJSONFormatOutputsValidJSON(t *testing.T) { func TestRenderCommandExplicitJSONFormatOutputsValidJSON(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello transcript"), 0o644); err != nil {
t.Fatal(err)
}
writePromptFile(t, promptDir, "prompt.render", "local-default") writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, renderCommand, []string{
var stderr bytes.Buffer "--prompt-dir", lib.promptDir,
code := renderCommand([]string{ "--profile-dir", lib.profileDir,
"--prompt-dir", promptDir,
"--profile-dir", profileDir,
"--prompt", "prompt.render", "--prompt", "prompt.render",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
"--format", "json", "--format", "json",
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
var payload map[string]any var payload map[string]any
if err := json.Unmarshal(stdout.Bytes(), &payload); err != nil { if err := json.Unmarshal([]byte(stdout), &payload); err != nil {
t.Fatalf("expected valid json output, got %v\nbody=%s", err, stdout.String()) t.Fatalf("expected valid json output, got %v\nbody=%s", err, stdout)
} }
if payload["prompt_id"] != "prompt.render" { if payload["prompt_id"] != "prompt.render" {
t.Fatalf("expected prompt_id, got %#v", payload["prompt_id"]) t.Fatalf("expected prompt_id, got %#v", payload["prompt_id"])
@@ -834,38 +874,25 @@ func TestRenderCommandUnknownFormatFailsClearly(t *testing.T) {
} }
func TestRenderCommandOutWritesToFile(t *testing.T) { func TestRenderCommandOutWritesToFile(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
profileDir := filepath.Join(tmp, "profiles") outPath := filepath.Join(lib.rootDir, "render.txt")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello transcript"), 0o644); err != nil {
t.Fatal(err)
}
outPath := filepath.Join(tmp, "render.txt")
writePromptFile(t, promptDir, "prompt.render", "local-default") writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, renderCommand, []string{
var stderr bytes.Buffer "--prompt-dir", lib.promptDir,
code := renderCommand([]string{ "--profile-dir", lib.profileDir,
"--prompt-dir", promptDir,
"--profile-dir", profileDir,
"--prompt", "prompt.render", "--prompt", "prompt.render",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
"--out", outPath, "--out", outPath,
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
if stdout.Len() != 0 { if stdout != "" {
t.Fatalf("expected empty stdout when --out is set, got %q", stdout.String()) t.Fatalf("expected empty stdout when --out is set, got %q", stdout)
} }
out, err := os.ReadFile(outPath) out, err := os.ReadFile(outPath)
if err != nil { if err != nil {
@@ -877,36 +904,23 @@ func TestRenderCommandOutWritesToFile(t *testing.T) {
} }
func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) { func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil {
t.Fatal(err)
}
writePromptFile(t, promptDir, "prompt.default", "local-default") writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model")
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, renderCommand, []string{
var stderr bytes.Buffer "--prompt-dir", lib.promptDir,
code := renderCommand([]string{ "--profile-dir", lib.profileDir,
"--prompt-dir", promptDir,
"--profile-dir", profileDir,
"--prompt", "prompt.default", "--prompt", "prompt.default",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
out := stdout.String() out := stdout
if !strings.Contains(out, "selected_profile_id: local-default") { if !strings.Contains(out, "selected_profile_id: local-default") {
t.Fatalf("expected prompt default profile in output, got %q", out) t.Fatalf("expected prompt default profile in output, got %q", out)
} }
@@ -916,38 +930,25 @@ func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
} }
func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) { func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil {
t.Fatal(err)
}
writePromptFile(t, promptDir, "prompt.default", "local-default") writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model")
writeProfileFile(t, profileDir, "quality", "http://127.0.0.1:1/v1", "quality-model") writeProfileFile(t, lib.profileDir, "quality", "http://127.0.0.1:1/v1", "quality-model")
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, renderCommand, []string{
var stderr bytes.Buffer "--prompt-dir", lib.promptDir,
code := renderCommand([]string{ "--profile-dir", lib.profileDir,
"--prompt-dir", promptDir,
"--profile-dir", profileDir,
"--prompt", "prompt.default", "--prompt", "prompt.default",
"--profile", "quality", "--profile", "quality",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
out := stdout.String() out := stdout
if !strings.Contains(out, "selected_profile_id: quality") { if !strings.Contains(out, "selected_profile_id: quality") {
t.Fatalf("expected explicit profile in output, got %q", out) t.Fatalf("expected explicit profile in output, got %q", out)
} }
@@ -957,104 +958,67 @@ func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
} }
func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) { func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil {
t.Fatal(err)
}
ts := newTestLLMServer("default-output", nil) ts := newTestLLMServer("default-output", nil)
defer ts.Close() defer ts.Close()
writePromptFile(t, promptDir, "prompt.default", "local-default") writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
writeProfileFile(t, profileDir, "local-default", ts.URL+"/v1", "profile-model") writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model")
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, runCommand, []string{
var stderr bytes.Buffer "--prompt-dir", lib.promptDir,
code := runCommand([]string{ "--profile-dir", lib.profileDir,
"--prompt-dir", promptDir,
"--profile-dir", profileDir,
"--prompt", "prompt.default", "--prompt", "prompt.default",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
if stdout.String() != "default-output" { if stdout != "default-output" {
t.Fatalf("unexpected stdout output: %q", stdout.String()) t.Fatalf("unexpected stdout output: %q", stdout)
} }
if !strings.Contains(stderr.String(), "selected_profile=local-default") { if !strings.Contains(stderr, "selected_profile=local-default") {
t.Fatalf("expected selected profile in summary, got %q", stderr.String()) t.Fatalf("expected selected profile in summary, got %q", stderr)
} }
} }
func TestRunCommandExplicitProfileOverridesPromptDefault(t *testing.T) { func TestRunCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil {
t.Fatal(err)
}
defaultServer := newTestLLMServer("from-default", nil) defaultServer := newTestLLMServer("from-default", nil)
defer defaultServer.Close() defer defaultServer.Close()
overrideServer := newTestLLMServer("from-override", nil) overrideServer := newTestLLMServer("from-override", nil)
defer overrideServer.Close() defer overrideServer.Close()
writePromptFile(t, promptDir, "prompt.default", "local-default") writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
writeProfileFile(t, profileDir, "local-default", defaultServer.URL+"/v1", "default-model") writeProfileFile(t, lib.profileDir, "local-default", defaultServer.URL+"/v1", "default-model")
writeProfileFile(t, profileDir, "quality", overrideServer.URL+"/v1", "quality-model") writeProfileFile(t, lib.profileDir, "quality", overrideServer.URL+"/v1", "quality-model")
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, runCommand, []string{
var stderr bytes.Buffer "--prompt-dir", lib.promptDir,
code := runCommand([]string{ "--profile-dir", lib.profileDir,
"--prompt-dir", promptDir,
"--profile-dir", profileDir,
"--prompt", "prompt.default", "--prompt", "prompt.default",
"--profile", "quality", "--profile", "quality",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
if stdout.String() != "from-override" { if stdout != "from-override" {
t.Fatalf("expected explicit profile output, got %q", stdout.String()) t.Fatalf("expected explicit profile output, got %q", stdout)
} }
if !strings.Contains(stderr.String(), "selected_profile=quality") { if !strings.Contains(stderr, "selected_profile=quality") {
t.Fatalf("expected selected profile quality, got %q", stderr.String()) t.Fatalf("expected selected profile quality, got %q", stderr)
} }
} }
func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) { func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil {
t.Fatal(err)
}
var baseHits int32 var baseHits int32
baseServer := newTestLLMServer("base", &baseHits) baseServer := newTestLLMServer("base", &baseHits)
@@ -1071,14 +1035,12 @@ func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) {
})) }))
defer overrideServer.Close() defer overrideServer.Close()
writePromptFile(t, promptDir, "prompt.default", "local-default") writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
writeProfileFile(t, profileDir, "local-default", baseServer.URL+"/v1", "profile-model") writeProfileFile(t, lib.profileDir, "local-default", baseServer.URL+"/v1", "profile-model")
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, runCommand, []string{
var stderr bytes.Buffer "--prompt-dir", lib.promptDir,
code := runCommand([]string{ "--profile-dir", lib.profileDir,
"--prompt-dir", promptDir,
"--profile-dir", profileDir,
"--prompt", "prompt.default", "--prompt", "prompt.default",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
"--llm-base-url", overrideServer.URL + "/v1", "--llm-base-url", overrideServer.URL + "/v1",
@@ -1087,9 +1049,9 @@ func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) {
"--max-tokens", "55", "--max-tokens", "55",
"--top-p", "0.2", "--top-p", "0.2",
"--timeout", "20s", "--timeout", "20s",
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
if atomic.LoadInt32(&baseHits) != 0 { if atomic.LoadInt32(&baseHits) != 0 {
t.Fatalf("expected base profile endpoint not to be hit, got %d", baseHits) t.Fatalf("expected base profile endpoint not to be hit, got %d", baseHits)
@@ -1097,8 +1059,8 @@ func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) {
if atomic.LoadInt32(&overrideHits) != 1 { if atomic.LoadInt32(&overrideHits) != 1 {
t.Fatalf("expected override endpoint to be hit once, got %d", overrideHits) t.Fatalf("expected override endpoint to be hit once, got %d", overrideHits)
} }
if stdout.String() != "override" { if stdout != "override" {
t.Fatalf("unexpected stdout output: %q", stdout.String()) t.Fatalf("unexpected stdout output: %q", stdout)
} }
if !strings.Contains(observedBody, `"model":"override-model"`) { if !strings.Contains(observedBody, `"model":"override-model"`) {
t.Fatalf("expected override model in request body, got %s", observedBody) t.Fatalf("expected override model in request body, got %s", observedBody)
@@ -1131,6 +1093,81 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
if !strings.Contains(stderr.String(), "prompt=p@1") { if !strings.Contains(stderr.String(), "prompt=p@1") {
t.Fatalf("expected summary on stderr, got %q", stderr.String()) t.Fatalf("expected summary on stderr, got %q", stderr.String())
} }
if strings.Contains(stderr.String(), "cached_tokens=") || strings.Contains(stderr.String(), "cache_write_tokens=") {
t.Fatalf("expected zero cache usage to be omitted from summary, got %q", stderr.String())
}
}
func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) {
var stderr bytes.Buffer
printSummary(&stderr, &domain.RunResult{
PromptID: "p",
PromptVersion: "1",
SelectedProfileID: "exec",
ModelName: "m",
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic},
RenderedPromptHash: "h",
InputHashes: map[string]string{"in": "x"},
Usage: domain.TokenUsage{
PromptTokens: 10,
CompletionTokens: 5,
TotalTokens: 15,
CachedTokens: 0,
CacheWriteTokens: 3,
},
})
summary := stderr.String()
if !strings.Contains(summary, "usage=10/5/15") {
t.Fatalf("expected base usage summary, got %q", summary)
}
if !strings.Contains(summary, "cached_tokens=0 cache_write_tokens=3") {
t.Fatalf("expected cache usage in summary, got %q", summary)
}
}
type cliTestLibrary struct {
rootDir string
promptDir string
profileDir string
}
func newCLITestLibrary(t *testing.T) *cliTestLibrary {
t.Helper()
root := t.TempDir()
lib := &cliTestLibrary{
rootDir: root,
promptDir: filepath.Join(root, "prompts"),
profileDir: filepath.Join(root, "profiles"),
}
if err := os.MkdirAll(lib.promptDir, 0o755); err != nil {
t.Fatalf("failed to create prompt fixture directory: %v", err)
}
if err := os.MkdirAll(lib.profileDir, 0o755); err != nil {
t.Fatalf("failed to create profile fixture directory: %v", err)
}
return lib
}
func (l *cliTestLibrary) writeInputFile(t *testing.T, name, body string) string {
t.Helper()
path := filepath.Join(l.rootDir, name)
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("failed to create input fixture directory: %v", err)
}
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
t.Fatalf("failed to write input fixture: %v", err)
}
return path
}
func runCLICommand(t *testing.T, command func([]string, io.Writer, io.Writer) int, args []string) (int, string, string) {
t.Helper()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := command(args, &stdout, &stderr)
return code, stdout.String(), stderr.String()
} }
func writePromptFile(t *testing.T, dir, id, defaultProfile string) { func writePromptFile(t *testing.T, dir, id, defaultProfile string) {

View File

@@ -21,15 +21,16 @@ type inputRefDTO struct {
} }
type modelOverrideRequestDTO struct { type modelOverrideRequestDTO struct {
Endpoint string `json:"endpoint,omitempty"` Endpoint string `json:"endpoint,omitempty"`
Model string `json:"model,omitempty"` Model string `json:"model,omitempty"`
Temperature float64 `json:"temperature,omitempty"` Temperature *float64 `json:"temperature,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"` MaxTokens *int `json:"max_tokens,omitempty"`
TopP float64 `json:"top_p,omitempty"` TopP *float64 `json:"top_p,omitempty"`
TimeoutSeconds int `json:"timeout_seconds,omitempty"` TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"` ServiceTier string `json:"service_tier,omitempty"`
APIKeyEnv string `json:"api_key_env,omitempty"` ReasoningEffort string `json:"reasoning_effort,omitempty"`
ExtraParams map[string]string `json:"extra_params,omitempty"` APIKeyEnv string `json:"api_key_env,omitempty"`
ExtraParams map[string]any `json:"extra_params,omitempty"`
} }
type runResponseDTO struct { type runResponseDTO struct {
@@ -69,21 +70,24 @@ type metadataDTO struct {
} }
type modelParamsDTO struct { type modelParamsDTO struct {
Endpoint string `json:"endpoint"` Endpoint string `json:"endpoint"`
Model string `json:"model"` Model string `json:"model"`
Temperature float64 `json:"temperature"` Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"` MaxTokens int `json:"max_tokens"`
TopP float64 `json:"top_p"` TopP float64 `json:"top_p"`
TimeoutSeconds int `json:"timeout_seconds"` TimeoutSeconds int `json:"timeout_seconds"`
ReasoningEffort string `json:"reasoning_effort,omitempty"` ServiceTier string `json:"service_tier,omitempty"`
APIKeyEnv string `json:"api_key_env,omitempty"` ReasoningEffort string `json:"reasoning_effort,omitempty"`
ExtraParams map[string]string `json:"extra_params,omitempty"` APIKeyEnv string `json:"api_key_env,omitempty"`
ExtraParams map[string]any `json:"extra_params,omitempty"`
} }
type tokenUsageDTO struct { type tokenUsageDTO struct {
PromptTokens int `json:"prompt_tokens"` PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"` CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"` TotalTokens int `json:"total_tokens"`
CachedTokens int `json:"cached_tokens"`
CacheWriteTokens int `json:"cache_write_tokens"`
} }
type validationDTO struct { type validationDTO struct {

View File

@@ -61,19 +61,9 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
} }
var model *domain.ExecutionTarget var model *domain.ExecutionTargetOverride
if req.Model != nil { if req.Model != nil {
model = &domain.ExecutionTarget{ model = executionTargetOverrideFromModelOverrideDTO(req.Model)
Endpoint: req.Model.Endpoint,
Model: req.Model.Model,
Temperature: req.Model.Temperature,
MaxTokens: req.Model.MaxTokens,
TopP: req.Model.TopP,
TimeoutSeconds: req.Model.TimeoutSeconds,
ReasoningEffort: req.Model.ReasoningEffort,
APIKeyEnv: req.Model.APIKeyEnv,
ExtraParams: req.Model.ExtraParams,
}
} }
res, err := h.runner.Run(r.Context(), domain.RunRequest{ res, err := h.runner.Run(r.Context(), domain.RunRequest{
@@ -109,22 +99,14 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
SelectedProfileID: res.SelectedProfileID, SelectedProfileID: res.SelectedProfileID,
ModelName: res.ModelName, ModelName: res.ModelName,
Endpoint: res.Endpoint, Endpoint: res.Endpoint,
ModelParams: modelParamsDTO{ ModelParams: modelParamsDTOFromExecutionTarget(res.EffectiveModelParams),
Endpoint: res.EffectiveModelParams.Endpoint, InputHashes: res.InputHashes,
Model: res.EffectiveModelParams.Model,
Temperature: res.EffectiveModelParams.Temperature,
MaxTokens: res.EffectiveModelParams.MaxTokens,
TopP: res.EffectiveModelParams.TopP,
TimeoutSeconds: res.EffectiveModelParams.TimeoutSeconds,
ReasoningEffort: res.EffectiveModelParams.ReasoningEffort,
APIKeyEnv: res.EffectiveModelParams.APIKeyEnv,
ExtraParams: res.EffectiveModelParams.ExtraParams,
},
InputHashes: res.InputHashes,
Usage: tokenUsageDTO{ Usage: tokenUsageDTO{
PromptTokens: res.Usage.PromptTokens, PromptTokens: res.Usage.PromptTokens,
CompletionTokens: res.Usage.CompletionTokens, CompletionTokens: res.Usage.CompletionTokens,
TotalTokens: res.Usage.TotalTokens, TotalTokens: res.Usage.TotalTokens,
CachedTokens: res.Usage.CachedTokens,
CacheWriteTokens: res.Usage.CacheWriteTokens,
}, },
StartTime: res.StartTime, StartTime: res.StartTime,
EndTime: res.EndTime, EndTime: res.EndTime,
@@ -141,6 +123,39 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, resp) writeJSON(w, http.StatusOK, resp)
} }
func executionTargetOverrideFromModelOverrideDTO(dto *modelOverrideRequestDTO) *domain.ExecutionTargetOverride {
if dto == nil {
return nil
}
return &domain.ExecutionTargetOverride{
Endpoint: dto.Endpoint,
Model: dto.Model,
Temperature: dto.Temperature,
MaxTokens: dto.MaxTokens,
TopP: dto.TopP,
TimeoutSeconds: dto.TimeoutSeconds,
ServiceTier: dto.ServiceTier,
ReasoningEffort: dto.ReasoningEffort,
APIKeyEnv: dto.APIKeyEnv,
ExtraParams: dto.ExtraParams,
}
}
func modelParamsDTOFromExecutionTarget(target domain.ExecutionTarget) modelParamsDTO {
return modelParamsDTO{
Endpoint: target.Endpoint,
Model: target.Model,
Temperature: target.Temperature,
MaxTokens: target.MaxTokens,
TopP: target.TopP,
TimeoutSeconds: target.TimeoutSeconds,
ServiceTier: target.ServiceTier,
ReasoningEffort: target.ReasoningEffort,
APIKeyEnv: target.APIKeyEnv,
ExtraParams: target.ExtraParams,
}
}
func mapValidation(v domain.ValidationResult) validationDTO { func mapValidation(v domain.ValidationResult) validationDTO {
return validationDTO{ return validationDTO{
Status: string(v.Status), Status: string(v.Status),
@@ -162,9 +177,9 @@ func mapRunError(err error) (int, string, string) {
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition" return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
case errors.Is(err, profile.ErrInvalidYAML), errors.Is(err, profile.ErrInvalidProfile): case errors.Is(err, profile.ErrInvalidYAML), errors.Is(err, profile.ErrInvalidProfile):
return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile" return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile"
case errors.Is(err, usecase.ErrInvalidRequest) && strings.Contains(err.Error(), "profile id is required either in request or prompt default_profile"): case errors.Is(err, usecase.ErrProfileRequired):
return http.StatusBadRequest, "profile_required", "profile_id is required when prompt default_profile is not set" return http.StatusBadRequest, "profile_required", "profile_id is required when prompt default_profile is not set"
case errors.Is(err, usecase.ErrInvalidRequest) && strings.Contains(err.Error(), "api key environment variable"): case errors.Is(err, usecase.ErrAPIKeyEnvMissing):
return http.StatusBadRequest, "api_key_env_missing", "api_key_env is set but the environment variable is missing" return http.StatusBadRequest, "api_key_env_missing", "api_key_env is set but the environment variable is missing"
case errors.Is(err, usecase.ErrInvalidRequest): case errors.Is(err, usecase.ErrInvalidRequest):
return http.StatusBadRequest, "invalid_request", "invalid run request" return http.StatusBadRequest, "invalid_request", "invalid run request"

View File

@@ -4,15 +4,16 @@ import (
"bytes" "bytes"
"context" "context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"reflect"
"strings" "strings"
"testing" "testing"
"time" "time"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile" "gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef" "gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase" "gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
@@ -32,6 +33,34 @@ func (f *fakeRunner) Run(ctx context.Context, req domain.RunRequest) (*domain.Ru
return f.result, nil return f.result, nil
} }
type handlerPromptRepo struct {
def *domain.PromptDefinition
}
func (r handlerPromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
return r.def, nil
}
type handlerProfileRepo struct {
profile *domain.ExecutionProfile
}
func (r handlerProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
return r.profile, nil
}
type handlerArtifactReader struct{}
func (handlerArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
return &domain.Artifact{Name: "input", Body: []byte("input"), Hash: "hash"}, nil
}
type handlerRenderer struct{}
func (handlerRenderer) Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
return &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, nil
}
func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) { func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
start := time.Now().UTC() start := time.Now().UTC()
end := start.Add(2 * time.Second) end := start.Add(2 * time.Second)
@@ -62,14 +91,21 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
MaxTokens: 42, MaxTokens: 42,
TopP: 0.9, TopP: 0.9,
TimeoutSeconds: 120, TimeoutSeconds: 120,
ServiceTier: "priority",
APIKeyEnv: envName, APIKeyEnv: envName,
}, },
InputHashes: map[string]string{"transcript": "h1"}, InputHashes: map[string]string{"transcript": "h1"},
Usage: domain.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3}, Usage: domain.TokenUsage{
StartTime: start, PromptTokens: 1,
EndTime: end, CompletionTokens: 2,
Duration: 2 * time.Second, TotalTokens: 3,
RawOutput: "hello", CachedTokens: 4,
CacheWriteTokens: 5,
},
StartTime: start,
EndTime: end,
Duration: 2 * time.Second,
RawOutput: "hello",
}} }}
h := NewHandler(r) h := NewHandler(r)
@@ -81,7 +117,7 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
"transcript": {"type": "file", "uri": "./t.md"} "transcript": {"type": "file", "uri": "./t.md"}
}, },
"vars": {"k": "v"}, "vars": {"k": "v"},
"model": {"model": "gpt-x", "timeout_seconds": 120, "api_key_env": "SCRIPTORIUM_API_KEY"} "model": {"model": "gpt-x", "timeout_seconds": 120, "service_tier": "flex", "api_key_env": "SCRIPTORIUM_API_KEY"}
}`) }`)
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewReader(body)) req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewReader(body))
w := httptest.NewRecorder() w := httptest.NewRecorder()
@@ -110,10 +146,20 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" { if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" {
t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"]) t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"])
} }
usage := metadata["usage"].(map[string]any)
if usage["prompt_tokens"] != float64(1) || usage["completion_tokens"] != float64(2) || usage["total_tokens"] != float64(3) {
t.Fatalf("unexpected base usage metadata: %#v", usage)
}
if usage["cached_tokens"] != float64(4) || usage["cache_write_tokens"] != float64(5) {
t.Fatalf("unexpected cache usage metadata: %#v", usage)
}
modelParams := metadata["model_params"].(map[string]any) modelParams := metadata["model_params"].(map[string]any)
if modelParams["api_key_env"] != envName { if modelParams["api_key_env"] != envName {
t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"]) t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"])
} }
if modelParams["service_tier"] != "priority" {
t.Fatalf("expected model_params.service_tier=priority, got %#v", modelParams["service_tier"])
}
if strings.Contains(w.Body.String(), secret) { if strings.Contains(w.Body.String(), secret) {
t.Fatalf("response leaked raw API key value: %s", w.Body.String()) t.Fatalf("response leaked raw API key value: %s", w.Body.String())
} }
@@ -130,9 +176,12 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
if r.last.Execution == nil || r.last.Execution.Model != "gpt-x" { if r.last.Execution == nil || r.last.Execution.Model != "gpt-x" {
t.Fatalf("expected model override, got %#v", r.last.Execution) t.Fatalf("expected model override, got %#v", r.last.Execution)
} }
if r.last.Execution.TimeoutSeconds != 120 { if r.last.Execution.TimeoutSeconds == nil || *r.last.Execution.TimeoutSeconds != 120 {
t.Fatalf("expected timeout_seconds override 120, got %#v", r.last.Execution) t.Fatalf("expected timeout_seconds override 120, got %#v", r.last.Execution)
} }
if r.last.Execution.ServiceTier != "flex" {
t.Fatalf("expected service_tier override flex, got %#v", r.last.Execution)
}
} }
func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) { func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
@@ -164,6 +213,265 @@ func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
if metadata["selected_profile_id"] != "prompt-default" { if metadata["selected_profile_id"] != "prompt-default" {
t.Fatalf("expected selected_profile_id from result, got %#v", metadata["selected_profile_id"]) t.Fatalf("expected selected_profile_id from result, got %#v", metadata["selected_profile_id"])
} }
usage := metadata["usage"].(map[string]any)
if usage["cached_tokens"] != float64(0) || usage["cache_write_tokens"] != float64(0) {
t.Fatalf("expected zero cache usage fields to be included, got %#v", usage)
}
}
func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
r := &fakeRunner{result: &domain.RunResult{
Artifact: domain.Artifact{Body: []byte("ok")},
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
}}
h := NewHandler(r)
reqBody := `{
"prompt_id": "prompt-1",
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
"model": {
"endpoint": "http://override/v1",
"model": "override-model",
"temperature": 0.6,
"max_tokens": 250,
"top_p": 0.85,
"timeout_seconds": 33,
"service_tier": "flex",
"reasoning_effort": "medium",
"api_key_env": "SCRIPTORIUM_API_KEY",
"extra_params": {"provider_option":"on"}
}
}`
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(reqBody))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
}
if r.last.Execution == nil {
t.Fatalf("expected execution override in run request")
}
got := r.last.Execution
if got.Endpoint != "http://override/v1" ||
got.Model != "override-model" ||
got.ServiceTier != "flex" ||
got.ReasoningEffort != "medium" ||
got.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
t.Fatalf("unexpected mapped execution target: %+v", got)
}
if got.Temperature == nil || *got.Temperature != 0.6 {
t.Fatalf("unexpected mapped temperature: %#v", got.Temperature)
}
if got.MaxTokens == nil || *got.MaxTokens != 250 {
t.Fatalf("unexpected mapped max_tokens: %#v", got.MaxTokens)
}
if got.TopP == nil || *got.TopP != 0.85 {
t.Fatalf("unexpected mapped top_p: %#v", got.TopP)
}
if got.TimeoutSeconds == nil || *got.TimeoutSeconds != 33 {
t.Fatalf("unexpected mapped timeout_seconds: %#v", got.TimeoutSeconds)
}
if !reflect.DeepEqual(got.ExtraParams, map[string]any{"provider_option": "on"}) {
t.Fatalf("unexpected mapped extra_params: %#v", got.ExtraParams)
}
}
func TestHandlerModelOverrideAcceptsJSONCompatibleExtraParams(t *testing.T) {
r := &fakeRunner{result: &domain.RunResult{
Artifact: domain.Artifact{Body: []byte("ok")},
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
}}
h := NewHandler(r)
reqBody := `{
"prompt_id": "prompt-1",
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
"model": {
"extra_params": {
"string_value": "enabled",
"number_value": 42,
"boolean_value": true,
"object_value": {"nested": "value", "count": 2},
"array_value": ["first", 3, false]
}
}
}`
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(reqBody))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
}
if r.last.Execution == nil {
t.Fatal("expected execution override in run request")
}
want := map[string]any{
"string_value": "enabled",
"number_value": float64(42),
"boolean_value": true,
"object_value": map[string]any{"nested": "value", "count": float64(2)},
"array_value": []any{"first", float64(3), false},
}
if !reflect.DeepEqual(r.last.Execution.ExtraParams, want) {
t.Fatalf("unexpected mapped extra_params:\ngot=%#v\nwant=%#v", r.last.Execution.ExtraParams, want)
}
}
func TestHandlerModelOverrideExplicitZeroTemperatureMapsAsPresent(t *testing.T) {
r := &fakeRunner{result: &domain.RunResult{
Artifact: domain.Artifact{Body: []byte("ok")},
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0},
}}
h := NewHandler(r)
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
"prompt_id": "prompt-1",
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
"model": {"temperature": 0}
}`))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
}
if r.last.Execution == nil || r.last.Execution.Temperature == nil {
t.Fatalf("expected temperature override to be present, got %#v", r.last.Execution)
}
if *r.last.Execution.Temperature != 0 {
t.Fatalf("expected zero temperature override, got %v", *r.last.Execution.Temperature)
}
}
func TestHandlerModelOverrideOmittedTemperatureMapsAsAbsent(t *testing.T) {
r := &fakeRunner{result: &domain.RunResult{
Artifact: domain.Artifact{Body: []byte("ok")},
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0.7},
}}
h := NewHandler(r)
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
"prompt_id": "prompt-1",
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
"model": {"model": "override-model"}
}`))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
}
if r.last.Execution == nil {
t.Fatal("expected model override")
}
if r.last.Execution.Temperature != nil {
t.Fatalf("expected omitted temperature to remain absent, got %#v", r.last.Execution.Temperature)
}
var resp map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("invalid JSON response: %v", err)
}
metadata := resp["metadata"].(map[string]any)
params := metadata["model_params"].(map[string]any)
if params["temperature"] != 0.7 {
t.Fatalf("expected effective profile/default temperature in response, got %#v", params["temperature"])
}
}
func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing.T) {
r := &fakeRunner{result: &domain.RunResult{
Artifact: domain.Artifact{
Name: "output",
ContentType: "text/plain",
Body: []byte("ok"),
Size: 2,
Hash: "abc",
},
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
EffectiveModelParams: domain.ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "gpt-test",
Temperature: 0.4,
MaxTokens: 321,
TopP: 0.7,
TimeoutSeconds: 45,
ServiceTier: "priority",
ReasoningEffort: "high",
APIKeyEnv: "SCRIPTORIUM_API_KEY",
ExtraParams: map[string]any{
"provider_option": "on",
"number_value": 42,
"object_value": map[string]any{"nested": "value"},
},
},
}}
h := NewHandler(r)
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
}
var resp map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("invalid JSON response: %v", err)
}
metadata := resp["metadata"].(map[string]any)
params := metadata["model_params"].(map[string]any)
if params["endpoint"] != "http://llm/v1" {
t.Fatalf("unexpected endpoint: %#v", params["endpoint"])
}
if params["model"] != "gpt-test" {
t.Fatalf("unexpected model: %#v", params["model"])
}
if params["temperature"] != 0.4 {
t.Fatalf("unexpected temperature: %#v", params["temperature"])
}
if params["max_tokens"] != float64(321) {
t.Fatalf("unexpected max_tokens: %#v", params["max_tokens"])
}
if params["top_p"] != 0.7 {
t.Fatalf("unexpected top_p: %#v", params["top_p"])
}
if params["timeout_seconds"] != float64(45) {
t.Fatalf("unexpected timeout_seconds: %#v", params["timeout_seconds"])
}
if params["service_tier"] != "priority" {
t.Fatalf("unexpected service_tier: %#v", params["service_tier"])
}
if params["reasoning_effort"] != "high" {
t.Fatalf("unexpected reasoning_effort: %#v", params["reasoning_effort"])
}
if params["api_key_env"] != "SCRIPTORIUM_API_KEY" {
t.Fatalf("unexpected api_key_env: %#v", params["api_key_env"])
}
extraParams, ok := params["extra_params"].(map[string]any)
if !ok {
t.Fatalf("expected extra_params object, got %#v", params["extra_params"])
}
if extraParams["provider_option"] != "on" {
t.Fatalf("unexpected extra_params.provider_option: %#v", extraParams["provider_option"])
}
if extraParams["number_value"] != float64(42) {
t.Fatalf("unexpected extra_params.number_value: %#v", extraParams["number_value"])
}
objectValue, ok := extraParams["object_value"].(map[string]any)
if !ok || objectValue["nested"] != "value" {
t.Fatalf("unexpected extra_params.object_value: %#v", extraParams["object_value"])
}
} }
func TestHandlerInvalidJSON(t *testing.T) { func TestHandlerInvalidJSON(t *testing.T) {
@@ -198,6 +506,54 @@ func TestHandlerMissingPromptID(t *testing.T) {
} }
} }
func TestHandlerReservedExtraParamsThroughRunnerMapsToInvalidRequest(t *testing.T) {
llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{})
if err != nil {
t.Fatal(err)
}
runner := usecase.NewRunner(
handlerPromptRepo{def: &domain.PromptDefinition{
ID: "p",
Version: "1",
DefaultProfile: "exec",
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "hi"}},
OutputFormat: domain.FormatText,
Validation: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone},
}},
handlerProfileRepo{profile: &domain.ExecutionProfile{
ID: "exec",
Endpoint: "http://example.invalid/v1",
Model: "model",
}},
handlerArtifactReader{},
handlerRenderer{},
llmClient,
nil,
)
h := NewHandler(runner)
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
"prompt_id":"p",
"inputs":{"x":{"type":"file","uri":"a"}},
"model":{"extra_params":{"model":"collision"}}
}`))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusBadRequest {
t.Fatalf("expected 400, got %d body=%s", w.Code, w.Body.String())
}
var resp map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("invalid JSON response: %v", err)
}
errBody := resp["error"].(map[string]any)
if errBody["code"] != "invalid_request" {
t.Fatalf("expected invalid_request code, got %#v", errBody["code"])
}
}
func TestHandlerUsecaseErrorMapping(t *testing.T) { func TestHandlerUsecaseErrorMapping(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
@@ -209,10 +565,10 @@ func TestHandlerUsecaseErrorMapping(t *testing.T) {
}{ }{
{name: "prompt not found", err: wrap(usecase.ErrProfileLoad, promptdef.ErrPromptDefinitionNotFound), status: http.StatusNotFound, code: "prompt_not_found", message: "prompt definition not found"}, {name: "prompt not found", err: wrap(usecase.ErrProfileLoad, promptdef.ErrPromptDefinitionNotFound), status: http.StatusNotFound, code: "prompt_not_found", message: "prompt definition not found"},
{name: "prompt load invalid", err: wrap(usecase.ErrProfileLoad, promptdef.ErrInvalidPromptDefinition), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition"}, {name: "prompt load invalid", err: wrap(usecase.ErrProfileLoad, promptdef.ErrInvalidPromptDefinition), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition"},
{name: "missing profile/default", err: wrap(usecase.ErrInvalidRequest, errors.New("profile id is required either in request or prompt default_profile")), status: http.StatusBadRequest, code: "profile_required", message: "profile_id is required when prompt default_profile is not set"}, {name: "missing profile/default", err: wrap(usecase.ErrInvalidRequest, usecase.ErrProfileRequired), status: http.StatusBadRequest, code: "profile_required", message: "profile_id is required when prompt default_profile is not set"},
{name: "profile not found", err: wrap(usecase.ErrProfileLoad, profile.ErrProfileNotFound), status: http.StatusNotFound, code: "profile_not_found", message: "execution profile not found"}, {name: "profile not found", err: wrap(usecase.ErrProfileLoad, profile.ErrProfileNotFound), status: http.StatusNotFound, code: "profile_not_found", message: "execution profile not found"},
{name: "profile invalid", err: wrap(usecase.ErrProfileLoad, profile.ErrInvalidProfile), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile"}, {name: "profile invalid", err: wrap(usecase.ErrProfileLoad, profile.ErrInvalidProfile), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile"},
{name: "api key env missing", err: wrap(usecase.ErrInvalidRequest, errors.New(`api key environment variable "SCRIPTORIUM_API_KEY" is not set`)), status: http.StatusBadRequest, code: "api_key_env_missing", message: "api_key_env is set but the environment variable is missing"}, {name: "api key env missing", err: wrap(usecase.ErrInvalidRequest, usecase.ErrAPIKeyEnvMissing), status: http.StatusBadRequest, code: "api_key_env_missing", message: "api_key_env is set but the environment variable is missing"},
{name: "artifact", err: wrap(usecase.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"}, {name: "artifact", err: wrap(usecase.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"},
{name: "prompt render", err: wrap(usecase.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"}, {name: "prompt render", err: wrap(usecase.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"},
{name: "llm", err: wrap(usecase.ErrLLMGenerate, fmt.Errorf("llm failed")), status: http.StatusBadGateway, code: "llm_failed", message: "model generation request failed", avoidCause: "llm failed"}, {name: "llm", err: wrap(usecase.ErrLLMGenerate, fmt.Errorf("llm failed")), status: http.StatusBadGateway, code: "llm_failed", message: "model generation request failed", avoidCause: "llm failed"},

View File

@@ -46,8 +46,8 @@ func TestCompositeReader_Read(t *testing.T) {
t.Run("unsupported ref type", func(t *testing.T) { t.Run("unsupported ref type", func(t *testing.T) {
ref := domain.ArtifactRef{ ref := domain.ArtifactRef{
Type: domain.ArtifactRefS3, Type: domain.ArtifactRefType("unsupported"),
URI: "s3://bucket/key", URI: "unsupported://bucket/key",
} }
_, err := reader.Read(ctx, ref) _, err := reader.Read(ctx, ref)
if !errors.Is(err, ErrUnsupportedRefType) { if !errors.Is(err, ErrUnsupportedRefType) {

View File

@@ -10,7 +10,6 @@ type ArtifactRefType string
const ( const (
ArtifactRefInline ArtifactRefType = "inline" ArtifactRefInline ArtifactRefType = "inline"
ArtifactRefFile ArtifactRefType = "file" ArtifactRefFile ArtifactRefType = "file"
ArtifactRefS3 ArtifactRefType = "s3"
) )
// OutputFormat defines the desired format of the generated artifact. // OutputFormat defines the desired format of the generated artifact.
@@ -41,6 +40,24 @@ const (
ValidationSkipped ValidationStatus = "skipped" ValidationSkipped ValidationStatus = "skipped"
) )
// CacheControlType defines provider cache behavior for prompt content.
type CacheControlType string
const (
CacheControlEphemeral CacheControlType = "ephemeral"
)
const (
// SessionIDMaxLength is OpenRouter's documented maximum session_id length.
SessionIDMaxLength = 256
)
// CacheControl describes provider cache metadata attached to prompt content.
type CacheControl struct {
Type CacheControlType `yaml:"type" json:"type"`
TTL string `yaml:"ttl,omitempty" json:"ttl,omitempty"`
}
// RunRequest represents a request to generate a single artifact. // RunRequest represents a request to generate a single artifact.
type RunRequest struct { type RunRequest struct {
PromptID string PromptID string
@@ -48,7 +65,7 @@ type RunRequest struct {
ProfileID string ProfileID string
Inputs map[string]ArtifactRef Inputs map[string]ArtifactRef
Vars map[string]string Vars map[string]string
Execution *ExecutionTarget Execution *ExecutionTargetOverride
Validation *OutputContract Validation *OutputContract
Metadata map[string]string Metadata map[string]string
} }
@@ -78,19 +95,21 @@ type RunResult struct {
// PreparedRun contains pre-LLM execution state from the prepare/render phase. // PreparedRun contains pre-LLM execution state from the prepare/render phase.
// It must never include resolved API key values, model output, or validation data. // It must never include resolved API key values, model output, or validation data.
type PreparedRun struct { type PreparedRun struct {
PromptID string `json:"prompt_id"` PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version,omitempty"` PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"` PromptHash string `json:"prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id"` SelectedProfileID string `json:"selected_profile_id"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"` EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
OutputContract OutputContract `json:"output_contract"` TargetPresence ExecutionTargetPresence `json:"-"`
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"` OutputContract OutputContract `json:"output_contract"`
InputHashes map[string]string `json:"input_hashes,omitempty"` StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
RenderedPromptHash string `json:"rendered_prompt_hash"` InputHashes map[string]string `json:"input_hashes,omitempty"`
Messages []RenderedMessage `json:"messages"` SessionID string `json:"session_id,omitempty"`
StartTime time.Time `json:"start_time,omitempty"` RenderedPromptHash string `json:"rendered_prompt_hash"`
EndTime time.Time `json:"end_time,omitempty"` Messages []RenderedMessage `json:"messages"`
DurationMS int64 `json:"duration_ms,omitempty"` StartTime time.Time `json:"start_time,omitempty"`
EndTime time.Time `json:"end_time,omitempty"`
DurationMS int64 `json:"duration_ms,omitempty"`
} }
// ArtifactRef represents a reference to an input artifact. // ArtifactRef represents a reference to an input artifact.
@@ -116,6 +135,7 @@ type PromptDefinition struct {
Version string `yaml:"version"` Version string `yaml:"version"`
DefaultProfile string `yaml:"default_profile"` DefaultProfile string `yaml:"default_profile"`
Description string `yaml:"description"` Description string `yaml:"description"`
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
Inputs []PromptInput `yaml:"inputs"` Inputs []PromptInput `yaml:"inputs"`
Templates []PromptMessageTemplate `yaml:"templates"` Templates []PromptMessageTemplate `yaml:"templates"`
OutputFormat OutputFormat `yaml:"output_format"` OutputFormat OutputFormat `yaml:"output_format"`
@@ -132,36 +152,62 @@ type PromptInput struct {
// PromptMessageTemplate defines a template for a chat message. // PromptMessageTemplate defines a template for a chat message.
type PromptMessageTemplate struct { type PromptMessageTemplate struct {
Role string `yaml:"role"` Role string `yaml:"role"`
Content string `yaml:"content"` Content string `yaml:"content"`
ContentFile string `yaml:"content_file"` ContentFile string `yaml:"content_file"`
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
} }
// ExecutionProfile describes how and where to execute a model. // ExecutionProfile describes how and where to execute a model.
type ExecutionProfile struct { type ExecutionProfile struct {
ID string `yaml:"id"` ID string `yaml:"id"`
Endpoint string `yaml:"endpoint"` Endpoint string `yaml:"endpoint"`
Model string `yaml:"model"` Model string `yaml:"model"`
Temperature float64 `yaml:"temperature"` Temperature float64 `yaml:"temperature"`
MaxTokens int `yaml:"max_tokens"` MaxTokens int `yaml:"max_tokens"`
TopP float64 `yaml:"top_p"` TopP float64 `yaml:"top_p"`
TimeoutSeconds int `yaml:"timeout_seconds"` TimeoutSeconds int `yaml:"timeout_seconds"`
ReasoningEffort string `yaml:"reasoning_effort"` ServiceTier string `yaml:"service_tier"`
APIKeyEnv string `yaml:"api_key_env"` ReasoningEffort string `yaml:"reasoning_effort"`
ExtraParams map[string]string `yaml:"extra_params"` APIKeyEnv string `yaml:"api_key_env"`
ExtraParams map[string]any `yaml:"extra_params"`
}
// ExecutionTargetOverride represents per-request runtime setting overrides.
type ExecutionTargetOverride struct {
Endpoint string `json:"endpoint,omitempty"`
Model string `json:"model,omitempty"`
Temperature *float64 `json:"temperature,omitempty"`
MaxTokens *int `json:"max_tokens,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
ServiceTier string `json:"service_tier,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"`
APIKeyEnv string `json:"api_key_env,omitempty"`
ExtraParams map[string]any `json:"extra_params,omitempty"`
}
// ExecutionTargetPresence tracks which effective runtime fields came from an
// explicit request override even when the resolved value is a zero value.
type ExecutionTargetPresence struct {
Temperature bool
MaxTokens bool
TopP bool
TimeoutSeconds bool
} }
// ExecutionTarget represents effective model runtime settings for a run. // ExecutionTarget represents effective model runtime settings for a run.
type ExecutionTarget struct { type ExecutionTarget struct {
Endpoint string `yaml:"endpoint" json:"endpoint"` Endpoint string `yaml:"endpoint" json:"endpoint"`
Model string `yaml:"model" json:"model"` Model string `yaml:"model" json:"model"`
Temperature float64 `yaml:"temperature" json:"temperature"` Temperature float64 `yaml:"temperature" json:"temperature"`
MaxTokens int `yaml:"max_tokens" json:"max_tokens"` MaxTokens int `yaml:"max_tokens" json:"max_tokens"`
TopP float64 `yaml:"top_p" json:"top_p"` TopP float64 `yaml:"top_p" json:"top_p"`
TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"` TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"`
ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"` ServiceTier string `yaml:"service_tier" json:"service_tier"`
APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"` ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"`
ExtraParams map[string]string `yaml:"extra_params" json:"extra_params"` APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"`
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
} }
// OutputContract defines the requirements for the output artifact. // OutputContract defines the requirements for the output artifact.
@@ -174,19 +220,22 @@ type OutputContract struct {
// RenderedPrompt represents the prompt after template application. // RenderedPrompt represents the prompt after template application.
type RenderedPrompt struct { type RenderedPrompt struct {
Messages []RenderedMessage `json:"messages"` SessionID string `json:"session_id,omitempty"`
Messages []RenderedMessage `json:"messages"`
} }
// RenderedMessage is a single message in a rendered prompt. // RenderedMessage is a single message in a rendered prompt.
type RenderedMessage struct { type RenderedMessage struct {
Role string `json:"role"` Role string `json:"role"`
Content string `json:"content"` Content string `json:"content"`
CacheControl *CacheControl `json:"cache_control,omitempty"`
} }
// GenerateRequest is the internal request passed to the LLM client. // GenerateRequest is the internal request passed to the LLM client.
type GenerateRequest struct { type GenerateRequest struct {
Prompt RenderedPrompt Prompt RenderedPrompt
Target ExecutionTarget Target ExecutionTarget
TargetPresence ExecutionTargetPresence
StructuredOutput *StructuredOutputSpec StructuredOutput *StructuredOutputSpec
} }
@@ -221,6 +270,8 @@ type TokenUsage struct {
PromptTokens int PromptTokens int
CompletionTokens int CompletionTokens int
TotalTokens int TotalTokens int
CachedTokens int
CacheWriteTokens int
} }
// ValidationResult represents the outcome of an output validation. // ValidationResult represents the outcome of an output validation.

View File

@@ -53,3 +53,88 @@ func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
} }
} }
} }
func TestPreparedRunJSONIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
prepared := PreparedRun{
PromptID: "prompt.id",
SelectedProfileID: "local-fast",
EffectiveModelParams: ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "gpt-test",
},
RenderedPromptHash: "rendered-hash",
Messages: []RenderedMessage{
{
Role: "system",
Content: "You are helpful.",
CacheControl: &CacheControl{
Type: CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "Summarize this."},
},
}
b, err := json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal failed: %v", err)
}
var decoded struct {
Messages []map[string]any `json:"messages"`
}
if err := json.Unmarshal(b, &decoded); err != nil {
t.Fatalf("unmarshal failed: %v", err)
}
if len(decoded.Messages) != 2 {
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
}
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
if !ok {
t.Fatalf("expected cache_control on first message, got %#v", decoded.Messages[0])
}
if cacheControl["type"] != string(CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
}
if _, ok := decoded.Messages[1]["cache_control"]; ok {
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
}
}
func TestPreparedRunJSONIncludesSessionIDOnlyWhenPresent(t *testing.T) {
prepared := PreparedRun{
PromptID: "prompt.id",
SelectedProfileID: "local-fast",
EffectiveModelParams: ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "gpt-test",
},
SessionID: "session-123",
RenderedPromptHash: "rendered-hash",
Messages: []RenderedMessage{{Role: "user", Content: "Summarize this."}},
}
b, err := json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal failed: %v", err)
}
var decoded map[string]any
if err := json.Unmarshal(b, &decoded); err != nil {
t.Fatalf("unmarshal failed: %v", err)
}
if decoded["session_id"] != "session-123" {
t.Fatalf("expected session_id in prepared run JSON, got %#v", decoded["session_id"])
}
prepared.SessionID = ""
b, err = json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal failed: %v", err)
}
if strings.Contains(string(b), "session_id") {
t.Fatalf("expected empty session_id to be omitted, got %s", b)
}
}

View File

@@ -0,0 +1,54 @@
package filecatalog
import (
"context"
"os"
"path/filepath"
"sort"
"strings"
)
// FindYAMLFiles returns sorted full paths for .yaml and .yml files under root.
func FindYAMLFiles(ctx context.Context, root string) ([]string, error) {
var files []string
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if d.IsDir() {
return nil
}
if !isYAMLFile(d.Name()) {
return nil
}
files = append(files, path)
return nil
})
sort.Strings(files)
return files, err
}
// RelativePath computes a clean relative path from root to path.
func RelativePath(root string, path string) string {
rel, err := filepath.Rel(root, path)
if err != nil {
return filepath.Clean(path)
}
return filepath.Clean(rel)
}
// Stem strips .yaml or .yml from a file name.
func Stem(name string) string {
name = strings.TrimSuffix(name, ".yaml")
name = strings.TrimSuffix(name, ".yml")
return name
}
func isYAMLFile(name string) bool {
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
}

View File

@@ -0,0 +1,84 @@
package filecatalog
import (
"context"
"errors"
"os"
"path/filepath"
"reflect"
"testing"
)
func TestFindYAMLFilesNestedSortedAndFiltered(t *testing.T) {
root := t.TempDir()
mustWriteFile(t, filepath.Join(root, "z", "prompt.yml"), "id: z")
mustWriteFile(t, filepath.Join(root, "a", "profile.yaml"), "id: a")
mustWriteFile(t, filepath.Join(root, "a", "ignore.txt"), "not yaml")
mustWriteFile(t, filepath.Join(root, "b", "ignore.yaml.bak"), "not yaml")
got, err := FindYAMLFiles(context.Background(), root)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
want := []string{
filepath.Join(root, "a", "profile.yaml"),
filepath.Join(root, "z", "prompt.yml"),
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("expected sorted YAML files %v, got %v", want, got)
}
}
func TestFindYAMLFilesHonorsContextCancellation(t *testing.T) {
root := t.TempDir()
mustWriteFile(t, filepath.Join(root, "one.yaml"), "id: one")
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := FindYAMLFiles(ctx, root)
if !errors.Is(err, context.Canceled) {
t.Fatalf("expected context.Canceled, got %v", err)
}
}
func TestRelativePathNested(t *testing.T) {
root := t.TempDir()
path := filepath.Join(root, "nested", "profiles", "local.yaml")
got := RelativePath(root, path)
want := filepath.Join("nested", "profiles", "local.yaml")
if got != want {
t.Fatalf("expected relative path %q, got %q", want, got)
}
}
func TestStemStripsYAMLExtensions(t *testing.T) {
tests := []struct {
name string
in string
want string
}{
{name: "yaml", in: "prompt.yaml", want: "prompt"},
{name: "yml", in: "profile.yml", want: "profile"},
{name: "other", in: "file.txt", want: "file.txt"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if got := Stem(tc.in); got != tc.want {
t.Fatalf("expected %q, got %q", tc.want, got)
}
})
}
}
func mustWriteFile(t *testing.T, path string, content string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("failed to create directory: %v", err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatalf("failed to write file %q: %v", path, err)
}
}

View File

@@ -96,6 +96,9 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
if prepared.PromptHash != "" { if prepared.PromptHash != "" {
fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash) fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash)
} }
if prepared.SessionID != "" {
fmt.Fprintf(&b, "session_id: %s\n", prepared.SessionID)
}
fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash) fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash)
target := prepared.EffectiveModelParams target := prepared.EffectiveModelParams
@@ -106,6 +109,9 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
fmt.Fprintf(&b, " max_tokens: %d\n", target.MaxTokens) fmt.Fprintf(&b, " max_tokens: %d\n", target.MaxTokens)
fmt.Fprintf(&b, " top_p: %g\n", target.TopP) fmt.Fprintf(&b, " top_p: %g\n", target.TopP)
fmt.Fprintf(&b, " timeout_seconds: %d\n", target.TimeoutSeconds) fmt.Fprintf(&b, " timeout_seconds: %d\n", target.TimeoutSeconds)
if target.ServiceTier != "" {
fmt.Fprintf(&b, " service_tier: %s\n", target.ServiceTier)
}
if target.ReasoningEffort != "" { if target.ReasoningEffort != "" {
fmt.Fprintf(&b, " reasoning_effort: %s\n", target.ReasoningEffort) fmt.Fprintf(&b, " reasoning_effort: %s\n", target.ReasoningEffort)
} }
@@ -120,7 +126,11 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
} }
sort.Strings(keys) sort.Strings(keys)
for _, k := range keys { for _, k := range keys {
fmt.Fprintf(&b, " %s: %s\n", k, target.ExtraParams[k]) renderedValue, err := formatExtraParamTextValue(target.ExtraParams[k])
if err != nil {
return nil, fmt.Errorf("failed to format extra_params.%s: %w", k, err)
}
fmt.Fprintf(&b, " %s: %s\n", k, renderedValue)
} }
} }
@@ -148,6 +158,13 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
messages := byRole[role] messages := byRole[role]
for i, msg := range messages { for i, msg := range messages {
fmt.Fprintf(&b, " - message: %d\n", i+1) fmt.Fprintf(&b, " - message: %d\n", i+1)
if msg.CacheControl != nil {
fmt.Fprintf(&b, " cache_control: %s", msg.CacheControl.Type)
if msg.CacheControl.TTL != "" {
fmt.Fprintf(&b, " ttl=%s", msg.CacheControl.TTL)
}
fmt.Fprintln(&b)
}
fmt.Fprintln(&b, " content: |") fmt.Fprintln(&b, " content: |")
content := msg.Content content := msg.Content
if content == "" { if content == "" {
@@ -162,3 +179,15 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
return b.Bytes(), nil return b.Bytes(), nil
} }
func formatExtraParamTextValue(value any) (string, error) {
if s, ok := value.(string); ok {
return s, nil
}
b, err := json.Marshal(value)
if err != nil {
return "", err
}
return string(b), nil
}

View File

@@ -28,6 +28,7 @@ func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) {
"max_tokens: 256", "max_tokens: 256",
"top_p: 0.8", "top_p: 0.8",
"timeout_seconds: 45", "timeout_seconds: 45",
"service_tier: priority",
"reasoning_effort: medium", "reasoning_effort: medium",
"api_key_env: SCRIPTORIUM_API_KEY", "api_key_env: SCRIPTORIUM_API_KEY",
"prompt_hash: prompt-hash", "prompt_hash: prompt-hash",
@@ -48,6 +49,36 @@ func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) {
} }
} }
func TestTextFormatterRendersExtraParamsDeterministically(t *testing.T) {
prepared := samplePreparedRun()
prepared.EffectiveModelParams.ExtraParams = map[string]any{
"z_string": "enabled",
"b_number": 42,
"a_object": map[string]any{
"nested": "value",
"count": 2,
},
"c_array": []any{"first", 3, false},
}
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
s := string(out)
want := strings.Join([]string{
" extra_params:",
" a_object: {\"count\":2,\"nested\":\"value\"}",
" b_number: 42",
" c_array: [\"first\",3,false]",
" z_string: enabled",
}, "\n")
if !strings.Contains(s, want) {
t.Fatalf("expected deterministic extra_params block %q, got:\n%s", want, s)
}
}
func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) { func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
const secret = "super-secret-api-key" const secret = "super-secret-api-key"
t.Setenv("SCRIPTORIUM_API_KEY", secret) t.Setenv("SCRIPTORIUM_API_KEY", secret)
@@ -61,8 +92,80 @@ func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
} }
} }
func TestTextFormatterIncludesMessageCacheControlBeforeContent(t *testing.T) {
prepared := samplePreparedRun()
prepared.Messages = []domain.RenderedMessage{
{
Role: "system",
Content: "System guidance.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "Summarize the transcript."},
}
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
s := string(out)
if !strings.Contains(s, " system:\n - message: 1\n cache_control: ephemeral ttl=1h\n content: |") {
t.Fatalf("expected system message cache control before content, got:\n%s", s)
}
if strings.Count(s, "cache_control:") != 1 {
t.Fatalf("expected exactly one cache_control line, got:\n%s", s)
}
}
func TestTextFormatterIncludesSessionIDWhenPresent(t *testing.T) {
prepared := samplePreparedRun()
prepared.SessionID = "session-123"
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if !strings.Contains(string(out), "session_id: session-123\n") {
t.Fatalf("expected session_id in text output, got:\n%s", out)
}
}
func TestTextFormatterOmitsEmptyCacheControlTTL(t *testing.T) {
prepared := samplePreparedRun()
prepared.Messages = []domain.RenderedMessage{
{
Role: "system",
Content: "System guidance.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
},
},
}
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
s := string(out)
if !strings.Contains(s, " cache_control: ephemeral\n") {
t.Fatalf("expected cache_control line without ttl, got:\n%s", s)
}
if strings.Contains(s, "ttl=") {
t.Fatalf("expected empty ttl to be omitted, got:\n%s", s)
}
}
func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) { func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
prepared := samplePreparedRun() prepared := samplePreparedRun()
prepared.SessionID = "session-123"
prepared.EffectiveModelParams.ExtraParams = map[string]any{
"number": 42,
"nested": map[string]any{
"enabled": true,
},
}
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON) out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
if err != nil { if err != nil {
@@ -86,9 +189,24 @@ func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
if decoded["rendered_prompt_hash"] != "rendered-hash" { if decoded["rendered_prompt_hash"] != "rendered-hash" {
t.Fatalf("expected rendered_prompt_hash in json output, got %#v", decoded["rendered_prompt_hash"]) t.Fatalf("expected rendered_prompt_hash in json output, got %#v", decoded["rendered_prompt_hash"])
} }
if _, ok := decoded["effective_model_params"]; !ok { if decoded["session_id"] != "session-123" {
t.Fatalf("expected session_id in json output, got %#v", decoded["session_id"])
}
modelParams, ok := decoded["effective_model_params"].(map[string]any)
if !ok {
t.Fatalf("expected effective_model_params in json output, got %#v", decoded) t.Fatalf("expected effective_model_params in json output, got %#v", decoded)
} }
extraParams, ok := modelParams["extra_params"].(map[string]any)
if !ok {
t.Fatalf("expected extra_params in json output, got %#v", modelParams["extra_params"])
}
if extraParams["number"] != float64(42) {
t.Fatalf("unexpected numeric extra param in json output: %#v", extraParams["number"])
}
nested, ok := extraParams["nested"].(map[string]any)
if !ok || nested["enabled"] != true {
t.Fatalf("unexpected nested extra param in json output: %#v", extraParams["nested"])
}
if _, ok := decoded["input_hashes"]; !ok { if _, ok := decoded["input_hashes"]; !ok {
t.Fatalf("expected input_hashes in json output, got %#v", decoded) t.Fatalf("expected input_hashes in json output, got %#v", decoded)
} }
@@ -97,6 +215,47 @@ func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
} }
} }
func TestJSONFormatterIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
prepared := samplePreparedRun()
prepared.Messages = []domain.RenderedMessage{
{
Role: "system",
Content: "System guidance.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "Summarize the transcript."},
}
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
var decoded struct {
Messages []map[string]any `json:"messages"`
}
if err := json.Unmarshal(out, &decoded); err != nil {
t.Fatalf("expected valid json output, got %v", err)
}
if len(decoded.Messages) != 2 {
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
}
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
if !ok {
t.Fatalf("expected first message cache_control, got %#v", decoded.Messages[0])
}
if cacheControl["type"] != string(domain.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
}
if _, ok := decoded.Messages[1]["cache_control"]; ok {
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
}
}
func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) { func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
const secret = "super-secret-api-key" const secret = "super-secret-api-key"
t.Setenv("SCRIPTORIUM_API_KEY", secret) t.Setenv("SCRIPTORIUM_API_KEY", secret)
@@ -168,6 +327,7 @@ func samplePreparedRun() *domain.PreparedRun {
MaxTokens: 256, MaxTokens: 256,
TopP: 0.8, TopP: 0.8,
TimeoutSeconds: 45, TimeoutSeconds: 45,
ServiceTier: "priority",
ReasoningEffort: "medium", ReasoningEffort: "medium",
APIKeyEnv: "SCRIPTORIUM_API_KEY", APIKeyEnv: "SCRIPTORIUM_API_KEY",
}, },

View File

@@ -12,6 +12,7 @@ import (
"os" "os"
"strings" "strings"
"time" "time"
"unicode/utf8"
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults" "gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
@@ -75,14 +76,6 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
return nil, fmt.Errorf("%w: timeout_seconds must be greater than or equal to 0", ErrInvalidRequest) return nil, fmt.Errorf("%w: timeout_seconds must be greater than or equal to 0", ErrInvalidRequest)
} }
model := strings.TrimSpace(req.Target.Model)
if model == "" {
model = strings.TrimSpace(c.defaultModel)
}
if model == "" {
return nil, fmt.Errorf("%w: model is required", ErrInvalidRequest)
}
endpoint := strings.TrimSpace(req.Target.Endpoint) endpoint := strings.TrimSpace(req.Target.Endpoint)
if endpoint == "" { if endpoint == "" {
endpoint = c.baseURL endpoint = c.baseURL
@@ -92,36 +85,17 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
} }
endpoint = strings.TrimRight(endpoint, "/") + defaults.OpenAIChatCompletionsPath endpoint = strings.TrimRight(endpoint, "/") + defaults.OpenAIChatCompletionsPath
wireReq := openAIChatRequest{ wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel)
Model: model, if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
} }
wireReq.Messages = make([]openAIChatMessage, 0, len(req.Prompt.Messages)) wirePayload, err := openAIChatRequestPayload(wireReq)
for _, msg := range req.Prompt.Messages { if err != nil {
wireReq.Messages = append(wireReq.Messages, openAIChatMessage{ return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
Role: msg.Role,
Content: msg.Content,
})
} }
if req.Target.Temperature != 0 { payload, err := json.Marshal(wirePayload)
wireReq.Temperature = &req.Target.Temperature
}
if req.Target.MaxTokens != 0 {
wireReq.MaxTokens = &req.Target.MaxTokens
}
if req.Target.TopP != 0 {
wireReq.TopP = &req.Target.TopP
}
if req.StructuredOutput != nil {
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
}
wireReq.ResponseFormat = responseFormat
}
payload, err := json.Marshal(wireReq)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err) return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
} }
@@ -142,6 +116,8 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
effectiveTimeout := c.timeout effectiveTimeout := c.timeout
if req.Target.TimeoutSeconds > 0 { if req.Target.TimeoutSeconds > 0 {
effectiveTimeout = time.Duration(req.Target.TimeoutSeconds) * time.Second effectiveTimeout = time.Duration(req.Target.TimeoutSeconds) * time.Second
} else if req.TargetPresence.TimeoutSeconds {
effectiveTimeout = 0
} }
httpClient := c.httpClient httpClient := c.httpClient
@@ -183,32 +159,166 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
PromptTokens: wireResp.Usage.PromptTokens, PromptTokens: wireResp.Usage.PromptTokens,
CompletionTokens: wireResp.Usage.CompletionTokens, CompletionTokens: wireResp.Usage.CompletionTokens,
TotalTokens: wireResp.Usage.TotalTokens, TotalTokens: wireResp.Usage.TotalTokens,
CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens,
CacheWriteTokens: wireResp.Usage.CacheWriteTokens,
}, },
}, nil }, nil
} }
type openAIChatRequest struct { func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) {
Model string `json:"model"` model := strings.TrimSpace(req.Target.Model)
Messages []openAIChatMessage `json:"messages"` if model == "" {
Temperature *float64 `json:"temperature,omitempty"` model = strings.TrimSpace(defaultModel)
MaxTokens *int `json:"max_tokens,omitempty"` }
TopP *float64 `json:"top_p,omitempty"` if model == "" {
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"` return openAIChatRequest{}, errors.New("model is required")
}
wireReq := openAIChatRequest{
Model: model,
}
if sessionID := strings.TrimSpace(req.Prompt.SessionID); sessionID != "" {
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
return openAIChatRequest{}, fmt.Errorf("session_id length %d exceeds maximum %d", n, domain.SessionIDMaxLength)
}
wireReq.SessionID = sessionID
}
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
for _, msg := range req.Prompt.Messages {
wireReq.Messages = append(wireReq.Messages, openAIChatRequestMessageFromRenderedMessage(msg))
}
if req.Target.Temperature != 0 || req.TargetPresence.Temperature {
wireReq.Temperature = &req.Target.Temperature
}
if req.Target.MaxTokens != 0 || req.TargetPresence.MaxTokens {
wireReq.MaxTokens = &req.Target.MaxTokens
}
if req.Target.TopP != 0 || req.TargetPresence.TopP {
wireReq.TopP = &req.Target.TopP
}
if strings.TrimSpace(req.Target.ServiceTier) != "" {
wireReq.ServiceTier = req.Target.ServiceTier
}
if strings.TrimSpace(req.Target.ReasoningEffort) != "" {
wireReq.ReasoningEffort = req.Target.ReasoningEffort
}
if len(req.Target.ExtraParams) > 0 {
wireReq.ExtraParams = req.Target.ExtraParams
}
if req.StructuredOutput != nil {
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
if err != nil {
return openAIChatRequest{}, err
}
wireReq.ResponseFormat = responseFormat
}
return wireReq, nil
} }
type openAIChatMessage struct { type openAIChatRequest struct {
Model string `json:"model"`
SessionID string `json:"session_id,omitempty"`
Messages []openAIChatRequestMessage `json:"messages"`
Temperature *float64 `json:"temperature,omitempty"`
MaxTokens *int `json:"max_tokens,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
ServiceTier string `json:"service_tier,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"`
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"`
ExtraParams map[string]any `json:"-"`
}
func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
out := map[string]any{
"model": req.Model,
"messages": req.Messages,
}
if req.SessionID != "" {
out["session_id"] = req.SessionID
}
if req.Temperature != nil {
out["temperature"] = *req.Temperature
}
if req.MaxTokens != nil {
out["max_tokens"] = *req.MaxTokens
}
if req.TopP != nil {
out["top_p"] = *req.TopP
}
if req.ServiceTier != "" {
out["service_tier"] = req.ServiceTier
}
if req.ReasoningEffort != "" {
out["reasoning_effort"] = req.ReasoningEffort
}
if req.ResponseFormat != nil {
out["response_format"] = req.ResponseFormat
}
for key, value := range req.ExtraParams {
if key == "" {
return nil, errors.New("extra_params key must not be empty")
}
if _, reserved := reservedOpenAIChatRequestFields[key]; reserved {
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
}
if _, err := json.Marshal(value); err != nil {
return nil, fmt.Errorf("extra_params.%s must be JSON-serializable: %w", key, err)
}
out[key] = value
}
return out, nil
}
var reservedOpenAIChatRequestFields = map[string]struct{}{
"model": {},
"session_id": {},
"messages": {},
"temperature": {},
"max_tokens": {},
"top_p": {},
"service_tier": {},
"reasoning_effort": {},
"response_format": {},
}
type openAIChatRequestMessage struct {
Role string `json:"role"`
Content any `json:"content"`
}
type openAIChatTextContentBlock struct {
Type string `json:"type"`
Text string `json:"text"`
CacheControl *openAICacheControl `json:"cache_control,omitempty"`
}
type openAICacheControl struct {
Type string `json:"type"`
TTL string `json:"ttl,omitempty"`
}
type openAIChatResponseMessage struct {
Role string `json:"role"` Role string `json:"role"`
Content string `json:"content"` Content string `json:"content"`
} }
type openAIChatResponse struct { type openAIChatResponse struct {
Choices []struct { Choices []struct {
Message openAIChatMessage `json:"message"` Message openAIChatResponseMessage `json:"message"`
} `json:"choices"` } `json:"choices"`
Usage struct { Usage struct {
PromptTokens int `json:"prompt_tokens"` PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"` CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"` TotalTokens int `json:"total_tokens"`
PromptTokensDetails struct {
CachedTokens int `json:"cached_tokens"`
} `json:"prompt_tokens_details"`
CacheWriteTokens int `json:"cache_write_tokens"`
} `json:"usage"` } `json:"usage"`
} }
@@ -223,6 +333,28 @@ type openAIJSONSchemaEnvelope struct {
Schema any `json:"schema"` Schema any `json:"schema"`
} }
func openAIChatRequestMessageFromRenderedMessage(msg domain.RenderedMessage) openAIChatRequestMessage {
wireMsg := openAIChatRequestMessage{
Role: msg.Role,
Content: msg.Content,
}
if msg.CacheControl == nil {
return wireMsg
}
wireMsg.Content = []openAIChatTextContentBlock{
{
Type: "text",
Text: msg.Content,
CacheControl: &openAICacheControl{
Type: string(msg.CacheControl.Type),
TTL: msg.CacheControl.TTL,
},
},
}
return wireMsg
}
func toOpenAIResponseFormat(spec *domain.StructuredOutputSpec) (*openAIResponseFormat, error) { func toOpenAIResponseFormat(spec *domain.StructuredOutputSpec) (*openAIResponseFormat, error) {
if spec == nil { if spec == nil {
return nil, nil return nil, nil

View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"math"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"strings" "strings"
@@ -61,6 +62,7 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
Temperature: 0.4, Temperature: 0.4,
MaxTokens: 123, MaxTokens: 123,
TopP: 0.7, TopP: 0.7,
ServiceTier: "priority",
APIKeyEnv: "SCRIPTORIUM_TEST_API_KEY", APIKeyEnv: "SCRIPTORIUM_TEST_API_KEY",
}, },
StructuredOutput: &domain.StructuredOutputSpec{ StructuredOutput: &domain.StructuredOutputSpec{
@@ -88,6 +90,9 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
if resp.Usage.PromptTokens != 11 || resp.Usage.CompletionTokens != 22 || resp.Usage.TotalTokens != 33 { if resp.Usage.PromptTokens != 11 || resp.Usage.CompletionTokens != 22 || resp.Usage.TotalTokens != 33 {
t.Fatalf("unexpected usage: %+v", resp.Usage) t.Fatalf("unexpected usage: %+v", resp.Usage)
} }
if resp.Usage.CachedTokens != 0 || resp.Usage.CacheWriteTokens != 0 {
t.Fatalf("expected absent cache usage fields to remain zero, got %+v", resp.Usage)
}
if obs.Authorization != "Bearer secret-key" { if obs.Authorization != "Bearer secret-key" {
t.Fatalf("unexpected Authorization header: %q", obs.Authorization) t.Fatalf("unexpected Authorization header: %q", obs.Authorization)
@@ -95,6 +100,18 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
if got, ok := obs.Body["model"].(string); !ok || got != "gpt-test" { if got, ok := obs.Body["model"].(string); !ok || got != "gpt-test" {
t.Fatalf("unexpected model payload: %#v", obs.Body["model"]) t.Fatalf("unexpected model payload: %#v", obs.Body["model"])
} }
if got, ok := obs.Body["temperature"].(float64); !ok || got != 0.4 {
t.Fatalf("unexpected temperature payload: %#v", obs.Body["temperature"])
}
if got, ok := obs.Body["max_tokens"].(float64); !ok || got != 123 {
t.Fatalf("unexpected max_tokens payload: %#v", obs.Body["max_tokens"])
}
if got, ok := obs.Body["top_p"].(float64); !ok || got != 0.7 {
t.Fatalf("unexpected top_p payload: %#v", obs.Body["top_p"])
}
if got, ok := obs.Body["service_tier"].(string); !ok || got != "priority" {
t.Fatalf("unexpected service_tier payload: %#v", obs.Body["service_tier"])
}
msgs, ok := obs.Body["messages"].([]any) msgs, ok := obs.Body["messages"].([]any)
if !ok || len(msgs) != 2 { if !ok || len(msgs) != 2 {
@@ -131,6 +148,245 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
} }
} }
func TestOpenAICompatibleClientSerializesCacheControlledMessageAsContentBlock(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "Stable instructions.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "Dynamic request."},
}},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
for _, forbidden := range []string{"cache_control", "extra_params"} {
if _, exists := observedBody[forbidden]; exists {
t.Fatalf("expected top-level %s to be omitted, got %#v", forbidden, observedBody[forbidden])
}
}
msgs, ok := observedBody["messages"].([]any)
if !ok || len(msgs) != 2 {
t.Fatalf("unexpected messages payload: %#v", observedBody["messages"])
}
msg0 := msgs[0].(map[string]any)
if msg0["role"] != "system" {
t.Fatalf("unexpected first message role: %#v", msg0["role"])
}
contentBlocks, ok := msg0["content"].([]any)
if !ok || len(contentBlocks) != 1 {
t.Fatalf("expected first message content block array, got %#v", msg0["content"])
}
block := contentBlocks[0].(map[string]any)
if block["type"] != "text" || block["text"] != "Stable instructions." {
t.Fatalf("unexpected text content block: %#v", block)
}
cacheControl, ok := block["cache_control"].(map[string]any)
if !ok {
t.Fatalf("expected cache_control on content block, got %#v", block)
}
if cacheControl["type"] != string(domain.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
}
msg1 := msgs[1].(map[string]any)
if msg1["role"] != "user" || msg1["content"] != "Dynamic request." {
t.Fatalf("expected uncached message to keep string content, got %#v", msg1)
}
}
func TestOpenAICompatibleClientOmitsEmptyCacheControlTTL(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "Stable instructions.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
},
},
}},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
msgs := observedBody["messages"].([]any)
msg0 := msgs[0].(map[string]any)
contentBlocks := msg0["content"].([]any)
block := contentBlocks[0].(map[string]any)
cacheControl := block["cache_control"].(map[string]any)
if cacheControl["type"] != string(domain.CacheControlEphemeral) {
t.Fatalf("unexpected cache_control type: %#v", cacheControl)
}
if _, exists := cacheControl["ttl"]; exists {
t.Fatalf("expected empty ttl to be omitted, got %#v", cacheControl)
}
}
func TestOpenAICompatibleClientSerializesSessionID(t *testing.T) {
var observedBody map[string]any
var observedSessionHeader string
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
observedSessionHeader = r.Header.Get("x-session-id")
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{
SessionID: " session-123 ",
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if observedBody["session_id"] != "session-123" {
t.Fatalf("expected top-level session_id, got %#v", observedBody["session_id"])
}
if observedSessionHeader != "" {
t.Fatalf("did not expect x-session-id header, got %q", observedSessionHeader)
}
}
func TestOpenAICompatibleClientOmitsEmptySessionID(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{
SessionID: " ",
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if _, exists := observedBody["session_id"]; exists {
t.Fatalf("expected empty session_id to be omitted, got %#v", observedBody["session_id"])
}
}
func TestOpenAICompatibleClientRejectsTooLongSessionID(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "http://example.com/v1",
Model: "model",
})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{
SessionID: strings.Repeat("x", domain.SessionIDMaxLength+1),
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
},
})
if err == nil {
t.Fatal("expected invalid request error")
}
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
}
func TestOpenAICompatibleClientParsesCacheUsage(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{
"choices": [{"message": {"role": "assistant", "content": "ok"}}],
"usage": {
"prompt_tokens": 100,
"completion_tokens": 20,
"total_tokens": 120,
"prompt_tokens_details": {"cached_tokens": 80},
"cache_write_tokens": 60
}
}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "model"})
if err != nil {
t.Fatal(err)
}
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if resp.Usage.PromptTokens != 100 || resp.Usage.CompletionTokens != 20 || resp.Usage.TotalTokens != 120 {
t.Fatalf("unexpected base usage fields: %+v", resp.Usage)
}
if resp.Usage.CachedTokens != 80 || resp.Usage.CacheWriteTokens != 60 {
t.Fatalf("unexpected cache usage fields: %+v", resp.Usage)
}
}
func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *testing.T) { func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *testing.T) {
var observedBody map[string]any var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -157,6 +413,284 @@ func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *test
if _, exists := observedBody["response_format"]; exists { if _, exists := observedBody["response_format"]; exists {
t.Fatalf("expected response_format omitted, got %#v", observedBody["response_format"]) t.Fatalf("expected response_format omitted, got %#v", observedBody["response_format"])
} }
if _, exists := observedBody["service_tier"]; exists {
t.Fatalf("expected service_tier omitted, got %#v", observedBody["service_tier"])
}
}
func TestOpenAICompatibleClientSerializesReasoningEffortAndExtraParams(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{
Model: "model",
ReasoningEffort: "high",
ExtraParams: map[string]any{
"string_value": "on",
"number_value": 42,
"boolean_value": true,
"object_value": map[string]any{"nested": "value", "count": 2},
"array_value": []any{"first", 3, false},
},
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if observedBody["reasoning_effort"] != "high" {
t.Fatalf("expected reasoning_effort high, got %#v", observedBody["reasoning_effort"])
}
if observedBody["string_value"] != "on" {
t.Fatalf("unexpected string extra param: %#v", observedBody["string_value"])
}
if observedBody["number_value"] != float64(42) {
t.Fatalf("unexpected number extra param: %#v", observedBody["number_value"])
}
if observedBody["boolean_value"] != true {
t.Fatalf("unexpected boolean extra param: %#v", observedBody["boolean_value"])
}
objectValue, ok := observedBody["object_value"].(map[string]any)
if !ok || objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
t.Fatalf("unexpected object extra param: %#v", observedBody["object_value"])
}
if _, exists := observedBody["extra_params"]; exists {
t.Fatalf("expected extra_params wrapper omitted, got %#v", observedBody["extra_params"])
}
arrayValue, ok := observedBody["array_value"].([]any)
if !ok || len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
t.Fatalf("unexpected array extra param: %#v", observedBody["array_value"])
}
}
func TestOpenAICompatibleClientOmitsReasoningEffortWhenUnset(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if _, exists := observedBody["reasoning_effort"]; exists {
t.Fatalf("expected reasoning_effort omitted, got %#v", observedBody["reasoning_effort"])
}
if _, exists := observedBody["extra_params"]; exists {
t.Fatalf("expected extra_params wrapper omitted, got %#v", observedBody["extra_params"])
}
}
func TestOpenAICompatibleClientSerializesExplicitZeroNumericOverrides(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model"},
TargetPresence: domain.ExecutionTargetPresence{
Temperature: true,
MaxTokens: true,
TopP: true,
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if observedBody["temperature"] != float64(0) {
t.Fatalf("expected explicit zero temperature, got %#v", observedBody["temperature"])
}
if observedBody["max_tokens"] != float64(0) {
t.Fatalf("expected explicit zero max_tokens, got %#v", observedBody["max_tokens"])
}
if observedBody["top_p"] != float64(0) {
t.Fatalf("expected explicit zero top_p, got %#v", observedBody["top_p"])
}
}
func TestOpenAICompatibleClientOmitsImplicitZeroNumericFields(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
for _, field := range []string{"temperature", "max_tokens", "top_p"} {
if _, exists := observedBody[field]; exists {
t.Fatalf("expected implicit zero field %q to be omitted, got body %#v", field, observedBody)
}
}
}
func TestOpenAICompatibleClientExplicitZeroTimeoutDisablesClientTimeout(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(20 * time.Millisecond)
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: ts.URL + "/v1",
Timeout: time.Nanosecond,
})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model", TimeoutSeconds: 0},
TargetPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true},
})
if err != nil {
t.Fatalf("expected explicit zero timeout to disable client timeout, got %v", err)
}
}
func TestOpenAICompatibleClientOmittedTimeoutUsesClientTimeout(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(20 * time.Millisecond)
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: ts.URL + "/v1",
Timeout: time.Nanosecond,
})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model", TimeoutSeconds: 0},
})
if err == nil {
t.Fatal("expected omitted timeout to use client timeout")
}
if !errors.Is(err, ErrRequestFailed) {
t.Fatalf("expected ErrRequestFailed, got %v", err)
}
}
func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *testing.T) {
tests := []struct {
name string
extraParams map[string]any
want string
}{
{name: "empty key", extraParams: map[string]any{"": "empty"}, want: "key must not be empty"},
{name: "unserializable value", extraParams: map[string]any{"bad": math.Inf(1)}, want: "JSON-serializable"},
}
for _, key := range []string{
"model",
"session_id",
"messages",
"temperature",
"max_tokens",
"top_p",
"service_tier",
"reasoning_effort",
"response_format",
} {
tests = append(tests, struct {
name string
extraParams map[string]any
want string
}{
name: "reserved key " + key,
extraParams: map[string]any{key: "collision"},
want: "reserved request field",
})
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
called := false
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
called = true
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model", ExtraParams: tc.extraParams},
})
if err == nil {
t.Fatal("expected invalid request error")
}
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
if !strings.Contains(err.Error(), tc.want) {
t.Fatalf("expected error to contain %q, got %v", tc.want, err)
}
if called {
t.Fatal("provider should not be called for invalid extra_params")
}
})
}
} }
func TestOpenAICompatibleClientNoAuthorizationHeaderWhenNoAPIKey(t *testing.T) { func TestOpenAICompatibleClientNoAuthorizationHeaderWhenNoAPIKey(t *testing.T) {

View File

@@ -10,6 +10,7 @@ import (
"strings" "strings"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/filecatalog"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
@@ -33,40 +34,40 @@ func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*doma
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile) return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
} }
files, err := os.ReadDir(r.dir) files, err := filecatalog.FindYAMLFiles(ctx, r.dir)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to read profile directory: %w", err) return nil, fmt.Errorf("failed to read profile directory: %w", err)
} }
for _, file := range files { var matches []profileMatch
for _, fullPath := range files {
select { select {
case <-ctx.Done(): case <-ctx.Done():
return nil, ctx.Err() return nil, ctx.Err()
default: default:
} }
if file.IsDir() || (!strings.HasSuffix(file.Name(), ".yaml") && !strings.HasSuffix(file.Name(), ".yml")) { relPath := filecatalog.RelativePath(r.dir, fullPath)
continue fileMatch := filecatalog.Stem(filepath.Base(fullPath)) == id
}
fullPath := filepath.Join(r.dir, file.Name())
data, err := os.ReadFile(fullPath) data, err := os.ReadFile(fullPath)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to read profile file %s: %w", file.Name(), err) return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
}
metadata := readProfileFileMetadata(data)
idMatch := fileMatch || metadata.id == id
if metadata.hasRawAPIKey {
if idMatch {
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath)
}
continue
} }
var prof domain.ExecutionProfile var prof domain.ExecutionProfile
decoder := yaml.NewDecoder(bytes.NewReader(data)) decoder := yaml.NewDecoder(bytes.NewReader(data))
decoder.KnownFields(true) decoder.KnownFields(true)
if err := decoder.Decode(&prof); err != nil { if err := decoder.Decode(&prof); err != nil {
if strings.Contains(err.Error(), "field api_key not found") { if idMatch {
if strings.TrimSuffix(strings.TrimSuffix(file.Name(), ".yaml"), ".yml") == id { return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, file.Name())
}
continue
}
if strings.TrimSuffix(strings.TrimSuffix(file.Name(), ".yaml"), ".yml") == id {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, file.Name(), err)
} }
continue continue
} }
@@ -76,16 +77,68 @@ func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*doma
} }
if err := validateProfile(&prof); err != nil { if err := validateProfile(&prof); err != nil {
if errors.Is(err, ErrRawAPIKeyNotAllowed) { if errors.Is(err, ErrRawAPIKeyNotAllowed) {
return nil, fmt.Errorf("%w: %s", err, file.Name()) return nil, fmt.Errorf("%w: %s", err, relPath)
} }
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, file.Name(), err) return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
} }
return &prof, nil matches = append(matches, profileMatch{
profile: &prof,
path: relPath,
})
}
if len(matches) > 1 {
paths := make([]string, 0, len(matches))
for _, match := range matches {
paths = append(paths, match.path)
}
return nil, fmt.Errorf("%w: duplicate execution profile id %q found in: %s", ErrInvalidProfile, id, strings.Join(paths, ", "))
}
if len(matches) == 1 {
return matches[0].profile, nil
} }
return nil, ErrProfileNotFound return nil, ErrProfileNotFound
} }
type profileMatch struct {
profile *domain.ExecutionProfile
path string
}
type profileFileMetadata struct {
id string
hasRawAPIKey bool
}
func readProfileFileMetadata(data []byte) profileFileMetadata {
var node yaml.Node
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&node); err != nil {
return profileFileMetadata{}
}
if node.Kind != yaml.DocumentNode || len(node.Content) == 0 {
return profileFileMetadata{}
}
mapping := node.Content[0]
if mapping.Kind != yaml.MappingNode {
return profileFileMetadata{}
}
var metadata profileFileMetadata
for i := 0; i+1 < len(mapping.Content); i += 2 {
key := mapping.Content[i]
value := mapping.Content[i+1]
switch key.Value {
case "id":
metadata.id = strings.TrimSpace(value.Value)
case "api_key":
metadata.hasRawAPIKey = true
}
}
return metadata
}
func validateProfile(p *domain.ExecutionProfile) error { func validateProfile(p *domain.ExecutionProfile) error {
if strings.TrimSpace(p.ID) == "" { if strings.TrimSpace(p.ID) == "" {
return errors.New("id is required") return errors.New("id is required")

View File

@@ -2,9 +2,11 @@ package profile
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"os" "os"
"path/filepath" "path/filepath"
"strings"
"testing" "testing"
) )
@@ -58,6 +60,149 @@ func TestFilesystemRepository_GetProfile(t *testing.T) {
if p.ReasoningEffort != "medium" { if p.ReasoningEffort != "medium" {
t.Fatalf("unexpected reasoning_effort: %q", p.ReasoningEffort) t.Fatalf("unexpected reasoning_effort: %q", p.ReasoningEffort)
} }
if p.ServiceTier != "priority" {
t.Fatalf("unexpected service_tier: %q", p.ServiceTier)
}
})
t.Run("valid nested profile", func(t *testing.T) {
nestedDir := filepath.Join(tmpDir, "local")
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
t.Fatal(err)
}
writeProfileTestFile(t, filepath.Join(nestedDir, "nested-local.yaml"), `
id: nested-local
endpoint: http://localhost:8000/v1
model: nested-model
temperature: 0.1
`)
p, err := repo.GetProfile(ctx, "nested-local")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.Model != "nested-model" {
t.Fatalf("unexpected model: %q", p.Model)
}
})
t.Run("valid profile with JSON-compatible extra params", func(t *testing.T) {
writeProfileTestFile(t, filepath.Join(tmpDir, "json-extra-params.yaml"), `
id: json-extra-params
endpoint: http://localhost:8000/v1
model: nested-model
extra_params:
string_value: enabled
number_value: 42
boolean_value: true
object_value:
nested: value
count: 2
array_value:
- first
- 3
- false
`)
p, err := repo.GetProfile(ctx, "json-extra-params")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
var got map[string]any
encoded, err := json.Marshal(p.ExtraParams)
if err != nil {
t.Fatalf("expected extra_params to marshal as JSON, got %v", err)
}
if err := json.Unmarshal(encoded, &got); err != nil {
t.Fatalf("expected extra_params JSON to decode, got %v", err)
}
if got["string_value"] != "enabled" {
t.Fatalf("unexpected string extra param: %#v", got["string_value"])
}
if got["number_value"] != float64(42) {
t.Fatalf("unexpected number extra param: %#v", got["number_value"])
}
if got["boolean_value"] != true {
t.Fatalf("unexpected boolean extra param: %#v", got["boolean_value"])
}
objectValue, ok := got["object_value"].(map[string]any)
if !ok {
t.Fatalf("expected object extra param, got %#v", got["object_value"])
}
if objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
t.Fatalf("unexpected object extra param: %#v", objectValue)
}
arrayValue, ok := got["array_value"].([]any)
if !ok {
t.Fatalf("expected array extra param, got %#v", got["array_value"])
}
if len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
t.Fatalf("unexpected array extra param: %#v", arrayValue)
}
})
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
id: duplicate-profile
endpoint: http://localhost:8000/v1
model: first-model
`)
nestedDir := filepath.Join(tmpDir, "duplicates")
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
t.Fatal(err)
}
writeProfileTestFile(t, filepath.Join(nestedDir, "duplicate-profile-b.yaml"), `
id: duplicate-profile
endpoint: http://localhost:8000/v1
model: second-model
`)
_, err := repo.GetProfile(ctx, "duplicate-profile")
if !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("expected duplicate profile to return ErrInvalidProfile, got %v", err)
}
for _, want := range []string{"duplicate execution profile id", "duplicate-profile-a.yaml", filepath.Join("duplicates", "duplicate-profile-b.yaml")} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("expected error to contain %q, got %v", want, err)
}
}
})
t.Run("nested raw api_key rejected for likely target file", func(t *testing.T) {
nestedDir := filepath.Join(tmpDir, "secure")
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
t.Fatal(err)
}
writeProfileTestFile(t, filepath.Join(nestedDir, "not_named_like_id.yaml"), `
id: nested_raw_api_key
endpoint: http://localhost:8000/v1
model: m
api_key: secret
`)
_, err := repo.GetProfile(ctx, "nested_raw_api_key")
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
}
if !strings.Contains(err.Error(), filepath.Join("secure", "not_named_like_id.yaml")) {
t.Fatalf("expected nested path in error, got %v", err)
}
})
t.Run("raw api_key in non-target profile is ignored", func(t *testing.T) {
writeProfileTestFile(t, filepath.Join(tmpDir, "raw-api-key-non-target.yaml"), `
id: raw-api-key-non-target
endpoint: http://localhost:8000/v1
model: m
api_key: secret
`)
_, err := repo.GetProfile(ctx, "does-not-exist-with-raw-key-nearby")
if !errors.Is(err, ErrProfileNotFound) {
t.Fatalf("expected ErrProfileNotFound for non-target raw api_key file, got %v", err)
}
}) })
t.Run("invalid yaml", func(t *testing.T) { t.Run("invalid yaml", func(t *testing.T) {
@@ -109,3 +254,10 @@ func TestFilesystemRepository_GetProfile(t *testing.T) {
} }
}) })
} }
func writeProfileTestFile(t *testing.T, path string, content string) {
t.Helper()
if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil {
t.Fatalf("failed to write profile test file %q: %v", path, err)
}
}

View File

@@ -2,6 +2,7 @@ id: local-secure
endpoint: http://localhost:8000/v1 endpoint: http://localhost:8000/v1
model: gpt-4o-mini model: gpt-4o-mini
api_key_env: SCRIPTORIUM_API_KEY api_key_env: SCRIPTORIUM_API_KEY
service_tier: priority
reasoning_effort: medium reasoning_effort: medium
extra_params: extra_params:
provider: local provider: local

View File

@@ -6,7 +6,9 @@ import (
"errors" "errors"
"fmt" "fmt"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"strings"
"text/template" "text/template"
"unicode/utf8"
) )
var ( var (
@@ -50,6 +52,11 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
}, },
} }
sessionID, err := renderSessionID(definition.SessionID, funcs, vars)
if err != nil {
return nil, err
}
var renderedMessages []domain.RenderedMessage var renderedMessages []domain.RenderedMessage
for i, tmplMsg := range definition.Templates { for i, tmplMsg := range definition.Templates {
@@ -75,12 +82,44 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
} }
renderedMessages = append(renderedMessages, domain.RenderedMessage{ renderedMessages = append(renderedMessages, domain.RenderedMessage{
Role: tmplMsg.Role, Role: tmplMsg.Role,
Content: buf.String(), Content: buf.String(),
CacheControl: cloneCacheControl(tmplMsg.CacheControl),
}) })
} }
return &domain.RenderedPrompt{ return &domain.RenderedPrompt{
Messages: renderedMessages, SessionID: sessionID,
Messages: renderedMessages,
}, nil }, nil
} }
func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
if strings.TrimSpace(raw) == "" {
return "", nil
}
tmpl, err := template.New("session_id").Funcs(funcs).Option("missingkey=error").Parse(raw)
if err != nil {
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, err)
}
var buf bytes.Buffer
if err := tmpl.Execute(&buf, vars); err != nil {
return "", fmt.Errorf("%w: session_id: %w", ErrRenderFailure, err)
}
sessionID := strings.TrimSpace(buf.String())
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
return "", fmt.Errorf("%w: session_id length %d exceeds maximum %d", ErrRenderFailure, n, domain.SessionIDMaxLength)
}
return sessionID, nil
}
func cloneCacheControl(in *domain.CacheControl) *domain.CacheControl {
if in == nil {
return nil
}
out := *in
return &out
}

View File

@@ -3,6 +3,7 @@ package prompt
import ( import (
"context" "context"
"errors" "errors"
"strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
@@ -78,6 +79,66 @@ func TestGoRenderer_Render(t *testing.T) {
} }
}) })
t.Run("copying cache control to rendered messages", func(t *testing.T) {
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{
Role: "system",
Content: "You are concise.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "Analyze this: {{input \"transcript\"}}"},
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(res.Messages) != 2 {
t.Fatalf("expected 2 messages, got %d", len(res.Messages))
}
if res.Messages[0].CacheControl == nil {
t.Fatal("expected rendered cache control")
}
if res.Messages[0].CacheControl.Type != domain.CacheControlEphemeral {
t.Fatalf("unexpected cache control type: %q", res.Messages[0].CacheControl.Type)
}
if res.Messages[0].CacheControl.TTL != "1h" {
t.Fatalf("unexpected cache control ttl: %q", res.Messages[0].CacheControl.TTL)
}
if res.Messages[1].CacheControl != nil {
t.Fatalf("expected no cache control on second message, got %#v", res.Messages[1].CacheControl)
}
})
t.Run("rendered cache control does not alias source template", func(t *testing.T) {
source := &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "You are concise.", CacheControl: source},
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if res.Messages[0].CacheControl == source {
t.Fatal("expected rendered cache control to be cloned")
}
res.Messages[0].CacheControl.TTL = ""
if source.TTL != "1h" {
t.Fatalf("source cache control was mutated, ttl=%q", source.TTL)
}
})
t.Run("accessing vars", func(t *testing.T) { t.Run("accessing vars", func(t *testing.T) {
def := &domain.PromptDefinition{ def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
@@ -95,6 +156,78 @@ func TestGoRenderer_Render(t *testing.T) {
} }
}) })
t.Run("rendering session id from vars", func(t *testing.T) {
def := &domain.PromptDefinition{
SessionID: " {{ .session_id }} ",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "Speak in a {{.tone}} tone."},
},
}
res, err := renderer.Render(ctx, def, inputs, map[string]string{
"tone": "concise",
"session_id": "agent-session-123",
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if res.SessionID != "agent-session-123" {
t.Fatalf("unexpected session id: %q", res.SessionID)
}
})
t.Run("empty rendered session id is omitted", func(t *testing.T) {
def := &domain.PromptDefinition{
SessionID: " ",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "Speak in a {{.tone}} tone."},
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if res.SessionID != "" {
t.Fatalf("expected empty session id, got %q", res.SessionID)
}
})
t.Run("missing session id var fails rendering", func(t *testing.T) {
def := &domain.PromptDefinition{
SessionID: "{{ .session_id }}",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "Speak in a {{.tone}} tone."},
},
}
_, err := renderer.Render(ctx, def, inputs, vars)
if !errors.Is(err, ErrRenderFailure) {
t.Fatalf("expected ErrRenderFailure, got %v", err)
}
})
t.Run("too long rendered session id fails rendering", func(t *testing.T) {
def := &domain.PromptDefinition{
SessionID: "{{ .session_id }}",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "Speak in a {{.tone}} tone."},
},
}
_, err := renderer.Render(ctx, def, inputs, map[string]string{
"tone": "concise",
"session_id": strings.Repeat("x", domain.SessionIDMaxLength+1),
})
if !errors.Is(err, ErrRenderFailure) {
t.Fatalf("expected ErrRenderFailure, got %v", err)
}
})
t.Run("inserting required input artifact", func(t *testing.T) { t.Run("inserting required input artifact", func(t *testing.T) {
def := &domain.PromptDefinition{ def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},

View File

@@ -10,6 +10,7 @@ import (
"strings" "strings"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/filecatalog"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
@@ -28,6 +29,7 @@ type promptDefinitionFile struct {
Version string `yaml:"version"` Version string `yaml:"version"`
DefaultProfile *string `yaml:"default_profile"` DefaultProfile *string `yaml:"default_profile"`
Description string `yaml:"description"` Description string `yaml:"description"`
SessionID string `yaml:"session_id"`
Inputs []promptInputFile `yaml:"inputs"` Inputs []promptInputFile `yaml:"inputs"`
Messages []promptMessageFile `yaml:"messages"` Messages []promptMessageFile `yaml:"messages"`
Output promptOutputContractFile `yaml:"output"` Output promptOutputContractFile `yaml:"output"`
@@ -41,9 +43,15 @@ type promptInputFile struct {
} }
type promptMessageFile struct { type promptMessageFile struct {
Role string `yaml:"role"` Role string `yaml:"role"`
Content string `yaml:"content"` Content string `yaml:"content"`
ContentFile string `yaml:"content_file"` ContentFile string `yaml:"content_file"`
CacheControl *cacheControlFile `yaml:"cache_control"`
}
type cacheControlFile struct {
Type string `yaml:"type"`
TTL string `yaml:"ttl"`
} }
type promptOutputContractFile struct { type promptOutputContractFile struct {
@@ -62,29 +70,26 @@ func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id strin
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition) return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
} }
files, err := os.ReadDir(r.dir) files, err := filecatalog.FindYAMLFiles(ctx, r.dir)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err) return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
} }
for _, file := range files { var matches []promptDefinitionMatch
for _, fullPath := range files {
select { select {
case <-ctx.Done(): case <-ctx.Done():
return nil, ctx.Err() return nil, ctx.Err()
default: default:
} }
if file.IsDir() || !isYAMLFile(file.Name()) { relPath := filecatalog.RelativePath(r.dir, fullPath)
continue fileMatch := filecatalog.Stem(filepath.Base(fullPath)) == id
}
fullPath := filepath.Join(r.dir, file.Name())
fileMatch := promptIDFromFileName(file.Name()) == id
raw, err := loadPromptDefinitionFile(fullPath) raw, err := loadPromptDefinitionFile(fullPath)
if err != nil { if err != nil {
if fileMatch { if fileMatch || promptDefinitionFileHasID(fullPath, id) {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, file.Name(), err) return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
} }
continue continue
} }
@@ -92,7 +97,7 @@ func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id strin
def, err := normalizePromptDefinition(raw, fullPath) def, err := normalizePromptDefinition(raw, fullPath)
if err != nil { if err != nil {
if fileMatch || strings.TrimSpace(raw.ID) == id { if fileMatch || strings.TrimSpace(raw.ID) == id {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, file.Name(), err) return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, relPath, err)
} }
continue continue
} }
@@ -103,12 +108,35 @@ func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id strin
if version != "" && def.Version != version { if version != "" && def.Version != version {
continue continue
} }
return def, nil matches = append(matches, promptDefinitionMatch{
def: def,
path: relPath,
})
}
if len(matches) > 1 {
paths := make([]string, 0, len(matches))
for _, match := range matches {
paths = append(paths, match.path)
}
if version != "" {
return nil, fmt.Errorf("%w: duplicate prompt definition id %q version %q found in: %s", ErrInvalidPromptDefinition, id, version, strings.Join(paths, ", "))
}
return nil, fmt.Errorf("%w: duplicate prompt definition id %q found in: %s", ErrInvalidPromptDefinition, id, strings.Join(paths, ", "))
}
if len(matches) == 1 {
return matches[0].def, nil
} }
return nil, ErrPromptDefinitionNotFound return nil, ErrPromptDefinitionNotFound
} }
type promptDefinitionMatch struct {
def *domain.PromptDefinition
path string
}
func loadPromptDefinitionFile(path string) (*promptDefinitionFile, error) { func loadPromptDefinitionFile(path string) (*promptDefinitionFile, error) {
data, err := os.ReadFile(path) data, err := os.ReadFile(path)
if err != nil { if err != nil {
@@ -124,6 +152,20 @@ func loadPromptDefinitionFile(path string) (*promptDefinitionFile, error) {
return &raw, nil return &raw, nil
} }
func promptDefinitionFileHasID(path string, id string) bool {
data, err := os.ReadFile(path)
if err != nil {
return false
}
var raw struct {
ID string `yaml:"id"`
}
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&raw); err != nil {
return false
}
return strings.TrimSpace(raw.ID) == id
}
func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*domain.PromptDefinition, error) { func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*domain.PromptDefinition, error) {
if raw == nil { if raw == nil {
return nil, errors.New("prompt definition is nil") return nil, errors.New("prompt definition is nil")
@@ -177,6 +219,11 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
return nil, fmt.Errorf("message %d (%s) must set exactly one of content or content_file", i, role) return nil, fmt.Errorf("message %d (%s) must set exactly one of content or content_file", i, role)
} }
cacheControl, err := normalizeCacheControl(msg.CacheControl)
if err != nil {
return nil, fmt.Errorf("message %d (%s) cache_control: %w", i, role, err)
}
templateContent := msg.Content templateContent := msg.Content
resolvedContentFile := "" resolvedContentFile := ""
if hasContentFile { if hasContentFile {
@@ -195,9 +242,10 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
} }
templates = append(templates, domain.PromptMessageTemplate{ templates = append(templates, domain.PromptMessageTemplate{
Role: role, Role: role,
Content: templateContent, Content: templateContent,
ContentFile: resolvedContentFile, ContentFile: resolvedContentFile,
CacheControl: cacheControl,
}) })
} }
@@ -227,6 +275,7 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
Version: version, Version: version,
DefaultProfile: defaultProfile, DefaultProfile: defaultProfile,
Description: strings.TrimSpace(raw.Description), Description: strings.TrimSpace(raw.Description),
SessionID: strings.TrimSpace(raw.SessionID),
Inputs: inputs, Inputs: inputs,
Templates: templates, Templates: templates,
OutputFormat: raw.Output.Format, OutputFormat: raw.Output.Format,
@@ -239,14 +288,28 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
}, nil }, nil
} }
func isYAMLFile(name string) bool { func normalizeCacheControl(raw *cacheControlFile) (*domain.CacheControl, error) {
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml") if raw == nil {
} return nil, nil
}
func promptIDFromFileName(name string) string { cacheType := strings.TrimSpace(raw.Type)
name = strings.TrimSuffix(name, ".yaml") if cacheType == "" {
name = strings.TrimSuffix(name, ".yml") return nil, errors.New("type is required")
return name }
if domain.CacheControlType(cacheType) != domain.CacheControlEphemeral {
return nil, fmt.Errorf("unsupported type %q", cacheType)
}
ttl := strings.TrimSpace(raw.TTL)
if ttl != "" && ttl != "1h" {
return nil, fmt.Errorf("unsupported ttl %q", ttl)
}
return &domain.CacheControl{
Type: domain.CacheControlType(cacheType),
TTL: ttl,
}, nil
} }
func isValidOutputFormat(f domain.OutputFormat) bool { func isValidOutputFormat(f domain.OutputFormat) bool {

View File

@@ -68,6 +68,77 @@ func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
} }
}) })
t.Run("valid cache control with ttl", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "valid-cache-control-ttl", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(p.Templates) != 2 {
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
}
assertCacheControl(t, p.Templates[0].CacheControl, domain.CacheControlEphemeral, "1h")
if p.Templates[1].CacheControl != nil {
t.Fatalf("expected second message cache control to be nil, got %#v", p.Templates[1].CacheControl)
}
})
t.Run("valid cache control without ttl", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "valid-cache-control-without-ttl", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(p.Templates) != 2 {
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
}
assertCacheControl(t, p.Templates[0].CacheControl, domain.CacheControlEphemeral, "")
if p.Templates[1].CacheControl != nil {
t.Fatalf("expected second message cache control to be nil, got %#v", p.Templates[1].CacheControl)
}
})
t.Run("valid session id template", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "valid-session-id", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.SessionID != "{{ .session_id }}" {
t.Fatalf("expected trimmed session_id template, got %q", p.SessionID)
}
})
t.Run("valid nested file-backed prompt resolves content file relative to nested YAML", func(t *testing.T) {
nestedDir := filepath.Join(tmpDir, "dnd", "recap")
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
t.Fatal(err)
}
writePromptTestFile(t, filepath.Join(nestedDir, "nested_recap.yaml"), `
id: nested-recap
version: "1.0.0"
messages:
- role: user
content_file: ./nested_recap.user.tmpl
output:
format: markdown
validation_mode: basic
repair_attempts: 0
`)
writePromptTestFile(t, filepath.Join(nestedDir, "nested_recap.user.tmpl"), `Nested recap: {{input "transcript"}}`)
p, err := repo.GetPromptDefinition(ctx, "nested-recap", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(p.Templates) != 1 {
t.Fatalf("expected one template, got %d", len(p.Templates))
}
if !strings.Contains(p.Templates[0].Content, "Nested recap") {
t.Fatalf("expected nested content file body, got %q", p.Templates[0].Content)
}
if !strings.Contains(p.Templates[0].ContentFile, filepath.Join("dnd", "recap", "nested_recap.user.tmpl")) {
t.Fatalf("expected nested content file path, got %q", p.Templates[0].ContentFile)
}
})
t.Run("prompt with default_profile", func(t *testing.T) { t.Run("prompt with default_profile", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "with-default-profile", "") p, err := repo.GetPromptDefinition(ctx, "with-default-profile", "")
if err != nil { if err != nil {
@@ -84,6 +155,124 @@ func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
} }
}) })
t.Run("duplicate prompt IDs fail as ambiguous", func(t *testing.T) {
writePromptTestFile(t, filepath.Join(tmpDir, "duplicate_a.yaml"), `
id: duplicate-prompt
version: "1.0.0"
messages:
- role: user
content: First duplicate.
output:
format: markdown
validation_mode: basic
repair_attempts: 0
`)
nestedDir := filepath.Join(tmpDir, "nested")
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
t.Fatal(err)
}
writePromptTestFile(t, filepath.Join(nestedDir, "duplicate_b.yaml"), `
id: duplicate-prompt
version: "2.0.0"
messages:
- role: user
content: Second duplicate.
output:
format: markdown
validation_mode: basic
repair_attempts: 0
`)
_, err := repo.GetPromptDefinition(ctx, "duplicate-prompt", "")
if !errors.Is(err, ErrInvalidPromptDefinition) {
t.Fatalf("expected duplicate prompt to return ErrInvalidPromptDefinition, got %v", err)
}
for _, want := range []string{"duplicate prompt definition id", "duplicate_a.yaml", filepath.Join("nested", "duplicate_b.yaml")} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("expected error to contain %q, got %v", want, err)
}
}
})
t.Run("duplicate prompt ID and requested version fails as ambiguous", func(t *testing.T) {
writePromptTestFile(t, filepath.Join(tmpDir, "version_duplicate_a.yaml"), `
id: duplicate-version-prompt
version: "1.0.0"
messages:
- role: user
content: First duplicate version.
output:
format: markdown
validation_mode: basic
repair_attempts: 0
`)
nestedDir := filepath.Join(tmpDir, "versioned")
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
t.Fatal(err)
}
writePromptTestFile(t, filepath.Join(nestedDir, "version_duplicate_b.yaml"), `
id: duplicate-version-prompt
version: "1.0.0"
messages:
- role: user
content: Second duplicate version.
output:
format: markdown
validation_mode: basic
repair_attempts: 0
`)
_, err := repo.GetPromptDefinition(ctx, "duplicate-version-prompt", "1.0.0")
if !errors.Is(err, ErrInvalidPromptDefinition) {
t.Fatalf("expected duplicate prompt version to return ErrInvalidPromptDefinition, got %v", err)
}
for _, want := range []string{"duplicate prompt definition id", "version \"1.0.0\"", "version_duplicate_a.yaml", filepath.Join("versioned", "version_duplicate_b.yaml")} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("expected error to contain %q, got %v", want, err)
}
}
})
t.Run("non-matching malformed nested prompt is ignored for not found lookup", func(t *testing.T) {
nestedDir := filepath.Join(tmpDir, "broken")
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
t.Fatal(err)
}
writePromptTestFile(t, filepath.Join(nestedDir, "unrelated.yaml"), "id: [")
_, err := repo.GetPromptDefinition(ctx, "does-not-exist-even-with-broken-nested-file", "")
if !errors.Is(err, ErrPromptDefinitionNotFound) {
t.Fatalf("expected ErrPromptDefinitionNotFound, got %v", err)
}
})
t.Run("strict decode failure in nested prompt matches by YAML ID", func(t *testing.T) {
nestedDir := filepath.Join(tmpDir, "strict")
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
t.Fatal(err)
}
writePromptTestFile(t, filepath.Join(nestedDir, "not_named_like_id.yaml"), `
id: nested-strict-error
version: "1.0.0"
unknown_field: true
messages:
- role: user
content: Invalid because of unknown field.
output:
format: markdown
validation_mode: basic
repair_attempts: 0
`)
_, err := repo.GetPromptDefinition(ctx, "nested-strict-error", "")
if !errors.Is(err, ErrInvalidYAML) {
t.Fatalf("expected ErrInvalidYAML, got %v", err)
}
if !strings.Contains(err.Error(), filepath.Join("strict", "not_named_like_id.yaml")) {
t.Fatalf("expected nested path in error, got %v", err)
}
})
t.Run("version lookup", func(t *testing.T) { t.Run("version lookup", func(t *testing.T) {
_, err := repo.GetPromptDefinition(ctx, "valid-inline", "9.9.9") _, err := repo.GetPromptDefinition(ctx, "valid-inline", "9.9.9")
if !errors.Is(err, ErrPromptDefinitionNotFound) { if !errors.Is(err, ErrPromptDefinitionNotFound) {
@@ -107,6 +296,10 @@ func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
{name: "invalid validation mode", id: "invalid_validation_mode", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"invalid validation mode"}}, {name: "invalid validation mode", id: "invalid_validation_mode", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"invalid validation mode"}},
{name: "json_schema without schema_path", id: "json_schema_without_schema_path", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"schema_path"}}, {name: "json_schema without schema_path", id: "json_schema_without_schema_path", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"schema_path"}},
{name: "unknown input field", id: "unknown_input_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unknown_input_setting not found"}}, {name: "unknown input field", id: "unknown_input_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unknown_input_setting not found"}},
{name: "empty cache control type", id: "empty_cache_control_type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "type is required"}},
{name: "unsupported cache control type", id: "unsupported_cache_control_type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported type"}},
{name: "unsupported cache control ttl", id: "unsupported_cache_control_ttl", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported ttl"}},
{name: "unknown cache control field", id: "unknown_cache_control_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unexpected not found"}},
} }
for _, tc := range cases { for _, tc := range cases {
@@ -131,6 +324,26 @@ func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
}) })
} }
func assertCacheControl(t *testing.T, got *domain.CacheControl, wantType domain.CacheControlType, wantTTL string) {
t.Helper()
if got == nil {
t.Fatal("expected cache control, got nil")
}
if got.Type != wantType {
t.Fatalf("unexpected cache control type: got %q want %q", got.Type, wantType)
}
if got.TTL != wantTTL {
t.Fatalf("unexpected cache control ttl: got %q want %q", got.TTL, wantTTL)
}
}
func writePromptTestFile(t *testing.T, path string, content string) {
t.Helper()
if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil {
t.Fatalf("failed to write prompt test file %q: %v", path, err)
}
}
func copyTree(src, dst string) error { func copyTree(src, dst string) error {
return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error {
if err != nil { if err != nil {

View File

@@ -0,0 +1,10 @@
id: empty-cache-control-type
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control: {}
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -0,0 +1,12 @@
id: unknown-cache-control-field
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: ephemeral
unexpected: true
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -0,0 +1,12 @@
id: unsupported-cache-control-ttl
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: ephemeral
ttl: 5m
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -0,0 +1,11 @@
id: unsupported-cache-control-type
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: persistent
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -0,0 +1,14 @@
id: valid-cache-control-ttl
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: ephemeral
ttl: 1h
- role: user
content: "Summarize the input."
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -0,0 +1,13 @@
id: valid-cache-control-without-ttl
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: ephemeral
- role: user
content: "Summarize the input."
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -0,0 +1,10 @@
id: valid-session-id
version: "1.0.0"
session_id: " {{ .session_id }} "
messages:
- role: user
content: Hello.
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -35,9 +35,9 @@ func TestRunnerIntegrationWithPromptAndProfileFixturesAndValidation(t *testing.T
t.Fatalf("failed to resolve repo root: %v", err) t.Fatalf("failed to resolve repo root: %v", err)
} }
promptsDir := filepath.Join(root, "prompts") promptsDir := filepath.Join(root, "examples", "prompts")
profilesDir := filepath.Join(root, "profiles") profilesDir := filepath.Join(root, "examples", "profiles")
schemasDir := filepath.Join(root, "schemas") schemasDir := filepath.Join(root, "examples", "schemas")
fixturesDir := filepath.Join(root, "examples", "fixtures") fixturesDir := filepath.Join(root, "examples", "fixtures")
t.Setenv("SCRIPTORIUM_API_KEY", "test-key") t.Setenv("SCRIPTORIUM_API_KEY", "test-key")

View File

@@ -24,12 +24,14 @@ import (
) )
var ( var (
ErrInvalidRequest = errors.New("invalid run request") ErrInvalidRequest = errors.New("invalid run request")
ErrProfileLoad = errors.New("failed to load prompt definition") ErrProfileRequired = errors.New("profile selection is required")
ErrArtifactLoad = errors.New("failed to load artifact") ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
ErrPromptRender = errors.New("failed to render prompt") ErrProfileLoad = errors.New("failed to load prompt definition")
ErrLLMGenerate = errors.New("failed to generate output") ErrArtifactLoad = errors.New("failed to load artifact")
ErrValidation = errors.New("failed to validate output") ErrPromptRender = errors.New("failed to render prompt")
ErrLLMGenerate = errors.New("failed to generate output")
ErrValidation = errors.New("failed to validate output")
) )
// Runner executes the Scriptorium core use case. // Runner executes the Scriptorium core use case.
@@ -88,11 +90,15 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
} }
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{ genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: prepared.Messages}, Prompt: domain.RenderedPrompt{SessionID: prepared.SessionID, Messages: prepared.Messages},
Target: prepared.EffectiveModelParams, Target: prepared.EffectiveModelParams,
TargetPresence: prepared.TargetPresence,
StructuredOutput: prepared.StructuredOutput, StructuredOutput: prepared.StructuredOutput,
}) })
if err != nil { if err != nil {
if errors.Is(err, llm.ErrInvalidRequest) {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err) return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err)
} }
@@ -177,7 +183,7 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
selectedProfileID = strings.TrimSpace(def.DefaultProfile) selectedProfileID = strings.TrimSpace(def.DefaultProfile)
} }
if selectedProfileID == "" { if selectedProfileID == "" {
return nil, fmt.Errorf("%w: profile id is required either in request or prompt default_profile", ErrInvalidRequest) return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired)
} }
execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID) execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID)
@@ -185,7 +191,10 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err) return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
} }
effectiveModel := resolveExecutionTarget(execProfile, req.Execution) effectiveModel, targetPresence, err := resolveExecutionTarget(execProfile, req.Execution)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
if strings.TrimSpace(effectiveModel.Endpoint) == "" { if strings.TrimSpace(effectiveModel.Endpoint) == "" {
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest) return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
} }
@@ -228,9 +237,11 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
PromptHash: promptDefinitionHash, PromptHash: promptDefinitionHash,
SelectedProfileID: selectedProfileID, SelectedProfileID: selectedProfileID,
EffectiveModelParams: effectiveModel, EffectiveModelParams: effectiveModel,
TargetPresence: targetPresence,
OutputContract: effectiveContract, OutputContract: effectiveContract,
StructuredOutput: structuredOutput, StructuredOutput: structuredOutput,
InputHashes: inputHashes, InputHashes: inputHashes,
SessionID: renderedPrompt.SessionID,
RenderedPromptHash: hashRenderedPrompt(*renderedPrompt), RenderedPromptHash: hashRenderedPrompt(*renderedPrompt),
Messages: renderedPrompt.Messages, Messages: renderedPrompt.Messages,
StartTime: start, StartTime: start,
@@ -342,6 +353,9 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution
if override.TimeoutSeconds != 0 { if override.TimeoutSeconds != 0 {
out.TimeoutSeconds = override.TimeoutSeconds out.TimeoutSeconds = override.TimeoutSeconds
} }
if strings.TrimSpace(override.ServiceTier) != "" {
out.ServiceTier = override.ServiceTier
}
if strings.TrimSpace(override.ReasoningEffort) != "" { if strings.TrimSpace(override.ReasoningEffort) != "" {
out.ReasoningEffort = override.ReasoningEffort out.ReasoningEffort = override.ReasoningEffort
} }
@@ -349,22 +363,75 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution
out.APIKeyEnv = override.APIKeyEnv out.APIKeyEnv = override.APIKeyEnv
} }
if len(override.ExtraParams) > 0 { if len(override.ExtraParams) > 0 {
cp := make(map[string]string, len(override.ExtraParams)) out.ExtraParams = copyExtraParams(override.ExtraParams)
for k, v := range override.ExtraParams {
cp[k] = v
}
out.ExtraParams = cp
} }
return out return out
} }
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTarget) domain.ExecutionTarget { func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
out := base
var presence domain.ExecutionTargetPresence
if override.Endpoint != "" {
out.Endpoint = override.Endpoint
}
if override.Model != "" {
out.Model = override.Model
}
if override.Temperature != nil {
if *override.Temperature < 0 || *override.Temperature > 2 {
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("temperature must be between 0 and 2")
}
out.Temperature = *override.Temperature
presence.Temperature = true
}
if override.MaxTokens != nil {
if *override.MaxTokens < 0 {
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("max_tokens must be greater than or equal to 0")
}
out.MaxTokens = *override.MaxTokens
presence.MaxTokens = true
}
if override.TopP != nil {
if *override.TopP < 0 || *override.TopP > 1 {
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("top_p must be between 0 and 1")
}
out.TopP = *override.TopP
presence.TopP = true
}
if override.TimeoutSeconds != nil {
if *override.TimeoutSeconds < 0 {
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("timeout_seconds must be greater than or equal to 0")
}
out.TimeoutSeconds = *override.TimeoutSeconds
presence.TimeoutSeconds = true
}
if strings.TrimSpace(override.ServiceTier) != "" {
out.ServiceTier = override.ServiceTier
}
if strings.TrimSpace(override.ReasoningEffort) != "" {
out.ReasoningEffort = override.ReasoningEffort
}
if strings.TrimSpace(override.APIKeyEnv) != "" {
out.APIKeyEnv = override.APIKeyEnv
}
if len(override.ExtraParams) > 0 {
out.ExtraParams = copyExtraParams(override.ExtraParams)
}
return out, presence, nil
}
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
out := defaults.ExecutionTargetDefault() out := defaults.ExecutionTargetDefault()
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue)) out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
var presence domain.ExecutionTargetPresence
if override != nil { if override != nil {
out = mergeExecutionTarget(out, *override) var err error
out, presence, err = mergeExecutionTargetOverride(out, *override)
if err != nil {
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, err
}
} }
return out return out, presence, nil
} }
func validateAPIKeyEnv(apiKeyEnv string) error { func validateAPIKeyEnv(apiKeyEnv string) error {
@@ -373,7 +440,7 @@ func validateAPIKeyEnv(apiKeyEnv string) error {
return nil return nil
} }
if strings.TrimSpace(os.Getenv(envName)) == "" { if strings.TrimSpace(os.Getenv(envName)) == "" {
return fmt.Errorf("api key environment variable %q is not set", envName) return fmt.Errorf("%w: api key environment variable %q is not set", ErrAPIKeyEnvMissing, envName)
} }
return nil return nil
} }
@@ -382,13 +449,6 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
if p == nil { if p == nil {
return domain.ExecutionTarget{} return domain.ExecutionTarget{}
} }
cp := map[string]string(nil)
if len(p.ExtraParams) > 0 {
cp = make(map[string]string, len(p.ExtraParams))
for k, v := range p.ExtraParams {
cp[k] = v
}
}
return domain.ExecutionTarget{ return domain.ExecutionTarget{
Endpoint: p.Endpoint, Endpoint: p.Endpoint,
Model: p.Model, Model: p.Model,
@@ -396,12 +456,24 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
MaxTokens: p.MaxTokens, MaxTokens: p.MaxTokens,
TopP: p.TopP, TopP: p.TopP,
TimeoutSeconds: p.TimeoutSeconds, TimeoutSeconds: p.TimeoutSeconds,
ServiceTier: p.ServiceTier,
ReasoningEffort: p.ReasoningEffort, ReasoningEffort: p.ReasoningEffort,
APIKeyEnv: p.APIKeyEnv, APIKeyEnv: p.APIKeyEnv,
ExtraParams: cp, ExtraParams: copyExtraParams(p.ExtraParams),
} }
} }
func copyExtraParams(src map[string]any) map[string]any {
if len(src) == 0 {
return nil
}
cp := make(map[string]any, len(src))
for k, v := range src {
cp[k] = v
}
return cp
}
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract { func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract {
contract := def.Validation contract := def.Validation
if contract.Format == "" { if contract.Format == "" {
@@ -418,10 +490,23 @@ func resolveOutputContract(def *domain.PromptDefinition, override *domain.Output
func hashRenderedPrompt(p domain.RenderedPrompt) string { func hashRenderedPrompt(p domain.RenderedPrompt) string {
var b strings.Builder var b strings.Builder
if p.SessionID != "" {
b.WriteString("session_id=")
b.WriteString(p.SessionID)
b.WriteString("\n---\n")
}
for _, msg := range p.Messages { for _, msg := range p.Messages {
b.WriteString(msg.Role) b.WriteString(msg.Role)
b.WriteByte('\n') b.WriteByte('\n')
b.WriteString(msg.Content) b.WriteString(msg.Content)
if msg.CacheControl != nil {
b.WriteString("\ncache_control.type=")
b.WriteString(string(msg.CacheControl.Type))
if msg.CacheControl.TTL != "" {
b.WriteString("\ncache_control.ttl=")
b.WriteString(msg.CacheControl.TTL)
}
}
b.WriteString("\n---\n") b.WriteString("\n---\n")
} }
h := sha256.Sum256([]byte(b.String())) h := sha256.Sum256([]byte(b.String()))

View File

@@ -14,6 +14,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults" "gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile" "gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt" "gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef" "gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
@@ -160,7 +161,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")}, "a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")}, "a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
}} }}
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{forbid: true} llmClient := &fakeLLM{forbid: true}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
@@ -172,7 +173,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"}, "glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
}, },
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90}, Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)},
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
@@ -198,6 +199,9 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
if len(prepared.Messages) != 2 { if len(prepared.Messages) != 2 {
t.Fatalf("expected two messages, got %d", len(prepared.Messages)) t.Fatalf("expected two messages, got %d", len(prepared.Messages))
} }
if prepared.SessionID != "session-123" {
t.Fatalf("expected prepared session id, got %q", prepared.SessionID)
}
if llmClient.calls != 0 { if llmClient.calls != 0 {
t.Fatalf("prepare should not call llm, calls=%d", llmClient.calls) t.Fatalf("prepare should not call llm, calls=%d", llmClient.calls)
} }
@@ -234,6 +238,9 @@ func TestRunnerPrepareMissingExplicitProfileAndMissingDefaultProfileFails(t *tes
if !errors.Is(err, ErrInvalidRequest) { if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err) t.Fatalf("expected ErrInvalidRequest, got %v", err)
} }
if !errors.Is(err, ErrProfileRequired) {
t.Fatalf("expected ErrProfileRequired, got %v", err)
}
} }
func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) { func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) {
@@ -257,6 +264,7 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
MaxTokens: 500, MaxTokens: 500,
TopP: 0.9, TopP: 0.9,
TimeoutSeconds: 120, TimeoutSeconds: 120,
ServiceTier: "priority",
}, },
}} }}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
@@ -265,11 +273,12 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
Inputs: singleInputRef(), Inputs: singleInputRef(),
Execution: &domain.ExecutionTarget{ Execution: &domain.ExecutionTargetOverride{
Endpoint: "http://override/v1", Endpoint: "http://override/v1",
Model: "override-model", Model: "override-model",
Temperature: 0.7, Temperature: float64Ptr(0.7),
TimeoutSeconds: 30, TimeoutSeconds: intPtr(30),
ServiceTier: "flex",
}, },
}) })
if err != nil { if err != nil {
@@ -281,6 +290,146 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
if prepared.EffectiveModelParams.TopP != 0.9 { if prepared.EffectiveModelParams.TopP != 0.9 {
t.Fatalf("expected profile top_p to remain, got %v", prepared.EffectiveModelParams.TopP) t.Fatalf("expected profile top_p to remain, got %v", prepared.EffectiveModelParams.TopP)
} }
if prepared.EffectiveModelParams.ServiceTier != "flex" {
t.Fatalf("expected service_tier override to win, got %q", prepared.EffectiveModelParams.ServiceTier)
}
}
func TestRunnerPrepareRequestNumericOverridePresence(t *testing.T) {
tests := []struct {
name string
override *domain.ExecutionTargetOverride
wantTemperature float64
wantMaxTokens int
wantTopP float64
wantTimeoutSecs int
wantPresence domain.ExecutionTargetPresence
}{
{
name: "omitted preserves profile values",
override: &domain.ExecutionTargetOverride{},
wantTemperature: 0.7,
wantMaxTokens: 321,
wantTopP: 0.8,
wantTimeoutSecs: 45,
},
{
name: "explicit zero temperature",
override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(0)},
wantTemperature: 0,
wantMaxTokens: 321,
wantTopP: 0.8,
wantTimeoutSecs: 45,
wantPresence: domain.ExecutionTargetPresence{Temperature: true},
},
{
name: "explicit zero max tokens",
override: &domain.ExecutionTargetOverride{MaxTokens: intPtr(0)},
wantTemperature: 0.7,
wantMaxTokens: 0,
wantTopP: 0.8,
wantTimeoutSecs: 45,
wantPresence: domain.ExecutionTargetPresence{MaxTokens: true},
},
{
name: "explicit zero top p",
override: &domain.ExecutionTargetOverride{TopP: float64Ptr(0)},
wantTemperature: 0.7,
wantMaxTokens: 321,
wantTopP: 0,
wantTimeoutSecs: 45,
wantPresence: domain.ExecutionTargetPresence{TopP: true},
},
{
name: "explicit zero timeout",
override: &domain.ExecutionTargetOverride{TimeoutSeconds: intPtr(0)},
wantTemperature: 0.7,
wantMaxTokens: 321,
wantTopP: 0.8,
wantTimeoutSecs: 0,
wantPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.7,
MaxTokens: 321,
TopP: 0.8,
TimeoutSeconds: 45,
},
}},
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
nil,
)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
Execution: tc.override,
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
got := prepared.EffectiveModelParams
if got.Temperature != tc.wantTemperature ||
got.MaxTokens != tc.wantMaxTokens ||
got.TopP != tc.wantTopP ||
got.TimeoutSeconds != tc.wantTimeoutSecs {
t.Fatalf("unexpected effective numeric settings: %+v", got)
}
if prepared.TargetPresence != tc.wantPresence {
t.Fatalf("unexpected target presence: got %+v want %+v", prepared.TargetPresence, tc.wantPresence)
}
})
}
}
func TestRunnerPrepareInvalidRequestNumericOverridesFail(t *testing.T) {
tests := []struct {
name string
override *domain.ExecutionTargetOverride
}{
{name: "temperature below range", override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(-0.1)}},
{name: "temperature above range", override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(2.1)}},
{name: "max tokens below range", override: &domain.ExecutionTargetOverride{MaxTokens: intPtr(-1)}},
{name: "top p below range", override: &domain.ExecutionTargetOverride{TopP: float64Ptr(-0.1)}},
{name: "top p above range", override: &domain.ExecutionTargetOverride{TopP: float64Ptr(1.1)}},
{name: "timeout below range", override: &domain.ExecutionTargetOverride{TimeoutSeconds: intPtr(-1)}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
nil,
)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
Execution: tc.override,
})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
})
}
} }
func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) { func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
@@ -292,6 +441,7 @@ func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
Model: "profile-model", Model: "profile-model",
TopP: 0.8, TopP: 0.8,
TimeoutSeconds: 90, TimeoutSeconds: 90,
ServiceTier: "priority",
}, },
}} }}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
@@ -310,6 +460,9 @@ func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
if prepared.EffectiveModelParams.TimeoutSeconds != 90 { if prepared.EffectiveModelParams.TimeoutSeconds != 90 {
t.Fatalf("expected profile timeout to beat default, got %d", prepared.EffectiveModelParams.TimeoutSeconds) t.Fatalf("expected profile timeout to beat default, got %d", prepared.EffectiveModelParams.TimeoutSeconds)
} }
if prepared.EffectiveModelParams.ServiceTier != "priority" {
t.Fatalf("expected profile service_tier to beat default, got %q", prepared.EffectiveModelParams.ServiceTier)
}
} }
func TestRunnerPrepareFileBackedPromptBodiesRenderCorrectly(t *testing.T) { func TestRunnerPrepareFileBackedPromptBodiesRenderCorrectly(t *testing.T) {
@@ -478,6 +631,35 @@ func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) {
} }
} }
func TestRunnerPrepareJSONSchemaSchemaLoadFailureReturnsValidationError(t *testing.T) {
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
def.Validation.SchemaPath = "missing.schema.json"
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
validator,
)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if !errors.Is(err, ErrValidation) {
t.Fatalf("expected ErrValidation, got %v", err)
}
if validator.schemaLoads != 1 {
t.Fatalf("expected one schema load attempt, got %d", validator.schemaLoads)
}
if validator.schemaLoadPath != "missing.schema.json" {
t.Fatalf("expected schema path missing.schema.json, got %q", validator.schemaLoadPath)
}
}
func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) { func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) {
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0) def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
def.Validation.SchemaPath = "missing.schema.json" def.Validation.SchemaPath = "missing.schema.json"
@@ -536,6 +718,100 @@ func TestDeriveStructuredSchemaName(t *testing.T) {
} }
} }
func TestHashRenderedPromptIncludesCacheControlWhenPresent(t *testing.T) {
uncached := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{Role: "system", Content: "sys"},
{Role: "user", Content: "usr"},
}}
wantLegacyHash := hashString("system\nsys\n---\nuser\nusr\n---\n")
if got := hashRenderedPrompt(uncached); got != wantLegacyHash {
t.Fatalf("expected no-cache hash to preserve legacy input, got %q want %q", got, wantLegacyHash)
}
withCache := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "sys",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "usr"},
}}
alsoWithCache := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "sys",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "usr"},
}}
withoutTTL := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "sys",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
},
},
{Role: "user", Content: "usr"},
}}
cachedHash := hashRenderedPrompt(withCache)
if cachedHash == hashRenderedPrompt(uncached) {
t.Fatal("expected cache control to change rendered prompt hash")
}
if cachedHash != hashRenderedPrompt(alsoWithCache) {
t.Fatal("expected identical cache control metadata to produce stable hash")
}
if cachedHash == hashRenderedPrompt(withoutTTL) {
t.Fatal("expected ttl changes to affect rendered prompt hash")
}
}
func TestHashRenderedPromptIncludesSessionIDWhenPresent(t *testing.T) {
withoutSession := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{Role: "system", Content: "sys"},
{Role: "user", Content: "usr"},
}}
withSession := domain.RenderedPrompt{
SessionID: "session-123",
Messages: []domain.RenderedMessage{
{Role: "system", Content: "sys"},
{Role: "user", Content: "usr"},
},
}
alsoWithSession := domain.RenderedPrompt{
SessionID: "session-123",
Messages: []domain.RenderedMessage{
{Role: "system", Content: "sys"},
{Role: "user", Content: "usr"},
},
}
otherSession := domain.RenderedPrompt{
SessionID: "session-456",
Messages: []domain.RenderedMessage{
{Role: "system", Content: "sys"},
{Role: "user", Content: "usr"},
},
}
sessionHash := hashRenderedPrompt(withSession)
if sessionHash == hashRenderedPrompt(withoutSession) {
t.Fatal("expected session_id to change rendered prompt hash")
}
if sessionHash != hashRenderedPrompt(alsoWithSession) {
t.Fatal("expected identical session_id to produce stable hash")
}
if sessionHash == hashRenderedPrompt(otherSession) {
t.Fatal("expected session_id value changes to affect rendered prompt hash")
}
}
func TestRunnerRunSuccessful(t *testing.T) { func TestRunnerRunSuccessful(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)} promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}
@@ -543,7 +819,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")}, "a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")}, "a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
}} }}
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
@@ -555,7 +831,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"}, "glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
}, },
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90}, Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)},
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
@@ -590,6 +866,48 @@ func TestRunnerRunSuccessful(t *testing.T) {
if llmClient.lastReq.Target.TimeoutSeconds != 90 { if llmClient.lastReq.Target.TimeoutSeconds != 90 {
t.Fatalf("expected timeout propagation, got %d", llmClient.lastReq.Target.TimeoutSeconds) t.Fatalf("expected timeout propagation, got %d", llmClient.lastReq.Target.TimeoutSeconds)
} }
if !llmClient.lastReq.TargetPresence.Temperature || !llmClient.lastReq.TargetPresence.TimeoutSeconds {
t.Fatalf("expected numeric override presence to be sent to llm, got %+v", llmClient.lastReq.TargetPresence)
}
if llmClient.lastReq.Prompt.SessionID != "session-123" {
t.Fatalf("expected session id to be sent to llm, got %q", llmClient.lastReq.Prompt.SessionID)
}
}
func TestRunnerRunPassesExtraParamsToGenerateRequestTarget(t *testing.T) {
extraParams := map[string]any{
"string_value": "enabled",
"number_value": 42,
"boolean_value": true,
"object_value": map[string]any{"nested": "value"},
"array_value": []any{"first", 3, false},
}
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
ExtraParams: extraParams,
},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if !reflect.DeepEqual(res.EffectiveModelParams.ExtraParams, extraParams) {
t.Fatalf("expected run result extra_params to match profile values, got %#v", res.EffectiveModelParams.ExtraParams)
}
if !reflect.DeepEqual(llmClient.lastReq.Target.ExtraParams, extraParams) {
t.Fatalf("expected generate request extra_params to match profile values, got %#v", llmClient.lastReq.Target.ExtraParams)
}
} }
func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T) { func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T) {
@@ -608,7 +926,7 @@ func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T)
Inputs: map[string]domain.ArtifactRef{ Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
}, },
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90}, Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)},
} }
prepared, err := runner.Prepare(context.Background(), req) prepared, err := runner.Prepare(context.Background(), req)
@@ -726,6 +1044,7 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T)
MaxTokens: 500, MaxTokens: 500,
TopP: 0.9, TopP: 0.9,
TimeoutSeconds: 120, TimeoutSeconds: 120,
ServiceTier: "priority",
}, },
}} }}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
@@ -735,11 +1054,12 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T)
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
Inputs: singleInputRef(), Inputs: singleInputRef(),
Execution: &domain.ExecutionTarget{ Execution: &domain.ExecutionTargetOverride{
Endpoint: "http://override/v1", Endpoint: "http://override/v1",
Model: "override-model", Model: "override-model",
Temperature: 0.7, Temperature: float64Ptr(0.7),
TimeoutSeconds: 30, TimeoutSeconds: intPtr(30),
ServiceTier: "flex",
}, },
}) })
if err != nil { if err != nil {
@@ -754,6 +1074,9 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T)
if res.EffectiveModelParams.TopP != 0.9 { if res.EffectiveModelParams.TopP != 0.9 {
t.Fatalf("expected non-overridden profile top_p to remain, got %v", res.EffectiveModelParams.TopP) t.Fatalf("expected non-overridden profile top_p to remain, got %v", res.EffectiveModelParams.TopP)
} }
if res.EffectiveModelParams.ServiceTier != "flex" {
t.Fatalf("expected service_tier override to win, got %q", res.EffectiveModelParams.ServiceTier)
}
} }
func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) { func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) {
@@ -765,6 +1088,7 @@ func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) {
Model: "profile-model", Model: "profile-model",
TopP: 0.8, TopP: 0.8,
TimeoutSeconds: 90, TimeoutSeconds: 90,
ServiceTier: "priority",
}, },
}} }}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
@@ -784,6 +1108,9 @@ func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) {
if res.EffectiveModelParams.TimeoutSeconds != 90 { if res.EffectiveModelParams.TimeoutSeconds != 90 {
t.Fatalf("expected profile timeout to beat default, got %d", res.EffectiveModelParams.TimeoutSeconds) t.Fatalf("expected profile timeout to beat default, got %d", res.EffectiveModelParams.TimeoutSeconds)
} }
if res.EffectiveModelParams.ServiceTier != "priority" {
t.Fatalf("expected profile service_tier to beat default, got %q", res.EffectiveModelParams.ServiceTier)
}
} }
func TestRunnerRunBuiltInDefaultsUsedWhenProfileOmitsOptionalFields(t *testing.T) { func TestRunnerRunBuiltInDefaultsUsedWhenProfileOmitsOptionalFields(t *testing.T) {
@@ -844,6 +1171,9 @@ func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) {
if !errors.Is(err, ErrInvalidRequest) { if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err) t.Fatalf("expected ErrInvalidRequest, got %v", err)
} }
if !errors.Is(err, ErrAPIKeyEnvMissing) {
t.Fatalf("expected ErrAPIKeyEnvMissing, got %v", err)
}
if !strings.Contains(err.Error(), "SCRIPTORIUM_MISSING_KEY") { if !strings.Contains(err.Error(), "SCRIPTORIUM_MISSING_KEY") {
t.Fatalf("expected missing env name in error, got %v", err) t.Fatalf("expected missing env name in error, got %v", err)
} }
@@ -863,7 +1193,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
Inputs: singleInputRef(), Inputs: singleInputRef(),
Execution: &domain.ExecutionTarget{APIKeyEnv: envName}, Execution: &domain.ExecutionTargetOverride{APIKeyEnv: envName},
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
@@ -888,7 +1218,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) {
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
Inputs: singleInputRef(), Inputs: singleInputRef(),
Execution: &domain.ExecutionTarget{APIKeyEnv: runtimeEnv}, Execution: &domain.ExecutionTargetOverride{APIKeyEnv: runtimeEnv},
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
@@ -987,6 +1317,28 @@ func TestRunnerRunLLMFailure(t *testing.T) {
} }
} }
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{err: llm.ErrInvalidRequest},
nil,
)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
if errors.Is(err, ErrLLMGenerate) {
t.Fatalf("did not expect ErrLLMGenerate, got %v", err)
}
}
func TestRunnerRunValidationStillWorks(t *testing.T) { func TestRunnerRunValidationStillWorks(t *testing.T) {
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}} validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
runner := NewRunner( runner := NewRunner(
@@ -1029,7 +1381,7 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
Inputs: singleInputRef(), Inputs: singleInputRef(),
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "override-model", TimeoutSeconds: 22}, Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "override-model", TimeoutSeconds: intPtr(22)},
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
@@ -1104,6 +1456,186 @@ func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
} }
} }
func TestExecutionProfileToTargetPopulatesAllFieldsAndCopiesExtraParams(t *testing.T) {
src := &domain.ExecutionProfile{
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.2,
MaxTokens: 123,
TopP: 0.75,
TimeoutSeconds: 90,
ServiceTier: "priority",
ReasoningEffort: "medium",
APIKeyEnv: "SCRIPTORIUM_API_KEY",
ExtraParams: map[string]any{
"provider_option": "on",
},
}
target := executionProfileToTarget(src)
if target.Endpoint != src.Endpoint ||
target.Model != src.Model ||
target.Temperature != src.Temperature ||
target.MaxTokens != src.MaxTokens ||
target.TopP != src.TopP ||
target.TimeoutSeconds != src.TimeoutSeconds ||
target.ServiceTier != src.ServiceTier ||
target.ReasoningEffort != src.ReasoningEffort ||
target.APIKeyEnv != src.APIKeyEnv {
t.Fatalf("expected all profile fields to populate target, got %+v", target)
}
if !reflect.DeepEqual(target.ExtraParams, src.ExtraParams) {
t.Fatalf("expected extra_params to match, got %#v", target.ExtraParams)
}
src.ExtraParams["provider_option"] = "changed"
if target.ExtraParams["provider_option"] != "on" {
t.Fatalf("expected extra_params copy to be independent, got %#v", target.ExtraParams)
}
}
func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testing.T) {
profileValue := &domain.ExecutionProfile{
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.3,
MaxTokens: 222,
TopP: 0.6,
TimeoutSeconds: 77,
ServiceTier: "priority",
ReasoningEffort: "low",
APIKeyEnv: "PROFILE_KEY",
ExtraParams: map[string]any{
"profile_option": "enabled",
},
}
target, presence, err := resolveExecutionTarget(profileValue, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if presence != (domain.ExecutionTargetPresence{}) {
t.Fatalf("expected no request override presence, got %+v", presence)
}
if target.Endpoint != profileValue.Endpoint ||
target.Model != profileValue.Model ||
target.Temperature != profileValue.Temperature ||
target.MaxTokens != profileValue.MaxTokens ||
target.TopP != profileValue.TopP ||
target.TimeoutSeconds != profileValue.TimeoutSeconds ||
target.ServiceTier != profileValue.ServiceTier ||
target.ReasoningEffort != profileValue.ReasoningEffort ||
target.APIKeyEnv != profileValue.APIKeyEnv {
t.Fatalf("expected profile values to populate target, got %+v", target)
}
if !reflect.DeepEqual(target.ExtraParams, profileValue.ExtraParams) {
t.Fatalf("expected profile extra_params in target, got %#v", target.ExtraParams)
}
}
func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFields(t *testing.T) {
profileValue := &domain.ExecutionProfile{
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.2,
MaxTokens: 200,
TopP: 0.8,
TimeoutSeconds: 90,
ServiceTier: "priority",
ReasoningEffort: "medium",
APIKeyEnv: "PROFILE_KEY",
ExtraParams: map[string]any{
"profile_only": "yes",
},
}
override := &domain.ExecutionTargetOverride{
Endpoint: "http://override/v1",
Model: "override-model",
Temperature: float64Ptr(0.9),
MaxTokens: intPtr(111),
TopP: float64Ptr(0.5),
TimeoutSeconds: intPtr(30),
ServiceTier: "flex",
ReasoningEffort: "high",
APIKeyEnv: "RUNTIME_KEY",
ExtraParams: map[string]any{
"runtime_only": "yes",
},
}
target, presence, err := resolveExecutionTarget(profileValue, override)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if presence != (domain.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}) {
t.Fatalf("unexpected override presence: %+v", presence)
}
if target.Endpoint != override.Endpoint ||
target.Model != override.Model ||
target.Temperature != *override.Temperature ||
target.MaxTokens != *override.MaxTokens ||
target.TopP != *override.TopP ||
target.TimeoutSeconds != *override.TimeoutSeconds ||
target.ServiceTier != override.ServiceTier ||
target.ReasoningEffort != override.ReasoningEffort ||
target.APIKeyEnv != override.APIKeyEnv {
t.Fatalf("expected runtime overrides to win for all fields, got %+v", target)
}
if !reflect.DeepEqual(target.ExtraParams, override.ExtraParams) {
t.Fatalf("expected runtime extra_params to replace profile extra_params, got %#v", target.ExtraParams)
}
}
func TestMergeExecutionTargetEmptyStringOverridesDoNotErase(t *testing.T) {
base := domain.ExecutionTarget{
Endpoint: "http://base/v1",
Model: "base-model",
ServiceTier: "priority",
ReasoningEffort: "medium",
APIKeyEnv: "BASE_KEY",
}
override := domain.ExecutionTarget{
Endpoint: "http://override/v1",
Model: "override-model",
ServiceTier: " ",
ReasoningEffort: " ",
APIKeyEnv: "",
}
merged := mergeExecutionTarget(base, override)
if merged.Endpoint != "http://override/v1" || merged.Model != "override-model" {
t.Fatalf("expected endpoint/model to override, got %+v", merged)
}
if merged.ServiceTier != "priority" {
t.Fatalf("expected empty service_tier override to be ignored, got %q", merged.ServiceTier)
}
if merged.ReasoningEffort != "medium" {
t.Fatalf("expected empty reasoning_effort override to be ignored, got %q", merged.ReasoningEffort)
}
if merged.APIKeyEnv != "BASE_KEY" {
t.Fatalf("expected empty api_key_env override to be ignored, got %q", merged.APIKeyEnv)
}
}
func TestMergeExecutionTargetEmptyExtraParamsDoesNotErase(t *testing.T) {
base := domain.ExecutionTarget{
ExtraParams: map[string]any{
"keep": "value",
},
}
override := domain.ExecutionTarget{
ExtraParams: map[string]any{},
}
merged := mergeExecutionTarget(base, override)
if !reflect.DeepEqual(merged.ExtraParams, base.ExtraParams) {
t.Fatalf("expected empty extra_params override not to erase base values, got %#v", merged.ExtraParams)
}
}
func TestBuildOutputArtifactDefaults(t *testing.T) { func TestBuildOutputArtifactDefaults(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
@@ -1171,6 +1703,14 @@ func singleInputRef() map[string]domain.ArtifactRef {
return map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}} return map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}}
} }
func float64Ptr(v float64) *float64 {
return &v
}
func intPtr(v int) *int {
return &v
}
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner { func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
return NewRunner( return NewRunner(
promptRepo, promptRepo,

View File

@@ -116,6 +116,49 @@ func TestStandardValidatorJSONSchemaSuccess(t *testing.T) {
} }
} }
func TestStandardValidatorJSONSchemaNestedSchemaPathSuccess(t *testing.T) {
tmp := t.TempDir()
nestedDir := filepath.Join(tmp, "dnd")
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(nestedDir, "schema.json"), []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"required": ["name"],
"properties": {
"name": {"type": "string"}
}
}`), 0644); err != nil {
t.Fatal(err)
}
v := NewStandardValidator(tmp)
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: filepath.Join("dnd", "schema.json"),
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.Status != domain.ValidationPassed || !res.IsValid {
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
}
}
func TestStandardValidatorJSONSchemaNestedSchemaPathMissing(t *testing.T) {
v := NewStandardValidator(t.TempDir())
_, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: filepath.Join("dnd", "missing.json"),
})
if err == nil {
t.Fatal("expected nested schema load error")
}
}
func TestStandardValidatorJSONSchemaFailure(t *testing.T) { func TestStandardValidatorJSONSchemaFailure(t *testing.T) {
tmp := t.TempDir() tmp := t.TempDir()
schemaPath := filepath.Join(tmp, "schema.json") schemaPath := filepath.Join(tmp, "schema.json")