25 Commits

Author SHA1 Message Date
23872dd742 Implement runtime parameter completion fixes
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-07-04 09:00:19 -05:00
7ffbf5f6ca Update woodpecker config to prepare only linux binaries 2026-07-04 08:53:30 -05:00
d0dc30fcc9 Document runtime provider parameters 2026-07-04 13:29:38 +00:00
b38f7b4dc3 Serialize runtime extra parameters outbound 2026-07-04 13:26:53 +00:00
0512995931 Allow JSON-compatible extra params 2026-07-04 13:24:27 +00:00
049a5feadb Make request execution overrides presence-aware 2026-07-04 13:20:39 +00:00
1798e9c575 Add a roadmap and implementation plan to support reasoning_effort and extra_params in outbound requests 2026-07-04 08:13:10 -05:00
5d4bc8c2b9 Remove completed feature roadmap docs 2026-07-02 20:12:10 -05:00
63fb8fc132 Implement support for OpenRouter sticky routing via a session_id variable 2026-07-02 20:08:44 -05:00
4d4bb7a121 Document prompt cache control behavior 2026-07-02 23:10:24 +00:00
5dcb3cd4fc Expose cache usage in adapters 2026-07-02 23:07:57 +00:00
efe346893c Serialize cache-controlled chat messages 2026-07-02 23:05:56 +00:00
c95d6fcfec Preserve cache control in rendered prompts 2026-07-02 23:03:50 +00:00
0badb4364d Add prompt cache control loading 2026-07-02 23:00:43 +00:00
1f63f8afbb Add a feature roadmap and implementation plan for cache_control values 2026-07-02 17:55:56 -05:00
bc099a31ad Finish cleanup roadmap follow-through
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-26 11:05:08 -05:00
4ff55221a3 Update internal docs for stable runner error reasons and serialized model fields 2026-05-26 15:00:06 +00:00
8d8024099f Complete final cleanup verification and align unsupported artifact test naming 2026-05-26 13:23:45 +00:00
18792fd8d1 Remove unsupported S3 artifact reference placeholder 2026-05-26 13:21:52 +00:00
8860aa033c Reduce CLI test fixture duplication with local setup helpers 2026-05-26 13:20:07 +00:00
3ca14d8b6e Add regression test for JSON schema load failure during prepare 2026-05-26 13:16:26 +00:00
099e9c4a3e Use stable usecase sentinels for HTTP invalid-request mapping 2026-05-26 13:14:57 +00:00
cfe6b9408a Refactor CLI command wiring with shared settings and runner helpers 2026-05-26 13:12:34 +00:00
6ececc749f Centralize YAML catalog scanning for prompt and profile repositories 2026-05-26 13:10:09 +00:00
79901fbb86 Refine execution target mapping helpers and coverage across usecase, HTTP, and LLM 2026-05-26 13:07:35 +00:00
41 changed files with 3721 additions and 1681 deletions

View File

@@ -28,10 +28,6 @@ steps:
build_binary linux amd64 "" build_binary linux amd64 ""
build_binary linux arm64 "" build_binary linux arm64 ""
build_binary darwin amd64 ""
build_binary darwin arm64 ""
build_binary windows amd64 ".exe"
build_binary windows arm64 ".exe"
- name: publish-release - name: publish-release
image: woodpeckerci/plugin-release image: woodpeckerci/plugin-release

View File

@@ -32,6 +32,8 @@ Integration references:
- an effective `prompt_dir` and `profile_dir` (from flags or config) - an effective `prompt_dir` and `profile_dir` (from flags or config)
- `serve` requires an effective `prompt_dir` and `profile_dir` (from flags or config). - `serve` requires an effective `prompt_dir` and `profile_dir` (from flags or config).
- Positional arguments are rejected. - Positional arguments are rejected.
- Prompt cache control is configured in prompt YAML (`messages[].cache_control`), not with CLI flags.
- Provider-specific `reasoning_effort` and `extra_params` are configured in profile YAML or HTTP model overrides, not with CLI flags.
## Flag Reference ## Flag Reference
@@ -56,6 +58,11 @@ Integration references:
- `--top-p <float>`: runtime top-p override. - `--top-p <float>`: runtime top-p override.
- `--timeout <duration>`: runtime timeout override (Go duration syntax, for example `30s`, `2m`). - `--timeout <duration>`: runtime timeout override (Go duration syntax, for example `30s`, `2m`).
Numeric runtime override flags are presence-aware:
- omitted numeric flags preserve the selected profile/default value
- explicit zero values override the selected profile/default value (`--temperature 0`, `--max-tokens 0`, `--top-p 0`, `--timeout 0s`)
### `scriptorium render` ### `scriptorium render`
- Supports the same flags as `run`, except: - Supports the same flags as `run`, except:
@@ -82,6 +89,7 @@ Notes:
- `--input name=path` maps prompt input names to local file paths. - `--input name=path` maps prompt input names to local file paths.
- `--var name=value` maps template variable names to values. - `--var name=value` maps template variable names to values.
- If a prompt defines `session_id: "{{ .session_id }}"`, pass the OpenRouter sticky-routing value with `--var session_id=<value>`.
- Both flags can be repeated. - Both flags can be repeated.
- Both flags also support comma-separated batches, for example: - Both flags also support comma-separated batches, for example:
- `--input transcript=./t.md,glossary=./g.yml` - `--input transcript=./t.md,glossary=./g.yml`
@@ -93,6 +101,7 @@ Notes:
- Writes generated artifact content to stdout by default. - Writes generated artifact content to stdout by default.
- Writes generated artifact content to `--out` when provided. - Writes generated artifact content to `--out` when provided.
- Prints run summary metadata to stderr on success. - Prints run summary metadata to stderr on success.
- Appends `cached_tokens=<n> cache_write_tokens=<n>` to the summary only when the provider reports non-zero cache usage.
- Prints errors to stderr on failure. - Prints errors to stderr on failure.
`render`: `render`:

View File

@@ -104,6 +104,7 @@ Field reference:
- `version` (required): prompt version. - `version` (required): prompt version.
- `default_profile` (optional): profile ID used when request does not provide `profile_id`. - `default_profile` (optional): profile ID used when request does not provide `profile_id`.
- `description` (optional): prompt description. - `description` (optional): prompt description.
- `session_id` (optional): Go-template string for OpenRouter sticky-routing `session_id`; rendered from request vars.
- `inputs` (optional list): expected named inputs. - `inputs` (optional list): expected named inputs.
- `messages` (required list): prompt message templates. - `messages` (required list): prompt message templates.
- `output` (required object): output contract. - `output` (required object): output contract.
@@ -119,6 +120,7 @@ Field reference:
- `role` (required) - `role` (required)
- `content` or `content_file` (exactly one is required) - `content` or `content_file` (exactly one is required)
- `cache_control` (optional object): provider prompt-cache metadata for this message
Message rules: Message rules:
@@ -128,6 +130,35 @@ Message rules:
- Prompt decoding is strict; unknown YAML fields are rejected. - Prompt decoding is strict; unknown YAML fields are rejected.
- Duplicate prompt IDs are invalid. If multiple files declare the requested prompt ID, Scriptorium fails instead of choosing one. - Duplicate prompt IDs are invalid. If multiple files declare the requested prompt ID, Scriptorium fails instead of choosing one.
`messages[].cache_control` fields:
- `type` (required when `cache_control` is present): currently only `ephemeral`.
- `ttl` (optional): currently only `1h`; omitted from outbound requests when unset.
Example cache-controlled message:
```yaml
messages:
- role: system
content_file: ./stable_context.md
cache_control:
type: ephemeral
ttl: 1h
- role: user
content: |
{{input "transcript"}}
```
Use cache control on stable reusable prompt content. Dynamic per-run inputs before the cache-controlled message change the provider cache key.
Example prompt-level session ID:
```yaml
session_id: "{{ .session_id }}"
```
When configured, `session_id` is rendered with the same variable context as messages. The rendered value is trimmed, omitted when empty, and rejected if longer than 256 characters. CLI callers pass the value through `--var session_id=<value>`; HTTP callers pass it through `"vars": {"session_id": "<value>"}`.
`output` fields: `output` fields:
- `format` (required): `text`, `markdown`, or `json`. - `format` (required): `text`, `markdown`, or `json`.
@@ -158,6 +189,11 @@ top_p: 1.0
timeout_seconds: 90 timeout_seconds: 90
api_key_env: SCRIPTORIUM_API_KEY api_key_env: SCRIPTORIUM_API_KEY
service_tier: priority service_tier: priority
reasoning_effort: medium
extra_params:
provider_route: primary
provider_options:
retry_budget: 2
``` ```
Field reference: Field reference:
@@ -170,9 +206,9 @@ Field reference:
- `top_p` (optional): range `0..1` - `top_p` (optional): range `0..1`
- `timeout_seconds` (optional): `>= 0` - `timeout_seconds` (optional): `>= 0`
- `service_tier` (optional): provider-specific request tier such as OpenRouter `flex` or `priority` - `service_tier` (optional): provider-specific request tier such as OpenRouter `flex` or `priority`
- `reasoning_effort` (optional) - `reasoning_effort` (optional): serialized as top-level `reasoning_effort` in outbound chat-completions requests
- `api_key_env` (optional) - `api_key_env` (optional)
- `extra_params` (optional map of strings) - `extra_params` (optional map): JSON-compatible provider-specific parameters. Values may be strings, numbers, booleans, objects, or arrays.
Profile rules: Profile rules:
@@ -180,11 +216,14 @@ Profile rules:
- Raw `api_key` is rejected; use `api_key_env`. - Raw `api_key` is rejected; use `api_key_env`.
- If `api_key_env` is set, that environment variable must be set when preparing/running. - If `api_key_env` is set, that environment variable must be set when preparing/running.
- Duplicate profile IDs are invalid. If multiple files declare the requested profile ID, Scriptorium fails instead of choosing one. - Duplicate profile IDs are invalid. If multiple files declare the requested profile ID, Scriptorium fails instead of choosing one.
- `extra_params` keys must not be empty and must not collide with reserved outbound request fields: `model`, `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, or `response_format`.
Current outbound request behavior: Current outbound request behavior:
- The OpenAI-compatible client currently serializes: `model`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, and optional `response_format` for `json_schema` prompts. - The OpenAI-compatible client currently serializes: `model`, optional `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, optional `response_format` for `json_schema` prompts, and `extra_params`.
- `reasoning_effort` and `extra_params` are parsed and carried in effective settings, but are not currently serialized into outbound chat-completions requests. - `extra_params` are flattened into provider-specific top-level JSON request fields. They are not wrapped in an `extra_params` object on the outbound provider request.
- Messages without `cache_control` serialize with string `content`.
- Messages with `cache_control` serialize as a single text content-block array containing `cache_control`.
## Schema Behavior ## Schema Behavior

View File

@@ -50,7 +50,10 @@ Copyable request example file:
"reasoning_effort": "medium", "reasoning_effort": "medium",
"api_key_env": "SCRIPTORIUM_API_KEY", "api_key_env": "SCRIPTORIUM_API_KEY",
"extra_params": { "extra_params": {
"route": "primary" "route": "primary",
"provider_options": {
"retry_budget": 2
}
} }
}, },
"include_raw_output": false "include_raw_output": false
@@ -67,6 +70,14 @@ Input reference types currently supported by runtime artifact loading:
- `file` - `file`
- `inline` - `inline`
Model override notes:
- Numeric model override fields distinguish omitted values from explicit zero values. For example, omitting `temperature` preserves the selected profile/default value, while `"temperature": 0` explicitly sets the effective temperature to zero.
- `extra_params` accepts JSON-compatible values: strings, numbers, booleans, objects, and arrays.
- `extra_params` are passed through effective model metadata and flattened into top-level provider request fields by the OpenAI-compatible client.
- `extra_params` keys must not be empty and must not collide with reserved outbound fields: `model`, `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, or `response_format`.
- Raw API-key values are not accepted. Use `api_key_env` to name an environment variable.
## Strict JSON Rules ## Strict JSON Rules
Request decoding uses strict JSON field checks: Request decoding uses strict JSON field checks:
@@ -119,7 +130,10 @@ Response shape:
"reasoning_effort": "medium", "reasoning_effort": "medium",
"api_key_env": "SCRIPTORIUM_API_KEY", "api_key_env": "SCRIPTORIUM_API_KEY",
"extra_params": { "extra_params": {
"route": "primary" "route": "primary",
"provider_options": {
"retry_budget": 2
}
} }
}, },
"input_hashes": { "input_hashes": {
@@ -128,7 +142,9 @@ Response shape:
"usage": { "usage": {
"prompt_tokens": 11, "prompt_tokens": 11,
"completion_tokens": 22, "completion_tokens": 22,
"total_tokens": 33 "total_tokens": 33,
"cached_tokens": 0,
"cache_write_tokens": 0
}, },
"start_time": "2026-05-04T12:00:00Z", "start_time": "2026-05-04T12:00:00Z",
"end_time": "2026-05-04T12:00:01Z", "end_time": "2026-05-04T12:00:01Z",
@@ -142,6 +158,8 @@ Response shape:
`raw_model_output` is omitted by default. `raw_model_output` is omitted by default.
`metadata.usage.cached_tokens` and `metadata.usage.cache_write_tokens` are always present as numbers. They are `0` when the provider omits compatible cache usage fields or reports no cache activity.
To include it, send: To include it, send:
- `"include_raw_output": true` - `"include_raw_output": true`

View File

@@ -26,15 +26,85 @@ Example:
Serialized JSON fields: Serialized JSON fields:
- `model` (required after fallback resolution) - `model` (required after fallback resolution)
- `messages` (role/content pairs from rendered prompt) - `session_id` (only when the rendered prompt includes a non-empty session ID)
- `temperature` (only when non-zero) - `messages` (rendered prompt messages)
- `max_tokens` (only when non-zero) - `temperature` (when non-zero, or when explicitly overridden to zero)
- `top_p` (only when non-zero) - `max_tokens` (when non-zero, or when explicitly overridden to zero)
- `top_p` (when non-zero, or when explicitly overridden to zero)
- `service_tier` (only when non-empty) - `service_tier` (only when non-empty)
- `reasoning_effort` (only when non-empty)
- `response_format` (only when structured output is provided) - `response_format` (only when structured output is provided)
- profile/request `extra_params` as additional provider-specific top-level fields
`service_tier` is provider-specific. OpenRouter currently documents request values such as `flex` and `priority`; Scriptorium forwards any non-empty configured value and lets the backend validate support. `service_tier` is provider-specific. OpenRouter currently documents request values such as `flex` and `priority`; Scriptorium forwards any non-empty configured value and lets the backend validate support.
`reasoning_effort` is provider-specific. Scriptorium forwards any non-empty configured value as top-level `reasoning_effort` and lets the backend validate support.
`extra_params` are flattened into the outbound JSON object. They are not wrapped in an `extra_params` object:
```json
{
"model": "gpt-4o-mini",
"messages": [
{
"role": "user",
"content": "rendered text"
}
],
"provider_route": "primary",
"provider_options": {
"retry_budget": 2
}
}
```
`extra_params` values must be JSON-compatible. Supported value shapes include strings, numbers, booleans, objects, and arrays.
Reserved `extra_params` keys are rejected before the HTTP request is made:
- `model`
- `session_id`
- `messages`
- `temperature`
- `max_tokens`
- `top_p`
- `service_tier`
- `reasoning_effort`
- `response_format`
Empty `extra_params` keys and values that cannot be encoded as JSON are also rejected before the HTTP request is made.
`session_id` is rendered from prompt YAML using request variables and serialized as a top-level JSON request field. Scriptorium does not send an `x-session-id` header. Empty rendered session IDs are omitted, and values longer than 256 characters are rejected before the HTTP request.
Messages without prompt cache control serialize with string `content`:
```json
{
"role": "system",
"content": "rendered text"
}
```
Messages with prompt cache control serialize as a single text content-block array:
```json
{
"role": "system",
"content": [
{
"type": "text",
"text": "rendered text",
"cache_control": {
"type": "ephemeral",
"ttl": "1h"
}
}
]
}
```
When cache-control `ttl` is unset in the prompt definition, `ttl` is omitted from the outbound payload.
Structured output is currently `json_schema` only, serialized as: Structured output is currently `json_schema` only, serialized as:
```json ```json
@@ -72,6 +142,7 @@ Base timeout comes from client configuration.
Per-request override: Per-request override:
- if `Target.TimeoutSeconds > 0`, use that value for request timeout - if `Target.TimeoutSeconds > 0`, use that value for request timeout
- if `Target.TimeoutSeconds == 0` and the value came from an explicit request override, disable the HTTP client timeout
- if `Target.TimeoutSeconds < 0`, request is rejected (`ErrInvalidRequest`) - if `Target.TimeoutSeconds < 0`, request is rejected (`ErrInvalidRequest`)
## Response Expectations ## Response Expectations
@@ -82,6 +153,13 @@ Expected successful response shape (subset used):
- `usage.prompt_tokens` - `usage.prompt_tokens`
- `usage.completion_tokens` - `usage.completion_tokens`
- `usage.total_tokens` - `usage.total_tokens`
- `usage.prompt_tokens_details.cached_tokens` (optional)
- `usage.cache_write_tokens` (optional)
Absent cache usage fields are treated as zero. Parsed cache usage is exposed through run results and adapter response surfaces as:
- `cached_tokens`
- `cache_write_tokens`
Malformed response conditions include: Malformed response conditions include:
@@ -99,10 +177,7 @@ Malformed responses return `ErrMalformedResponse`.
## Unsupported Or Non-Serialized Fields ## Unsupported Or Non-Serialized Fields
The following fields may exist in profile/effective settings but are not currently serialized into outbound chat-completions payloads: The client does not serialize top-level `cache_control`.
- `reasoning_effort`
- `extra_params`
No built-in retries, tool-calls, or multi-request payload modes are implemented in this client. No built-in retries, tool-calls, or multi-request payload modes are implemented in this client.

View File

@@ -22,11 +22,13 @@ CLI adapter:
- Input: process args, filesystem config/assets, environment. - Input: process args, filesystem config/assets, environment.
- Output: exit code, stdout artifact/prepared output, stderr summaries/errors. - Output: exit code, stdout artifact/prepared output, stderr summaries/errors.
- `run` summaries include cache usage counters only when either parsed cache counter is non-zero.
HTTP adapter: HTTP adapter:
- Input: JSON request body (`runRequestDTO`). - Input: JSON request body (`runRequestDTO`).
- Output: JSON success/error body with mapped status codes. - Output: JSON success/error body with mapped status codes.
- Success metadata includes token usage plus cache usage counters.
Filesystem repositories: Filesystem repositories:
@@ -67,6 +69,8 @@ Primary app settings consumed by adapters:
Execution profile/request settings used through runner: Execution profile/request settings used through runner:
- `endpoint`, `model`, `temperature`, `max_tokens`, `top_p`, `timeout_seconds`, `service_tier`, `api_key_env`, `reasoning_effort`, `extra_params` - `endpoint`, `model`, `temperature`, `max_tokens`, `top_p`, `timeout_seconds`, `service_tier`, `api_key_env`, `reasoning_effort`, `extra_params`
- CLI and HTTP request adapters preserve caller intent for numeric runtime overrides. Omitted values remain absent; explicit zero values are mapped as explicit overrides.
- HTTP `extra_params` accepts JSON-compatible values and maps them to domain request overrides without provider-specific adapter logic.
## External Dependencies ## External Dependencies
@@ -93,6 +97,13 @@ Artifact refs:
LLM adapter: LLM adapter:
- endpoint appends `/chat/completions`. - endpoint appends `/chat/completions`.
- rendered messages without cache control serialize with string `content`.
- rendered messages with cache control serialize as one text content block with `cache_control`.
- non-empty `reasoning_effort` serializes as a top-level provider request field.
- `extra_params` flatten into provider-specific top-level JSON request fields.
- reserved `extra_params` keys are rejected before the provider call: `model`, `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, and `response_format`.
- empty `extra_params` keys and values that cannot be JSON-encoded are rejected before the provider call.
- compatible cache usage response fields are parsed into domain token usage.
- non-2xx responses map to request failure errors. - non-2xx responses map to request failure errors.
- malformed responses (including missing/empty first choice content) are errors. - malformed responses (including missing/empty first choice content) are errors.
@@ -105,6 +116,7 @@ Validator:
HTTP error mapping: HTTP error mapping:
- maps domain/use-case errors to stable HTTP code + error code/message. - maps domain/use-case errors to stable HTTP code + error code/message.
- distinguishes missing profile selection and missing `api_key_env` variable using stable use-case sentinel errors.
- avoids returning internal wrapped-cause details in response payload. - avoids returning internal wrapped-cause details in response payload.
## CLI Adapter Semantics ## CLI Adapter Semantics
@@ -140,4 +152,5 @@ Behavior highlights:
- Adapter packages do not own runner decision logic. - Adapter packages do not own runner decision logic.
- External request/response strictness is part of contract stability. - External request/response strictness is part of contract stability.
- Prepared-render output never includes resolved API key values. - Prepared-render output never includes resolved API key values.
- Outbound OpenAI-compatible request includes only currently serialized fields (`model`, `messages`, optional `temperature`, `max_tokens`, `top_p`, optional `response_format`). - Outbound OpenAI-compatible request includes currently serialized first-class fields (`model`, optional `session_id`, `messages`, optional `temperature`, `max_tokens`, `top_p`, optional `service_tier`, optional `reasoning_effort`, optional `response_format`) plus validated `extra_params` flattened as provider-specific top-level fields.
- Outbound cache control is message-level only; no top-level cache-control field is serialized.

View File

@@ -66,14 +66,21 @@ It receives fully constructed repositories/readers/validators from adapters. Eff
## Failure Behavior ## Failure Behavior
Key error classes surfaced from `Runner`: Primary runner error classes:
- `ErrInvalidRequest`: invalid prompt/profile/request/runtime/API-key-env prerequisites. - `ErrInvalidRequest`: invalid run request envelope.
- `ErrProfileLoad`: prompt or profile load failures. - `ErrProfileRequired`: specific invalid-request reason when neither request `profile_id` nor prompt `default_profile` is available.
- `ErrAPIKeyEnvMissing`: specific invalid-request reason when `api_key_env` is set but the named environment variable is unset/empty.
- `ErrProfileLoad`: prompt/profile repository load failures.
- `ErrArtifactLoad`: artifact read failures. - `ErrArtifactLoad`: artifact read failures.
- `ErrPromptRender`: template render failures. - `ErrPromptRender`: template render failures.
- `ErrLLMGenerate`: model request failures. - `ErrLLMGenerate`: outbound model request failures.
- `ErrValidation`: validation runtime failures (including schema load/compile failures). - `ErrValidation`: validation runtime failures (including structured-output schema load/compile failures).
Reason sentinel behavior:
- `ErrProfileRequired` and `ErrAPIKeyEnvMissing` are wrapped with `ErrInvalidRequest`.
- Adapters can use `errors.Is` for stable reason mapping without matching runner prose.
Validation content failures are not run errors: Validation content failures are not run errors:
@@ -90,20 +97,33 @@ Validation content failures are not run errors:
3. select profile ID: 3. select profile ID:
- explicit request profile ID - explicit request profile ID
- prompt `default_profile` - prompt `default_profile`
- otherwise request error - otherwise return an invalid request with `ErrProfileRequired`
4. load execution profile. 4. load execution profile.
5. merge effective runtime target: 5. merge effective runtime target:
- built-in execution defaults - built-in execution defaults
- selected profile values - selected profile values
- request overrides - request overrides
6. verify required `api_key_env` environment variable (name only; value is not returned). - request numeric overrides are presence-aware, so omitted values preserve the current effective value and explicit zero values override it
6. verify required `api_key_env` environment variable:
- missing/empty env value returns an invalid request with `ErrAPIKeyEnvMissing`
- only the environment-variable name is retained; secret value is never returned
7. resolve output contract and structured-output schema payload when `json_schema` mode is active. 7. resolve output contract and structured-output schema payload when `json_schema` mode is active.
8. read input artifacts. 8. read input artifacts.
9. render prompt messages. 9. render prompt messages, including any normalized message cache-control metadata.
10. compute prompt/input/render hashes and return `PreparedRun`. 10. compute prompt/input/render hashes and return `PreparedRun`.
`rendered_prompt_hash` includes cache-control metadata when present because it affects the outbound provider request. Prompts without cache control keep the role/content hash behavior.
`Prepare` does not call the LLM. `Prepare` does not call the LLM.
Runtime target notes:
- Profile `extra_params` and request `extra_params` carry JSON-compatible values through prepared output, run metadata, and `domain.GenerateRequest.Target`.
- The OpenAI-compatible client serializes non-empty `reasoning_effort` as a top-level provider request field.
- The OpenAI-compatible client flattens `extra_params` into provider-specific top-level JSON request fields.
- Empty `extra_params` keys, reserved outbound field names, and values that cannot be JSON-encoded fail before the provider request.
- Resolved API-key values are never stored in `PreparedRun`, `RunResult`, logs, or HTTP responses.
## Run Flow ## Run Flow
`Run` performs: `Run` performs:
@@ -114,7 +134,7 @@ Validation content failures are not run errors:
4. build output artifact content type from output format. 4. build output artifact content type from output format.
5. validate output. 5. validate output.
6. optionally attempt bounded repair when repairer is injected and contract allows it. 6. optionally attempt bounded repair when repairer is injected and contract allows it.
7. return `RunResult` with artifact, raw output, validation, hashes, profile/model metadata, usage, and timestamps. 7. return `RunResult` with artifact, raw output, validation, hashes, profile/model metadata, token/cache usage, and timestamps.
## Repair Hook Boundary ## Repair Hook Boundary

View File

@@ -1,519 +0,0 @@
# Code Quality And Deduplication Audit
## Executive Summary
Overall code quality is solid for the current project size. The repository follows the documented shape: thin CLI/HTTP adapters, a central `Runner`, strict config/prompt/profile loading, narrow filesystem and LLM adapters, and package tests around the important public behaviors.
The top three refactoring targets are:
1. centralize execution-target field mapping across profiles, CLI overrides, HTTP DTOs, prepared output, and LLM request serialization;
2. share filesystem repository scanning helpers between prompt and profile repositories;
3. reduce CLI command wiring duplication for app settings, runner construction, and common preflight behavior.
The codebase appears ready for a limited cleanup pass. No major architectural rewrite is warranted. The main risk before a release is maintenance drift when adding fields or policy to execution settings, repositories, or command setup.
## Repository Map Reviewed
Main areas inspected:
- `cmd/scriptorium`: process entrypoint path, via repository layout and command docs.
- `internal/adapter/cli`: `run`, `render`, `serve` parsing, config loading, app wiring, stdout/stderr behavior, summary output, exit-code policy.
- `internal/adapter/http`: request DTOs, strict JSON decoding, domain mapping, response metadata mapping, error mapping.
- `internal/config`: app config loading, defaults, strict YAML decoding, CLI override precedence.
- `internal/defaults`: built-in runtime/config/content-type defaults.
- `internal/domain`: domain request/result/profile/target/artifact/validation types.
- `internal/usecase`: `Runner.Prepare`, `Runner.Run`, execution target merge, schema structured-output setup, validation and repair policy.
- `internal/promptdef`: recursive prompt YAML loading, prompt normalization, content-file resolution, prompt output-contract validation.
- `internal/profile`: recursive profile YAML loading, strict decoding, duplicate ID handling, profile validation, raw API-key rejection.
- `internal/artifact`: `inline` and `file` artifact resolution.
- `internal/prompt`: Go template rendering and required input checks.
- `internal/llm`: OpenAI-compatible request construction, timeout/API-key handling, response parsing.
- `internal/validate`: basic/JSON/JSON Schema validation and schema document loading.
- `internal/format`: prepared-run text/JSON output.
- `examples/`: maintained example config, prompts, profiles, schemas, fixtures, HTTP/render examples.
- Package tests under `internal/**`.
Major execution paths reviewed:
- CLI `run`: parse flags/config, build `RunRequest`, construct runner, call LLM, write artifact, print summary, choose exit code.
- CLI `render`: parse flags/config, build `RunRequest`, construct runner without LLM, prepare only, format prepared run.
- CLI `serve`: parse flags/config, construct runner, expose HTTP `POST /v1/runs`.
- HTTP `POST /v1/runs`: strict decode DTO, map to `RunRequest`, map `RunResult` to JSON response.
- Runner `Prepare` and `Run`: prompt/profile/artifact/schema resolution, render, LLM generate, validation, repair hook.
Areas not deeply inspected:
- No `internal/app`, `internal/stage`, `internal/modules`, `internal/storage`, `internal/manifest`, `pkg`, or external test-suite directories exist in the current repository.
- Full test execution was not run because this is a report-only pass; inspection was static.
## High-Confidence Deduplication Opportunities
### 1. Execution Target Field Mapping Is Repeated Across Boundaries
Affected files/packages:
- `internal/domain/domain.go`
- `internal/usecase/runner.go`
- `internal/adapter/cli/run.go`
- `internal/adapter/http/dto.go`
- `internal/adapter/http/handler.go`
- `internal/llm/openai_compatible_client.go`
- `internal/format/prepared_run.go`
- tests in `internal/usecase`, `internal/adapter/http`, `internal/adapter/cli`, `internal/llm`, and `internal/format`
Duplicated or near-duplicated behavior:
- Execution settings fields are listed and copied in several places: profile-to-target conversion, target merge, CLI runtime override construction, HTTP request mapping, HTTP metadata mapping, prepared-run text output, and OpenAI-compatible request construction.
- Adding `service_tier` required coordinated edits across many of these sites, which is a strong signal that field-level mapping is too scattered.
Why it matters:
- A future profile/runtime field can easily be parsed but not sent, displayed but not merged, accepted over HTTP but not included in metadata, or tested in one command but not another.
- This is public-interface drift risk because CLI, HTTP, render output, and provider requests all expose different slices of the same effective execution target.
Recommended refactor:
- Keep transport DTOs local to adapters, but add small mapping helpers at each boundary:
- a use-case/domain helper for profile-to-target copy and execution-target merge;
- an HTTP helper such as `executionTargetFromModelOverrideDTO` and `modelParamsDTOFromExecutionTarget`;
- an LLM helper such as `openAIChatRequestFromGenerateRequest` for outbound serialization.
- Add a table-driven test that constructs an `ExecutionTarget` with every supported field and verifies the HTTP metadata DTO and LLM request payload retain the intended fields.
- Avoid reflection-based generic copying; explicit mapping is still clearer here.
Suggested tests:
- Expand runner merge tests to use an all-fields target.
- Add HTTP DTO mapping tests that fail when a domain execution field is omitted from request or metadata mapping.
- Add LLM request serialization tests for all serialized fields and explicit omission of non-serialized fields.
Risk level: high. The refactor itself is low-to-medium implementation risk, but the duplicated behavior has high drift risk.
### 2. Prompt And Profile Filesystem Repository Scanning Is Duplicated
Affected files/packages:
- `internal/promptdef/filesystem_repository.go`
- `internal/profile/filesystem_repository.go`
- repository tests in `internal/promptdef` and `internal/profile`
Duplicated or near-duplicated behavior:
- Both repositories recursively walk configured directories, filter `.yaml`/`.yml`, sort paths, compute relative paths, derive likely IDs from filenames, use strict YAML decoding, partially decode IDs to decide whether to surface malformed likely-target files, detect duplicate IDs, and include relative paths in errors.
Why it matters:
- Prompt/profile repository policy should remain aligned: nested scanning, stable ordering, duplicate ID errors, likely-target malformed file behavior, and relative-path diagnostics.
- Future changes to repository scanning or extension rules would need to be made in two packages.
Recommended refactor:
- Introduce a narrow internal helper for filesystem catalog behavior, for example `internal/filecatalog` or another small package whose scope is only:
- recursive YAML file discovery with context cancellation;
- stable sorting;
- relative path formatting;
- YAML extension checks;
- filename stem extraction.
- Keep prompt/profile-specific normalization and validation in their current packages.
- Do not make a generic repository framework.
Suggested tests:
- Add helper-level tests for recursive YAML discovery, ordering, extension filtering, and relative path output.
- Keep existing prompt/profile behavior tests unchanged to confirm public error behavior survives.
Risk level: high for drift prevention; low implementation risk if the helper stays small.
### 3. CLI Command Setup And Runner Wiring Are Repeated
Affected files/packages:
- `internal/adapter/cli/run.go`
- `internal/adapter/cli/run_test.go`
Duplicated or near-duplicated behavior:
- `run`, `render`, and `serve` each construct similar runner dependencies.
- `run` and `serve` both create an OpenAI-compatible client with default timeout.
- `run`, `render`, and `serve` all resolve app settings, clean dirs, validate required prompt/profile dirs, and create filesystem repositories/validator/renderer/readers.
- `run` and `render` share most execution request flags and request construction, with command-specific differences around `--schema-dir`, `--format`, and LLM use.
Why it matters:
- Adding or changing a dependency, default, or preflight check can drift between commands.
- The current duplication is still readable, but it is large enough that future command additions or option changes will be error-prone.
Recommended refactor:
- Add a small CLI-local wiring helper, such as `newRunnerFromDirs(promptDir, profileDir, schemaDir string, llmClient llm.Client) *usecase.Runner`.
- Add a small CLI-local resolved settings struct for common `prompt_dir`, `profile_dir`, `schema_dir`, and render-format handling.
- Preserve the existing command-specific parse functions and flag surfaces; do not hide command behavior behind a generic command framework.
Suggested tests:
- Keep parser tests for each command.
- Add one test that verifies `run`, `render`, and `serve` use the same configured prompt/profile/schema dirs by exercising config-derived dirs.
- Keep command-level success tests for `run` and `render`.
Risk level: medium. Behavior is public, but a small CLI-local helper can be behavior-preserving.
### 4. HTTP Error Mapping Depends On Error Message Substrings
Affected files/packages:
- `internal/usecase/runner.go`
- `internal/adapter/http/handler.go`
- `internal/adapter/http/handler_test.go`
Duplicated or near-duplicated behavior:
- Runner creates `ErrInvalidRequest` with human-readable details for cases such as missing profile selection and missing API-key env.
- HTTP maps some specific invalid-request cases by checking `strings.Contains(err.Error(), ...)`.
Why it matters:
- User-facing HTTP error codes can drift if a runner error message is clarified.
- This crosses package boundaries in a brittle way: HTTP should depend on stable error identity, not exact prose from `internal/usecase`.
Recommended refactor:
- Add narrow sentinel errors or typed invalid-request reasons in `internal/usecase`, for example profile-required and API-key-env-missing.
- Keep HTTP response messages stable and adapter-owned.
- Do not expose HTTP-specific error codes from the runner.
Suggested tests:
- HTTP error mapping tests should assert `profile_required` and `api_key_env_missing` via sentinel wrapping, not via message matching.
- Runner tests should assert `errors.Is` for the new reason errors.
Risk level: high for public API stability; low-to-medium implementation risk.
## Medium-Confidence Opportunities
### 1. Strict YAML Decode Setup Is Repeated
Affected files/packages:
- `internal/config/config.go`
- `internal/promptdef/filesystem_repository.go`
- `internal/profile/filesystem_repository.go`
Duplicated or near-duplicated behavior:
- Each package constructs a YAML decoder and enables `KnownFields(true)`.
Semantic differences that may be intentional:
- Prompt/profile loaders need likely-target behavior and raw `api_key` handling.
- Config loading has its own explicit/implicit file search policy.
Why it matters:
- Strict decoding is an architectural invariant. A future YAML loader could forget to enable it.
Recommended refactor:
- Consider a tiny helper for strict YAML decode from bytes.
- Keep package-specific error wrapping and partial ID decode logic local.
Suggested tests:
- Existing unknown-field tests in config, promptdef, and profile should remain.
- Add any new YAML-consuming package with an unknown-field test.
Risk level: medium.
### 2. Schema Path Resolution Is Centralized, But Schema Loading Happens Twice
Affected files/packages:
- `internal/usecase/runner.go`
- `internal/validate/standard_validator.go`
Duplicated or near-duplicated behavior:
- For `json_schema`, `Runner.Prepare` asks the validator to load the schema document for provider-level structured output.
- Later validation resolves and compiles the same schema path again.
Semantic differences that may be intentional:
- Provider structured-output payload needs the raw JSON schema document.
- Runtime validation needs a compiled schema.
Why it matters:
- The current behavior is correct, but schema file errors can surface at different phases and the same file is read more than once during `Run`.
- Future caching or schema behavior changes should have one clear owner.
Recommended refactor:
- Keep path resolution inside `internal/validate`.
- Consider a schema service/loader interface that can return both raw document and compiled schema from one path, only if schema-related work grows.
- Do not add caching unless repeated schema loads become a measured cost.
Suggested tests:
- Keep existing nested schema path tests.
- Add a test that `Prepare` fails before LLM call when structured-output schema cannot load.
Risk level: medium.
### 3. Artifact Hashing And Output Artifact Construction Are Split
Affected files/packages:
- `internal/artifact/reader.go`
- `internal/usecase/runner.go`
Duplicated or near-duplicated behavior:
- Input artifacts and generated output artifacts both compute SHA-256 hashes and fill size/body/content-type fields.
Semantic differences that may be intentional:
- Input artifacts derive content type from source extension or inline default.
- Output artifacts derive content type from prompt output format and always use the default output artifact name.
Why it matters:
- Hash algorithm and artifact metadata policy should remain consistent.
Recommended refactor:
- Consider an `artifact.Build` or `artifact.HashBody` helper only for shared hash/size construction.
- Keep source-specific content type and naming decisions local.
Suggested tests:
- Existing artifact reader tests and runner output content-type tests should cover behavior.
- Add a small helper test if a shared hash function is introduced.
Risk level: medium-low.
### 4. Domain Contains An Unsupported `s3` Artifact Reference Constant
Affected files/packages:
- `internal/domain/domain.go`
- `internal/artifact/reader.go`
- docs that state only `inline` and `file` are implemented
Duplicated or near-duplicated behavior:
- Not a duplication issue. This is a cleanup issue: `domain.ArtifactRefS3` exists, but no reader supports it and docs do not document it as implemented.
Semantic differences that may be intentional:
- The constant may be a placeholder for a future adapter.
Why it matters:
- Future-facing code outside `docs/roadmap/` can confuse maintainers and tests. It also weakens the otherwise clean "implemented behavior only" policy.
Recommended refactor:
- Remove the constant if no near-term S3 implementation is planned.
- If kept, add a code comment that it is intentionally unsupported today and ensure docs continue to state only `inline` and `file` are supported.
Suggested tests:
- Existing unsupported artifact type tests should continue to pass.
Risk level: medium-low.
### 5. Test Fixture Setup Is Repeated In CLI And Use-Case Tests
Affected files/packages:
- `internal/adapter/cli/run_test.go`
- `internal/usecase/runner_test.go`
- `internal/usecase/integration_test.go`
Duplicated or near-duplicated behavior:
- Tests repeatedly construct temp prompt/profile dirs, write prompt/profile YAML, set up fake LLMs/readers/renderers/validators, and build runners.
Semantic differences that may be intentional:
- Package-local tests avoid exporting test helpers and keep each package independent.
Why it matters:
- Repeated setup makes behavior-preserving refactors noisier and can obscure the exact behavior under test.
Recommended refactor:
- Add package-local helper builders where duplication is highest, especially in CLI command tests.
- Avoid a cross-package test utility package unless multiple packages need the same public fixture contract.
Suggested tests:
- This is test cleanup only; existing test assertions should remain equivalent.
Risk level: low.
## Boundary And Responsibility Concerns
- HTTP error mapping currently depends on runner error message text. Stable reason identity should live in `internal/usecase`; HTTP status/code/message mapping should remain in `internal/adapter/http`.
- Execution-target merge policy lives in `internal/usecase`, which fits the architecture. The concern is not placement but incomplete centralization of field mapping around that policy.
- CLI app wiring currently constructs concrete repositories/readers/renderers/validators inline in each command. That is acceptable adapter responsibility, but repeated wiring should be centralized within the CLI adapter package.
- Prompt/profile filesystem traversal is duplicated in two repository packages. A narrow filesystem catalog helper would fit the architecture because it would not own prompt/profile policy.
- LLM provider request shape is properly isolated in `internal/llm`. It should remain there; do not move OpenAI/OpenRouter request-field policy into runner or prompt definitions.
## Path, Key, And Naming Construction Review
Path and naming construction is mostly explicit and low-risk:
- Config paths are cleaned in `internal/config` and again in CLI finalization.
- Prompt `content_file` paths are resolved relative to the prompt YAML file in `internal/promptdef`.
- Schema paths are resolved through `internal/validate.StandardValidator`.
- OpenAI-compatible endpoint paths use `defaults.OpenAIChatCompletionsPath`.
- Output artifact name comes from `defaults.OutputArtifactName`.
Areas needing cleanup:
- Prompt/profile recursive YAML discovery and relative-path formatting should share one helper.
- CLI path cleaning after config resolution is repeated and should be consolidated with common command settings finalization.
- Structured schema names are derived in `internal/usecase`; this is currently one place and should stay there unless structured-output support expands.
No remote keys, cache paths, manifest paths, lock files, or generated report paths exist in the current implementation.
## Resolution And Catalog Review
Named concept resolution is mostly consistent:
- Prompt resolution uses YAML `id` and optional `version`, not file path.
- Profile resolution uses YAML `id`, not file path.
- Prompt/profile subdirectories are organizational only.
- Duplicate prompt/profile IDs fail instead of choosing first match.
- Schema resolution is explicit path-based relative to `schema_dir`; no basename search.
- Input artifacts are resolved by `artifact.Reader`, with `inline` and `file` as implemented types.
- Prompt required-input checks happen in the renderer.
Recommended centralization:
- Share only filesystem catalog mechanics between prompt/profile repositories.
- Keep prompt ID/version, profile ID, schema path, and input resolution policies in their current owning packages.
## Config And Command-Loading Review
Config precedence is consistent with policy:
1. built-in defaults;
2. config file values;
3. CLI overrides.
Intentional differences:
- `render` supports `--format`; `run` does not.
- `serve` supports `--addr`; `run`/`render` do not.
- `serve` rejects runtime model override flags.
- `render` does not expose `--schema-dir`, even though it can prepare `json_schema` prompts through config/default schema settings.
- CLI runtime model override flags are narrower than HTTP model override fields; this is currently documented by omission in CLI docs.
Likely cleanup areas:
- Common command settings finalization can be made smaller and less repetitive.
- Runner dependency construction should be a CLI-local helper.
- The `runConfig` flag-set booleans work, but adding more runtime override fields will continue to require touching several fields and the override condition.
## State, Manifest, Or Progress Handling Review
Scriptorium has no durable state, manifests, checkpoints, progress records, cache state, or resume behavior. This matches `docs/policy/architecture.md` and `docs/operations.md`.
There is no drift affecting resume, retry, force, dry-run, or audit behavior because those features do not exist. Validation repair hooks exist in the runner but are not wired by CLI/HTTP today; that boundary is documented.
## Refactors To Avoid
- Do not introduce a generic workflow/stage engine. Scriptorium intentionally executes one prompt request.
- Do not build a plugin architecture for prompt/profile/schema/artifact backends before another backend exists.
- Do not replace explicit CLI parse functions with a broad command framework.
- Do not merge CLI and HTTP adapters. Their public interfaces and failure semantics differ.
- Do not create a generic reflection-based mapper for domain/DTO/request structs.
- Do not centralize prompt/profile normalization into one generic YAML repository; their validation and error semantics are different.
- Do not add schema caching or a manifest/state store as part of deduplication.
- Do not document or implement unsupported artifact backends as cleanup.
## Recommended Implementation Sequence
1. Execution-target mapping cleanup
- Goal: reduce drift when adding profile/runtime/provider fields.
- Files to update: `internal/usecase`, `internal/adapter/http`, `internal/llm`, targeted tests.
- Acceptance criteria: all existing field mapping tests pass; a new all-fields test fails if a mapped execution field is omitted.
- Suggested validation: `go test ./internal/usecase ./internal/adapter/http ./internal/llm ./internal/format`.
- Small enough for one implementation prompt: yes.
2. Prompt/profile filesystem catalog helper
- Goal: centralize recursive YAML discovery, path sorting, extension filtering, and relative path formatting.
- Files to update: prompt/profile repositories plus new narrow helper package.
- Acceptance criteria: existing repository tests pass unchanged; new helper tests cover nested discovery and stable ordering.
- Suggested validation: `go test ./internal/promptdef ./internal/profile`.
- Small enough for one implementation prompt: yes.
3. CLI wiring and config finalization helper
- Goal: centralize common command settings resolution and runner construction.
- Files to update: `internal/adapter/cli/run.go` and CLI tests.
- Acceptance criteria: no flag behavior changes; run/render/serve config precedence tests pass.
- Suggested validation: `go test ./internal/adapter/cli`.
- Small enough for one implementation prompt: yes.
4. Stable use-case invalid-request reasons
- Goal: remove HTTP dependency on runner error message substrings.
- Files to update: `internal/usecase/runner.go`, `internal/adapter/http/handler.go`, tests.
- Acceptance criteria: HTTP error codes/messages remain unchanged; runner exposes stable `errors.Is` reason errors.
- Suggested validation: `go test ./internal/usecase ./internal/adapter/http`.
- Small enough for one implementation prompt: yes.
5. Test helper cleanup
- Goal: reduce repeated fixture construction after behavior-preserving refactors are complete.
- Files to update: primarily `internal/adapter/cli/run_test.go`, optionally `internal/usecase/runner_test.go`.
- Acceptance criteria: test intent remains clear; no cross-package helper package unless strongly justified.
- Suggested validation: `go test ./internal/adapter/cli ./internal/usecase`.
- Small enough for one implementation prompt: yes.
6. Dead-code/legacy sweep
- Goal: remove or clearly mark unsupported placeholders such as `ArtifactRefS3`.
- Files to update: domain/artifact tests/docs only if needed.
- Acceptance criteria: docs continue to describe only implemented behavior outside roadmap.
- Suggested validation: `go test ./internal/domain ./internal/artifact`.
- Small enough for one implementation prompt: yes.
## Test Strategy
Tests to add before or during cleanup:
- Execution target all-fields mapping tests:
- runner profile/default/request merge;
- HTTP request DTO to domain target;
- HTTP domain result to metadata DTO;
- LLM domain target to outbound JSON payload.
- Repository catalog helper tests:
- recursive scan;
- `.yaml` and `.yml` filtering;
- stable sorted paths;
- relative clean path formatting;
- context cancellation if the helper preserves current behavior.
- HTTP error mapping tests:
- profile-required and API-key-env-missing should rely on sentinel errors, not message text.
- CLI wiring regression tests:
- `run`, `render`, and `serve` preserve config precedence and required directory checks.
- Schema behavior tests:
- `Prepare` fails before LLM generation when `json_schema` structured-output schema cannot be loaded.
Validation commands for cleanup work:
- `go test ./internal/usecase ./internal/adapter/http ./internal/llm ./internal/format`
- `go test ./internal/promptdef ./internal/profile`
- `go test ./internal/adapter/cli`
- `go test ./...` before merging broader cleanup.
No automated docs or link checker is currently present.
## Appendix: Findings Not Worth Acting On
- Repeated `select { case <-ctx.Done(): ... }` checks are acceptable. They are local, simple, and appear at IO/loop boundaries where behavior is easy to read.
- CLI and HTTP request validation should remain separate. Their external contracts differ, and centralizing all request validation would blur adapter responsibilities.
- Prompt and profile validation should not be merged. They both use YAML and IDs, but their schemas, normalization rules, and error policies differ.
- Prepared-run text formatting is verbose but intentionally presentation-specific. Avoid abstracting it until another formatter needs the same layout policy.
- Hashing appears in artifact loading and runner metadata, but not all hashes represent the same thing. A tiny body-hash helper may be useful later; a generic hashing subsystem is not justified.
- HTTP DTOs duplicate domain field names by design. They should remain transport-owned so JSON compatibility can evolve deliberately.
- Config `applyConfig` and `ApplyCLIOverrides` look similar, but they apply different source labels and validation contexts. A broad merge abstraction would likely reduce clarity.

View File

@@ -1,504 +0,0 @@
# Cleanup Implementation Roadmap
## Purpose
This roadmap turns the findings in [audit.md](audit.md) into a staged, decision-complete cleanup plan for Scriptorium.
Audience: LLM coding agents implementing the cleanup in order.
Controlling policies:
- [Documentation policy](../policy/documentation.md)
- [Architecture policy](../policy/architecture.md)
- [Development guide](../policy/development.md)
## Global Implementation Rules
- Implement stages in order.
- Keep public CLI flags, HTTP request/response shapes, config precedence, prompt/profile ID semantics, and validation behavior stable unless a stage explicitly says otherwise.
- Keep adapters thin and use-case policy in `internal/usecase`.
- Prefer explicit helpers over reflection, generic workflow abstractions, or broad framework-style rewrites.
- Update non-roadmap docs only when implemented behavior changes or when a stale implemented-behavior statement is found during a stage.
- Do not document future behavior outside `docs/roadmap/`.
- Run the stage-specific tests before moving to the next stage.
- Run `go test ./...` after the final stage.
## Stage 1: Execution Target Mapping Cleanup
### Goal
Reduce drift when adding or changing execution/runtime fields such as `service_tier`, `api_key_env`, `reasoning_effort`, or future provider request keys.
### Scope
Update only explicit execution-target mapping and serialization paths. Do not add new CLI flags or new provider features.
### Implementation
In `internal/usecase`:
- Keep execution-target merge policy in `internal/usecase`.
- Add focused helper coverage around `resolveExecutionTarget`, `mergeExecutionTarget`, and profile-to-target conversion.
- Keep the existing semantics:
- built-in defaults first;
- profile values override defaults;
- request overrides override profile values;
- zero numeric values do not override;
- empty/whitespace string values do not override;
- non-empty `ExtraParams` replaces the previous map with a copy.
In `internal/adapter/http`:
- Add local helper functions:
- `executionTargetFromModelOverrideDTO(*modelOverrideRequestDTO) *domain.ExecutionTarget`
- `modelParamsDTOFromExecutionTarget(domain.ExecutionTarget) modelParamsDTO`
- Use those helpers in `handler.go`.
- Keep DTO types unexported and transport-owned.
- Keep HTTP response field names and omission behavior unchanged.
In `internal/llm`:
- Add a local helper such as `openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error)` or an equivalent small function.
- Keep endpoint construction, HTTP client timeout handling, API-key environment lookup, and response parsing in `Generate`.
- Keep outbound serialization behavior unchanged:
- send `model` and `messages`;
- send `temperature`, `max_tokens`, `top_p`, and `service_tier` only when currently sent;
- send `response_format` only when structured output is present;
- do not serialize `reasoning_effort` or `extra_params`.
Do not:
- use reflection to copy fields;
- move HTTP DTOs into `internal/domain`;
- add generic mapper packages;
- change prepared-run JSON tags.
### Tests
Add or update tests so an all-fields `domain.ExecutionTarget` catches omissions.
Required tests:
- Runner merge/profile conversion:
- profile values populate all supported execution fields;
- runtime overrides beat profile values for all overrideable fields;
- empty string overrides do not erase profile values;
- empty `ExtraParams` does not erase profile values.
- HTTP adapter:
- request `model` object maps every supported field into `RunRequest.Execution`;
- response `metadata.model_params` includes every supported field according to current DTO tags.
- LLM client:
- outbound JSON includes every serialized execution field;
- outbound JSON omits `service_tier` when empty;
- outbound JSON still omits `reasoning_effort` and `extra_params`.
### Validation
Run:
```bash
go test ./internal/usecase ./internal/adapter/http ./internal/llm ./internal/format
```
### Acceptance Criteria
- No public behavior changes.
- Adding a new execution target field later has obvious mapping/test locations.
- Existing HTTP and LLM behavior remains stable.
- Stage is small enough for one implementation prompt.
## Stage 2: Prompt/Profile Filesystem Catalog Helper
### Goal
Centralize shared recursive YAML discovery mechanics while preserving prompt/profile-specific validation and error behavior.
### Scope
Create a narrow helper package for filesystem catalog mechanics only.
Recommended package:
- `internal/filecatalog`
### Implementation
Add helper functions with explicit, small responsibilities:
- recursively find YAML files under a root directory;
- honor context cancellation during walking;
- accept `.yaml` and `.yml`;
- return stable sorted full paths;
- compute clean relative paths from a root;
- return filename stems with `.yaml`/`.yml` stripped.
Use the helper in:
- `internal/promptdef/filesystem_repository.go`
- `internal/profile/filesystem_repository.go`
Preserve existing behavior:
- prompt/profile lookup uses YAML `id`, not file path;
- subdirectories are organizational only;
- duplicate prompt/profile IDs are invalid;
- malformed likely-target files still surface errors;
- relative nested paths still appear in errors;
- prompt `content_file` resolution remains relative to the prompt YAML file;
- prompt/profile strict YAML and validation stay in their existing packages.
Do not:
- create a generic repository framework;
- merge prompt and profile normalization;
- move prompt/profile domain policy into the helper;
- change error messages except for unavoidable wording caused by helper extraction.
### Tests
Add tests for `internal/filecatalog`:
- nested YAML discovery;
- `.yaml` and `.yml` accepted;
- non-YAML files ignored;
- returned paths sorted deterministically;
- relative path formatting works for nested files;
- filename stem stripping handles both extensions.
Keep existing prompt/profile repository tests passing.
### Validation
Run:
```bash
go test ./internal/filecatalog ./internal/promptdef ./internal/profile
```
### Acceptance Criteria
- Prompt/profile repository tests pass without behavior expectation changes.
- Shared filesystem scanning logic exists in one place.
- Prompt/profile packages still own their own validation and normalization.
- Stage is small enough for one implementation prompt.
## Stage 3: CLI Wiring And Settings Finalization Cleanup
### Goal
Reduce duplicated command setup while preserving each command's public flag surface and behavior.
### Scope
Clean up `internal/adapter/cli` only, except for tests.
### Implementation
Add CLI-local helpers. Recommended helpers:
- `commonCommandSettings` or similar struct containing resolved `promptDir`, `profileDir`, `schemaDir`, `serverAddr`, and `defaultRenderFormat` where applicable.
- `resolveCommonSettings(fs *flag.FlagSet, configPath string, overrides appconfig.CLIOverrides) (commonCommandSettings, error)`.
- `validateRequiredLibraryDirs(promptDir, profileDir string) error`.
- `newRunner(promptDir, profileDir, schemaDir string, llmClient llm.Client) *usecase.Runner`.
- Optionally `newOpenAIClient() (*llm.OpenAICompatibleClient, error)` if it removes exact duplication without obscuring command behavior.
Preserve command differences:
- `run` exposes runtime model override flags and `--schema-dir`;
- `render` exposes runtime model override flags and `--format`, but not `--schema-dir`;
- `serve` exposes `--addr` and `--schema-dir`, but no runtime model override flags;
- `render` default format comes from `defaults.render_format` unless `--format` is set;
- deprecated `--prompt-id` and `--profile-id` aliases remain accepted.
Keep existing parse functions:
- `parseRunArgs`
- `parseRenderArgs`
- `parseServeArgs`
Do not:
- replace the standard library `flag` package;
- introduce a command framework;
- make `serve` accept runtime model override flags;
- change error prefixes such as `run parse error`, `render parse error`, or `serve parse error`;
- change CLI output behavior.
### Tests
Required regression tests:
- `run`, `render`, and `serve` still apply config precedence correctly.
- Missing effective `prompt_dir` and `profile_dir` still return the same guidance.
- `render` still uses config default render format and explicit `--format` override.
- `serve` still rejects runtime model override flags.
- `run` and `render` still build equivalent runtime override requests for shared flags.
### Validation
Run:
```bash
go test ./internal/adapter/cli
```
### Acceptance Criteria
- No CLI flag, output, exit-code, or precedence changes.
- Runner dependency construction is centralized inside the CLI adapter.
- Command-specific behavior remains easy to read.
- Stage is small enough for one implementation prompt.
## Stage 4: Stable Use-Case Error Reasons For HTTP Mapping
### Goal
Remove HTTP error mapping's dependency on runner error message substrings.
### Scope
Change error identity, not public HTTP error responses.
### Implementation
In `internal/usecase`:
- Add stable sentinel errors for invalid-request reasons that HTTP currently distinguishes by message text.
- Required sentinels:
- missing profile selection, for the case where neither request profile nor prompt `default_profile` is available;
- missing API-key environment value, for the case where `api_key_env` is set but the named environment variable is unset or empty.
- Wrap these sentinels with `ErrInvalidRequest` so existing broad invalid-request checks keep working.
- Preserve clear human-readable runner errors.
In `internal/adapter/http`:
- Replace `strings.Contains(err.Error(), ...)` checks for these cases with `errors.Is`.
- Keep current HTTP status codes, error codes, and response messages:
- `400 profile_required`;
- `400 api_key_env_missing`.
Do not:
- expose HTTP-specific error codes from `internal/usecase`;
- change the HTTP JSON error body shape;
- remove broad fallback handling for `usecase.ErrInvalidRequest`.
### Tests
Required tests:
- Runner tests assert `errors.Is(err, usecase.ErrProfileRequired)` or the chosen sentinel name for missing profile selection.
- Runner tests assert `errors.Is(err, usecase.ErrAPIKeyEnvMissing)` or the chosen sentinel name for missing API-key environment value.
- HTTP handler tests still assert unchanged status/code/message for both cases.
- HTTP handler tests should not construct errors by relying on exact runner prose for these two cases.
### Validation
Run:
```bash
go test ./internal/usecase ./internal/adapter/http
```
### Acceptance Criteria
- HTTP mapping no longer depends on runner message substrings for the two distinguished invalid-request cases.
- Public HTTP behavior is unchanged.
- Runner errors remain clear in CLI output.
- Stage is small enough for one implementation prompt.
## Stage 5: Schema Failure Regression Coverage
### Goal
Protect the current structured-output invariant before future schema cleanup: `Prepare` must load a `json_schema` document before any LLM call.
### Scope
Add regression coverage only. Do not add schema caching or change schema loading architecture in this stage.
### Implementation
In `internal/usecase/runner_test.go` or an appropriate package test:
- Add a test where a prompt uses `validation_mode: json_schema` with a missing or failing schema document.
- Assert `Runner.Prepare` fails with `ErrValidation`.
- Assert no LLM call is made for `Runner.Run` when structured-output schema loading fails.
If existing tests already cover part of this behavior, consolidate assertions without making the test suite harder to read.
Do not:
- cache compiled schemas;
- change `validate.StandardValidator` behavior;
- introduce a schema service abstraction.
### Validation
Run:
```bash
go test ./internal/usecase ./internal/validate
```
### Acceptance Criteria
- Missing structured-output schema fails before generation.
- Existing JSON Schema validation behavior is unchanged.
- Stage is small enough for one implementation prompt.
## Stage 6: Test Fixture Cleanup
### Goal
Reduce repeated test setup after behavior-preserving production refactors are complete.
### Scope
Prefer package-local test helpers. Avoid cross-package test utility packages unless a helper is needed by more than two packages and represents a stable public fixture contract.
### Implementation
In `internal/adapter/cli/run_test.go`:
- Consolidate repeated temp prompt/profile/input setup into local helper functions.
- Keep helper names behavior-focused, for example:
- `newCLITestLibrary`
- `writePromptFileWithDefaultProfile`
- `writeProfileFile`
- `runCLICommand`
- Do not hide assertions inside helpers unless the assertion is truly setup validation.
In `internal/usecase/runner_test.go`:
- Keep existing fake interfaces package-local.
- Remove only high-volume duplication that obscures test intent.
Do not:
- move package-private fake types into production code;
- create a broad `internal/testutil` package unless a later cleanup stage proves it necessary;
- rewrite tests into table-driven form when cases have meaningfully different setup.
### Validation
Run:
```bash
go test ./internal/adapter/cli ./internal/usecase
```
### Acceptance Criteria
- Test intent is at least as clear as before.
- No production behavior changes.
- Test fixture setup has less repeated boilerplate in CLI tests.
- Stage is small enough for one implementation prompt.
## Stage 7: Unsupported Placeholder Sweep
### Goal
Remove code that suggests unimplemented artifact behavior outside roadmap documentation.
### Scope
Remove unsupported placeholders only when they are not needed by current tests or public docs.
### Implementation
Remove `domain.ArtifactRefS3` from `internal/domain/domain.go` unless new evidence shows it is intentionally needed by implemented code.
Preserve current behavior:
- supported artifact reference types remain `inline` and `file`;
- unsupported artifact reference types still return `artifact.ErrUnsupportedRefType`;
- docs continue to describe only `inline` and `file` outside roadmap files.
Update tests only if they reference the removed constant. Prefer testing unsupported artifact behavior with a literal custom type such as `domain.ArtifactRefType("s3")` or `domain.ArtifactRefType("unsupported")`.
Do not:
- add S3 support;
- document S3 as implemented;
- add future-backend placeholders elsewhere.
### Validation
Run:
```bash
go test ./internal/domain ./internal/artifact
```
### Acceptance Criteria
- Unsupported placeholder constant is removed.
- Unsupported artifact-type behavior remains covered.
- No non-roadmap doc claims unimplemented artifact support.
- Stage is small enough for one implementation prompt.
## Stage 8: Final Verification And Documentation Alignment
### Goal
Confirm the cleanup sequence preserved behavior and documentation accuracy.
### Implementation
Run the full test suite:
```bash
go test ./...
```
Run the maintained render smoke command:
```bash
go run ./cmd/scriptorium render --config ./examples/config.yml --prompt generic.markdown_summary --input transcript=./examples/fixtures/transcript.md --input glossary=./examples/fixtures/glossary.yml --format json
```
Search for stale or unsupported terms:
```bash
rg -n "ArtifactRefS3|s3|strings\\.Contains\\(err\\.Error\\(\\)|TODO|future|planned" internal docs README.md examples
```
Review results manually:
- `s3` should not appear as an implemented artifact type.
- `strings.Contains(err.Error())` should not be used for stable use-case reason mapping.
- Any `future` or `planned` wording outside `docs/roadmap/` must describe current boundaries, not aspirational behavior.
Update docs only if cleanup changed implemented behavior or if the search reveals stale implemented-behavior docs.
### Acceptance Criteria
- Full test suite passes.
- Maintained render smoke command succeeds.
- No stale unsupported artifact placeholder remains.
- Non-roadmap docs describe implemented behavior only.
- Working tree contains only intentional cleanup changes.
## Deferred Work
Do not implement these during the staged cleanup unless a later audit makes them high-confidence:
- schema caching or a combined raw/compiled schema service;
- artifact hash/build helper beyond a small helper introduced opportunistically during touched code;
- generic YAML repository framework;
- generic CLI command framework;
- plugin architecture for future prompt/profile/schema/artifact backends;
- durable state, manifests, checkpoints, or resume behavior.
## Completion Criteria
The cleanup roadmap is complete when all stages have been implemented in order, the final verification passes, and the resulting code still satisfies:
- `Runner.Run` reuses `Runner.Prepare`;
- CLI and HTTP adapters instantiate `Runner` without a repairer;
- unknown config/prompt/profile YAML and HTTP JSON fields are rejected;
- raw API key values are not accepted or emitted;
- prompt/profile subdirectories remain organizational only;
- schema paths remain explicit and relative to `schema_dir` when not absolute;
- public CLI and HTTP behavior remains stable.

View File

@@ -0,0 +1,262 @@
# Runtime Parameter Implementation Plan
This plan implements the target state in `docs/roadmap/params.md`.
Audience: LLM coding agents implementing the feature in order. Follow `docs/policy/architecture.md`, `docs/policy/development.md`, and `docs/policy/documentation.md` before changing code.
## Constraints
- Keep adapters thin. CLI and HTTP should capture caller intent and map it into domain request types; merge decisions belong in `internal/usecase`.
- Keep external decoding strict. Unknown YAML/JSON fields must continue to fail.
- Do not accept or emit raw API key values.
- Do not add dependencies unless there is a clear need. This feature should use the standard library plus existing dependencies.
- Do not expand the HTTP API surface beyond `POST /v1/runs`.
- Do not add provider-specific adapter packages.
- Keep each stage passing `go test ./...` before moving to the next stage.
## Stage 1: Presence-Aware Request Overrides
Goal: make per-request numeric execution overrides presence-aware while keeping resolved execution settings concrete.
### Domain Changes
1. In `internal/domain/domain.go`, add a request-only type:
```go
type ExecutionTargetOverride struct {
Endpoint string `json:"endpoint,omitempty"`
Model string `json:"model,omitempty"`
Temperature *float64 `json:"temperature,omitempty"`
MaxTokens *int `json:"max_tokens,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
ServiceTier string `json:"service_tier,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"`
APIKeyEnv string `json:"api_key_env,omitempty"`
ExtraParams map[string]any `json:"extra_params,omitempty"`
}
```
2. Change `domain.RunRequest.Execution` from `*ExecutionTarget` to `*ExecutionTargetOverride`.
3. Change `ExecutionProfile.ExtraParams` and `ExecutionTarget.ExtraParams` from `map[string]string` to `map[string]any`.
4. Keep `ExecutionTarget` concrete. It represents the resolved effective runtime target after defaults, profile, and request overrides are merged.
### Runner Changes
1. Update `internal/usecase/runner.go` so profile values still merge over built-in defaults and request overrides merge over that result.
2. Keep the existing concrete profile merge semantics for profile numeric fields.
3. Add a separate request override merge path that uses pointer presence:
- `nil` numeric pointer means omitted; preserve the current value.
- non-nil numeric pointer means explicit override, even when the value is `0`.
4. Validate request override numeric values before or during merge:
- `temperature`: `0 <= value <= 2`
- `max_tokens`: `value >= 0`
- `top_p`: `0 <= value <= 1`
- `timeout_seconds`: `value >= 0`
5. Preserve existing validation after merge:
- effective endpoint required
- effective model required
- `api_key_env`, when set, must name a non-empty environment variable
6. Preserve secret handling. The resolved API key value must never be stored in `PreparedRun`, `RunResult`, logs, or HTTP responses.
### CLI Changes
1. Update `internal/adapter/cli/run.go` request construction to build `domain.ExecutionTargetOverride`.
2. Use the existing `flagWasSet` booleans to populate numeric pointers only when the user provided the flag.
3. Required behavior:
- omitted `--temperature` preserves profile/default temperature;
- `--temperature 0` explicitly sets temperature to zero;
- omitted `--top-p` preserves profile/default top-p;
- `--top-p 0` explicitly sets top-p to zero;
- omitted `--max-tokens` preserves profile/default max tokens;
- `--max-tokens 0` explicitly sets max tokens to zero;
- omitted `--timeout` preserves profile/default timeout;
- `--timeout 0s` explicitly sets timeout seconds to zero.
4. Do not add new CLI flags in this stage.
### HTTP Changes
1. Update `internal/adapter/http/dto.go` so numeric model override fields are pointers:
- `Temperature *float64`
- `MaxTokens *int`
- `TopP *float64`
- `TimeoutSeconds *int`
2. Update DTO mapping in `internal/adapter/http/handler.go` to build `domain.ExecutionTargetOverride`.
3. Preserve strict JSON decoding and existing error mapping.
4. Required behavior:
- omitted numeric JSON fields preserve profile/default values;
- explicit numeric zero JSON fields override profile/default values.
### Tests
Add or update tests in:
- `internal/usecase/runner_test.go`
- `internal/adapter/cli/run_test.go`
- `internal/adapter/http/handler_test.go`
Required test coverage:
- Runner preserves profile value when request numeric override is omitted.
- Runner applies explicit zero request override for `temperature`.
- Runner applies explicit zero request override for `top_p`.
- Runner applies explicit zero request override for `max_tokens`.
- Runner applies explicit zero request override for `timeout_seconds`.
- Invalid request override ranges fail as invalid request errors.
- CLI `--temperature 0` reaches effective settings as zero.
- HTTP `"temperature": 0` reaches effective settings as zero.
- HTTP omitted `temperature` preserves profile/default value.
### Verification
Run:
```bash
go test ./...
```
## Stage 2: JSON-Compatible `extra_params`
Goal: allow provider-specific parameters to carry JSON-compatible values throughout profile, HTTP, prepared output, metadata, and LLM request construction.
### Domain And Loader Changes
1. Complete all compile fixes from changing `ExtraParams` to `map[string]any`.
2. Ensure `internal/profile/filesystem_repository.go` continues to decode profiles strictly while allowing nested JSON-compatible values under `extra_params`.
3. Add profile repository tests for `extra_params` containing:
- string
- number
- boolean
- nested object or array
4. Ensure formatter output remains deterministic:
- keep sorting `extra_params` keys in `internal/format/prepared_run.go`;
- render non-string values with stable JSON encoding in text output.
5. Preserve JSON formatter behavior through normal `encoding/json` output.
### HTTP Changes
1. Change HTTP model override `ExtraParams` to `map[string]any`.
2. Add handler tests proving HTTP accepts JSON-compatible `extra_params` values.
3. Preserve strict rejection of unknown fields and raw API-key payload fields.
### Verification
Run:
```bash
go test ./...
```
## Stage 3: Outbound Serialization
Goal: serialize `reasoning_effort` and `extra_params` to the OpenAI-compatible chat-completions request.
### LLM Adapter Changes
1. In `internal/llm/openai_compatible_client.go`, add first-class outbound support for `reasoning_effort`.
2. Add `extra_params` support by flattening `domain.ExecutionTarget.ExtraParams` into additional top-level JSON request fields.
3. Implement reserved-field collision checks before the HTTP request is made.
4. Reserved keys must include:
- `model`
- `session_id`
- `messages`
- `temperature`
- `max_tokens`
- `top_p`
- `service_tier`
- `reasoning_effort`
- `response_format`
5. Reject empty `extra_params` keys.
6. Ensure each `extra_params` value can be marshaled as JSON. If marshaling fails, return `ErrInvalidRequest` with context.
7. Keep existing request behavior unchanged when `reasoning_effort` and `extra_params` are unset.
### Recommended Implementation Shape
Use a custom marshal path for the outbound chat request rather than string manipulation.
One acceptable shape:
- Add `ReasoningEffort string` and `ExtraParams map[string]any` to the internal `openAIChatRequest`.
- Add a helper that converts `openAIChatRequest` into `map[string]any`, inserts first-class fields when set, then inserts `ExtraParams` after collision validation.
- Marshal that map with `encoding/json`.
Do not construct outbound JSON with manual string concatenation.
### Tests
Update `internal/llm/openai_compatible_client_test.go`.
Required test coverage:
- outbound JSON includes `reasoning_effort` when set;
- outbound JSON omits `reasoning_effort` when unset;
- outbound JSON includes string, number, boolean, object, and array `extra_params`;
- reserved `extra_params` keys fail before provider call;
- empty `extra_params` keys fail before provider call;
- existing message, cache-control, service-tier, response-format, and usage parsing tests continue to pass.
### Verification
Run:
```bash
go test ./...
```
## Stage 4: Documentation And Examples
Goal: move implemented behavior from roadmap to canonical docs after code is complete.
Update only after Stages 1 through 3 are implemented.
### Required Docs
Update:
- `docs/config.md`
- `docs/cli.md`
- `docs/integrations/http-api.md`
- `docs/integrations/openai-compatible-chat.md`
- `docs/internal/runner.md`
- `docs/internal/adapters.md`
Required documentation content:
- `reasoning_effort` is serialized outbound when set.
- `extra_params` serializes as provider-specific top-level outbound JSON fields.
- `extra_params` supports JSON-compatible values.
- reserved `extra_params` fields are rejected.
- per-request numeric overrides distinguish omitted values from explicit zero values.
- CLI explicit zero behavior for existing numeric flags.
- HTTP explicit zero behavior for model override numeric fields.
- no raw API-key values are accepted or emitted.
### Examples
Update examples only if needed to keep them accurate and runnable.
If adding an `extra_params` example, keep it secret-free and simple. Prefer a harmless provider-routing example over a vendor-specific feature that requires special credentials.
### Verification
Run:
```bash
go test ./...
go run ./cmd/scriptorium render \
--config ./examples/config.yml \
--prompt generic.markdown_summary \
--input transcript=./examples/fixtures/transcript.md \
--input glossary=./examples/fixtures/glossary.yml \
--format json
```
## Final Checks
Before considering the feature complete:
1. Confirm `git diff` contains only intended code, test, doc, and example changes.
2. Confirm all non-roadmap docs describe implemented behavior only.
3. Confirm no output path exposes raw API key values.
4. Confirm `go test ./...` passes.
5. Confirm the render smoke command passes.

96
docs/roadmap/params.md Normal file
View File

@@ -0,0 +1,96 @@
# Runtime Parameter Feature Roadmap
This roadmap defines the target behavior for runtime model parameters.
Current behavior has two limitations:
- `reasoning_effort` and `extra_params` are parsed into effective execution settings but are not serialized into outbound OpenAI-compatible chat-completions requests.
- Per-request numeric execution overrides use zero-value merge semantics, so callers cannot reliably override a profile value with an explicit zero such as `temperature: 0`.
The implementation plan for this feature lives in `docs/roadmap/implementation.md`.
## Target State
Scriptorium should preserve the existing separation between prompt definitions, execution profiles, and per-request execution overrides while making runtime parameter behavior explicit and predictable.
Expected end state:
- Effective execution settings remain visible in prepared-run output, run metadata, and HTTP metadata without exposing raw secret values.
- `reasoning_effort` is treated as a first-class effective execution setting and is serialized to the outbound OpenAI-compatible request when set.
- `extra_params` supports provider-specific OpenAI-compatible request fields.
- `extra_params` is serialized as additional top-level outbound JSON fields.
- `extra_params` values support JSON-compatible scalar, object, and array values.
- `extra_params` cannot override first-class outbound request fields.
- Per-request numeric overrides preserve caller intent, including explicit zero values.
- Omitted per-request numeric overrides continue to inherit the selected profile and built-in defaults.
- External decoding remains strict for config, prompt, profile, and HTTP request payloads.
## Policy Decisions
### `extra_params`
`extra_params` should serialize as additional top-level outbound JSON fields in the OpenAI-compatible chat-completions request.
Reasoning:
Most OpenAI-compatible providers expose vendor-specific chat-completions parameters as top-level fields. This keeps Scriptorium's adapter compatible with that ecosystem without adding first-class fields for every provider option.
`extra_params` must not silently override Scriptorium-owned fields. Reserved outbound fields include at least:
- `model`
- `session_id`
- `messages`
- `temperature`
- `max_tokens`
- `top_p`
- `service_tier`
- `reasoning_effort`
- `response_format`
If a caller supplies a reserved key through `extra_params`, Scriptorium should fail before making the outbound HTTP request.
`extra_params` should use JSON-compatible values rather than only strings.
Reasoning:
Provider-specific parameters commonly need booleans, numbers, objects, or arrays. String-only values would force awkward encoding and would likely require a later compatibility break.
### Presence-Aware Overrides
Per-request execution overrides should use a presence-aware type with pointer fields for optional numeric values.
Reasoning:
The resolved execution target should remain a concrete value used by prepared runs, generated requests, and metadata. Optionality matters at the request boundary, not after the runner has resolved the effective target.
This keeps adapter and merge logic precise while avoiding nil checks in formatter, metadata, and LLM serialization paths.
## Scope
In scope:
- Runtime merge behavior for per-request execution overrides.
- HTTP model override decoding for explicit zero numeric values.
- CLI execution override handling for explicit zero numeric flags.
- Outbound serialization of `reasoning_effort`.
- Outbound serialization of JSON-compatible `extra_params`.
- Tests and documentation for the changed implemented behavior.
Out of scope:
- Expanding the HTTP API beyond `POST /v1/runs`.
- Adding built-in HTTP authentication or authorization.
- Adding durable run state, run history, or multi-step orchestration.
- Adding broad provider-specific adapter packages.
- Adding new CLI flags for every provider-specific parameter.
## Acceptance Criteria
- A profile containing `reasoning_effort: medium` produces an outbound request with `reasoning_effort`.
- HTTP callers can pass `reasoning_effort` through the existing `model` override object and have it appear outbound.
- A profile or HTTP request containing JSON-compatible `extra_params` produces outbound top-level JSON fields according to the reserved-field policy.
- Reserved `extra_params` collisions fail before the outbound provider call.
- CLI callers can pass `--temperature 0` and observe `temperature: 0` in rendered/effective settings and outbound requests.
- HTTP callers can send `"temperature": 0` and observe the same behavior.
- Omitting `temperature` continues to preserve the selected profile/default value.
- Raw API key values remain unsupported in config, profiles, CLI flags, HTTP payloads, logs, and rendered output.

View File

@@ -252,6 +252,39 @@ Relevant links:
- [Configuration reference](config.md) - [Configuration reference](config.md)
- [Operations guide](operations.md) - [Operations guide](operations.md)
## Prompt Cache Misses Or No Cache Usage
Symptom:
- CLI run summary omits `cached_tokens` / `cache_write_tokens`.
- HTTP `metadata.usage.cached_tokens` and `metadata.usage.cache_write_tokens` are both `0`.
- Provider cost or latency does not improve after repeated similar runs.
Likely cause:
- The selected prompt has no `messages[].cache_control`.
- Dynamic per-run input appears before the cache-controlled message and changes the provider cache key.
- The provider does not support the serialized cache-control shape for the selected model.
- The provider imposes minimum token thresholds or cache-breakpoint limits.
Diagnostic step:
- Run `render --format json` and verify the intended rendered message includes `cache_control`.
- Confirm stable reusable context appears before the cache-controlled message, with dynamic input after it.
- Check provider docs/logs for model support, minimum token thresholds, and breakpoint limits.
Safe fix:
- Move stable reusable context before the cache-controlled message.
- Move highly dynamic input after the cache breakpoint.
- Keep `cache_control.type: ephemeral` and, when using `ttl`, set `ttl: 1h`.
- Use CLI cache counters or HTTP cache usage fields to verify cache reads/writes after rerunning.
Relevant links:
- [Configuration reference](config.md)
- [OpenAI-compatible chat integration](integrations/openai-compatible-chat.md)
## Validation Status Failed (`run` Exit 2 Or HTTP 200 With Failed Status) ## Validation Status Failed (`run` Exit 2 Or HTTP 200 With Failed Status)
Symptom: Symptom:

View File

@@ -81,6 +81,14 @@ type serveConfig struct {
schemaDir string schemaDir string
} }
type commonCommandSettings struct {
promptDir string
profileDir string
schemaDir string
serverAddr string
defaultRenderFormat renderformat.PreparedRunOutputFormat
}
type listFlag []string type listFlag []string
func (l *listFlag) String() string { func (l *listFlag) String() string {
@@ -125,22 +133,13 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
return ExitRuntimeError return ExitRuntimeError
} }
llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{ llmClient, err := newOpenAIClient()
Timeout: defaults.LLMRequestTimeoutDefault,
})
if err != nil { if err != nil {
fmt.Fprintf(stderr, "llm client error: %v\n", err) fmt.Fprintf(stderr, "llm client error: %v\n", err)
return ExitRuntimeError return ExitRuntimeError
} }
runner := usecase.NewRunner( runner := newRunner(cfg.promptDir, cfg.profileDir, cfg.schemaDir, llmClient)
promptdef.NewFilesystemRepository(cfg.promptDir),
profile.NewFilesystemRepository(cfg.profileDir),
artifactadapter.NewCompositeReader(),
prompt.NewGoRenderer(),
llmClient,
validate.NewStandardValidator(cfg.schemaDir),
)
res, runErr := runner.Run(context.Background(), req) res, runErr := runner.Run(context.Background(), req)
if runErr != nil { if runErr != nil {
@@ -170,14 +169,7 @@ func renderCommand(args []string, stdout, stderr io.Writer) int {
return ExitRuntimeError return ExitRuntimeError
} }
runner := usecase.NewRunner( runner := newRunner(cfg.promptDir, cfg.profileDir, cfg.schemaDir, nil)
promptdef.NewFilesystemRepository(cfg.promptDir),
profile.NewFilesystemRepository(cfg.profileDir),
artifactadapter.NewCompositeReader(),
prompt.NewGoRenderer(),
nil,
validate.NewStandardValidator(cfg.schemaDir),
)
prepared, prepErr := runner.Prepare(context.Background(), req) prepared, prepErr := runner.Prepare(context.Background(), req)
if prepErr != nil { if prepErr != nil {
@@ -205,22 +197,13 @@ func serveCommand(args []string, stderr io.Writer) int {
return ExitRuntimeError return ExitRuntimeError
} }
llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{ llmClient, err := newOpenAIClient()
Timeout: defaults.LLMRequestTimeoutDefault,
})
if err != nil { if err != nil {
fmt.Fprintf(stderr, "llm client error: %v\n", err) fmt.Fprintf(stderr, "llm client error: %v\n", err)
return ExitRuntimeError return ExitRuntimeError
} }
runner := usecase.NewRunner( runner := newRunner(cfg.promptDir, cfg.profileDir, cfg.schemaDir, llmClient)
promptdef.NewFilesystemRepository(cfg.promptDir),
profile.NewFilesystemRepository(cfg.profileDir),
artifactadapter.NewCompositeReader(),
prompt.NewGoRenderer(),
llmClient,
validate.NewStandardValidator(cfg.schemaDir),
)
h := httpadapter.NewHandler(runner) h := httpadapter.NewHandler(runner)
srv := &http.Server{ srv := &http.Server{
@@ -307,7 +290,7 @@ func parseServeArgs(args []string) (*serveConfig, error) {
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args()) return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
} }
settings, err := resolveAppSettings(fs, cfg.configPath, appconfig.CLIOverrides{ settings, err := resolveCommonSettings(fs, cfg.configPath, appconfig.CLIOverrides{
PromptDir: cfg.promptDirIfSet(fs), PromptDir: cfg.promptDirIfSet(fs),
ProfileDir: cfg.profileDirIfSet(fs), ProfileDir: cfg.profileDirIfSet(fs),
SchemaDir: cfg.schemaDirIfSet(fs), SchemaDir: cfg.schemaDirIfSet(fs),
@@ -317,16 +300,13 @@ func parseServeArgs(args []string) (*serveConfig, error) {
return nil, err return nil, err
} }
cfg.promptDir = settings.PromptDir cfg.promptDir = settings.promptDir
cfg.profileDir = settings.ProfileDir cfg.profileDir = settings.profileDir
cfg.schemaDir = settings.SchemaDir cfg.schemaDir = settings.schemaDir
cfg.addr = settings.ServerAddr cfg.addr = settings.serverAddr
if strings.TrimSpace(cfg.promptDir) == "" { if err := validateRequiredLibraryDirs(cfg.promptDir, cfg.profileDir); err != nil {
return nil, errors.New(errPromptDirRequired) return nil, err
}
if strings.TrimSpace(cfg.profileDir) == "" {
return nil, errors.New(errProfileDirRequired)
} }
cfg.promptDir = filepath.Clean(cfg.promptDir) cfg.promptDir = filepath.Clean(cfg.promptDir)
@@ -359,7 +339,7 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
return fmt.Errorf("unexpected positional args: %v", fs.Args()) return fmt.Errorf("unexpected positional args: %v", fs.Args())
} }
settings, err := resolveAppSettings(fs, cfg.configPath, appconfig.CLIOverrides{ settings, err := resolveCommonSettings(fs, cfg.configPath, appconfig.CLIOverrides{
PromptDir: cfg.promptDirIfSet(fs), PromptDir: cfg.promptDirIfSet(fs),
ProfileDir: cfg.profileDirIfSet(fs), ProfileDir: cfg.profileDirIfSet(fs),
SchemaDir: cfg.schemaDirIfSet(fs), SchemaDir: cfg.schemaDirIfSet(fs),
@@ -368,16 +348,13 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
return err return err
} }
cfg.promptDir = settings.PromptDir cfg.promptDir = settings.promptDir
cfg.profileDir = settings.ProfileDir cfg.profileDir = settings.profileDir
cfg.schemaDir = settings.SchemaDir cfg.schemaDir = settings.schemaDir
cfg.defaultRenderFormat = settings.DefaultRenderFormat cfg.defaultRenderFormat = settings.defaultRenderFormat
if strings.TrimSpace(cfg.promptDir) == "" { if err := validateRequiredLibraryDirs(cfg.promptDir, cfg.profileDir); err != nil {
return errors.New(errPromptDirRequired) return err
}
if strings.TrimSpace(cfg.profileDir) == "" {
return errors.New(errProfileDirRequired)
} }
if strings.TrimSpace(cfg.promptID) == "" { if strings.TrimSpace(cfg.promptID) == "" {
return errors.New("--prompt is required") return errors.New("--prompt is required")
@@ -476,6 +453,47 @@ func resolveAppSettings(fs *flag.FlagSet, configPath string, overrides appconfig
return merged, nil return merged, nil
} }
func resolveCommonSettings(fs *flag.FlagSet, configPath string, overrides appconfig.CLIOverrides) (commonCommandSettings, error) {
settings, err := resolveAppSettings(fs, configPath, overrides)
if err != nil {
return commonCommandSettings{}, err
}
return commonCommandSettings{
promptDir: settings.PromptDir,
profileDir: settings.ProfileDir,
schemaDir: settings.SchemaDir,
serverAddr: settings.ServerAddr,
defaultRenderFormat: settings.DefaultRenderFormat,
}, nil
}
func validateRequiredLibraryDirs(promptDir, profileDir string) error {
if strings.TrimSpace(promptDir) == "" {
return errors.New(errPromptDirRequired)
}
if strings.TrimSpace(profileDir) == "" {
return errors.New(errProfileDirRequired)
}
return nil
}
func newRunner(promptDir, profileDir, schemaDir string, llmClient llm.Client) *usecase.Runner {
return usecase.NewRunner(
promptdef.NewFilesystemRepository(promptDir),
profile.NewFilesystemRepository(profileDir),
artifactadapter.NewCompositeReader(),
prompt.NewGoRenderer(),
llmClient,
validate.NewStandardValidator(schemaDir),
)
}
func newOpenAIClient() (*llm.OpenAICompatibleClient, error) {
return llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
Timeout: defaults.LLMRequestTimeoutDefault,
})
}
func buildRunRequestFromConfig(cfg *runConfig) (domain.RunRequest, error) { func buildRunRequestFromConfig(cfg *runConfig) (domain.RunRequest, error) {
inputMappings, err := parseMappings(cfg.inputRaw, false) inputMappings, err := parseMappings(cfg.inputRaw, false)
if err != nil { if err != nil {
@@ -495,18 +513,25 @@ func buildRunRequestFromConfig(cfg *runConfig) (domain.RunRequest, error) {
inputs[name] = domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path} inputs[name] = domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path}
} }
var modelOverride *domain.ExecutionTarget var modelOverride *domain.ExecutionTargetOverride
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.apiKeyEnvSet || cfg.timeoutSet { if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.apiKeyEnvSet || cfg.timeoutSet {
modelOverride = &domain.ExecutionTarget{ modelOverride = &domain.ExecutionTargetOverride{
Endpoint: cfg.llmBaseURL, Endpoint: cfg.llmBaseURL,
Model: cfg.model, Model: cfg.model,
Temperature: cfg.temperature, APIKeyEnv: cfg.apiKeyEnv,
MaxTokens: cfg.maxTokens, }
TopP: cfg.topP, if cfg.temperatureSet {
APIKeyEnv: cfg.apiKeyEnv, modelOverride.Temperature = &cfg.temperature
}
if cfg.maxTokensSet {
modelOverride.MaxTokens = &cfg.maxTokens
}
if cfg.topPSet {
modelOverride.TopP = &cfg.topP
} }
if cfg.timeoutSet { if cfg.timeoutSet {
modelOverride.TimeoutSeconds = int(cfg.timeout.Seconds()) timeoutSeconds := int(cfg.timeout.Seconds())
modelOverride.TimeoutSeconds = &timeoutSeconds
} }
} }
@@ -588,7 +613,7 @@ func printSummary(stderr io.Writer, res *domain.RunResult) {
if res == nil { if res == nil {
return return
} }
fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d\n", fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d",
res.PromptID, res.PromptID,
res.PromptVersion, res.PromptVersion,
res.SelectedProfileID, res.SelectedProfileID,
@@ -602,6 +627,10 @@ func printSummary(stderr io.Writer, res *domain.RunResult) {
res.Usage.CompletionTokens, res.Usage.CompletionTokens,
res.Usage.TotalTokens, res.Usage.TotalTokens,
) )
if res.Usage.CachedTokens != 0 || res.Usage.CacheWriteTokens != 0 {
fmt.Fprintf(stderr, " cached_tokens=%d cache_write_tokens=%d", res.Usage.CachedTokens, res.Usage.CacheWriteTokens)
}
fmt.Fprintln(stderr)
} }
func printUsage(w io.Writer) { func printUsage(w io.Writer) {

View File

@@ -11,6 +11,7 @@ import (
"net/http/httptest" "net/http/httptest"
"os" "os"
"path/filepath" "path/filepath"
"reflect"
"strings" "strings"
"sync/atomic" "sync/atomic"
"testing" "testing"
@@ -410,6 +411,28 @@ defaults:
} }
} }
func TestParseRenderArgsExplicitFormatOverridesConfigDefaultFormat(t *testing.T) {
configPath := writeAppConfigFile(t, `
prompt_dir: ./from-config/prompts
profile_dir: ./from-config/profiles
defaults:
render_format: json
`)
cfg, err := parseRenderArgs([]string{
"--config", configPath,
"--prompt", "p",
"--input", "a=b",
"--format", "text",
})
if err != nil {
t.Fatalf("expected valid args, got %v", err)
}
if cfg.outputFormat != renderformat.PreparedRunFormatText {
t.Fatalf("expected explicit --format text to override config default, got %q", cfg.outputFormat)
}
}
func TestParseServeArgsWithExplicitConfigLoadsSettingsAndCLIAddrOverrides(t *testing.T) { func TestParseServeArgsWithExplicitConfigLoadsSettingsAndCLIAddrOverrides(t *testing.T) {
configPath := writeAppConfigFile(t, ` configPath := writeAppConfigFile(t, `
prompt_dir: ./from-config/prompts prompt_dir: ./from-config/prompts
@@ -471,6 +494,59 @@ server:
} }
} }
func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *testing.T) {
runCfg, err := parseRunArgs([]string{
"--prompt-dir", "./prompts",
"--profile-dir", "./profiles",
"--prompt", "prompt-1",
"--profile", "profile-1",
"--input", "transcript=./transcript.md",
"--var", "session_date=2026-05-01",
"--llm-base-url", "http://localhost:8000/v1",
"--model", "model-x",
"--temperature", "0.8",
"--max-tokens", "123",
"--top-p", "0.6",
"--timeout", "90s",
"--api-key-env", "SCRIPTORIUM_API_KEY",
})
if err != nil {
t.Fatalf("expected valid run args, got %v", err)
}
renderCfg, err := parseRenderArgs([]string{
"--prompt-dir", "./prompts",
"--profile-dir", "./profiles",
"--prompt", "prompt-1",
"--profile", "profile-1",
"--input", "transcript=./transcript.md",
"--var", "session_date=2026-05-01",
"--llm-base-url", "http://localhost:8000/v1",
"--model", "model-x",
"--temperature", "0.8",
"--max-tokens", "123",
"--top-p", "0.6",
"--timeout", "90s",
"--api-key-env", "SCRIPTORIUM_API_KEY",
})
if err != nil {
t.Fatalf("expected valid render args, got %v", err)
}
runReq, err := buildRunRequestFromConfig(runCfg)
if err != nil {
t.Fatalf("expected run request build success, got %v", err)
}
renderReq, err := buildRunRequestFromConfig(&renderCfg.runConfig)
if err != nil {
t.Fatalf("expected render request build success, got %v", err)
}
if !reflect.DeepEqual(runReq, renderReq) {
t.Fatalf("expected run/render shared flag requests to match.\nrun=%#v\nrender=%#v", runReq, renderReq)
}
}
func TestParseRunArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) { func TestParseRunArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) {
configPath := writeAppConfigFile(t, ` configPath := writeAppConfigFile(t, `
profile_dir: ./profiles profile_dir: ./profiles
@@ -586,42 +662,29 @@ func TestRunCommandVarsOptional(t *testing.T) {
} }
func TestRunCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) { func TestRunCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil {
t.Fatal(err)
}
ts := newTestLLMServer("from-config-dirs", nil) ts := newTestLLMServer("from-config-dirs", nil)
defer ts.Close() defer ts.Close()
writePromptFile(t, promptDir, "prompt.default", "local-default") writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
writeProfileFile(t, profileDir, "local-default", ts.URL+"/v1", "profile-model") writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model")
configPath := writeAppConfigFile(t, fmt.Sprintf(` configPath := writeAppConfigFile(t, fmt.Sprintf(`
prompt_dir: %s prompt_dir: %s
profile_dir: %s profile_dir: %s
`, promptDir, profileDir)) `, lib.promptDir, lib.profileDir))
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, runCommand, []string{
var stderr bytes.Buffer
code := runCommand([]string{
"--config", configPath, "--config", configPath,
"--prompt", "prompt.default", "--prompt", "prompt.default",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
if stdout.String() != "from-config-dirs" { if stdout != "from-config-dirs" {
t.Fatalf("unexpected stdout output: %q", stdout.String()) t.Fatalf("unexpected stdout output: %q", stdout)
} }
} }
@@ -630,28 +693,15 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te
const secret = "super-secret-render-key" const secret = "super-secret-render-key"
t.Setenv(envName, secret) t.Setenv(envName, secret)
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello transcript"), 0o644); err != nil {
t.Fatal(err)
}
writePromptFileWithTemplate(t, promptDir, "prompt.render", "local-default", "Date {{.session_date}} - Summarize: {{input \"transcript\"}}") writePromptFileWithTemplate(t, lib.promptDir, "prompt.render", "local-default", "Date {{.session_date}} - Summarize: {{input \"transcript\"}}")
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, renderCommand, []string{
var stderr bytes.Buffer "--prompt-dir", lib.promptDir,
code := renderCommand([]string{ "--profile-dir", lib.profileDir,
"--prompt-dir", promptDir,
"--profile-dir", profileDir,
"--prompt", "prompt.render", "--prompt", "prompt.render",
"--profile", "local-default", "--profile", "local-default",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
@@ -663,15 +713,15 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te
"--top-p", "0.2", "--top-p", "0.2",
"--timeout", "20s", "--timeout", "20s",
"--api-key-env", envName, "--api-key-env", envName,
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
if stderr.Len() != 0 { if stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", stderr.String()) t.Fatalf("expected empty stderr on success, got %q", stderr)
} }
out := stdout.String() out := stdout
for _, want := range []string{ for _, want := range []string{
"prompt: prompt.render", "prompt: prompt.render",
"selected_profile_id: local-default", "selected_profile_id: local-default",
@@ -697,112 +747,102 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te
} }
} }
func TestRenderCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) { func TestRenderCommandExplicitZeroTemperatureReachesEffectiveSettings(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil { writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
t.Fatal(err) profile := `id: local-default
} endpoint: http://127.0.0.1:1/v1
if err := os.MkdirAll(profileDir, 0o755); err != nil { model: profile-model
t.Fatal(err) temperature: 0.7
} `
inputPath := filepath.Join(tmp, "transcript.md") if err := os.WriteFile(filepath.Join(lib.profileDir, "local-default.yaml"), []byte(profile), 0o644); err != nil {
if err := os.WriteFile(inputPath, []byte("hello transcript"), 0o644); err != nil { t.Fatalf("failed to write profile fixture: %v", err)
t.Fatal(err)
} }
writePromptFile(t, promptDir, "prompt.render", "local-default") code, stdout, stderr := runCLICommand(t, renderCommand, []string{
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") "--prompt-dir", lib.promptDir,
"--profile-dir", lib.profileDir,
"--prompt", "prompt.render",
"--input", "transcript=" + inputPath,
"--temperature", "0",
})
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
if !strings.Contains(stdout, "\n temperature: 0\n") {
t.Fatalf("expected explicit zero temperature in effective settings, got:\n%s", stdout)
}
}
func TestRenderCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
lib := newCLITestLibrary(t)
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
configPath := writeAppConfigFile(t, fmt.Sprintf(` configPath := writeAppConfigFile(t, fmt.Sprintf(`
prompt_dir: %s prompt_dir: %s
profile_dir: %s profile_dir: %s
`, promptDir, profileDir)) `, lib.promptDir, lib.profileDir))
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, renderCommand, []string{
var stderr bytes.Buffer
code := renderCommand([]string{
"--config", configPath, "--config", configPath,
"--prompt", "prompt.render", "--prompt", "prompt.render",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
if !strings.Contains(stdout.String(), "prompt: prompt.render") { if !strings.Contains(stdout, "prompt: prompt.render") {
t.Fatalf("expected rendered output, got %q", stdout.String()) t.Fatalf("expected rendered output, got %q", stdout)
} }
} }
func TestRenderCommandExplicitTextFormatWorks(t *testing.T) { func TestRenderCommandExplicitTextFormatWorks(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello transcript"), 0o644); err != nil {
t.Fatal(err)
}
writePromptFile(t, promptDir, "prompt.render", "local-default") writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, renderCommand, []string{
var stderr bytes.Buffer "--prompt-dir", lib.promptDir,
code := renderCommand([]string{ "--profile-dir", lib.profileDir,
"--prompt-dir", promptDir,
"--profile-dir", profileDir,
"--prompt", "prompt.render", "--prompt", "prompt.render",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
"--format", "text", "--format", "text",
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
if !strings.Contains(stdout.String(), "prompt: prompt.render") { if !strings.Contains(stdout, "prompt: prompt.render") {
t.Fatalf("expected text output for explicit --format text, got %q", stdout.String()) t.Fatalf("expected text output for explicit --format text, got %q", stdout)
} }
} }
func TestRenderCommandExplicitJSONFormatOutputsValidJSON(t *testing.T) { func TestRenderCommandExplicitJSONFormatOutputsValidJSON(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello transcript"), 0o644); err != nil {
t.Fatal(err)
}
writePromptFile(t, promptDir, "prompt.render", "local-default") writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, renderCommand, []string{
var stderr bytes.Buffer "--prompt-dir", lib.promptDir,
code := renderCommand([]string{ "--profile-dir", lib.profileDir,
"--prompt-dir", promptDir,
"--profile-dir", profileDir,
"--prompt", "prompt.render", "--prompt", "prompt.render",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
"--format", "json", "--format", "json",
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
var payload map[string]any var payload map[string]any
if err := json.Unmarshal(stdout.Bytes(), &payload); err != nil { if err := json.Unmarshal([]byte(stdout), &payload); err != nil {
t.Fatalf("expected valid json output, got %v\nbody=%s", err, stdout.String()) t.Fatalf("expected valid json output, got %v\nbody=%s", err, stdout)
} }
if payload["prompt_id"] != "prompt.render" { if payload["prompt_id"] != "prompt.render" {
t.Fatalf("expected prompt_id, got %#v", payload["prompt_id"]) t.Fatalf("expected prompt_id, got %#v", payload["prompt_id"])
@@ -834,38 +874,25 @@ func TestRenderCommandUnknownFormatFailsClearly(t *testing.T) {
} }
func TestRenderCommandOutWritesToFile(t *testing.T) { func TestRenderCommandOutWritesToFile(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
profileDir := filepath.Join(tmp, "profiles") outPath := filepath.Join(lib.rootDir, "render.txt")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello transcript"), 0o644); err != nil {
t.Fatal(err)
}
outPath := filepath.Join(tmp, "render.txt")
writePromptFile(t, promptDir, "prompt.render", "local-default") writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, renderCommand, []string{
var stderr bytes.Buffer "--prompt-dir", lib.promptDir,
code := renderCommand([]string{ "--profile-dir", lib.profileDir,
"--prompt-dir", promptDir,
"--profile-dir", profileDir,
"--prompt", "prompt.render", "--prompt", "prompt.render",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
"--out", outPath, "--out", outPath,
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
if stdout.Len() != 0 { if stdout != "" {
t.Fatalf("expected empty stdout when --out is set, got %q", stdout.String()) t.Fatalf("expected empty stdout when --out is set, got %q", stdout)
} }
out, err := os.ReadFile(outPath) out, err := os.ReadFile(outPath)
if err != nil { if err != nil {
@@ -877,36 +904,23 @@ func TestRenderCommandOutWritesToFile(t *testing.T) {
} }
func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) { func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil {
t.Fatal(err)
}
writePromptFile(t, promptDir, "prompt.default", "local-default") writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model")
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, renderCommand, []string{
var stderr bytes.Buffer "--prompt-dir", lib.promptDir,
code := renderCommand([]string{ "--profile-dir", lib.profileDir,
"--prompt-dir", promptDir,
"--profile-dir", profileDir,
"--prompt", "prompt.default", "--prompt", "prompt.default",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
out := stdout.String() out := stdout
if !strings.Contains(out, "selected_profile_id: local-default") { if !strings.Contains(out, "selected_profile_id: local-default") {
t.Fatalf("expected prompt default profile in output, got %q", out) t.Fatalf("expected prompt default profile in output, got %q", out)
} }
@@ -916,38 +930,25 @@ func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
} }
func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) { func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil {
t.Fatal(err)
}
writePromptFile(t, promptDir, "prompt.default", "local-default") writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model")
writeProfileFile(t, profileDir, "quality", "http://127.0.0.1:1/v1", "quality-model") writeProfileFile(t, lib.profileDir, "quality", "http://127.0.0.1:1/v1", "quality-model")
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, renderCommand, []string{
var stderr bytes.Buffer "--prompt-dir", lib.promptDir,
code := renderCommand([]string{ "--profile-dir", lib.profileDir,
"--prompt-dir", promptDir,
"--profile-dir", profileDir,
"--prompt", "prompt.default", "--prompt", "prompt.default",
"--profile", "quality", "--profile", "quality",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
out := stdout.String() out := stdout
if !strings.Contains(out, "selected_profile_id: quality") { if !strings.Contains(out, "selected_profile_id: quality") {
t.Fatalf("expected explicit profile in output, got %q", out) t.Fatalf("expected explicit profile in output, got %q", out)
} }
@@ -957,104 +958,67 @@ func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
} }
func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) { func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil {
t.Fatal(err)
}
ts := newTestLLMServer("default-output", nil) ts := newTestLLMServer("default-output", nil)
defer ts.Close() defer ts.Close()
writePromptFile(t, promptDir, "prompt.default", "local-default") writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
writeProfileFile(t, profileDir, "local-default", ts.URL+"/v1", "profile-model") writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model")
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, runCommand, []string{
var stderr bytes.Buffer "--prompt-dir", lib.promptDir,
code := runCommand([]string{ "--profile-dir", lib.profileDir,
"--prompt-dir", promptDir,
"--profile-dir", profileDir,
"--prompt", "prompt.default", "--prompt", "prompt.default",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
if stdout.String() != "default-output" { if stdout != "default-output" {
t.Fatalf("unexpected stdout output: %q", stdout.String()) t.Fatalf("unexpected stdout output: %q", stdout)
} }
if !strings.Contains(stderr.String(), "selected_profile=local-default") { if !strings.Contains(stderr, "selected_profile=local-default") {
t.Fatalf("expected selected profile in summary, got %q", stderr.String()) t.Fatalf("expected selected profile in summary, got %q", stderr)
} }
} }
func TestRunCommandExplicitProfileOverridesPromptDefault(t *testing.T) { func TestRunCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil {
t.Fatal(err)
}
defaultServer := newTestLLMServer("from-default", nil) defaultServer := newTestLLMServer("from-default", nil)
defer defaultServer.Close() defer defaultServer.Close()
overrideServer := newTestLLMServer("from-override", nil) overrideServer := newTestLLMServer("from-override", nil)
defer overrideServer.Close() defer overrideServer.Close()
writePromptFile(t, promptDir, "prompt.default", "local-default") writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
writeProfileFile(t, profileDir, "local-default", defaultServer.URL+"/v1", "default-model") writeProfileFile(t, lib.profileDir, "local-default", defaultServer.URL+"/v1", "default-model")
writeProfileFile(t, profileDir, "quality", overrideServer.URL+"/v1", "quality-model") writeProfileFile(t, lib.profileDir, "quality", overrideServer.URL+"/v1", "quality-model")
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, runCommand, []string{
var stderr bytes.Buffer "--prompt-dir", lib.promptDir,
code := runCommand([]string{ "--profile-dir", lib.profileDir,
"--prompt-dir", promptDir,
"--profile-dir", profileDir,
"--prompt", "prompt.default", "--prompt", "prompt.default",
"--profile", "quality", "--profile", "quality",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
if stdout.String() != "from-override" { if stdout != "from-override" {
t.Fatalf("expected explicit profile output, got %q", stdout.String()) t.Fatalf("expected explicit profile output, got %q", stdout)
} }
if !strings.Contains(stderr.String(), "selected_profile=quality") { if !strings.Contains(stderr, "selected_profile=quality") {
t.Fatalf("expected selected profile quality, got %q", stderr.String()) t.Fatalf("expected selected profile quality, got %q", stderr)
} }
} }
func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) { func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) {
tmp := t.TempDir() lib := newCLITestLibrary(t)
promptDir := filepath.Join(tmp, "prompts") inputPath := lib.writeInputFile(t, "transcript.md", "hello")
profileDir := filepath.Join(tmp, "profiles")
if err := os.MkdirAll(promptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(profileDir, 0o755); err != nil {
t.Fatal(err)
}
inputPath := filepath.Join(tmp, "transcript.md")
if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil {
t.Fatal(err)
}
var baseHits int32 var baseHits int32
baseServer := newTestLLMServer("base", &baseHits) baseServer := newTestLLMServer("base", &baseHits)
@@ -1071,14 +1035,12 @@ func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) {
})) }))
defer overrideServer.Close() defer overrideServer.Close()
writePromptFile(t, promptDir, "prompt.default", "local-default") writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
writeProfileFile(t, profileDir, "local-default", baseServer.URL+"/v1", "profile-model") writeProfileFile(t, lib.profileDir, "local-default", baseServer.URL+"/v1", "profile-model")
var stdout bytes.Buffer code, stdout, stderr := runCLICommand(t, runCommand, []string{
var stderr bytes.Buffer "--prompt-dir", lib.promptDir,
code := runCommand([]string{ "--profile-dir", lib.profileDir,
"--prompt-dir", promptDir,
"--profile-dir", profileDir,
"--prompt", "prompt.default", "--prompt", "prompt.default",
"--input", "transcript=" + inputPath, "--input", "transcript=" + inputPath,
"--llm-base-url", overrideServer.URL + "/v1", "--llm-base-url", overrideServer.URL + "/v1",
@@ -1087,9 +1049,9 @@ func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) {
"--max-tokens", "55", "--max-tokens", "55",
"--top-p", "0.2", "--top-p", "0.2",
"--timeout", "20s", "--timeout", "20s",
}, &stdout, &stderr) })
if code != ExitOK { if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
} }
if atomic.LoadInt32(&baseHits) != 0 { if atomic.LoadInt32(&baseHits) != 0 {
t.Fatalf("expected base profile endpoint not to be hit, got %d", baseHits) t.Fatalf("expected base profile endpoint not to be hit, got %d", baseHits)
@@ -1097,8 +1059,8 @@ func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) {
if atomic.LoadInt32(&overrideHits) != 1 { if atomic.LoadInt32(&overrideHits) != 1 {
t.Fatalf("expected override endpoint to be hit once, got %d", overrideHits) t.Fatalf("expected override endpoint to be hit once, got %d", overrideHits)
} }
if stdout.String() != "override" { if stdout != "override" {
t.Fatalf("unexpected stdout output: %q", stdout.String()) t.Fatalf("unexpected stdout output: %q", stdout)
} }
if !strings.Contains(observedBody, `"model":"override-model"`) { if !strings.Contains(observedBody, `"model":"override-model"`) {
t.Fatalf("expected override model in request body, got %s", observedBody) t.Fatalf("expected override model in request body, got %s", observedBody)
@@ -1131,6 +1093,81 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
if !strings.Contains(stderr.String(), "prompt=p@1") { if !strings.Contains(stderr.String(), "prompt=p@1") {
t.Fatalf("expected summary on stderr, got %q", stderr.String()) t.Fatalf("expected summary on stderr, got %q", stderr.String())
} }
if strings.Contains(stderr.String(), "cached_tokens=") || strings.Contains(stderr.String(), "cache_write_tokens=") {
t.Fatalf("expected zero cache usage to be omitted from summary, got %q", stderr.String())
}
}
func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) {
var stderr bytes.Buffer
printSummary(&stderr, &domain.RunResult{
PromptID: "p",
PromptVersion: "1",
SelectedProfileID: "exec",
ModelName: "m",
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic},
RenderedPromptHash: "h",
InputHashes: map[string]string{"in": "x"},
Usage: domain.TokenUsage{
PromptTokens: 10,
CompletionTokens: 5,
TotalTokens: 15,
CachedTokens: 0,
CacheWriteTokens: 3,
},
})
summary := stderr.String()
if !strings.Contains(summary, "usage=10/5/15") {
t.Fatalf("expected base usage summary, got %q", summary)
}
if !strings.Contains(summary, "cached_tokens=0 cache_write_tokens=3") {
t.Fatalf("expected cache usage in summary, got %q", summary)
}
}
type cliTestLibrary struct {
rootDir string
promptDir string
profileDir string
}
func newCLITestLibrary(t *testing.T) *cliTestLibrary {
t.Helper()
root := t.TempDir()
lib := &cliTestLibrary{
rootDir: root,
promptDir: filepath.Join(root, "prompts"),
profileDir: filepath.Join(root, "profiles"),
}
if err := os.MkdirAll(lib.promptDir, 0o755); err != nil {
t.Fatalf("failed to create prompt fixture directory: %v", err)
}
if err := os.MkdirAll(lib.profileDir, 0o755); err != nil {
t.Fatalf("failed to create profile fixture directory: %v", err)
}
return lib
}
func (l *cliTestLibrary) writeInputFile(t *testing.T, name, body string) string {
t.Helper()
path := filepath.Join(l.rootDir, name)
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("failed to create input fixture directory: %v", err)
}
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
t.Fatalf("failed to write input fixture: %v", err)
}
return path
}
func runCLICommand(t *testing.T, command func([]string, io.Writer, io.Writer) int, args []string) (int, string, string) {
t.Helper()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := command(args, &stdout, &stderr)
return code, stdout.String(), stderr.String()
} }
func writePromptFile(t *testing.T, dir, id, defaultProfile string) { func writePromptFile(t *testing.T, dir, id, defaultProfile string) {

View File

@@ -21,16 +21,16 @@ type inputRefDTO struct {
} }
type modelOverrideRequestDTO struct { type modelOverrideRequestDTO struct {
Endpoint string `json:"endpoint,omitempty"` Endpoint string `json:"endpoint,omitempty"`
Model string `json:"model,omitempty"` Model string `json:"model,omitempty"`
Temperature float64 `json:"temperature,omitempty"` Temperature *float64 `json:"temperature,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"` MaxTokens *int `json:"max_tokens,omitempty"`
TopP float64 `json:"top_p,omitempty"` TopP *float64 `json:"top_p,omitempty"`
TimeoutSeconds int `json:"timeout_seconds,omitempty"` TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
ServiceTier string `json:"service_tier,omitempty"` ServiceTier string `json:"service_tier,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"` ReasoningEffort string `json:"reasoning_effort,omitempty"`
APIKeyEnv string `json:"api_key_env,omitempty"` APIKeyEnv string `json:"api_key_env,omitempty"`
ExtraParams map[string]string `json:"extra_params,omitempty"` ExtraParams map[string]any `json:"extra_params,omitempty"`
} }
type runResponseDTO struct { type runResponseDTO struct {
@@ -70,22 +70,24 @@ type metadataDTO struct {
} }
type modelParamsDTO struct { type modelParamsDTO struct {
Endpoint string `json:"endpoint"` Endpoint string `json:"endpoint"`
Model string `json:"model"` Model string `json:"model"`
Temperature float64 `json:"temperature"` Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"` MaxTokens int `json:"max_tokens"`
TopP float64 `json:"top_p"` TopP float64 `json:"top_p"`
TimeoutSeconds int `json:"timeout_seconds"` TimeoutSeconds int `json:"timeout_seconds"`
ServiceTier string `json:"service_tier,omitempty"` ServiceTier string `json:"service_tier,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"` ReasoningEffort string `json:"reasoning_effort,omitempty"`
APIKeyEnv string `json:"api_key_env,omitempty"` APIKeyEnv string `json:"api_key_env,omitempty"`
ExtraParams map[string]string `json:"extra_params,omitempty"` ExtraParams map[string]any `json:"extra_params,omitempty"`
} }
type tokenUsageDTO struct { type tokenUsageDTO struct {
PromptTokens int `json:"prompt_tokens"` PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"` CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"` TotalTokens int `json:"total_tokens"`
CachedTokens int `json:"cached_tokens"`
CacheWriteTokens int `json:"cache_write_tokens"`
} }
type validationDTO struct { type validationDTO struct {

View File

@@ -61,20 +61,9 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
} }
var model *domain.ExecutionTarget var model *domain.ExecutionTargetOverride
if req.Model != nil { if req.Model != nil {
model = &domain.ExecutionTarget{ model = executionTargetOverrideFromModelOverrideDTO(req.Model)
Endpoint: req.Model.Endpoint,
Model: req.Model.Model,
Temperature: req.Model.Temperature,
MaxTokens: req.Model.MaxTokens,
TopP: req.Model.TopP,
TimeoutSeconds: req.Model.TimeoutSeconds,
ServiceTier: req.Model.ServiceTier,
ReasoningEffort: req.Model.ReasoningEffort,
APIKeyEnv: req.Model.APIKeyEnv,
ExtraParams: req.Model.ExtraParams,
}
} }
res, err := h.runner.Run(r.Context(), domain.RunRequest{ res, err := h.runner.Run(r.Context(), domain.RunRequest{
@@ -110,23 +99,14 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
SelectedProfileID: res.SelectedProfileID, SelectedProfileID: res.SelectedProfileID,
ModelName: res.ModelName, ModelName: res.ModelName,
Endpoint: res.Endpoint, Endpoint: res.Endpoint,
ModelParams: modelParamsDTO{ ModelParams: modelParamsDTOFromExecutionTarget(res.EffectiveModelParams),
Endpoint: res.EffectiveModelParams.Endpoint, InputHashes: res.InputHashes,
Model: res.EffectiveModelParams.Model,
Temperature: res.EffectiveModelParams.Temperature,
MaxTokens: res.EffectiveModelParams.MaxTokens,
TopP: res.EffectiveModelParams.TopP,
TimeoutSeconds: res.EffectiveModelParams.TimeoutSeconds,
ServiceTier: res.EffectiveModelParams.ServiceTier,
ReasoningEffort: res.EffectiveModelParams.ReasoningEffort,
APIKeyEnv: res.EffectiveModelParams.APIKeyEnv,
ExtraParams: res.EffectiveModelParams.ExtraParams,
},
InputHashes: res.InputHashes,
Usage: tokenUsageDTO{ Usage: tokenUsageDTO{
PromptTokens: res.Usage.PromptTokens, PromptTokens: res.Usage.PromptTokens,
CompletionTokens: res.Usage.CompletionTokens, CompletionTokens: res.Usage.CompletionTokens,
TotalTokens: res.Usage.TotalTokens, TotalTokens: res.Usage.TotalTokens,
CachedTokens: res.Usage.CachedTokens,
CacheWriteTokens: res.Usage.CacheWriteTokens,
}, },
StartTime: res.StartTime, StartTime: res.StartTime,
EndTime: res.EndTime, EndTime: res.EndTime,
@@ -143,6 +123,39 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, resp) writeJSON(w, http.StatusOK, resp)
} }
func executionTargetOverrideFromModelOverrideDTO(dto *modelOverrideRequestDTO) *domain.ExecutionTargetOverride {
if dto == nil {
return nil
}
return &domain.ExecutionTargetOverride{
Endpoint: dto.Endpoint,
Model: dto.Model,
Temperature: dto.Temperature,
MaxTokens: dto.MaxTokens,
TopP: dto.TopP,
TimeoutSeconds: dto.TimeoutSeconds,
ServiceTier: dto.ServiceTier,
ReasoningEffort: dto.ReasoningEffort,
APIKeyEnv: dto.APIKeyEnv,
ExtraParams: dto.ExtraParams,
}
}
func modelParamsDTOFromExecutionTarget(target domain.ExecutionTarget) modelParamsDTO {
return modelParamsDTO{
Endpoint: target.Endpoint,
Model: target.Model,
Temperature: target.Temperature,
MaxTokens: target.MaxTokens,
TopP: target.TopP,
TimeoutSeconds: target.TimeoutSeconds,
ServiceTier: target.ServiceTier,
ReasoningEffort: target.ReasoningEffort,
APIKeyEnv: target.APIKeyEnv,
ExtraParams: target.ExtraParams,
}
}
func mapValidation(v domain.ValidationResult) validationDTO { func mapValidation(v domain.ValidationResult) validationDTO {
return validationDTO{ return validationDTO{
Status: string(v.Status), Status: string(v.Status),
@@ -164,9 +177,9 @@ func mapRunError(err error) (int, string, string) {
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition" return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
case errors.Is(err, profile.ErrInvalidYAML), errors.Is(err, profile.ErrInvalidProfile): case errors.Is(err, profile.ErrInvalidYAML), errors.Is(err, profile.ErrInvalidProfile):
return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile" return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile"
case errors.Is(err, usecase.ErrInvalidRequest) && strings.Contains(err.Error(), "profile id is required either in request or prompt default_profile"): case errors.Is(err, usecase.ErrProfileRequired):
return http.StatusBadRequest, "profile_required", "profile_id is required when prompt default_profile is not set" return http.StatusBadRequest, "profile_required", "profile_id is required when prompt default_profile is not set"
case errors.Is(err, usecase.ErrInvalidRequest) && strings.Contains(err.Error(), "api key environment variable"): case errors.Is(err, usecase.ErrAPIKeyEnvMissing):
return http.StatusBadRequest, "api_key_env_missing", "api_key_env is set but the environment variable is missing" return http.StatusBadRequest, "api_key_env_missing", "api_key_env is set but the environment variable is missing"
case errors.Is(err, usecase.ErrInvalidRequest): case errors.Is(err, usecase.ErrInvalidRequest):
return http.StatusBadRequest, "invalid_request", "invalid run request" return http.StatusBadRequest, "invalid_request", "invalid run request"

View File

@@ -4,15 +4,16 @@ import (
"bytes" "bytes"
"context" "context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"reflect"
"strings" "strings"
"testing" "testing"
"time" "time"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile" "gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef" "gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase" "gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
@@ -32,6 +33,34 @@ func (f *fakeRunner) Run(ctx context.Context, req domain.RunRequest) (*domain.Ru
return f.result, nil return f.result, nil
} }
type handlerPromptRepo struct {
def *domain.PromptDefinition
}
func (r handlerPromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
return r.def, nil
}
type handlerProfileRepo struct {
profile *domain.ExecutionProfile
}
func (r handlerProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
return r.profile, nil
}
type handlerArtifactReader struct{}
func (handlerArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
return &domain.Artifact{Name: "input", Body: []byte("input"), Hash: "hash"}, nil
}
type handlerRenderer struct{}
func (handlerRenderer) Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
return &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, nil
}
func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) { func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
start := time.Now().UTC() start := time.Now().UTC()
end := start.Add(2 * time.Second) end := start.Add(2 * time.Second)
@@ -66,11 +95,17 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
APIKeyEnv: envName, APIKeyEnv: envName,
}, },
InputHashes: map[string]string{"transcript": "h1"}, InputHashes: map[string]string{"transcript": "h1"},
Usage: domain.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3}, Usage: domain.TokenUsage{
StartTime: start, PromptTokens: 1,
EndTime: end, CompletionTokens: 2,
Duration: 2 * time.Second, TotalTokens: 3,
RawOutput: "hello", CachedTokens: 4,
CacheWriteTokens: 5,
},
StartTime: start,
EndTime: end,
Duration: 2 * time.Second,
RawOutput: "hello",
}} }}
h := NewHandler(r) h := NewHandler(r)
@@ -111,6 +146,13 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" { if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" {
t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"]) t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"])
} }
usage := metadata["usage"].(map[string]any)
if usage["prompt_tokens"] != float64(1) || usage["completion_tokens"] != float64(2) || usage["total_tokens"] != float64(3) {
t.Fatalf("unexpected base usage metadata: %#v", usage)
}
if usage["cached_tokens"] != float64(4) || usage["cache_write_tokens"] != float64(5) {
t.Fatalf("unexpected cache usage metadata: %#v", usage)
}
modelParams := metadata["model_params"].(map[string]any) modelParams := metadata["model_params"].(map[string]any)
if modelParams["api_key_env"] != envName { if modelParams["api_key_env"] != envName {
t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"]) t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"])
@@ -134,7 +176,7 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
if r.last.Execution == nil || r.last.Execution.Model != "gpt-x" { if r.last.Execution == nil || r.last.Execution.Model != "gpt-x" {
t.Fatalf("expected model override, got %#v", r.last.Execution) t.Fatalf("expected model override, got %#v", r.last.Execution)
} }
if r.last.Execution.TimeoutSeconds != 120 { if r.last.Execution.TimeoutSeconds == nil || *r.last.Execution.TimeoutSeconds != 120 {
t.Fatalf("expected timeout_seconds override 120, got %#v", r.last.Execution) t.Fatalf("expected timeout_seconds override 120, got %#v", r.last.Execution)
} }
if r.last.Execution.ServiceTier != "flex" { if r.last.Execution.ServiceTier != "flex" {
@@ -171,6 +213,265 @@ func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
if metadata["selected_profile_id"] != "prompt-default" { if metadata["selected_profile_id"] != "prompt-default" {
t.Fatalf("expected selected_profile_id from result, got %#v", metadata["selected_profile_id"]) t.Fatalf("expected selected_profile_id from result, got %#v", metadata["selected_profile_id"])
} }
usage := metadata["usage"].(map[string]any)
if usage["cached_tokens"] != float64(0) || usage["cache_write_tokens"] != float64(0) {
t.Fatalf("expected zero cache usage fields to be included, got %#v", usage)
}
}
func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
r := &fakeRunner{result: &domain.RunResult{
Artifact: domain.Artifact{Body: []byte("ok")},
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
}}
h := NewHandler(r)
reqBody := `{
"prompt_id": "prompt-1",
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
"model": {
"endpoint": "http://override/v1",
"model": "override-model",
"temperature": 0.6,
"max_tokens": 250,
"top_p": 0.85,
"timeout_seconds": 33,
"service_tier": "flex",
"reasoning_effort": "medium",
"api_key_env": "SCRIPTORIUM_API_KEY",
"extra_params": {"provider_option":"on"}
}
}`
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(reqBody))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
}
if r.last.Execution == nil {
t.Fatalf("expected execution override in run request")
}
got := r.last.Execution
if got.Endpoint != "http://override/v1" ||
got.Model != "override-model" ||
got.ServiceTier != "flex" ||
got.ReasoningEffort != "medium" ||
got.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
t.Fatalf("unexpected mapped execution target: %+v", got)
}
if got.Temperature == nil || *got.Temperature != 0.6 {
t.Fatalf("unexpected mapped temperature: %#v", got.Temperature)
}
if got.MaxTokens == nil || *got.MaxTokens != 250 {
t.Fatalf("unexpected mapped max_tokens: %#v", got.MaxTokens)
}
if got.TopP == nil || *got.TopP != 0.85 {
t.Fatalf("unexpected mapped top_p: %#v", got.TopP)
}
if got.TimeoutSeconds == nil || *got.TimeoutSeconds != 33 {
t.Fatalf("unexpected mapped timeout_seconds: %#v", got.TimeoutSeconds)
}
if !reflect.DeepEqual(got.ExtraParams, map[string]any{"provider_option": "on"}) {
t.Fatalf("unexpected mapped extra_params: %#v", got.ExtraParams)
}
}
func TestHandlerModelOverrideAcceptsJSONCompatibleExtraParams(t *testing.T) {
r := &fakeRunner{result: &domain.RunResult{
Artifact: domain.Artifact{Body: []byte("ok")},
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
}}
h := NewHandler(r)
reqBody := `{
"prompt_id": "prompt-1",
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
"model": {
"extra_params": {
"string_value": "enabled",
"number_value": 42,
"boolean_value": true,
"object_value": {"nested": "value", "count": 2},
"array_value": ["first", 3, false]
}
}
}`
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(reqBody))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
}
if r.last.Execution == nil {
t.Fatal("expected execution override in run request")
}
want := map[string]any{
"string_value": "enabled",
"number_value": float64(42),
"boolean_value": true,
"object_value": map[string]any{"nested": "value", "count": float64(2)},
"array_value": []any{"first", float64(3), false},
}
if !reflect.DeepEqual(r.last.Execution.ExtraParams, want) {
t.Fatalf("unexpected mapped extra_params:\ngot=%#v\nwant=%#v", r.last.Execution.ExtraParams, want)
}
}
func TestHandlerModelOverrideExplicitZeroTemperatureMapsAsPresent(t *testing.T) {
r := &fakeRunner{result: &domain.RunResult{
Artifact: domain.Artifact{Body: []byte("ok")},
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0},
}}
h := NewHandler(r)
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
"prompt_id": "prompt-1",
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
"model": {"temperature": 0}
}`))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
}
if r.last.Execution == nil || r.last.Execution.Temperature == nil {
t.Fatalf("expected temperature override to be present, got %#v", r.last.Execution)
}
if *r.last.Execution.Temperature != 0 {
t.Fatalf("expected zero temperature override, got %v", *r.last.Execution.Temperature)
}
}
func TestHandlerModelOverrideOmittedTemperatureMapsAsAbsent(t *testing.T) {
r := &fakeRunner{result: &domain.RunResult{
Artifact: domain.Artifact{Body: []byte("ok")},
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0.7},
}}
h := NewHandler(r)
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
"prompt_id": "prompt-1",
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
"model": {"model": "override-model"}
}`))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
}
if r.last.Execution == nil {
t.Fatal("expected model override")
}
if r.last.Execution.Temperature != nil {
t.Fatalf("expected omitted temperature to remain absent, got %#v", r.last.Execution.Temperature)
}
var resp map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("invalid JSON response: %v", err)
}
metadata := resp["metadata"].(map[string]any)
params := metadata["model_params"].(map[string]any)
if params["temperature"] != 0.7 {
t.Fatalf("expected effective profile/default temperature in response, got %#v", params["temperature"])
}
}
func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing.T) {
r := &fakeRunner{result: &domain.RunResult{
Artifact: domain.Artifact{
Name: "output",
ContentType: "text/plain",
Body: []byte("ok"),
Size: 2,
Hash: "abc",
},
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
EffectiveModelParams: domain.ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "gpt-test",
Temperature: 0.4,
MaxTokens: 321,
TopP: 0.7,
TimeoutSeconds: 45,
ServiceTier: "priority",
ReasoningEffort: "high",
APIKeyEnv: "SCRIPTORIUM_API_KEY",
ExtraParams: map[string]any{
"provider_option": "on",
"number_value": 42,
"object_value": map[string]any{"nested": "value"},
},
},
}}
h := NewHandler(r)
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
}
var resp map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("invalid JSON response: %v", err)
}
metadata := resp["metadata"].(map[string]any)
params := metadata["model_params"].(map[string]any)
if params["endpoint"] != "http://llm/v1" {
t.Fatalf("unexpected endpoint: %#v", params["endpoint"])
}
if params["model"] != "gpt-test" {
t.Fatalf("unexpected model: %#v", params["model"])
}
if params["temperature"] != 0.4 {
t.Fatalf("unexpected temperature: %#v", params["temperature"])
}
if params["max_tokens"] != float64(321) {
t.Fatalf("unexpected max_tokens: %#v", params["max_tokens"])
}
if params["top_p"] != 0.7 {
t.Fatalf("unexpected top_p: %#v", params["top_p"])
}
if params["timeout_seconds"] != float64(45) {
t.Fatalf("unexpected timeout_seconds: %#v", params["timeout_seconds"])
}
if params["service_tier"] != "priority" {
t.Fatalf("unexpected service_tier: %#v", params["service_tier"])
}
if params["reasoning_effort"] != "high" {
t.Fatalf("unexpected reasoning_effort: %#v", params["reasoning_effort"])
}
if params["api_key_env"] != "SCRIPTORIUM_API_KEY" {
t.Fatalf("unexpected api_key_env: %#v", params["api_key_env"])
}
extraParams, ok := params["extra_params"].(map[string]any)
if !ok {
t.Fatalf("expected extra_params object, got %#v", params["extra_params"])
}
if extraParams["provider_option"] != "on" {
t.Fatalf("unexpected extra_params.provider_option: %#v", extraParams["provider_option"])
}
if extraParams["number_value"] != float64(42) {
t.Fatalf("unexpected extra_params.number_value: %#v", extraParams["number_value"])
}
objectValue, ok := extraParams["object_value"].(map[string]any)
if !ok || objectValue["nested"] != "value" {
t.Fatalf("unexpected extra_params.object_value: %#v", extraParams["object_value"])
}
} }
func TestHandlerInvalidJSON(t *testing.T) { func TestHandlerInvalidJSON(t *testing.T) {
@@ -205,6 +506,54 @@ func TestHandlerMissingPromptID(t *testing.T) {
} }
} }
func TestHandlerReservedExtraParamsThroughRunnerMapsToInvalidRequest(t *testing.T) {
llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{})
if err != nil {
t.Fatal(err)
}
runner := usecase.NewRunner(
handlerPromptRepo{def: &domain.PromptDefinition{
ID: "p",
Version: "1",
DefaultProfile: "exec",
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "hi"}},
OutputFormat: domain.FormatText,
Validation: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone},
}},
handlerProfileRepo{profile: &domain.ExecutionProfile{
ID: "exec",
Endpoint: "http://example.invalid/v1",
Model: "model",
}},
handlerArtifactReader{},
handlerRenderer{},
llmClient,
nil,
)
h := NewHandler(runner)
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
"prompt_id":"p",
"inputs":{"x":{"type":"file","uri":"a"}},
"model":{"extra_params":{"model":"collision"}}
}`))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusBadRequest {
t.Fatalf("expected 400, got %d body=%s", w.Code, w.Body.String())
}
var resp map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("invalid JSON response: %v", err)
}
errBody := resp["error"].(map[string]any)
if errBody["code"] != "invalid_request" {
t.Fatalf("expected invalid_request code, got %#v", errBody["code"])
}
}
func TestHandlerUsecaseErrorMapping(t *testing.T) { func TestHandlerUsecaseErrorMapping(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
@@ -216,10 +565,10 @@ func TestHandlerUsecaseErrorMapping(t *testing.T) {
}{ }{
{name: "prompt not found", err: wrap(usecase.ErrProfileLoad, promptdef.ErrPromptDefinitionNotFound), status: http.StatusNotFound, code: "prompt_not_found", message: "prompt definition not found"}, {name: "prompt not found", err: wrap(usecase.ErrProfileLoad, promptdef.ErrPromptDefinitionNotFound), status: http.StatusNotFound, code: "prompt_not_found", message: "prompt definition not found"},
{name: "prompt load invalid", err: wrap(usecase.ErrProfileLoad, promptdef.ErrInvalidPromptDefinition), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition"}, {name: "prompt load invalid", err: wrap(usecase.ErrProfileLoad, promptdef.ErrInvalidPromptDefinition), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition"},
{name: "missing profile/default", err: wrap(usecase.ErrInvalidRequest, errors.New("profile id is required either in request or prompt default_profile")), status: http.StatusBadRequest, code: "profile_required", message: "profile_id is required when prompt default_profile is not set"}, {name: "missing profile/default", err: wrap(usecase.ErrInvalidRequest, usecase.ErrProfileRequired), status: http.StatusBadRequest, code: "profile_required", message: "profile_id is required when prompt default_profile is not set"},
{name: "profile not found", err: wrap(usecase.ErrProfileLoad, profile.ErrProfileNotFound), status: http.StatusNotFound, code: "profile_not_found", message: "execution profile not found"}, {name: "profile not found", err: wrap(usecase.ErrProfileLoad, profile.ErrProfileNotFound), status: http.StatusNotFound, code: "profile_not_found", message: "execution profile not found"},
{name: "profile invalid", err: wrap(usecase.ErrProfileLoad, profile.ErrInvalidProfile), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile"}, {name: "profile invalid", err: wrap(usecase.ErrProfileLoad, profile.ErrInvalidProfile), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile"},
{name: "api key env missing", err: wrap(usecase.ErrInvalidRequest, errors.New(`api key environment variable "SCRIPTORIUM_API_KEY" is not set`)), status: http.StatusBadRequest, code: "api_key_env_missing", message: "api_key_env is set but the environment variable is missing"}, {name: "api key env missing", err: wrap(usecase.ErrInvalidRequest, usecase.ErrAPIKeyEnvMissing), status: http.StatusBadRequest, code: "api_key_env_missing", message: "api_key_env is set but the environment variable is missing"},
{name: "artifact", err: wrap(usecase.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"}, {name: "artifact", err: wrap(usecase.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"},
{name: "prompt render", err: wrap(usecase.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"}, {name: "prompt render", err: wrap(usecase.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"},
{name: "llm", err: wrap(usecase.ErrLLMGenerate, fmt.Errorf("llm failed")), status: http.StatusBadGateway, code: "llm_failed", message: "model generation request failed", avoidCause: "llm failed"}, {name: "llm", err: wrap(usecase.ErrLLMGenerate, fmt.Errorf("llm failed")), status: http.StatusBadGateway, code: "llm_failed", message: "model generation request failed", avoidCause: "llm failed"},

View File

@@ -46,8 +46,8 @@ func TestCompositeReader_Read(t *testing.T) {
t.Run("unsupported ref type", func(t *testing.T) { t.Run("unsupported ref type", func(t *testing.T) {
ref := domain.ArtifactRef{ ref := domain.ArtifactRef{
Type: domain.ArtifactRefS3, Type: domain.ArtifactRefType("unsupported"),
URI: "s3://bucket/key", URI: "unsupported://bucket/key",
} }
_, err := reader.Read(ctx, ref) _, err := reader.Read(ctx, ref)
if !errors.Is(err, ErrUnsupportedRefType) { if !errors.Is(err, ErrUnsupportedRefType) {

View File

@@ -10,7 +10,6 @@ type ArtifactRefType string
const ( const (
ArtifactRefInline ArtifactRefType = "inline" ArtifactRefInline ArtifactRefType = "inline"
ArtifactRefFile ArtifactRefType = "file" ArtifactRefFile ArtifactRefType = "file"
ArtifactRefS3 ArtifactRefType = "s3"
) )
// OutputFormat defines the desired format of the generated artifact. // OutputFormat defines the desired format of the generated artifact.
@@ -41,6 +40,24 @@ const (
ValidationSkipped ValidationStatus = "skipped" ValidationSkipped ValidationStatus = "skipped"
) )
// CacheControlType defines provider cache behavior for prompt content.
type CacheControlType string
const (
CacheControlEphemeral CacheControlType = "ephemeral"
)
const (
// SessionIDMaxLength is OpenRouter's documented maximum session_id length.
SessionIDMaxLength = 256
)
// CacheControl describes provider cache metadata attached to prompt content.
type CacheControl struct {
Type CacheControlType `yaml:"type" json:"type"`
TTL string `yaml:"ttl,omitempty" json:"ttl,omitempty"`
}
// RunRequest represents a request to generate a single artifact. // RunRequest represents a request to generate a single artifact.
type RunRequest struct { type RunRequest struct {
PromptID string PromptID string
@@ -48,7 +65,7 @@ type RunRequest struct {
ProfileID string ProfileID string
Inputs map[string]ArtifactRef Inputs map[string]ArtifactRef
Vars map[string]string Vars map[string]string
Execution *ExecutionTarget Execution *ExecutionTargetOverride
Validation *OutputContract Validation *OutputContract
Metadata map[string]string Metadata map[string]string
} }
@@ -78,19 +95,21 @@ type RunResult struct {
// PreparedRun contains pre-LLM execution state from the prepare/render phase. // PreparedRun contains pre-LLM execution state from the prepare/render phase.
// It must never include resolved API key values, model output, or validation data. // It must never include resolved API key values, model output, or validation data.
type PreparedRun struct { type PreparedRun struct {
PromptID string `json:"prompt_id"` PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version,omitempty"` PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"` PromptHash string `json:"prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id"` SelectedProfileID string `json:"selected_profile_id"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"` EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
OutputContract OutputContract `json:"output_contract"` TargetPresence ExecutionTargetPresence `json:"-"`
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"` OutputContract OutputContract `json:"output_contract"`
InputHashes map[string]string `json:"input_hashes,omitempty"` StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
RenderedPromptHash string `json:"rendered_prompt_hash"` InputHashes map[string]string `json:"input_hashes,omitempty"`
Messages []RenderedMessage `json:"messages"` SessionID string `json:"session_id,omitempty"`
StartTime time.Time `json:"start_time,omitempty"` RenderedPromptHash string `json:"rendered_prompt_hash"`
EndTime time.Time `json:"end_time,omitempty"` Messages []RenderedMessage `json:"messages"`
DurationMS int64 `json:"duration_ms,omitempty"` StartTime time.Time `json:"start_time,omitempty"`
EndTime time.Time `json:"end_time,omitempty"`
DurationMS int64 `json:"duration_ms,omitempty"`
} }
// ArtifactRef represents a reference to an input artifact. // ArtifactRef represents a reference to an input artifact.
@@ -116,6 +135,7 @@ type PromptDefinition struct {
Version string `yaml:"version"` Version string `yaml:"version"`
DefaultProfile string `yaml:"default_profile"` DefaultProfile string `yaml:"default_profile"`
Description string `yaml:"description"` Description string `yaml:"description"`
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
Inputs []PromptInput `yaml:"inputs"` Inputs []PromptInput `yaml:"inputs"`
Templates []PromptMessageTemplate `yaml:"templates"` Templates []PromptMessageTemplate `yaml:"templates"`
OutputFormat OutputFormat `yaml:"output_format"` OutputFormat OutputFormat `yaml:"output_format"`
@@ -132,38 +152,62 @@ type PromptInput struct {
// PromptMessageTemplate defines a template for a chat message. // PromptMessageTemplate defines a template for a chat message.
type PromptMessageTemplate struct { type PromptMessageTemplate struct {
Role string `yaml:"role"` Role string `yaml:"role"`
Content string `yaml:"content"` Content string `yaml:"content"`
ContentFile string `yaml:"content_file"` ContentFile string `yaml:"content_file"`
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
} }
// ExecutionProfile describes how and where to execute a model. // ExecutionProfile describes how and where to execute a model.
type ExecutionProfile struct { type ExecutionProfile struct {
ID string `yaml:"id"` ID string `yaml:"id"`
Endpoint string `yaml:"endpoint"` Endpoint string `yaml:"endpoint"`
Model string `yaml:"model"` Model string `yaml:"model"`
Temperature float64 `yaml:"temperature"` Temperature float64 `yaml:"temperature"`
MaxTokens int `yaml:"max_tokens"` MaxTokens int `yaml:"max_tokens"`
TopP float64 `yaml:"top_p"` TopP float64 `yaml:"top_p"`
TimeoutSeconds int `yaml:"timeout_seconds"` TimeoutSeconds int `yaml:"timeout_seconds"`
ServiceTier string `yaml:"service_tier"` ServiceTier string `yaml:"service_tier"`
ReasoningEffort string `yaml:"reasoning_effort"` ReasoningEffort string `yaml:"reasoning_effort"`
APIKeyEnv string `yaml:"api_key_env"` APIKeyEnv string `yaml:"api_key_env"`
ExtraParams map[string]string `yaml:"extra_params"` ExtraParams map[string]any `yaml:"extra_params"`
}
// ExecutionTargetOverride represents per-request runtime setting overrides.
type ExecutionTargetOverride struct {
Endpoint string `json:"endpoint,omitempty"`
Model string `json:"model,omitempty"`
Temperature *float64 `json:"temperature,omitempty"`
MaxTokens *int `json:"max_tokens,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
ServiceTier string `json:"service_tier,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"`
APIKeyEnv string `json:"api_key_env,omitempty"`
ExtraParams map[string]any `json:"extra_params,omitempty"`
}
// ExecutionTargetPresence tracks which effective runtime fields came from an
// explicit request override even when the resolved value is a zero value.
type ExecutionTargetPresence struct {
Temperature bool
MaxTokens bool
TopP bool
TimeoutSeconds bool
} }
// ExecutionTarget represents effective model runtime settings for a run. // ExecutionTarget represents effective model runtime settings for a run.
type ExecutionTarget struct { type ExecutionTarget struct {
Endpoint string `yaml:"endpoint" json:"endpoint"` Endpoint string `yaml:"endpoint" json:"endpoint"`
Model string `yaml:"model" json:"model"` Model string `yaml:"model" json:"model"`
Temperature float64 `yaml:"temperature" json:"temperature"` Temperature float64 `yaml:"temperature" json:"temperature"`
MaxTokens int `yaml:"max_tokens" json:"max_tokens"` MaxTokens int `yaml:"max_tokens" json:"max_tokens"`
TopP float64 `yaml:"top_p" json:"top_p"` TopP float64 `yaml:"top_p" json:"top_p"`
TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"` TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"`
ServiceTier string `yaml:"service_tier" json:"service_tier"` ServiceTier string `yaml:"service_tier" json:"service_tier"`
ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"` ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"`
APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"` APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"`
ExtraParams map[string]string `yaml:"extra_params" json:"extra_params"` ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
} }
// OutputContract defines the requirements for the output artifact. // OutputContract defines the requirements for the output artifact.
@@ -176,19 +220,22 @@ type OutputContract struct {
// RenderedPrompt represents the prompt after template application. // RenderedPrompt represents the prompt after template application.
type RenderedPrompt struct { type RenderedPrompt struct {
Messages []RenderedMessage `json:"messages"` SessionID string `json:"session_id,omitempty"`
Messages []RenderedMessage `json:"messages"`
} }
// RenderedMessage is a single message in a rendered prompt. // RenderedMessage is a single message in a rendered prompt.
type RenderedMessage struct { type RenderedMessage struct {
Role string `json:"role"` Role string `json:"role"`
Content string `json:"content"` Content string `json:"content"`
CacheControl *CacheControl `json:"cache_control,omitempty"`
} }
// GenerateRequest is the internal request passed to the LLM client. // GenerateRequest is the internal request passed to the LLM client.
type GenerateRequest struct { type GenerateRequest struct {
Prompt RenderedPrompt Prompt RenderedPrompt
Target ExecutionTarget Target ExecutionTarget
TargetPresence ExecutionTargetPresence
StructuredOutput *StructuredOutputSpec StructuredOutput *StructuredOutputSpec
} }
@@ -223,6 +270,8 @@ type TokenUsage struct {
PromptTokens int PromptTokens int
CompletionTokens int CompletionTokens int
TotalTokens int TotalTokens int
CachedTokens int
CacheWriteTokens int
} }
// ValidationResult represents the outcome of an output validation. // ValidationResult represents the outcome of an output validation.

View File

@@ -53,3 +53,88 @@ func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
} }
} }
} }
func TestPreparedRunJSONIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
prepared := PreparedRun{
PromptID: "prompt.id",
SelectedProfileID: "local-fast",
EffectiveModelParams: ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "gpt-test",
},
RenderedPromptHash: "rendered-hash",
Messages: []RenderedMessage{
{
Role: "system",
Content: "You are helpful.",
CacheControl: &CacheControl{
Type: CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "Summarize this."},
},
}
b, err := json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal failed: %v", err)
}
var decoded struct {
Messages []map[string]any `json:"messages"`
}
if err := json.Unmarshal(b, &decoded); err != nil {
t.Fatalf("unmarshal failed: %v", err)
}
if len(decoded.Messages) != 2 {
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
}
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
if !ok {
t.Fatalf("expected cache_control on first message, got %#v", decoded.Messages[0])
}
if cacheControl["type"] != string(CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
}
if _, ok := decoded.Messages[1]["cache_control"]; ok {
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
}
}
func TestPreparedRunJSONIncludesSessionIDOnlyWhenPresent(t *testing.T) {
prepared := PreparedRun{
PromptID: "prompt.id",
SelectedProfileID: "local-fast",
EffectiveModelParams: ExecutionTarget{
Endpoint: "http://llm/v1",
Model: "gpt-test",
},
SessionID: "session-123",
RenderedPromptHash: "rendered-hash",
Messages: []RenderedMessage{{Role: "user", Content: "Summarize this."}},
}
b, err := json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal failed: %v", err)
}
var decoded map[string]any
if err := json.Unmarshal(b, &decoded); err != nil {
t.Fatalf("unmarshal failed: %v", err)
}
if decoded["session_id"] != "session-123" {
t.Fatalf("expected session_id in prepared run JSON, got %#v", decoded["session_id"])
}
prepared.SessionID = ""
b, err = json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal failed: %v", err)
}
if strings.Contains(string(b), "session_id") {
t.Fatalf("expected empty session_id to be omitted, got %s", b)
}
}

View File

@@ -0,0 +1,54 @@
package filecatalog
import (
"context"
"os"
"path/filepath"
"sort"
"strings"
)
// FindYAMLFiles returns sorted full paths for .yaml and .yml files under root.
func FindYAMLFiles(ctx context.Context, root string) ([]string, error) {
var files []string
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if d.IsDir() {
return nil
}
if !isYAMLFile(d.Name()) {
return nil
}
files = append(files, path)
return nil
})
sort.Strings(files)
return files, err
}
// RelativePath computes a clean relative path from root to path.
func RelativePath(root string, path string) string {
rel, err := filepath.Rel(root, path)
if err != nil {
return filepath.Clean(path)
}
return filepath.Clean(rel)
}
// Stem strips .yaml or .yml from a file name.
func Stem(name string) string {
name = strings.TrimSuffix(name, ".yaml")
name = strings.TrimSuffix(name, ".yml")
return name
}
func isYAMLFile(name string) bool {
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
}

View File

@@ -0,0 +1,84 @@
package filecatalog
import (
"context"
"errors"
"os"
"path/filepath"
"reflect"
"testing"
)
func TestFindYAMLFilesNestedSortedAndFiltered(t *testing.T) {
root := t.TempDir()
mustWriteFile(t, filepath.Join(root, "z", "prompt.yml"), "id: z")
mustWriteFile(t, filepath.Join(root, "a", "profile.yaml"), "id: a")
mustWriteFile(t, filepath.Join(root, "a", "ignore.txt"), "not yaml")
mustWriteFile(t, filepath.Join(root, "b", "ignore.yaml.bak"), "not yaml")
got, err := FindYAMLFiles(context.Background(), root)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
want := []string{
filepath.Join(root, "a", "profile.yaml"),
filepath.Join(root, "z", "prompt.yml"),
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("expected sorted YAML files %v, got %v", want, got)
}
}
func TestFindYAMLFilesHonorsContextCancellation(t *testing.T) {
root := t.TempDir()
mustWriteFile(t, filepath.Join(root, "one.yaml"), "id: one")
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := FindYAMLFiles(ctx, root)
if !errors.Is(err, context.Canceled) {
t.Fatalf("expected context.Canceled, got %v", err)
}
}
func TestRelativePathNested(t *testing.T) {
root := t.TempDir()
path := filepath.Join(root, "nested", "profiles", "local.yaml")
got := RelativePath(root, path)
want := filepath.Join("nested", "profiles", "local.yaml")
if got != want {
t.Fatalf("expected relative path %q, got %q", want, got)
}
}
func TestStemStripsYAMLExtensions(t *testing.T) {
tests := []struct {
name string
in string
want string
}{
{name: "yaml", in: "prompt.yaml", want: "prompt"},
{name: "yml", in: "profile.yml", want: "profile"},
{name: "other", in: "file.txt", want: "file.txt"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if got := Stem(tc.in); got != tc.want {
t.Fatalf("expected %q, got %q", tc.want, got)
}
})
}
}
func mustWriteFile(t *testing.T, path string, content string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("failed to create directory: %v", err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatalf("failed to write file %q: %v", path, err)
}
}

View File

@@ -96,6 +96,9 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
if prepared.PromptHash != "" { if prepared.PromptHash != "" {
fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash) fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash)
} }
if prepared.SessionID != "" {
fmt.Fprintf(&b, "session_id: %s\n", prepared.SessionID)
}
fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash) fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash)
target := prepared.EffectiveModelParams target := prepared.EffectiveModelParams
@@ -123,7 +126,11 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
} }
sort.Strings(keys) sort.Strings(keys)
for _, k := range keys { for _, k := range keys {
fmt.Fprintf(&b, " %s: %s\n", k, target.ExtraParams[k]) renderedValue, err := formatExtraParamTextValue(target.ExtraParams[k])
if err != nil {
return nil, fmt.Errorf("failed to format extra_params.%s: %w", k, err)
}
fmt.Fprintf(&b, " %s: %s\n", k, renderedValue)
} }
} }
@@ -151,6 +158,13 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
messages := byRole[role] messages := byRole[role]
for i, msg := range messages { for i, msg := range messages {
fmt.Fprintf(&b, " - message: %d\n", i+1) fmt.Fprintf(&b, " - message: %d\n", i+1)
if msg.CacheControl != nil {
fmt.Fprintf(&b, " cache_control: %s", msg.CacheControl.Type)
if msg.CacheControl.TTL != "" {
fmt.Fprintf(&b, " ttl=%s", msg.CacheControl.TTL)
}
fmt.Fprintln(&b)
}
fmt.Fprintln(&b, " content: |") fmt.Fprintln(&b, " content: |")
content := msg.Content content := msg.Content
if content == "" { if content == "" {
@@ -165,3 +179,15 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
return b.Bytes(), nil return b.Bytes(), nil
} }
func formatExtraParamTextValue(value any) (string, error) {
if s, ok := value.(string); ok {
return s, nil
}
b, err := json.Marshal(value)
if err != nil {
return "", err
}
return string(b), nil
}

View File

@@ -49,6 +49,36 @@ func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) {
} }
} }
func TestTextFormatterRendersExtraParamsDeterministically(t *testing.T) {
prepared := samplePreparedRun()
prepared.EffectiveModelParams.ExtraParams = map[string]any{
"z_string": "enabled",
"b_number": 42,
"a_object": map[string]any{
"nested": "value",
"count": 2,
},
"c_array": []any{"first", 3, false},
}
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
s := string(out)
want := strings.Join([]string{
" extra_params:",
" a_object: {\"count\":2,\"nested\":\"value\"}",
" b_number: 42",
" c_array: [\"first\",3,false]",
" z_string: enabled",
}, "\n")
if !strings.Contains(s, want) {
t.Fatalf("expected deterministic extra_params block %q, got:\n%s", want, s)
}
}
func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) { func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
const secret = "super-secret-api-key" const secret = "super-secret-api-key"
t.Setenv("SCRIPTORIUM_API_KEY", secret) t.Setenv("SCRIPTORIUM_API_KEY", secret)
@@ -62,8 +92,80 @@ func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
} }
} }
func TestTextFormatterIncludesMessageCacheControlBeforeContent(t *testing.T) {
prepared := samplePreparedRun()
prepared.Messages = []domain.RenderedMessage{
{
Role: "system",
Content: "System guidance.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "Summarize the transcript."},
}
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
s := string(out)
if !strings.Contains(s, " system:\n - message: 1\n cache_control: ephemeral ttl=1h\n content: |") {
t.Fatalf("expected system message cache control before content, got:\n%s", s)
}
if strings.Count(s, "cache_control:") != 1 {
t.Fatalf("expected exactly one cache_control line, got:\n%s", s)
}
}
func TestTextFormatterIncludesSessionIDWhenPresent(t *testing.T) {
prepared := samplePreparedRun()
prepared.SessionID = "session-123"
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if !strings.Contains(string(out), "session_id: session-123\n") {
t.Fatalf("expected session_id in text output, got:\n%s", out)
}
}
func TestTextFormatterOmitsEmptyCacheControlTTL(t *testing.T) {
prepared := samplePreparedRun()
prepared.Messages = []domain.RenderedMessage{
{
Role: "system",
Content: "System guidance.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
},
},
}
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
s := string(out)
if !strings.Contains(s, " cache_control: ephemeral\n") {
t.Fatalf("expected cache_control line without ttl, got:\n%s", s)
}
if strings.Contains(s, "ttl=") {
t.Fatalf("expected empty ttl to be omitted, got:\n%s", s)
}
}
func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) { func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
prepared := samplePreparedRun() prepared := samplePreparedRun()
prepared.SessionID = "session-123"
prepared.EffectiveModelParams.ExtraParams = map[string]any{
"number": 42,
"nested": map[string]any{
"enabled": true,
},
}
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON) out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
if err != nil { if err != nil {
@@ -87,9 +189,24 @@ func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
if decoded["rendered_prompt_hash"] != "rendered-hash" { if decoded["rendered_prompt_hash"] != "rendered-hash" {
t.Fatalf("expected rendered_prompt_hash in json output, got %#v", decoded["rendered_prompt_hash"]) t.Fatalf("expected rendered_prompt_hash in json output, got %#v", decoded["rendered_prompt_hash"])
} }
if _, ok := decoded["effective_model_params"]; !ok { if decoded["session_id"] != "session-123" {
t.Fatalf("expected session_id in json output, got %#v", decoded["session_id"])
}
modelParams, ok := decoded["effective_model_params"].(map[string]any)
if !ok {
t.Fatalf("expected effective_model_params in json output, got %#v", decoded) t.Fatalf("expected effective_model_params in json output, got %#v", decoded)
} }
extraParams, ok := modelParams["extra_params"].(map[string]any)
if !ok {
t.Fatalf("expected extra_params in json output, got %#v", modelParams["extra_params"])
}
if extraParams["number"] != float64(42) {
t.Fatalf("unexpected numeric extra param in json output: %#v", extraParams["number"])
}
nested, ok := extraParams["nested"].(map[string]any)
if !ok || nested["enabled"] != true {
t.Fatalf("unexpected nested extra param in json output: %#v", extraParams["nested"])
}
if _, ok := decoded["input_hashes"]; !ok { if _, ok := decoded["input_hashes"]; !ok {
t.Fatalf("expected input_hashes in json output, got %#v", decoded) t.Fatalf("expected input_hashes in json output, got %#v", decoded)
} }
@@ -98,6 +215,47 @@ func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
} }
} }
func TestJSONFormatterIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
prepared := samplePreparedRun()
prepared.Messages = []domain.RenderedMessage{
{
Role: "system",
Content: "System guidance.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "Summarize the transcript."},
}
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
var decoded struct {
Messages []map[string]any `json:"messages"`
}
if err := json.Unmarshal(out, &decoded); err != nil {
t.Fatalf("expected valid json output, got %v", err)
}
if len(decoded.Messages) != 2 {
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
}
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
if !ok {
t.Fatalf("expected first message cache_control, got %#v", decoded.Messages[0])
}
if cacheControl["type"] != string(domain.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
}
if _, ok := decoded.Messages[1]["cache_control"]; ok {
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
}
}
func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) { func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
const secret = "super-secret-api-key" const secret = "super-secret-api-key"
t.Setenv("SCRIPTORIUM_API_KEY", secret) t.Setenv("SCRIPTORIUM_API_KEY", secret)

View File

@@ -12,6 +12,7 @@ import (
"os" "os"
"strings" "strings"
"time" "time"
"unicode/utf8"
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults" "gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
@@ -75,14 +76,6 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
return nil, fmt.Errorf("%w: timeout_seconds must be greater than or equal to 0", ErrInvalidRequest) return nil, fmt.Errorf("%w: timeout_seconds must be greater than or equal to 0", ErrInvalidRequest)
} }
model := strings.TrimSpace(req.Target.Model)
if model == "" {
model = strings.TrimSpace(c.defaultModel)
}
if model == "" {
return nil, fmt.Errorf("%w: model is required", ErrInvalidRequest)
}
endpoint := strings.TrimSpace(req.Target.Endpoint) endpoint := strings.TrimSpace(req.Target.Endpoint)
if endpoint == "" { if endpoint == "" {
endpoint = c.baseURL endpoint = c.baseURL
@@ -92,39 +85,17 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
} }
endpoint = strings.TrimRight(endpoint, "/") + defaults.OpenAIChatCompletionsPath endpoint = strings.TrimRight(endpoint, "/") + defaults.OpenAIChatCompletionsPath
wireReq := openAIChatRequest{ wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel)
Model: model, if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
} }
wireReq.Messages = make([]openAIChatMessage, 0, len(req.Prompt.Messages)) wirePayload, err := openAIChatRequestPayload(wireReq)
for _, msg := range req.Prompt.Messages { if err != nil {
wireReq.Messages = append(wireReq.Messages, openAIChatMessage{ return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
Role: msg.Role,
Content: msg.Content,
})
} }
if req.Target.Temperature != 0 { payload, err := json.Marshal(wirePayload)
wireReq.Temperature = &req.Target.Temperature
}
if req.Target.MaxTokens != 0 {
wireReq.MaxTokens = &req.Target.MaxTokens
}
if req.Target.TopP != 0 {
wireReq.TopP = &req.Target.TopP
}
if strings.TrimSpace(req.Target.ServiceTier) != "" {
wireReq.ServiceTier = req.Target.ServiceTier
}
if req.StructuredOutput != nil {
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
}
wireReq.ResponseFormat = responseFormat
}
payload, err := json.Marshal(wireReq)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err) return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
} }
@@ -145,6 +116,8 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
effectiveTimeout := c.timeout effectiveTimeout := c.timeout
if req.Target.TimeoutSeconds > 0 { if req.Target.TimeoutSeconds > 0 {
effectiveTimeout = time.Duration(req.Target.TimeoutSeconds) * time.Second effectiveTimeout = time.Duration(req.Target.TimeoutSeconds) * time.Second
} else if req.TargetPresence.TimeoutSeconds {
effectiveTimeout = 0
} }
httpClient := c.httpClient httpClient := c.httpClient
@@ -186,33 +159,166 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
PromptTokens: wireResp.Usage.PromptTokens, PromptTokens: wireResp.Usage.PromptTokens,
CompletionTokens: wireResp.Usage.CompletionTokens, CompletionTokens: wireResp.Usage.CompletionTokens,
TotalTokens: wireResp.Usage.TotalTokens, TotalTokens: wireResp.Usage.TotalTokens,
CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens,
CacheWriteTokens: wireResp.Usage.CacheWriteTokens,
}, },
}, nil }, nil
} }
type openAIChatRequest struct { func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) {
Model string `json:"model"` model := strings.TrimSpace(req.Target.Model)
Messages []openAIChatMessage `json:"messages"` if model == "" {
Temperature *float64 `json:"temperature,omitempty"` model = strings.TrimSpace(defaultModel)
MaxTokens *int `json:"max_tokens,omitempty"` }
TopP *float64 `json:"top_p,omitempty"` if model == "" {
ServiceTier string `json:"service_tier,omitempty"` return openAIChatRequest{}, errors.New("model is required")
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"` }
wireReq := openAIChatRequest{
Model: model,
}
if sessionID := strings.TrimSpace(req.Prompt.SessionID); sessionID != "" {
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
return openAIChatRequest{}, fmt.Errorf("session_id length %d exceeds maximum %d", n, domain.SessionIDMaxLength)
}
wireReq.SessionID = sessionID
}
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
for _, msg := range req.Prompt.Messages {
wireReq.Messages = append(wireReq.Messages, openAIChatRequestMessageFromRenderedMessage(msg))
}
if req.Target.Temperature != 0 || req.TargetPresence.Temperature {
wireReq.Temperature = &req.Target.Temperature
}
if req.Target.MaxTokens != 0 || req.TargetPresence.MaxTokens {
wireReq.MaxTokens = &req.Target.MaxTokens
}
if req.Target.TopP != 0 || req.TargetPresence.TopP {
wireReq.TopP = &req.Target.TopP
}
if strings.TrimSpace(req.Target.ServiceTier) != "" {
wireReq.ServiceTier = req.Target.ServiceTier
}
if strings.TrimSpace(req.Target.ReasoningEffort) != "" {
wireReq.ReasoningEffort = req.Target.ReasoningEffort
}
if len(req.Target.ExtraParams) > 0 {
wireReq.ExtraParams = req.Target.ExtraParams
}
if req.StructuredOutput != nil {
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
if err != nil {
return openAIChatRequest{}, err
}
wireReq.ResponseFormat = responseFormat
}
return wireReq, nil
} }
type openAIChatMessage struct { type openAIChatRequest struct {
Model string `json:"model"`
SessionID string `json:"session_id,omitempty"`
Messages []openAIChatRequestMessage `json:"messages"`
Temperature *float64 `json:"temperature,omitempty"`
MaxTokens *int `json:"max_tokens,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
ServiceTier string `json:"service_tier,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"`
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"`
ExtraParams map[string]any `json:"-"`
}
func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
out := map[string]any{
"model": req.Model,
"messages": req.Messages,
}
if req.SessionID != "" {
out["session_id"] = req.SessionID
}
if req.Temperature != nil {
out["temperature"] = *req.Temperature
}
if req.MaxTokens != nil {
out["max_tokens"] = *req.MaxTokens
}
if req.TopP != nil {
out["top_p"] = *req.TopP
}
if req.ServiceTier != "" {
out["service_tier"] = req.ServiceTier
}
if req.ReasoningEffort != "" {
out["reasoning_effort"] = req.ReasoningEffort
}
if req.ResponseFormat != nil {
out["response_format"] = req.ResponseFormat
}
for key, value := range req.ExtraParams {
if key == "" {
return nil, errors.New("extra_params key must not be empty")
}
if _, reserved := reservedOpenAIChatRequestFields[key]; reserved {
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
}
if _, err := json.Marshal(value); err != nil {
return nil, fmt.Errorf("extra_params.%s must be JSON-serializable: %w", key, err)
}
out[key] = value
}
return out, nil
}
var reservedOpenAIChatRequestFields = map[string]struct{}{
"model": {},
"session_id": {},
"messages": {},
"temperature": {},
"max_tokens": {},
"top_p": {},
"service_tier": {},
"reasoning_effort": {},
"response_format": {},
}
type openAIChatRequestMessage struct {
Role string `json:"role"`
Content any `json:"content"`
}
type openAIChatTextContentBlock struct {
Type string `json:"type"`
Text string `json:"text"`
CacheControl *openAICacheControl `json:"cache_control,omitempty"`
}
type openAICacheControl struct {
Type string `json:"type"`
TTL string `json:"ttl,omitempty"`
}
type openAIChatResponseMessage struct {
Role string `json:"role"` Role string `json:"role"`
Content string `json:"content"` Content string `json:"content"`
} }
type openAIChatResponse struct { type openAIChatResponse struct {
Choices []struct { Choices []struct {
Message openAIChatMessage `json:"message"` Message openAIChatResponseMessage `json:"message"`
} `json:"choices"` } `json:"choices"`
Usage struct { Usage struct {
PromptTokens int `json:"prompt_tokens"` PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"` CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"` TotalTokens int `json:"total_tokens"`
PromptTokensDetails struct {
CachedTokens int `json:"cached_tokens"`
} `json:"prompt_tokens_details"`
CacheWriteTokens int `json:"cache_write_tokens"`
} `json:"usage"` } `json:"usage"`
} }
@@ -227,6 +333,28 @@ type openAIJSONSchemaEnvelope struct {
Schema any `json:"schema"` Schema any `json:"schema"`
} }
func openAIChatRequestMessageFromRenderedMessage(msg domain.RenderedMessage) openAIChatRequestMessage {
wireMsg := openAIChatRequestMessage{
Role: msg.Role,
Content: msg.Content,
}
if msg.CacheControl == nil {
return wireMsg
}
wireMsg.Content = []openAIChatTextContentBlock{
{
Type: "text",
Text: msg.Content,
CacheControl: &openAICacheControl{
Type: string(msg.CacheControl.Type),
TTL: msg.CacheControl.TTL,
},
},
}
return wireMsg
}
func toOpenAIResponseFormat(spec *domain.StructuredOutputSpec) (*openAIResponseFormat, error) { func toOpenAIResponseFormat(spec *domain.StructuredOutputSpec) (*openAIResponseFormat, error) {
if spec == nil { if spec == nil {
return nil, nil return nil, nil

View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"math"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"strings" "strings"
@@ -89,6 +90,9 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
if resp.Usage.PromptTokens != 11 || resp.Usage.CompletionTokens != 22 || resp.Usage.TotalTokens != 33 { if resp.Usage.PromptTokens != 11 || resp.Usage.CompletionTokens != 22 || resp.Usage.TotalTokens != 33 {
t.Fatalf("unexpected usage: %+v", resp.Usage) t.Fatalf("unexpected usage: %+v", resp.Usage)
} }
if resp.Usage.CachedTokens != 0 || resp.Usage.CacheWriteTokens != 0 {
t.Fatalf("expected absent cache usage fields to remain zero, got %+v", resp.Usage)
}
if obs.Authorization != "Bearer secret-key" { if obs.Authorization != "Bearer secret-key" {
t.Fatalf("unexpected Authorization header: %q", obs.Authorization) t.Fatalf("unexpected Authorization header: %q", obs.Authorization)
@@ -96,6 +100,15 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
if got, ok := obs.Body["model"].(string); !ok || got != "gpt-test" { if got, ok := obs.Body["model"].(string); !ok || got != "gpt-test" {
t.Fatalf("unexpected model payload: %#v", obs.Body["model"]) t.Fatalf("unexpected model payload: %#v", obs.Body["model"])
} }
if got, ok := obs.Body["temperature"].(float64); !ok || got != 0.4 {
t.Fatalf("unexpected temperature payload: %#v", obs.Body["temperature"])
}
if got, ok := obs.Body["max_tokens"].(float64); !ok || got != 123 {
t.Fatalf("unexpected max_tokens payload: %#v", obs.Body["max_tokens"])
}
if got, ok := obs.Body["top_p"].(float64); !ok || got != 0.7 {
t.Fatalf("unexpected top_p payload: %#v", obs.Body["top_p"])
}
if got, ok := obs.Body["service_tier"].(string); !ok || got != "priority" { if got, ok := obs.Body["service_tier"].(string); !ok || got != "priority" {
t.Fatalf("unexpected service_tier payload: %#v", obs.Body["service_tier"]) t.Fatalf("unexpected service_tier payload: %#v", obs.Body["service_tier"])
} }
@@ -135,6 +148,245 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
} }
} }
func TestOpenAICompatibleClientSerializesCacheControlledMessageAsContentBlock(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "Stable instructions.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "Dynamic request."},
}},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
for _, forbidden := range []string{"cache_control", "extra_params"} {
if _, exists := observedBody[forbidden]; exists {
t.Fatalf("expected top-level %s to be omitted, got %#v", forbidden, observedBody[forbidden])
}
}
msgs, ok := observedBody["messages"].([]any)
if !ok || len(msgs) != 2 {
t.Fatalf("unexpected messages payload: %#v", observedBody["messages"])
}
msg0 := msgs[0].(map[string]any)
if msg0["role"] != "system" {
t.Fatalf("unexpected first message role: %#v", msg0["role"])
}
contentBlocks, ok := msg0["content"].([]any)
if !ok || len(contentBlocks) != 1 {
t.Fatalf("expected first message content block array, got %#v", msg0["content"])
}
block := contentBlocks[0].(map[string]any)
if block["type"] != "text" || block["text"] != "Stable instructions." {
t.Fatalf("unexpected text content block: %#v", block)
}
cacheControl, ok := block["cache_control"].(map[string]any)
if !ok {
t.Fatalf("expected cache_control on content block, got %#v", block)
}
if cacheControl["type"] != string(domain.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
}
msg1 := msgs[1].(map[string]any)
if msg1["role"] != "user" || msg1["content"] != "Dynamic request." {
t.Fatalf("expected uncached message to keep string content, got %#v", msg1)
}
}
func TestOpenAICompatibleClientOmitsEmptyCacheControlTTL(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "Stable instructions.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
},
},
}},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
msgs := observedBody["messages"].([]any)
msg0 := msgs[0].(map[string]any)
contentBlocks := msg0["content"].([]any)
block := contentBlocks[0].(map[string]any)
cacheControl := block["cache_control"].(map[string]any)
if cacheControl["type"] != string(domain.CacheControlEphemeral) {
t.Fatalf("unexpected cache_control type: %#v", cacheControl)
}
if _, exists := cacheControl["ttl"]; exists {
t.Fatalf("expected empty ttl to be omitted, got %#v", cacheControl)
}
}
func TestOpenAICompatibleClientSerializesSessionID(t *testing.T) {
var observedBody map[string]any
var observedSessionHeader string
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
observedSessionHeader = r.Header.Get("x-session-id")
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{
SessionID: " session-123 ",
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if observedBody["session_id"] != "session-123" {
t.Fatalf("expected top-level session_id, got %#v", observedBody["session_id"])
}
if observedSessionHeader != "" {
t.Fatalf("did not expect x-session-id header, got %q", observedSessionHeader)
}
}
func TestOpenAICompatibleClientOmitsEmptySessionID(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{
SessionID: " ",
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if _, exists := observedBody["session_id"]; exists {
t.Fatalf("expected empty session_id to be omitted, got %#v", observedBody["session_id"])
}
}
func TestOpenAICompatibleClientRejectsTooLongSessionID(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "http://example.com/v1",
Model: "model",
})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{
SessionID: strings.Repeat("x", domain.SessionIDMaxLength+1),
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
},
})
if err == nil {
t.Fatal("expected invalid request error")
}
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
}
func TestOpenAICompatibleClientParsesCacheUsage(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{
"choices": [{"message": {"role": "assistant", "content": "ok"}}],
"usage": {
"prompt_tokens": 100,
"completion_tokens": 20,
"total_tokens": 120,
"prompt_tokens_details": {"cached_tokens": 80},
"cache_write_tokens": 60
}
}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "model"})
if err != nil {
t.Fatal(err)
}
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if resp.Usage.PromptTokens != 100 || resp.Usage.CompletionTokens != 20 || resp.Usage.TotalTokens != 120 {
t.Fatalf("unexpected base usage fields: %+v", resp.Usage)
}
if resp.Usage.CachedTokens != 80 || resp.Usage.CacheWriteTokens != 60 {
t.Fatalf("unexpected cache usage fields: %+v", resp.Usage)
}
}
func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *testing.T) { func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *testing.T) {
var observedBody map[string]any var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -166,6 +418,281 @@ func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *test
} }
} }
func TestOpenAICompatibleClientSerializesReasoningEffortAndExtraParams(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{
Model: "model",
ReasoningEffort: "high",
ExtraParams: map[string]any{
"string_value": "on",
"number_value": 42,
"boolean_value": true,
"object_value": map[string]any{"nested": "value", "count": 2},
"array_value": []any{"first", 3, false},
},
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if observedBody["reasoning_effort"] != "high" {
t.Fatalf("expected reasoning_effort high, got %#v", observedBody["reasoning_effort"])
}
if observedBody["string_value"] != "on" {
t.Fatalf("unexpected string extra param: %#v", observedBody["string_value"])
}
if observedBody["number_value"] != float64(42) {
t.Fatalf("unexpected number extra param: %#v", observedBody["number_value"])
}
if observedBody["boolean_value"] != true {
t.Fatalf("unexpected boolean extra param: %#v", observedBody["boolean_value"])
}
objectValue, ok := observedBody["object_value"].(map[string]any)
if !ok || objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
t.Fatalf("unexpected object extra param: %#v", observedBody["object_value"])
}
if _, exists := observedBody["extra_params"]; exists {
t.Fatalf("expected extra_params wrapper omitted, got %#v", observedBody["extra_params"])
}
arrayValue, ok := observedBody["array_value"].([]any)
if !ok || len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
t.Fatalf("unexpected array extra param: %#v", observedBody["array_value"])
}
}
func TestOpenAICompatibleClientOmitsReasoningEffortWhenUnset(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if _, exists := observedBody["reasoning_effort"]; exists {
t.Fatalf("expected reasoning_effort omitted, got %#v", observedBody["reasoning_effort"])
}
if _, exists := observedBody["extra_params"]; exists {
t.Fatalf("expected extra_params wrapper omitted, got %#v", observedBody["extra_params"])
}
}
func TestOpenAICompatibleClientSerializesExplicitZeroNumericOverrides(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model"},
TargetPresence: domain.ExecutionTargetPresence{
Temperature: true,
MaxTokens: true,
TopP: true,
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if observedBody["temperature"] != float64(0) {
t.Fatalf("expected explicit zero temperature, got %#v", observedBody["temperature"])
}
if observedBody["max_tokens"] != float64(0) {
t.Fatalf("expected explicit zero max_tokens, got %#v", observedBody["max_tokens"])
}
if observedBody["top_p"] != float64(0) {
t.Fatalf("expected explicit zero top_p, got %#v", observedBody["top_p"])
}
}
func TestOpenAICompatibleClientOmitsImplicitZeroNumericFields(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
t.Fatalf("failed to decode request body: %v", err)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model"},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
for _, field := range []string{"temperature", "max_tokens", "top_p"} {
if _, exists := observedBody[field]; exists {
t.Fatalf("expected implicit zero field %q to be omitted, got body %#v", field, observedBody)
}
}
}
func TestOpenAICompatibleClientExplicitZeroTimeoutDisablesClientTimeout(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(20 * time.Millisecond)
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: ts.URL + "/v1",
Timeout: time.Nanosecond,
})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model", TimeoutSeconds: 0},
TargetPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true},
})
if err != nil {
t.Fatalf("expected explicit zero timeout to disable client timeout, got %v", err)
}
}
func TestOpenAICompatibleClientOmittedTimeoutUsesClientTimeout(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(20 * time.Millisecond)
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: ts.URL + "/v1",
Timeout: time.Nanosecond,
})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model", TimeoutSeconds: 0},
})
if err == nil {
t.Fatal("expected omitted timeout to use client timeout")
}
if !errors.Is(err, ErrRequestFailed) {
t.Fatalf("expected ErrRequestFailed, got %v", err)
}
}
func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *testing.T) {
tests := []struct {
name string
extraParams map[string]any
want string
}{
{name: "empty key", extraParams: map[string]any{"": "empty"}, want: "key must not be empty"},
{name: "unserializable value", extraParams: map[string]any{"bad": math.Inf(1)}, want: "JSON-serializable"},
}
for _, key := range []string{
"model",
"session_id",
"messages",
"temperature",
"max_tokens",
"top_p",
"service_tier",
"reasoning_effort",
"response_format",
} {
tests = append(tests, struct {
name string
extraParams map[string]any
want string
}{
name: "reserved key " + key,
extraParams: map[string]any{key: "collision"},
want: "reserved request field",
})
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
called := false
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
called = true
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model", ExtraParams: tc.extraParams},
})
if err == nil {
t.Fatal("expected invalid request error")
}
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
if !strings.Contains(err.Error(), tc.want) {
t.Fatalf("expected error to contain %q, got %v", tc.want, err)
}
if called {
t.Fatal("provider should not be called for invalid extra_params")
}
})
}
}
func TestOpenAICompatibleClientNoAuthorizationHeaderWhenNoAPIKey(t *testing.T) { func TestOpenAICompatibleClientNoAuthorizationHeaderWhenNoAPIKey(t *testing.T) {
hadAuth := false hadAuth := false
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

View File

@@ -7,10 +7,10 @@ import (
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"sort"
"strings" "strings"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/filecatalog"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
@@ -34,7 +34,7 @@ func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*doma
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile) return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
} }
files, err := r.yamlFiles(ctx) files, err := filecatalog.FindYAMLFiles(ctx, r.dir)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to read profile directory: %w", err) return nil, fmt.Errorf("failed to read profile directory: %w", err)
} }
@@ -47,24 +47,25 @@ func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*doma
default: default:
} }
relPath := r.relativePath(fullPath) relPath := filecatalog.RelativePath(r.dir, fullPath)
fileMatch := profileIDFromFileName(filepath.Base(fullPath)) == id fileMatch := filecatalog.Stem(filepath.Base(fullPath)) == id
data, err := os.ReadFile(fullPath) data, err := os.ReadFile(fullPath)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err) return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
} }
metadata := readProfileFileMetadata(data)
idMatch := fileMatch || metadata.id == id
if metadata.hasRawAPIKey {
if idMatch {
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath)
}
continue
}
var prof domain.ExecutionProfile var prof domain.ExecutionProfile
decoder := yaml.NewDecoder(bytes.NewReader(data)) decoder := yaml.NewDecoder(bytes.NewReader(data))
decoder.KnownFields(true) decoder.KnownFields(true)
if err := decoder.Decode(&prof); err != nil { if err := decoder.Decode(&prof); err != nil {
idMatch := fileMatch || profileFileHasID(data, id)
if strings.Contains(err.Error(), "field api_key not found") {
if idMatch {
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath)
}
continue
}
if idMatch { if idMatch {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err) return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
} }
@@ -106,56 +107,36 @@ type profileMatch struct {
path string path string
} }
func (r *filesystemRepository) yamlFiles(ctx context.Context) ([]string, error) { type profileFileMetadata struct {
var files []string id string
err := filepath.WalkDir(r.dir, func(path string, d os.DirEntry, err error) error { hasRawAPIKey bool
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if d.IsDir() {
return nil
}
if !isYAMLFile(d.Name()) {
return nil
}
files = append(files, path)
return nil
})
sort.Strings(files)
return files, err
} }
func (r *filesystemRepository) relativePath(path string) string { func readProfileFileMetadata(data []byte) profileFileMetadata {
rel, err := filepath.Rel(r.dir, path) var node yaml.Node
if err != nil { if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&node); err != nil {
return filepath.Clean(path) return profileFileMetadata{}
} }
return filepath.Clean(rel) if node.Kind != yaml.DocumentNode || len(node.Content) == 0 {
} return profileFileMetadata{}
func isYAMLFile(name string) bool {
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
}
func profileIDFromFileName(name string) string {
name = strings.TrimSuffix(name, ".yaml")
name = strings.TrimSuffix(name, ".yml")
return name
}
func profileFileHasID(data []byte, id string) bool {
var raw struct {
ID string `yaml:"id"`
} }
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&raw); err != nil { mapping := node.Content[0]
return false if mapping.Kind != yaml.MappingNode {
return profileFileMetadata{}
} }
return strings.TrimSpace(raw.ID) == id
var metadata profileFileMetadata
for i := 0; i+1 < len(mapping.Content); i += 2 {
key := mapping.Content[i]
value := mapping.Content[i+1]
switch key.Value {
case "id":
metadata.id = strings.TrimSpace(value.Value)
case "api_key":
metadata.hasRawAPIKey = true
}
}
return metadata
} }
func validateProfile(p *domain.ExecutionProfile) error { func validateProfile(p *domain.ExecutionProfile) error {

View File

@@ -2,6 +2,7 @@ package profile
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"os" "os"
"path/filepath" "path/filepath"
@@ -85,6 +86,63 @@ temperature: 0.1
} }
}) })
t.Run("valid profile with JSON-compatible extra params", func(t *testing.T) {
writeProfileTestFile(t, filepath.Join(tmpDir, "json-extra-params.yaml"), `
id: json-extra-params
endpoint: http://localhost:8000/v1
model: nested-model
extra_params:
string_value: enabled
number_value: 42
boolean_value: true
object_value:
nested: value
count: 2
array_value:
- first
- 3
- false
`)
p, err := repo.GetProfile(ctx, "json-extra-params")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
var got map[string]any
encoded, err := json.Marshal(p.ExtraParams)
if err != nil {
t.Fatalf("expected extra_params to marshal as JSON, got %v", err)
}
if err := json.Unmarshal(encoded, &got); err != nil {
t.Fatalf("expected extra_params JSON to decode, got %v", err)
}
if got["string_value"] != "enabled" {
t.Fatalf("unexpected string extra param: %#v", got["string_value"])
}
if got["number_value"] != float64(42) {
t.Fatalf("unexpected number extra param: %#v", got["number_value"])
}
if got["boolean_value"] != true {
t.Fatalf("unexpected boolean extra param: %#v", got["boolean_value"])
}
objectValue, ok := got["object_value"].(map[string]any)
if !ok {
t.Fatalf("expected object extra param, got %#v", got["object_value"])
}
if objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
t.Fatalf("unexpected object extra param: %#v", objectValue)
}
arrayValue, ok := got["array_value"].([]any)
if !ok {
t.Fatalf("expected array extra param, got %#v", got["array_value"])
}
if len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
t.Fatalf("unexpected array extra param: %#v", arrayValue)
}
})
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) { t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), ` writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
id: duplicate-profile id: duplicate-profile
@@ -133,6 +191,20 @@ api_key: secret
} }
}) })
t.Run("raw api_key in non-target profile is ignored", func(t *testing.T) {
writeProfileTestFile(t, filepath.Join(tmpDir, "raw-api-key-non-target.yaml"), `
id: raw-api-key-non-target
endpoint: http://localhost:8000/v1
model: m
api_key: secret
`)
_, err := repo.GetProfile(ctx, "does-not-exist-with-raw-key-nearby")
if !errors.Is(err, ErrProfileNotFound) {
t.Fatalf("expected ErrProfileNotFound for non-target raw api_key file, got %v", err)
}
})
t.Run("invalid yaml", func(t *testing.T) { t.Run("invalid yaml", func(t *testing.T) {
_, err := repo.GetProfile(ctx, "invalid_yaml") _, err := repo.GetProfile(ctx, "invalid_yaml")
if !errors.Is(err, ErrInvalidYAML) { if !errors.Is(err, ErrInvalidYAML) {

View File

@@ -6,7 +6,9 @@ import (
"errors" "errors"
"fmt" "fmt"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"strings"
"text/template" "text/template"
"unicode/utf8"
) )
var ( var (
@@ -50,6 +52,11 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
}, },
} }
sessionID, err := renderSessionID(definition.SessionID, funcs, vars)
if err != nil {
return nil, err
}
var renderedMessages []domain.RenderedMessage var renderedMessages []domain.RenderedMessage
for i, tmplMsg := range definition.Templates { for i, tmplMsg := range definition.Templates {
@@ -75,12 +82,44 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
} }
renderedMessages = append(renderedMessages, domain.RenderedMessage{ renderedMessages = append(renderedMessages, domain.RenderedMessage{
Role: tmplMsg.Role, Role: tmplMsg.Role,
Content: buf.String(), Content: buf.String(),
CacheControl: cloneCacheControl(tmplMsg.CacheControl),
}) })
} }
return &domain.RenderedPrompt{ return &domain.RenderedPrompt{
Messages: renderedMessages, SessionID: sessionID,
Messages: renderedMessages,
}, nil }, nil
} }
func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
if strings.TrimSpace(raw) == "" {
return "", nil
}
tmpl, err := template.New("session_id").Funcs(funcs).Option("missingkey=error").Parse(raw)
if err != nil {
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, err)
}
var buf bytes.Buffer
if err := tmpl.Execute(&buf, vars); err != nil {
return "", fmt.Errorf("%w: session_id: %w", ErrRenderFailure, err)
}
sessionID := strings.TrimSpace(buf.String())
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
return "", fmt.Errorf("%w: session_id length %d exceeds maximum %d", ErrRenderFailure, n, domain.SessionIDMaxLength)
}
return sessionID, nil
}
func cloneCacheControl(in *domain.CacheControl) *domain.CacheControl {
if in == nil {
return nil
}
out := *in
return &out
}

View File

@@ -3,6 +3,7 @@ package prompt
import ( import (
"context" "context"
"errors" "errors"
"strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
@@ -78,6 +79,66 @@ func TestGoRenderer_Render(t *testing.T) {
} }
}) })
t.Run("copying cache control to rendered messages", func(t *testing.T) {
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{
Role: "system",
Content: "You are concise.",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "Analyze this: {{input \"transcript\"}}"},
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(res.Messages) != 2 {
t.Fatalf("expected 2 messages, got %d", len(res.Messages))
}
if res.Messages[0].CacheControl == nil {
t.Fatal("expected rendered cache control")
}
if res.Messages[0].CacheControl.Type != domain.CacheControlEphemeral {
t.Fatalf("unexpected cache control type: %q", res.Messages[0].CacheControl.Type)
}
if res.Messages[0].CacheControl.TTL != "1h" {
t.Fatalf("unexpected cache control ttl: %q", res.Messages[0].CacheControl.TTL)
}
if res.Messages[1].CacheControl != nil {
t.Fatalf("expected no cache control on second message, got %#v", res.Messages[1].CacheControl)
}
})
t.Run("rendered cache control does not alias source template", func(t *testing.T) {
source := &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}
def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "You are concise.", CacheControl: source},
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if res.Messages[0].CacheControl == source {
t.Fatal("expected rendered cache control to be cloned")
}
res.Messages[0].CacheControl.TTL = ""
if source.TTL != "1h" {
t.Fatalf("source cache control was mutated, ttl=%q", source.TTL)
}
})
t.Run("accessing vars", func(t *testing.T) { t.Run("accessing vars", func(t *testing.T) {
def := &domain.PromptDefinition{ def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
@@ -95,6 +156,78 @@ func TestGoRenderer_Render(t *testing.T) {
} }
}) })
t.Run("rendering session id from vars", func(t *testing.T) {
def := &domain.PromptDefinition{
SessionID: " {{ .session_id }} ",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "Speak in a {{.tone}} tone."},
},
}
res, err := renderer.Render(ctx, def, inputs, map[string]string{
"tone": "concise",
"session_id": "agent-session-123",
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if res.SessionID != "agent-session-123" {
t.Fatalf("unexpected session id: %q", res.SessionID)
}
})
t.Run("empty rendered session id is omitted", func(t *testing.T) {
def := &domain.PromptDefinition{
SessionID: " ",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "Speak in a {{.tone}} tone."},
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if res.SessionID != "" {
t.Fatalf("expected empty session id, got %q", res.SessionID)
}
})
t.Run("missing session id var fails rendering", func(t *testing.T) {
def := &domain.PromptDefinition{
SessionID: "{{ .session_id }}",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "Speak in a {{.tone}} tone."},
},
}
_, err := renderer.Render(ctx, def, inputs, vars)
if !errors.Is(err, ErrRenderFailure) {
t.Fatalf("expected ErrRenderFailure, got %v", err)
}
})
t.Run("too long rendered session id fails rendering", func(t *testing.T) {
def := &domain.PromptDefinition{
SessionID: "{{ .session_id }}",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: "Speak in a {{.tone}} tone."},
},
}
_, err := renderer.Render(ctx, def, inputs, map[string]string{
"tone": "concise",
"session_id": strings.Repeat("x", domain.SessionIDMaxLength+1),
})
if !errors.Is(err, ErrRenderFailure) {
t.Fatalf("expected ErrRenderFailure, got %v", err)
}
})
t.Run("inserting required input artifact", func(t *testing.T) { t.Run("inserting required input artifact", func(t *testing.T) {
def := &domain.PromptDefinition{ def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},

View File

@@ -7,10 +7,10 @@ import (
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"sort"
"strings" "strings"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/filecatalog"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
@@ -29,6 +29,7 @@ type promptDefinitionFile struct {
Version string `yaml:"version"` Version string `yaml:"version"`
DefaultProfile *string `yaml:"default_profile"` DefaultProfile *string `yaml:"default_profile"`
Description string `yaml:"description"` Description string `yaml:"description"`
SessionID string `yaml:"session_id"`
Inputs []promptInputFile `yaml:"inputs"` Inputs []promptInputFile `yaml:"inputs"`
Messages []promptMessageFile `yaml:"messages"` Messages []promptMessageFile `yaml:"messages"`
Output promptOutputContractFile `yaml:"output"` Output promptOutputContractFile `yaml:"output"`
@@ -42,9 +43,15 @@ type promptInputFile struct {
} }
type promptMessageFile struct { type promptMessageFile struct {
Role string `yaml:"role"` Role string `yaml:"role"`
Content string `yaml:"content"` Content string `yaml:"content"`
ContentFile string `yaml:"content_file"` ContentFile string `yaml:"content_file"`
CacheControl *cacheControlFile `yaml:"cache_control"`
}
type cacheControlFile struct {
Type string `yaml:"type"`
TTL string `yaml:"ttl"`
} }
type promptOutputContractFile struct { type promptOutputContractFile struct {
@@ -63,7 +70,7 @@ func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id strin
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition) return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
} }
files, err := r.yamlFiles(ctx) files, err := filecatalog.FindYAMLFiles(ctx, r.dir)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err) return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
} }
@@ -76,8 +83,8 @@ func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id strin
default: default:
} }
relPath := r.relativePath(fullPath) relPath := filecatalog.RelativePath(r.dir, fullPath)
fileMatch := promptIDFromFileName(filepath.Base(fullPath)) == id fileMatch := filecatalog.Stem(filepath.Base(fullPath)) == id
raw, err := loadPromptDefinitionFile(fullPath) raw, err := loadPromptDefinitionFile(fullPath)
if err != nil { if err != nil {
@@ -130,38 +137,6 @@ type promptDefinitionMatch struct {
path string path string
} }
func (r *filesystemRepository) yamlFiles(ctx context.Context) ([]string, error) {
var files []string
err := filepath.WalkDir(r.dir, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if d.IsDir() {
return nil
}
if !isYAMLFile(d.Name()) {
return nil
}
files = append(files, path)
return nil
})
sort.Strings(files)
return files, err
}
func (r *filesystemRepository) relativePath(path string) string {
rel, err := filepath.Rel(r.dir, path)
if err != nil {
return filepath.Clean(path)
}
return filepath.Clean(rel)
}
func loadPromptDefinitionFile(path string) (*promptDefinitionFile, error) { func loadPromptDefinitionFile(path string) (*promptDefinitionFile, error) {
data, err := os.ReadFile(path) data, err := os.ReadFile(path)
if err != nil { if err != nil {
@@ -244,6 +219,11 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
return nil, fmt.Errorf("message %d (%s) must set exactly one of content or content_file", i, role) return nil, fmt.Errorf("message %d (%s) must set exactly one of content or content_file", i, role)
} }
cacheControl, err := normalizeCacheControl(msg.CacheControl)
if err != nil {
return nil, fmt.Errorf("message %d (%s) cache_control: %w", i, role, err)
}
templateContent := msg.Content templateContent := msg.Content
resolvedContentFile := "" resolvedContentFile := ""
if hasContentFile { if hasContentFile {
@@ -262,9 +242,10 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
} }
templates = append(templates, domain.PromptMessageTemplate{ templates = append(templates, domain.PromptMessageTemplate{
Role: role, Role: role,
Content: templateContent, Content: templateContent,
ContentFile: resolvedContentFile, ContentFile: resolvedContentFile,
CacheControl: cacheControl,
}) })
} }
@@ -294,6 +275,7 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
Version: version, Version: version,
DefaultProfile: defaultProfile, DefaultProfile: defaultProfile,
Description: strings.TrimSpace(raw.Description), Description: strings.TrimSpace(raw.Description),
SessionID: strings.TrimSpace(raw.SessionID),
Inputs: inputs, Inputs: inputs,
Templates: templates, Templates: templates,
OutputFormat: raw.Output.Format, OutputFormat: raw.Output.Format,
@@ -306,14 +288,28 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
}, nil }, nil
} }
func isYAMLFile(name string) bool { func normalizeCacheControl(raw *cacheControlFile) (*domain.CacheControl, error) {
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml") if raw == nil {
} return nil, nil
}
func promptIDFromFileName(name string) string { cacheType := strings.TrimSpace(raw.Type)
name = strings.TrimSuffix(name, ".yaml") if cacheType == "" {
name = strings.TrimSuffix(name, ".yml") return nil, errors.New("type is required")
return name }
if domain.CacheControlType(cacheType) != domain.CacheControlEphemeral {
return nil, fmt.Errorf("unsupported type %q", cacheType)
}
ttl := strings.TrimSpace(raw.TTL)
if ttl != "" && ttl != "1h" {
return nil, fmt.Errorf("unsupported ttl %q", ttl)
}
return &domain.CacheControl{
Type: domain.CacheControlType(cacheType),
TTL: ttl,
}, nil
} }
func isValidOutputFormat(f domain.OutputFormat) bool { func isValidOutputFormat(f domain.OutputFormat) bool {

View File

@@ -68,6 +68,44 @@ func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
} }
}) })
t.Run("valid cache control with ttl", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "valid-cache-control-ttl", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(p.Templates) != 2 {
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
}
assertCacheControl(t, p.Templates[0].CacheControl, domain.CacheControlEphemeral, "1h")
if p.Templates[1].CacheControl != nil {
t.Fatalf("expected second message cache control to be nil, got %#v", p.Templates[1].CacheControl)
}
})
t.Run("valid cache control without ttl", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "valid-cache-control-without-ttl", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(p.Templates) != 2 {
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
}
assertCacheControl(t, p.Templates[0].CacheControl, domain.CacheControlEphemeral, "")
if p.Templates[1].CacheControl != nil {
t.Fatalf("expected second message cache control to be nil, got %#v", p.Templates[1].CacheControl)
}
})
t.Run("valid session id template", func(t *testing.T) {
p, err := repo.GetPromptDefinition(ctx, "valid-session-id", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.SessionID != "{{ .session_id }}" {
t.Fatalf("expected trimmed session_id template, got %q", p.SessionID)
}
})
t.Run("valid nested file-backed prompt resolves content file relative to nested YAML", func(t *testing.T) { t.Run("valid nested file-backed prompt resolves content file relative to nested YAML", func(t *testing.T) {
nestedDir := filepath.Join(tmpDir, "dnd", "recap") nestedDir := filepath.Join(tmpDir, "dnd", "recap")
if err := os.MkdirAll(nestedDir, 0o755); err != nil { if err := os.MkdirAll(nestedDir, 0o755); err != nil {
@@ -258,6 +296,10 @@ output:
{name: "invalid validation mode", id: "invalid_validation_mode", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"invalid validation mode"}}, {name: "invalid validation mode", id: "invalid_validation_mode", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"invalid validation mode"}},
{name: "json_schema without schema_path", id: "json_schema_without_schema_path", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"schema_path"}}, {name: "json_schema without schema_path", id: "json_schema_without_schema_path", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"schema_path"}},
{name: "unknown input field", id: "unknown_input_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unknown_input_setting not found"}}, {name: "unknown input field", id: "unknown_input_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unknown_input_setting not found"}},
{name: "empty cache control type", id: "empty_cache_control_type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "type is required"}},
{name: "unsupported cache control type", id: "unsupported_cache_control_type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported type"}},
{name: "unsupported cache control ttl", id: "unsupported_cache_control_ttl", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported ttl"}},
{name: "unknown cache control field", id: "unknown_cache_control_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unexpected not found"}},
} }
for _, tc := range cases { for _, tc := range cases {
@@ -282,6 +324,19 @@ output:
}) })
} }
func assertCacheControl(t *testing.T, got *domain.CacheControl, wantType domain.CacheControlType, wantTTL string) {
t.Helper()
if got == nil {
t.Fatal("expected cache control, got nil")
}
if got.Type != wantType {
t.Fatalf("unexpected cache control type: got %q want %q", got.Type, wantType)
}
if got.TTL != wantTTL {
t.Fatalf("unexpected cache control ttl: got %q want %q", got.TTL, wantTTL)
}
}
func writePromptTestFile(t *testing.T, path string, content string) { func writePromptTestFile(t *testing.T, path string, content string) {
t.Helper() t.Helper()
if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil { if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil {

View File

@@ -0,0 +1,10 @@
id: empty-cache-control-type
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control: {}
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -0,0 +1,12 @@
id: unknown-cache-control-field
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: ephemeral
unexpected: true
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -0,0 +1,12 @@
id: unsupported-cache-control-ttl
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: ephemeral
ttl: 5m
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -0,0 +1,11 @@
id: unsupported-cache-control-type
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: persistent
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -0,0 +1,14 @@
id: valid-cache-control-ttl
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: ephemeral
ttl: 1h
- role: user
content: "Summarize the input."
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -0,0 +1,13 @@
id: valid-cache-control-without-ttl
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: ephemeral
- role: user
content: "Summarize the input."
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -0,0 +1,10 @@
id: valid-session-id
version: "1.0.0"
session_id: " {{ .session_id }} "
messages:
- role: user
content: Hello.
output:
format: markdown
validation_mode: basic
repair_attempts: 0

View File

@@ -24,12 +24,14 @@ import (
) )
var ( var (
ErrInvalidRequest = errors.New("invalid run request") ErrInvalidRequest = errors.New("invalid run request")
ErrProfileLoad = errors.New("failed to load prompt definition") ErrProfileRequired = errors.New("profile selection is required")
ErrArtifactLoad = errors.New("failed to load artifact") ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
ErrPromptRender = errors.New("failed to render prompt") ErrProfileLoad = errors.New("failed to load prompt definition")
ErrLLMGenerate = errors.New("failed to generate output") ErrArtifactLoad = errors.New("failed to load artifact")
ErrValidation = errors.New("failed to validate output") ErrPromptRender = errors.New("failed to render prompt")
ErrLLMGenerate = errors.New("failed to generate output")
ErrValidation = errors.New("failed to validate output")
) )
// Runner executes the Scriptorium core use case. // Runner executes the Scriptorium core use case.
@@ -88,11 +90,15 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
} }
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{ genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: prepared.Messages}, Prompt: domain.RenderedPrompt{SessionID: prepared.SessionID, Messages: prepared.Messages},
Target: prepared.EffectiveModelParams, Target: prepared.EffectiveModelParams,
TargetPresence: prepared.TargetPresence,
StructuredOutput: prepared.StructuredOutput, StructuredOutput: prepared.StructuredOutput,
}) })
if err != nil { if err != nil {
if errors.Is(err, llm.ErrInvalidRequest) {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err) return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err)
} }
@@ -177,7 +183,7 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
selectedProfileID = strings.TrimSpace(def.DefaultProfile) selectedProfileID = strings.TrimSpace(def.DefaultProfile)
} }
if selectedProfileID == "" { if selectedProfileID == "" {
return nil, fmt.Errorf("%w: profile id is required either in request or prompt default_profile", ErrInvalidRequest) return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired)
} }
execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID) execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID)
@@ -185,7 +191,10 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err) return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
} }
effectiveModel := resolveExecutionTarget(execProfile, req.Execution) effectiveModel, targetPresence, err := resolveExecutionTarget(execProfile, req.Execution)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
if strings.TrimSpace(effectiveModel.Endpoint) == "" { if strings.TrimSpace(effectiveModel.Endpoint) == "" {
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest) return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
} }
@@ -228,9 +237,11 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
PromptHash: promptDefinitionHash, PromptHash: promptDefinitionHash,
SelectedProfileID: selectedProfileID, SelectedProfileID: selectedProfileID,
EffectiveModelParams: effectiveModel, EffectiveModelParams: effectiveModel,
TargetPresence: targetPresence,
OutputContract: effectiveContract, OutputContract: effectiveContract,
StructuredOutput: structuredOutput, StructuredOutput: structuredOutput,
InputHashes: inputHashes, InputHashes: inputHashes,
SessionID: renderedPrompt.SessionID,
RenderedPromptHash: hashRenderedPrompt(*renderedPrompt), RenderedPromptHash: hashRenderedPrompt(*renderedPrompt),
Messages: renderedPrompt.Messages, Messages: renderedPrompt.Messages,
StartTime: start, StartTime: start,
@@ -352,22 +363,75 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution
out.APIKeyEnv = override.APIKeyEnv out.APIKeyEnv = override.APIKeyEnv
} }
if len(override.ExtraParams) > 0 { if len(override.ExtraParams) > 0 {
cp := make(map[string]string, len(override.ExtraParams)) out.ExtraParams = copyExtraParams(override.ExtraParams)
for k, v := range override.ExtraParams {
cp[k] = v
}
out.ExtraParams = cp
} }
return out return out
} }
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTarget) domain.ExecutionTarget { func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
out := base
var presence domain.ExecutionTargetPresence
if override.Endpoint != "" {
out.Endpoint = override.Endpoint
}
if override.Model != "" {
out.Model = override.Model
}
if override.Temperature != nil {
if *override.Temperature < 0 || *override.Temperature > 2 {
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("temperature must be between 0 and 2")
}
out.Temperature = *override.Temperature
presence.Temperature = true
}
if override.MaxTokens != nil {
if *override.MaxTokens < 0 {
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("max_tokens must be greater than or equal to 0")
}
out.MaxTokens = *override.MaxTokens
presence.MaxTokens = true
}
if override.TopP != nil {
if *override.TopP < 0 || *override.TopP > 1 {
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("top_p must be between 0 and 1")
}
out.TopP = *override.TopP
presence.TopP = true
}
if override.TimeoutSeconds != nil {
if *override.TimeoutSeconds < 0 {
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("timeout_seconds must be greater than or equal to 0")
}
out.TimeoutSeconds = *override.TimeoutSeconds
presence.TimeoutSeconds = true
}
if strings.TrimSpace(override.ServiceTier) != "" {
out.ServiceTier = override.ServiceTier
}
if strings.TrimSpace(override.ReasoningEffort) != "" {
out.ReasoningEffort = override.ReasoningEffort
}
if strings.TrimSpace(override.APIKeyEnv) != "" {
out.APIKeyEnv = override.APIKeyEnv
}
if len(override.ExtraParams) > 0 {
out.ExtraParams = copyExtraParams(override.ExtraParams)
}
return out, presence, nil
}
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
out := defaults.ExecutionTargetDefault() out := defaults.ExecutionTargetDefault()
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue)) out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
var presence domain.ExecutionTargetPresence
if override != nil { if override != nil {
out = mergeExecutionTarget(out, *override) var err error
out, presence, err = mergeExecutionTargetOverride(out, *override)
if err != nil {
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, err
}
} }
return out return out, presence, nil
} }
func validateAPIKeyEnv(apiKeyEnv string) error { func validateAPIKeyEnv(apiKeyEnv string) error {
@@ -376,7 +440,7 @@ func validateAPIKeyEnv(apiKeyEnv string) error {
return nil return nil
} }
if strings.TrimSpace(os.Getenv(envName)) == "" { if strings.TrimSpace(os.Getenv(envName)) == "" {
return fmt.Errorf("api key environment variable %q is not set", envName) return fmt.Errorf("%w: api key environment variable %q is not set", ErrAPIKeyEnvMissing, envName)
} }
return nil return nil
} }
@@ -385,13 +449,6 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
if p == nil { if p == nil {
return domain.ExecutionTarget{} return domain.ExecutionTarget{}
} }
cp := map[string]string(nil)
if len(p.ExtraParams) > 0 {
cp = make(map[string]string, len(p.ExtraParams))
for k, v := range p.ExtraParams {
cp[k] = v
}
}
return domain.ExecutionTarget{ return domain.ExecutionTarget{
Endpoint: p.Endpoint, Endpoint: p.Endpoint,
Model: p.Model, Model: p.Model,
@@ -402,10 +459,21 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
ServiceTier: p.ServiceTier, ServiceTier: p.ServiceTier,
ReasoningEffort: p.ReasoningEffort, ReasoningEffort: p.ReasoningEffort,
APIKeyEnv: p.APIKeyEnv, APIKeyEnv: p.APIKeyEnv,
ExtraParams: cp, ExtraParams: copyExtraParams(p.ExtraParams),
} }
} }
func copyExtraParams(src map[string]any) map[string]any {
if len(src) == 0 {
return nil
}
cp := make(map[string]any, len(src))
for k, v := range src {
cp[k] = v
}
return cp
}
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract { func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract {
contract := def.Validation contract := def.Validation
if contract.Format == "" { if contract.Format == "" {
@@ -422,10 +490,23 @@ func resolveOutputContract(def *domain.PromptDefinition, override *domain.Output
func hashRenderedPrompt(p domain.RenderedPrompt) string { func hashRenderedPrompt(p domain.RenderedPrompt) string {
var b strings.Builder var b strings.Builder
if p.SessionID != "" {
b.WriteString("session_id=")
b.WriteString(p.SessionID)
b.WriteString("\n---\n")
}
for _, msg := range p.Messages { for _, msg := range p.Messages {
b.WriteString(msg.Role) b.WriteString(msg.Role)
b.WriteByte('\n') b.WriteByte('\n')
b.WriteString(msg.Content) b.WriteString(msg.Content)
if msg.CacheControl != nil {
b.WriteString("\ncache_control.type=")
b.WriteString(string(msg.CacheControl.Type))
if msg.CacheControl.TTL != "" {
b.WriteString("\ncache_control.ttl=")
b.WriteString(msg.CacheControl.TTL)
}
}
b.WriteString("\n---\n") b.WriteString("\n---\n")
} }
h := sha256.Sum256([]byte(b.String())) h := sha256.Sum256([]byte(b.String()))

View File

@@ -14,6 +14,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults" "gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile" "gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt" "gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef" "gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
@@ -160,7 +161,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")}, "a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")}, "a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
}} }}
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{forbid: true} llmClient := &fakeLLM{forbid: true}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
@@ -172,7 +173,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"}, "glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
}, },
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90}, Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)},
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
@@ -198,6 +199,9 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
if len(prepared.Messages) != 2 { if len(prepared.Messages) != 2 {
t.Fatalf("expected two messages, got %d", len(prepared.Messages)) t.Fatalf("expected two messages, got %d", len(prepared.Messages))
} }
if prepared.SessionID != "session-123" {
t.Fatalf("expected prepared session id, got %q", prepared.SessionID)
}
if llmClient.calls != 0 { if llmClient.calls != 0 {
t.Fatalf("prepare should not call llm, calls=%d", llmClient.calls) t.Fatalf("prepare should not call llm, calls=%d", llmClient.calls)
} }
@@ -234,6 +238,9 @@ func TestRunnerPrepareMissingExplicitProfileAndMissingDefaultProfileFails(t *tes
if !errors.Is(err, ErrInvalidRequest) { if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err) t.Fatalf("expected ErrInvalidRequest, got %v", err)
} }
if !errors.Is(err, ErrProfileRequired) {
t.Fatalf("expected ErrProfileRequired, got %v", err)
}
} }
func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) { func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) {
@@ -266,11 +273,11 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
Inputs: singleInputRef(), Inputs: singleInputRef(),
Execution: &domain.ExecutionTarget{ Execution: &domain.ExecutionTargetOverride{
Endpoint: "http://override/v1", Endpoint: "http://override/v1",
Model: "override-model", Model: "override-model",
Temperature: 0.7, Temperature: float64Ptr(0.7),
TimeoutSeconds: 30, TimeoutSeconds: intPtr(30),
ServiceTier: "flex", ServiceTier: "flex",
}, },
}) })
@@ -288,6 +295,143 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
} }
} }
func TestRunnerPrepareRequestNumericOverridePresence(t *testing.T) {
tests := []struct {
name string
override *domain.ExecutionTargetOverride
wantTemperature float64
wantMaxTokens int
wantTopP float64
wantTimeoutSecs int
wantPresence domain.ExecutionTargetPresence
}{
{
name: "omitted preserves profile values",
override: &domain.ExecutionTargetOverride{},
wantTemperature: 0.7,
wantMaxTokens: 321,
wantTopP: 0.8,
wantTimeoutSecs: 45,
},
{
name: "explicit zero temperature",
override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(0)},
wantTemperature: 0,
wantMaxTokens: 321,
wantTopP: 0.8,
wantTimeoutSecs: 45,
wantPresence: domain.ExecutionTargetPresence{Temperature: true},
},
{
name: "explicit zero max tokens",
override: &domain.ExecutionTargetOverride{MaxTokens: intPtr(0)},
wantTemperature: 0.7,
wantMaxTokens: 0,
wantTopP: 0.8,
wantTimeoutSecs: 45,
wantPresence: domain.ExecutionTargetPresence{MaxTokens: true},
},
{
name: "explicit zero top p",
override: &domain.ExecutionTargetOverride{TopP: float64Ptr(0)},
wantTemperature: 0.7,
wantMaxTokens: 321,
wantTopP: 0,
wantTimeoutSecs: 45,
wantPresence: domain.ExecutionTargetPresence{TopP: true},
},
{
name: "explicit zero timeout",
override: &domain.ExecutionTargetOverride{TimeoutSeconds: intPtr(0)},
wantTemperature: 0.7,
wantMaxTokens: 321,
wantTopP: 0.8,
wantTimeoutSecs: 0,
wantPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.7,
MaxTokens: 321,
TopP: 0.8,
TimeoutSeconds: 45,
},
}},
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
nil,
)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
Execution: tc.override,
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
got := prepared.EffectiveModelParams
if got.Temperature != tc.wantTemperature ||
got.MaxTokens != tc.wantMaxTokens ||
got.TopP != tc.wantTopP ||
got.TimeoutSeconds != tc.wantTimeoutSecs {
t.Fatalf("unexpected effective numeric settings: %+v", got)
}
if prepared.TargetPresence != tc.wantPresence {
t.Fatalf("unexpected target presence: got %+v want %+v", prepared.TargetPresence, tc.wantPresence)
}
})
}
}
func TestRunnerPrepareInvalidRequestNumericOverridesFail(t *testing.T) {
tests := []struct {
name string
override *domain.ExecutionTargetOverride
}{
{name: "temperature below range", override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(-0.1)}},
{name: "temperature above range", override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(2.1)}},
{name: "max tokens below range", override: &domain.ExecutionTargetOverride{MaxTokens: intPtr(-1)}},
{name: "top p below range", override: &domain.ExecutionTargetOverride{TopP: float64Ptr(-0.1)}},
{name: "top p above range", override: &domain.ExecutionTargetOverride{TopP: float64Ptr(1.1)}},
{name: "timeout below range", override: &domain.ExecutionTargetOverride{TimeoutSeconds: intPtr(-1)}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
nil,
)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
Execution: tc.override,
})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
})
}
}
func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) { func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
@@ -487,6 +631,35 @@ func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) {
} }
} }
func TestRunnerPrepareJSONSchemaSchemaLoadFailureReturnsValidationError(t *testing.T) {
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
def.Validation.SchemaPath = "missing.schema.json"
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
validator,
)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if !errors.Is(err, ErrValidation) {
t.Fatalf("expected ErrValidation, got %v", err)
}
if validator.schemaLoads != 1 {
t.Fatalf("expected one schema load attempt, got %d", validator.schemaLoads)
}
if validator.schemaLoadPath != "missing.schema.json" {
t.Fatalf("expected schema path missing.schema.json, got %q", validator.schemaLoadPath)
}
}
func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) { func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) {
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0) def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
def.Validation.SchemaPath = "missing.schema.json" def.Validation.SchemaPath = "missing.schema.json"
@@ -545,6 +718,100 @@ func TestDeriveStructuredSchemaName(t *testing.T) {
} }
} }
func TestHashRenderedPromptIncludesCacheControlWhenPresent(t *testing.T) {
uncached := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{Role: "system", Content: "sys"},
{Role: "user", Content: "usr"},
}}
wantLegacyHash := hashString("system\nsys\n---\nuser\nusr\n---\n")
if got := hashRenderedPrompt(uncached); got != wantLegacyHash {
t.Fatalf("expected no-cache hash to preserve legacy input, got %q want %q", got, wantLegacyHash)
}
withCache := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "sys",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "usr"},
}}
alsoWithCache := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "sys",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
TTL: "1h",
},
},
{Role: "user", Content: "usr"},
}}
withoutTTL := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "sys",
CacheControl: &domain.CacheControl{
Type: domain.CacheControlEphemeral,
},
},
{Role: "user", Content: "usr"},
}}
cachedHash := hashRenderedPrompt(withCache)
if cachedHash == hashRenderedPrompt(uncached) {
t.Fatal("expected cache control to change rendered prompt hash")
}
if cachedHash != hashRenderedPrompt(alsoWithCache) {
t.Fatal("expected identical cache control metadata to produce stable hash")
}
if cachedHash == hashRenderedPrompt(withoutTTL) {
t.Fatal("expected ttl changes to affect rendered prompt hash")
}
}
func TestHashRenderedPromptIncludesSessionIDWhenPresent(t *testing.T) {
withoutSession := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{Role: "system", Content: "sys"},
{Role: "user", Content: "usr"},
}}
withSession := domain.RenderedPrompt{
SessionID: "session-123",
Messages: []domain.RenderedMessage{
{Role: "system", Content: "sys"},
{Role: "user", Content: "usr"},
},
}
alsoWithSession := domain.RenderedPrompt{
SessionID: "session-123",
Messages: []domain.RenderedMessage{
{Role: "system", Content: "sys"},
{Role: "user", Content: "usr"},
},
}
otherSession := domain.RenderedPrompt{
SessionID: "session-456",
Messages: []domain.RenderedMessage{
{Role: "system", Content: "sys"},
{Role: "user", Content: "usr"},
},
}
sessionHash := hashRenderedPrompt(withSession)
if sessionHash == hashRenderedPrompt(withoutSession) {
t.Fatal("expected session_id to change rendered prompt hash")
}
if sessionHash != hashRenderedPrompt(alsoWithSession) {
t.Fatal("expected identical session_id to produce stable hash")
}
if sessionHash == hashRenderedPrompt(otherSession) {
t.Fatal("expected session_id value changes to affect rendered prompt hash")
}
}
func TestRunnerRunSuccessful(t *testing.T) { func TestRunnerRunSuccessful(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)} promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}
@@ -552,7 +819,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")}, "a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")}, "a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
}} }}
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
@@ -564,7 +831,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"}, "glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
}, },
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90}, Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)},
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
@@ -599,6 +866,48 @@ func TestRunnerRunSuccessful(t *testing.T) {
if llmClient.lastReq.Target.TimeoutSeconds != 90 { if llmClient.lastReq.Target.TimeoutSeconds != 90 {
t.Fatalf("expected timeout propagation, got %d", llmClient.lastReq.Target.TimeoutSeconds) t.Fatalf("expected timeout propagation, got %d", llmClient.lastReq.Target.TimeoutSeconds)
} }
if !llmClient.lastReq.TargetPresence.Temperature || !llmClient.lastReq.TargetPresence.TimeoutSeconds {
t.Fatalf("expected numeric override presence to be sent to llm, got %+v", llmClient.lastReq.TargetPresence)
}
if llmClient.lastReq.Prompt.SessionID != "session-123" {
t.Fatalf("expected session id to be sent to llm, got %q", llmClient.lastReq.Prompt.SessionID)
}
}
func TestRunnerRunPassesExtraParamsToGenerateRequestTarget(t *testing.T) {
extraParams := map[string]any{
"string_value": "enabled",
"number_value": 42,
"boolean_value": true,
"object_value": map[string]any{"nested": "value"},
"array_value": []any{"first", 3, false},
}
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
ExtraParams: extraParams,
},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if !reflect.DeepEqual(res.EffectiveModelParams.ExtraParams, extraParams) {
t.Fatalf("expected run result extra_params to match profile values, got %#v", res.EffectiveModelParams.ExtraParams)
}
if !reflect.DeepEqual(llmClient.lastReq.Target.ExtraParams, extraParams) {
t.Fatalf("expected generate request extra_params to match profile values, got %#v", llmClient.lastReq.Target.ExtraParams)
}
} }
func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T) { func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T) {
@@ -617,7 +926,7 @@ func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T)
Inputs: map[string]domain.ArtifactRef{ Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
}, },
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90}, Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)},
} }
prepared, err := runner.Prepare(context.Background(), req) prepared, err := runner.Prepare(context.Background(), req)
@@ -745,11 +1054,11 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T)
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
Inputs: singleInputRef(), Inputs: singleInputRef(),
Execution: &domain.ExecutionTarget{ Execution: &domain.ExecutionTargetOverride{
Endpoint: "http://override/v1", Endpoint: "http://override/v1",
Model: "override-model", Model: "override-model",
Temperature: 0.7, Temperature: float64Ptr(0.7),
TimeoutSeconds: 30, TimeoutSeconds: intPtr(30),
ServiceTier: "flex", ServiceTier: "flex",
}, },
}) })
@@ -862,6 +1171,9 @@ func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) {
if !errors.Is(err, ErrInvalidRequest) { if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err) t.Fatalf("expected ErrInvalidRequest, got %v", err)
} }
if !errors.Is(err, ErrAPIKeyEnvMissing) {
t.Fatalf("expected ErrAPIKeyEnvMissing, got %v", err)
}
if !strings.Contains(err.Error(), "SCRIPTORIUM_MISSING_KEY") { if !strings.Contains(err.Error(), "SCRIPTORIUM_MISSING_KEY") {
t.Fatalf("expected missing env name in error, got %v", err) t.Fatalf("expected missing env name in error, got %v", err)
} }
@@ -881,7 +1193,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
Inputs: singleInputRef(), Inputs: singleInputRef(),
Execution: &domain.ExecutionTarget{APIKeyEnv: envName}, Execution: &domain.ExecutionTargetOverride{APIKeyEnv: envName},
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
@@ -906,7 +1218,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) {
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
Inputs: singleInputRef(), Inputs: singleInputRef(),
Execution: &domain.ExecutionTarget{APIKeyEnv: runtimeEnv}, Execution: &domain.ExecutionTargetOverride{APIKeyEnv: runtimeEnv},
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
@@ -1005,6 +1317,28 @@ func TestRunnerRunLLMFailure(t *testing.T) {
} }
} }
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{err: llm.ErrInvalidRequest},
nil,
)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
if errors.Is(err, ErrLLMGenerate) {
t.Fatalf("did not expect ErrLLMGenerate, got %v", err)
}
}
func TestRunnerRunValidationStillWorks(t *testing.T) { func TestRunnerRunValidationStillWorks(t *testing.T) {
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}} validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
runner := NewRunner( runner := NewRunner(
@@ -1047,7 +1381,7 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
Inputs: singleInputRef(), Inputs: singleInputRef(),
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "override-model", TimeoutSeconds: 22}, Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "override-model", TimeoutSeconds: intPtr(22)},
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
@@ -1122,6 +1456,186 @@ func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
} }
} }
func TestExecutionProfileToTargetPopulatesAllFieldsAndCopiesExtraParams(t *testing.T) {
src := &domain.ExecutionProfile{
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.2,
MaxTokens: 123,
TopP: 0.75,
TimeoutSeconds: 90,
ServiceTier: "priority",
ReasoningEffort: "medium",
APIKeyEnv: "SCRIPTORIUM_API_KEY",
ExtraParams: map[string]any{
"provider_option": "on",
},
}
target := executionProfileToTarget(src)
if target.Endpoint != src.Endpoint ||
target.Model != src.Model ||
target.Temperature != src.Temperature ||
target.MaxTokens != src.MaxTokens ||
target.TopP != src.TopP ||
target.TimeoutSeconds != src.TimeoutSeconds ||
target.ServiceTier != src.ServiceTier ||
target.ReasoningEffort != src.ReasoningEffort ||
target.APIKeyEnv != src.APIKeyEnv {
t.Fatalf("expected all profile fields to populate target, got %+v", target)
}
if !reflect.DeepEqual(target.ExtraParams, src.ExtraParams) {
t.Fatalf("expected extra_params to match, got %#v", target.ExtraParams)
}
src.ExtraParams["provider_option"] = "changed"
if target.ExtraParams["provider_option"] != "on" {
t.Fatalf("expected extra_params copy to be independent, got %#v", target.ExtraParams)
}
}
func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testing.T) {
profileValue := &domain.ExecutionProfile{
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.3,
MaxTokens: 222,
TopP: 0.6,
TimeoutSeconds: 77,
ServiceTier: "priority",
ReasoningEffort: "low",
APIKeyEnv: "PROFILE_KEY",
ExtraParams: map[string]any{
"profile_option": "enabled",
},
}
target, presence, err := resolveExecutionTarget(profileValue, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if presence != (domain.ExecutionTargetPresence{}) {
t.Fatalf("expected no request override presence, got %+v", presence)
}
if target.Endpoint != profileValue.Endpoint ||
target.Model != profileValue.Model ||
target.Temperature != profileValue.Temperature ||
target.MaxTokens != profileValue.MaxTokens ||
target.TopP != profileValue.TopP ||
target.TimeoutSeconds != profileValue.TimeoutSeconds ||
target.ServiceTier != profileValue.ServiceTier ||
target.ReasoningEffort != profileValue.ReasoningEffort ||
target.APIKeyEnv != profileValue.APIKeyEnv {
t.Fatalf("expected profile values to populate target, got %+v", target)
}
if !reflect.DeepEqual(target.ExtraParams, profileValue.ExtraParams) {
t.Fatalf("expected profile extra_params in target, got %#v", target.ExtraParams)
}
}
func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFields(t *testing.T) {
profileValue := &domain.ExecutionProfile{
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.2,
MaxTokens: 200,
TopP: 0.8,
TimeoutSeconds: 90,
ServiceTier: "priority",
ReasoningEffort: "medium",
APIKeyEnv: "PROFILE_KEY",
ExtraParams: map[string]any{
"profile_only": "yes",
},
}
override := &domain.ExecutionTargetOverride{
Endpoint: "http://override/v1",
Model: "override-model",
Temperature: float64Ptr(0.9),
MaxTokens: intPtr(111),
TopP: float64Ptr(0.5),
TimeoutSeconds: intPtr(30),
ServiceTier: "flex",
ReasoningEffort: "high",
APIKeyEnv: "RUNTIME_KEY",
ExtraParams: map[string]any{
"runtime_only": "yes",
},
}
target, presence, err := resolveExecutionTarget(profileValue, override)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if presence != (domain.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}) {
t.Fatalf("unexpected override presence: %+v", presence)
}
if target.Endpoint != override.Endpoint ||
target.Model != override.Model ||
target.Temperature != *override.Temperature ||
target.MaxTokens != *override.MaxTokens ||
target.TopP != *override.TopP ||
target.TimeoutSeconds != *override.TimeoutSeconds ||
target.ServiceTier != override.ServiceTier ||
target.ReasoningEffort != override.ReasoningEffort ||
target.APIKeyEnv != override.APIKeyEnv {
t.Fatalf("expected runtime overrides to win for all fields, got %+v", target)
}
if !reflect.DeepEqual(target.ExtraParams, override.ExtraParams) {
t.Fatalf("expected runtime extra_params to replace profile extra_params, got %#v", target.ExtraParams)
}
}
func TestMergeExecutionTargetEmptyStringOverridesDoNotErase(t *testing.T) {
base := domain.ExecutionTarget{
Endpoint: "http://base/v1",
Model: "base-model",
ServiceTier: "priority",
ReasoningEffort: "medium",
APIKeyEnv: "BASE_KEY",
}
override := domain.ExecutionTarget{
Endpoint: "http://override/v1",
Model: "override-model",
ServiceTier: " ",
ReasoningEffort: " ",
APIKeyEnv: "",
}
merged := mergeExecutionTarget(base, override)
if merged.Endpoint != "http://override/v1" || merged.Model != "override-model" {
t.Fatalf("expected endpoint/model to override, got %+v", merged)
}
if merged.ServiceTier != "priority" {
t.Fatalf("expected empty service_tier override to be ignored, got %q", merged.ServiceTier)
}
if merged.ReasoningEffort != "medium" {
t.Fatalf("expected empty reasoning_effort override to be ignored, got %q", merged.ReasoningEffort)
}
if merged.APIKeyEnv != "BASE_KEY" {
t.Fatalf("expected empty api_key_env override to be ignored, got %q", merged.APIKeyEnv)
}
}
func TestMergeExecutionTargetEmptyExtraParamsDoesNotErase(t *testing.T) {
base := domain.ExecutionTarget{
ExtraParams: map[string]any{
"keep": "value",
},
}
override := domain.ExecutionTarget{
ExtraParams: map[string]any{},
}
merged := mergeExecutionTarget(base, override)
if !reflect.DeepEqual(merged.ExtraParams, base.ExtraParams) {
t.Fatalf("expected empty extra_params override not to erase base values, got %#v", merged.ExtraParams)
}
}
func TestBuildOutputArtifactDefaults(t *testing.T) { func TestBuildOutputArtifactDefaults(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
@@ -1189,6 +1703,14 @@ func singleInputRef() map[string]domain.ArtifactRef {
return map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}} return map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}}
} }
func float64Ptr(v float64) *float64 {
return &v
}
func intPtr(v int) *int {
return &v
}
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner { func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
return NewRunner( return NewRunner(
promptRepo, promptRepo,