Compare commits
25 Commits
75fa0a030a
...
v0.10.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 23872dd742 | |||
| 7ffbf5f6ca | |||
| d0dc30fcc9 | |||
| b38f7b4dc3 | |||
| 0512995931 | |||
| 049a5feadb | |||
| 1798e9c575 | |||
| 5d4bc8c2b9 | |||
| 63fb8fc132 | |||
| 4d4bb7a121 | |||
| 5dcb3cd4fc | |||
| efe346893c | |||
| c95d6fcfec | |||
| 0badb4364d | |||
| 1f63f8afbb | |||
| bc099a31ad | |||
| 4ff55221a3 | |||
| 8d8024099f | |||
| 18792fd8d1 | |||
| 8860aa033c | |||
| 3ca14d8b6e | |||
| 099e9c4a3e | |||
| cfe6b9408a | |||
| 6ececc749f | |||
| 79901fbb86 |
@@ -28,10 +28,6 @@ steps:
|
|||||||
|
|
||||||
build_binary linux amd64 ""
|
build_binary linux amd64 ""
|
||||||
build_binary linux arm64 ""
|
build_binary linux arm64 ""
|
||||||
build_binary darwin amd64 ""
|
|
||||||
build_binary darwin arm64 ""
|
|
||||||
build_binary windows amd64 ".exe"
|
|
||||||
build_binary windows arm64 ".exe"
|
|
||||||
|
|
||||||
- name: publish-release
|
- name: publish-release
|
||||||
image: woodpeckerci/plugin-release
|
image: woodpeckerci/plugin-release
|
||||||
|
|||||||
@@ -32,6 +32,8 @@ Integration references:
|
|||||||
- an effective `prompt_dir` and `profile_dir` (from flags or config)
|
- an effective `prompt_dir` and `profile_dir` (from flags or config)
|
||||||
- `serve` requires an effective `prompt_dir` and `profile_dir` (from flags or config).
|
- `serve` requires an effective `prompt_dir` and `profile_dir` (from flags or config).
|
||||||
- Positional arguments are rejected.
|
- Positional arguments are rejected.
|
||||||
|
- Prompt cache control is configured in prompt YAML (`messages[].cache_control`), not with CLI flags.
|
||||||
|
- Provider-specific `reasoning_effort` and `extra_params` are configured in profile YAML or HTTP model overrides, not with CLI flags.
|
||||||
|
|
||||||
## Flag Reference
|
## Flag Reference
|
||||||
|
|
||||||
@@ -56,6 +58,11 @@ Integration references:
|
|||||||
- `--top-p <float>`: runtime top-p override.
|
- `--top-p <float>`: runtime top-p override.
|
||||||
- `--timeout <duration>`: runtime timeout override (Go duration syntax, for example `30s`, `2m`).
|
- `--timeout <duration>`: runtime timeout override (Go duration syntax, for example `30s`, `2m`).
|
||||||
|
|
||||||
|
Numeric runtime override flags are presence-aware:
|
||||||
|
|
||||||
|
- omitted numeric flags preserve the selected profile/default value
|
||||||
|
- explicit zero values override the selected profile/default value (`--temperature 0`, `--max-tokens 0`, `--top-p 0`, `--timeout 0s`)
|
||||||
|
|
||||||
### `scriptorium render`
|
### `scriptorium render`
|
||||||
|
|
||||||
- Supports the same flags as `run`, except:
|
- Supports the same flags as `run`, except:
|
||||||
@@ -82,6 +89,7 @@ Notes:
|
|||||||
|
|
||||||
- `--input name=path` maps prompt input names to local file paths.
|
- `--input name=path` maps prompt input names to local file paths.
|
||||||
- `--var name=value` maps template variable names to values.
|
- `--var name=value` maps template variable names to values.
|
||||||
|
- If a prompt defines `session_id: "{{ .session_id }}"`, pass the OpenRouter sticky-routing value with `--var session_id=<value>`.
|
||||||
- Both flags can be repeated.
|
- Both flags can be repeated.
|
||||||
- Both flags also support comma-separated batches, for example:
|
- Both flags also support comma-separated batches, for example:
|
||||||
- `--input transcript=./t.md,glossary=./g.yml`
|
- `--input transcript=./t.md,glossary=./g.yml`
|
||||||
@@ -93,6 +101,7 @@ Notes:
|
|||||||
- Writes generated artifact content to stdout by default.
|
- Writes generated artifact content to stdout by default.
|
||||||
- Writes generated artifact content to `--out` when provided.
|
- Writes generated artifact content to `--out` when provided.
|
||||||
- Prints run summary metadata to stderr on success.
|
- Prints run summary metadata to stderr on success.
|
||||||
|
- Appends `cached_tokens=<n> cache_write_tokens=<n>` to the summary only when the provider reports non-zero cache usage.
|
||||||
- Prints errors to stderr on failure.
|
- Prints errors to stderr on failure.
|
||||||
|
|
||||||
`render`:
|
`render`:
|
||||||
|
|||||||
@@ -104,6 +104,7 @@ Field reference:
|
|||||||
- `version` (required): prompt version.
|
- `version` (required): prompt version.
|
||||||
- `default_profile` (optional): profile ID used when request does not provide `profile_id`.
|
- `default_profile` (optional): profile ID used when request does not provide `profile_id`.
|
||||||
- `description` (optional): prompt description.
|
- `description` (optional): prompt description.
|
||||||
|
- `session_id` (optional): Go-template string for OpenRouter sticky-routing `session_id`; rendered from request vars.
|
||||||
- `inputs` (optional list): expected named inputs.
|
- `inputs` (optional list): expected named inputs.
|
||||||
- `messages` (required list): prompt message templates.
|
- `messages` (required list): prompt message templates.
|
||||||
- `output` (required object): output contract.
|
- `output` (required object): output contract.
|
||||||
@@ -119,6 +120,7 @@ Field reference:
|
|||||||
|
|
||||||
- `role` (required)
|
- `role` (required)
|
||||||
- `content` or `content_file` (exactly one is required)
|
- `content` or `content_file` (exactly one is required)
|
||||||
|
- `cache_control` (optional object): provider prompt-cache metadata for this message
|
||||||
|
|
||||||
Message rules:
|
Message rules:
|
||||||
|
|
||||||
@@ -128,6 +130,35 @@ Message rules:
|
|||||||
- Prompt decoding is strict; unknown YAML fields are rejected.
|
- Prompt decoding is strict; unknown YAML fields are rejected.
|
||||||
- Duplicate prompt IDs are invalid. If multiple files declare the requested prompt ID, Scriptorium fails instead of choosing one.
|
- Duplicate prompt IDs are invalid. If multiple files declare the requested prompt ID, Scriptorium fails instead of choosing one.
|
||||||
|
|
||||||
|
`messages[].cache_control` fields:
|
||||||
|
|
||||||
|
- `type` (required when `cache_control` is present): currently only `ephemeral`.
|
||||||
|
- `ttl` (optional): currently only `1h`; omitted from outbound requests when unset.
|
||||||
|
|
||||||
|
Example cache-controlled message:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content_file: ./stable_context.md
|
||||||
|
cache_control:
|
||||||
|
type: ephemeral
|
||||||
|
ttl: 1h
|
||||||
|
- role: user
|
||||||
|
content: |
|
||||||
|
{{input "transcript"}}
|
||||||
|
```
|
||||||
|
|
||||||
|
Use cache control on stable reusable prompt content. Dynamic per-run inputs before the cache-controlled message change the provider cache key.
|
||||||
|
|
||||||
|
Example prompt-level session ID:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
session_id: "{{ .session_id }}"
|
||||||
|
```
|
||||||
|
|
||||||
|
When configured, `session_id` is rendered with the same variable context as messages. The rendered value is trimmed, omitted when empty, and rejected if longer than 256 characters. CLI callers pass the value through `--var session_id=<value>`; HTTP callers pass it through `"vars": {"session_id": "<value>"}`.
|
||||||
|
|
||||||
`output` fields:
|
`output` fields:
|
||||||
|
|
||||||
- `format` (required): `text`, `markdown`, or `json`.
|
- `format` (required): `text`, `markdown`, or `json`.
|
||||||
@@ -158,6 +189,11 @@ top_p: 1.0
|
|||||||
timeout_seconds: 90
|
timeout_seconds: 90
|
||||||
api_key_env: SCRIPTORIUM_API_KEY
|
api_key_env: SCRIPTORIUM_API_KEY
|
||||||
service_tier: priority
|
service_tier: priority
|
||||||
|
reasoning_effort: medium
|
||||||
|
extra_params:
|
||||||
|
provider_route: primary
|
||||||
|
provider_options:
|
||||||
|
retry_budget: 2
|
||||||
```
|
```
|
||||||
|
|
||||||
Field reference:
|
Field reference:
|
||||||
@@ -170,9 +206,9 @@ Field reference:
|
|||||||
- `top_p` (optional): range `0..1`
|
- `top_p` (optional): range `0..1`
|
||||||
- `timeout_seconds` (optional): `>= 0`
|
- `timeout_seconds` (optional): `>= 0`
|
||||||
- `service_tier` (optional): provider-specific request tier such as OpenRouter `flex` or `priority`
|
- `service_tier` (optional): provider-specific request tier such as OpenRouter `flex` or `priority`
|
||||||
- `reasoning_effort` (optional)
|
- `reasoning_effort` (optional): serialized as top-level `reasoning_effort` in outbound chat-completions requests
|
||||||
- `api_key_env` (optional)
|
- `api_key_env` (optional)
|
||||||
- `extra_params` (optional map of strings)
|
- `extra_params` (optional map): JSON-compatible provider-specific parameters. Values may be strings, numbers, booleans, objects, or arrays.
|
||||||
|
|
||||||
Profile rules:
|
Profile rules:
|
||||||
|
|
||||||
@@ -180,11 +216,14 @@ Profile rules:
|
|||||||
- Raw `api_key` is rejected; use `api_key_env`.
|
- Raw `api_key` is rejected; use `api_key_env`.
|
||||||
- If `api_key_env` is set, that environment variable must be set when preparing/running.
|
- If `api_key_env` is set, that environment variable must be set when preparing/running.
|
||||||
- Duplicate profile IDs are invalid. If multiple files declare the requested profile ID, Scriptorium fails instead of choosing one.
|
- Duplicate profile IDs are invalid. If multiple files declare the requested profile ID, Scriptorium fails instead of choosing one.
|
||||||
|
- `extra_params` keys must not be empty and must not collide with reserved outbound request fields: `model`, `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, or `response_format`.
|
||||||
|
|
||||||
Current outbound request behavior:
|
Current outbound request behavior:
|
||||||
|
|
||||||
- The OpenAI-compatible client currently serializes: `model`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, and optional `response_format` for `json_schema` prompts.
|
- The OpenAI-compatible client currently serializes: `model`, optional `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, optional `response_format` for `json_schema` prompts, and `extra_params`.
|
||||||
- `reasoning_effort` and `extra_params` are parsed and carried in effective settings, but are not currently serialized into outbound chat-completions requests.
|
- `extra_params` are flattened into provider-specific top-level JSON request fields. They are not wrapped in an `extra_params` object on the outbound provider request.
|
||||||
|
- Messages without `cache_control` serialize with string `content`.
|
||||||
|
- Messages with `cache_control` serialize as a single text content-block array containing `cache_control`.
|
||||||
|
|
||||||
## Schema Behavior
|
## Schema Behavior
|
||||||
|
|
||||||
|
|||||||
@@ -50,7 +50,10 @@ Copyable request example file:
|
|||||||
"reasoning_effort": "medium",
|
"reasoning_effort": "medium",
|
||||||
"api_key_env": "SCRIPTORIUM_API_KEY",
|
"api_key_env": "SCRIPTORIUM_API_KEY",
|
||||||
"extra_params": {
|
"extra_params": {
|
||||||
"route": "primary"
|
"route": "primary",
|
||||||
|
"provider_options": {
|
||||||
|
"retry_budget": 2
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"include_raw_output": false
|
"include_raw_output": false
|
||||||
@@ -67,6 +70,14 @@ Input reference types currently supported by runtime artifact loading:
|
|||||||
- `file`
|
- `file`
|
||||||
- `inline`
|
- `inline`
|
||||||
|
|
||||||
|
Model override notes:
|
||||||
|
|
||||||
|
- Numeric model override fields distinguish omitted values from explicit zero values. For example, omitting `temperature` preserves the selected profile/default value, while `"temperature": 0` explicitly sets the effective temperature to zero.
|
||||||
|
- `extra_params` accepts JSON-compatible values: strings, numbers, booleans, objects, and arrays.
|
||||||
|
- `extra_params` are passed through effective model metadata and flattened into top-level provider request fields by the OpenAI-compatible client.
|
||||||
|
- `extra_params` keys must not be empty and must not collide with reserved outbound fields: `model`, `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, or `response_format`.
|
||||||
|
- Raw API-key values are not accepted. Use `api_key_env` to name an environment variable.
|
||||||
|
|
||||||
## Strict JSON Rules
|
## Strict JSON Rules
|
||||||
|
|
||||||
Request decoding uses strict JSON field checks:
|
Request decoding uses strict JSON field checks:
|
||||||
@@ -119,7 +130,10 @@ Response shape:
|
|||||||
"reasoning_effort": "medium",
|
"reasoning_effort": "medium",
|
||||||
"api_key_env": "SCRIPTORIUM_API_KEY",
|
"api_key_env": "SCRIPTORIUM_API_KEY",
|
||||||
"extra_params": {
|
"extra_params": {
|
||||||
"route": "primary"
|
"route": "primary",
|
||||||
|
"provider_options": {
|
||||||
|
"retry_budget": 2
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"input_hashes": {
|
"input_hashes": {
|
||||||
@@ -128,7 +142,9 @@ Response shape:
|
|||||||
"usage": {
|
"usage": {
|
||||||
"prompt_tokens": 11,
|
"prompt_tokens": 11,
|
||||||
"completion_tokens": 22,
|
"completion_tokens": 22,
|
||||||
"total_tokens": 33
|
"total_tokens": 33,
|
||||||
|
"cached_tokens": 0,
|
||||||
|
"cache_write_tokens": 0
|
||||||
},
|
},
|
||||||
"start_time": "2026-05-04T12:00:00Z",
|
"start_time": "2026-05-04T12:00:00Z",
|
||||||
"end_time": "2026-05-04T12:00:01Z",
|
"end_time": "2026-05-04T12:00:01Z",
|
||||||
@@ -142,6 +158,8 @@ Response shape:
|
|||||||
|
|
||||||
`raw_model_output` is omitted by default.
|
`raw_model_output` is omitted by default.
|
||||||
|
|
||||||
|
`metadata.usage.cached_tokens` and `metadata.usage.cache_write_tokens` are always present as numbers. They are `0` when the provider omits compatible cache usage fields or reports no cache activity.
|
||||||
|
|
||||||
To include it, send:
|
To include it, send:
|
||||||
|
|
||||||
- `"include_raw_output": true`
|
- `"include_raw_output": true`
|
||||||
|
|||||||
@@ -26,15 +26,85 @@ Example:
|
|||||||
Serialized JSON fields:
|
Serialized JSON fields:
|
||||||
|
|
||||||
- `model` (required after fallback resolution)
|
- `model` (required after fallback resolution)
|
||||||
- `messages` (role/content pairs from rendered prompt)
|
- `session_id` (only when the rendered prompt includes a non-empty session ID)
|
||||||
- `temperature` (only when non-zero)
|
- `messages` (rendered prompt messages)
|
||||||
- `max_tokens` (only when non-zero)
|
- `temperature` (when non-zero, or when explicitly overridden to zero)
|
||||||
- `top_p` (only when non-zero)
|
- `max_tokens` (when non-zero, or when explicitly overridden to zero)
|
||||||
|
- `top_p` (when non-zero, or when explicitly overridden to zero)
|
||||||
- `service_tier` (only when non-empty)
|
- `service_tier` (only when non-empty)
|
||||||
|
- `reasoning_effort` (only when non-empty)
|
||||||
- `response_format` (only when structured output is provided)
|
- `response_format` (only when structured output is provided)
|
||||||
|
- profile/request `extra_params` as additional provider-specific top-level fields
|
||||||
|
|
||||||
`service_tier` is provider-specific. OpenRouter currently documents request values such as `flex` and `priority`; Scriptorium forwards any non-empty configured value and lets the backend validate support.
|
`service_tier` is provider-specific. OpenRouter currently documents request values such as `flex` and `priority`; Scriptorium forwards any non-empty configured value and lets the backend validate support.
|
||||||
|
|
||||||
|
`reasoning_effort` is provider-specific. Scriptorium forwards any non-empty configured value as top-level `reasoning_effort` and lets the backend validate support.
|
||||||
|
|
||||||
|
`extra_params` are flattened into the outbound JSON object. They are not wrapped in an `extra_params` object:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"model": "gpt-4o-mini",
|
||||||
|
"messages": [
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": "rendered text"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"provider_route": "primary",
|
||||||
|
"provider_options": {
|
||||||
|
"retry_budget": 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`extra_params` values must be JSON-compatible. Supported value shapes include strings, numbers, booleans, objects, and arrays.
|
||||||
|
|
||||||
|
Reserved `extra_params` keys are rejected before the HTTP request is made:
|
||||||
|
|
||||||
|
- `model`
|
||||||
|
- `session_id`
|
||||||
|
- `messages`
|
||||||
|
- `temperature`
|
||||||
|
- `max_tokens`
|
||||||
|
- `top_p`
|
||||||
|
- `service_tier`
|
||||||
|
- `reasoning_effort`
|
||||||
|
- `response_format`
|
||||||
|
|
||||||
|
Empty `extra_params` keys and values that cannot be encoded as JSON are also rejected before the HTTP request is made.
|
||||||
|
|
||||||
|
`session_id` is rendered from prompt YAML using request variables and serialized as a top-level JSON request field. Scriptorium does not send an `x-session-id` header. Empty rendered session IDs are omitted, and values longer than 256 characters are rejected before the HTTP request.
|
||||||
|
|
||||||
|
Messages without prompt cache control serialize with string `content`:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"role": "system",
|
||||||
|
"content": "rendered text"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Messages with prompt cache control serialize as a single text content-block array:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"role": "system",
|
||||||
|
"content": [
|
||||||
|
{
|
||||||
|
"type": "text",
|
||||||
|
"text": "rendered text",
|
||||||
|
"cache_control": {
|
||||||
|
"type": "ephemeral",
|
||||||
|
"ttl": "1h"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
When cache-control `ttl` is unset in the prompt definition, `ttl` is omitted from the outbound payload.
|
||||||
|
|
||||||
Structured output is currently `json_schema` only, serialized as:
|
Structured output is currently `json_schema` only, serialized as:
|
||||||
|
|
||||||
```json
|
```json
|
||||||
@@ -72,6 +142,7 @@ Base timeout comes from client configuration.
|
|||||||
Per-request override:
|
Per-request override:
|
||||||
|
|
||||||
- if `Target.TimeoutSeconds > 0`, use that value for request timeout
|
- if `Target.TimeoutSeconds > 0`, use that value for request timeout
|
||||||
|
- if `Target.TimeoutSeconds == 0` and the value came from an explicit request override, disable the HTTP client timeout
|
||||||
- if `Target.TimeoutSeconds < 0`, request is rejected (`ErrInvalidRequest`)
|
- if `Target.TimeoutSeconds < 0`, request is rejected (`ErrInvalidRequest`)
|
||||||
|
|
||||||
## Response Expectations
|
## Response Expectations
|
||||||
@@ -82,6 +153,13 @@ Expected successful response shape (subset used):
|
|||||||
- `usage.prompt_tokens`
|
- `usage.prompt_tokens`
|
||||||
- `usage.completion_tokens`
|
- `usage.completion_tokens`
|
||||||
- `usage.total_tokens`
|
- `usage.total_tokens`
|
||||||
|
- `usage.prompt_tokens_details.cached_tokens` (optional)
|
||||||
|
- `usage.cache_write_tokens` (optional)
|
||||||
|
|
||||||
|
Absent cache usage fields are treated as zero. Parsed cache usage is exposed through run results and adapter response surfaces as:
|
||||||
|
|
||||||
|
- `cached_tokens`
|
||||||
|
- `cache_write_tokens`
|
||||||
|
|
||||||
Malformed response conditions include:
|
Malformed response conditions include:
|
||||||
|
|
||||||
@@ -99,10 +177,7 @@ Malformed responses return `ErrMalformedResponse`.
|
|||||||
|
|
||||||
## Unsupported Or Non-Serialized Fields
|
## Unsupported Or Non-Serialized Fields
|
||||||
|
|
||||||
The following fields may exist in profile/effective settings but are not currently serialized into outbound chat-completions payloads:
|
The client does not serialize top-level `cache_control`.
|
||||||
|
|
||||||
- `reasoning_effort`
|
|
||||||
- `extra_params`
|
|
||||||
|
|
||||||
No built-in retries, tool-calls, or multi-request payload modes are implemented in this client.
|
No built-in retries, tool-calls, or multi-request payload modes are implemented in this client.
|
||||||
|
|
||||||
|
|||||||
@@ -22,11 +22,13 @@ CLI adapter:
|
|||||||
|
|
||||||
- Input: process args, filesystem config/assets, environment.
|
- Input: process args, filesystem config/assets, environment.
|
||||||
- Output: exit code, stdout artifact/prepared output, stderr summaries/errors.
|
- Output: exit code, stdout artifact/prepared output, stderr summaries/errors.
|
||||||
|
- `run` summaries include cache usage counters only when either parsed cache counter is non-zero.
|
||||||
|
|
||||||
HTTP adapter:
|
HTTP adapter:
|
||||||
|
|
||||||
- Input: JSON request body (`runRequestDTO`).
|
- Input: JSON request body (`runRequestDTO`).
|
||||||
- Output: JSON success/error body with mapped status codes.
|
- Output: JSON success/error body with mapped status codes.
|
||||||
|
- Success metadata includes token usage plus cache usage counters.
|
||||||
|
|
||||||
Filesystem repositories:
|
Filesystem repositories:
|
||||||
|
|
||||||
@@ -67,6 +69,8 @@ Primary app settings consumed by adapters:
|
|||||||
Execution profile/request settings used through runner:
|
Execution profile/request settings used through runner:
|
||||||
|
|
||||||
- `endpoint`, `model`, `temperature`, `max_tokens`, `top_p`, `timeout_seconds`, `service_tier`, `api_key_env`, `reasoning_effort`, `extra_params`
|
- `endpoint`, `model`, `temperature`, `max_tokens`, `top_p`, `timeout_seconds`, `service_tier`, `api_key_env`, `reasoning_effort`, `extra_params`
|
||||||
|
- CLI and HTTP request adapters preserve caller intent for numeric runtime overrides. Omitted values remain absent; explicit zero values are mapped as explicit overrides.
|
||||||
|
- HTTP `extra_params` accepts JSON-compatible values and maps them to domain request overrides without provider-specific adapter logic.
|
||||||
|
|
||||||
## External Dependencies
|
## External Dependencies
|
||||||
|
|
||||||
@@ -93,6 +97,13 @@ Artifact refs:
|
|||||||
LLM adapter:
|
LLM adapter:
|
||||||
|
|
||||||
- endpoint appends `/chat/completions`.
|
- endpoint appends `/chat/completions`.
|
||||||
|
- rendered messages without cache control serialize with string `content`.
|
||||||
|
- rendered messages with cache control serialize as one text content block with `cache_control`.
|
||||||
|
- non-empty `reasoning_effort` serializes as a top-level provider request field.
|
||||||
|
- `extra_params` flatten into provider-specific top-level JSON request fields.
|
||||||
|
- reserved `extra_params` keys are rejected before the provider call: `model`, `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, and `response_format`.
|
||||||
|
- empty `extra_params` keys and values that cannot be JSON-encoded are rejected before the provider call.
|
||||||
|
- compatible cache usage response fields are parsed into domain token usage.
|
||||||
- non-2xx responses map to request failure errors.
|
- non-2xx responses map to request failure errors.
|
||||||
- malformed responses (including missing/empty first choice content) are errors.
|
- malformed responses (including missing/empty first choice content) are errors.
|
||||||
|
|
||||||
@@ -105,6 +116,7 @@ Validator:
|
|||||||
HTTP error mapping:
|
HTTP error mapping:
|
||||||
|
|
||||||
- maps domain/use-case errors to stable HTTP code + error code/message.
|
- maps domain/use-case errors to stable HTTP code + error code/message.
|
||||||
|
- distinguishes missing profile selection and missing `api_key_env` variable using stable use-case sentinel errors.
|
||||||
- avoids returning internal wrapped-cause details in response payload.
|
- avoids returning internal wrapped-cause details in response payload.
|
||||||
|
|
||||||
## CLI Adapter Semantics
|
## CLI Adapter Semantics
|
||||||
@@ -140,4 +152,5 @@ Behavior highlights:
|
|||||||
- Adapter packages do not own runner decision logic.
|
- Adapter packages do not own runner decision logic.
|
||||||
- External request/response strictness is part of contract stability.
|
- External request/response strictness is part of contract stability.
|
||||||
- Prepared-render output never includes resolved API key values.
|
- Prepared-render output never includes resolved API key values.
|
||||||
- Outbound OpenAI-compatible request includes only currently serialized fields (`model`, `messages`, optional `temperature`, `max_tokens`, `top_p`, optional `response_format`).
|
- Outbound OpenAI-compatible request includes currently serialized first-class fields (`model`, optional `session_id`, `messages`, optional `temperature`, `max_tokens`, `top_p`, optional `service_tier`, optional `reasoning_effort`, optional `response_format`) plus validated `extra_params` flattened as provider-specific top-level fields.
|
||||||
|
- Outbound cache control is message-level only; no top-level cache-control field is serialized.
|
||||||
|
|||||||
@@ -66,14 +66,21 @@ It receives fully constructed repositories/readers/validators from adapters. Eff
|
|||||||
|
|
||||||
## Failure Behavior
|
## Failure Behavior
|
||||||
|
|
||||||
Key error classes surfaced from `Runner`:
|
Primary runner error classes:
|
||||||
|
|
||||||
- `ErrInvalidRequest`: invalid prompt/profile/request/runtime/API-key-env prerequisites.
|
- `ErrInvalidRequest`: invalid run request envelope.
|
||||||
- `ErrProfileLoad`: prompt or profile load failures.
|
- `ErrProfileRequired`: specific invalid-request reason when neither request `profile_id` nor prompt `default_profile` is available.
|
||||||
|
- `ErrAPIKeyEnvMissing`: specific invalid-request reason when `api_key_env` is set but the named environment variable is unset/empty.
|
||||||
|
- `ErrProfileLoad`: prompt/profile repository load failures.
|
||||||
- `ErrArtifactLoad`: artifact read failures.
|
- `ErrArtifactLoad`: artifact read failures.
|
||||||
- `ErrPromptRender`: template render failures.
|
- `ErrPromptRender`: template render failures.
|
||||||
- `ErrLLMGenerate`: model request failures.
|
- `ErrLLMGenerate`: outbound model request failures.
|
||||||
- `ErrValidation`: validation runtime failures (including schema load/compile failures).
|
- `ErrValidation`: validation runtime failures (including structured-output schema load/compile failures).
|
||||||
|
|
||||||
|
Reason sentinel behavior:
|
||||||
|
|
||||||
|
- `ErrProfileRequired` and `ErrAPIKeyEnvMissing` are wrapped with `ErrInvalidRequest`.
|
||||||
|
- Adapters can use `errors.Is` for stable reason mapping without matching runner prose.
|
||||||
|
|
||||||
Validation content failures are not run errors:
|
Validation content failures are not run errors:
|
||||||
|
|
||||||
@@ -90,20 +97,33 @@ Validation content failures are not run errors:
|
|||||||
3. select profile ID:
|
3. select profile ID:
|
||||||
- explicit request profile ID
|
- explicit request profile ID
|
||||||
- prompt `default_profile`
|
- prompt `default_profile`
|
||||||
- otherwise request error
|
- otherwise return an invalid request with `ErrProfileRequired`
|
||||||
4. load execution profile.
|
4. load execution profile.
|
||||||
5. merge effective runtime target:
|
5. merge effective runtime target:
|
||||||
- built-in execution defaults
|
- built-in execution defaults
|
||||||
- selected profile values
|
- selected profile values
|
||||||
- request overrides
|
- request overrides
|
||||||
6. verify required `api_key_env` environment variable (name only; value is not returned).
|
- request numeric overrides are presence-aware, so omitted values preserve the current effective value and explicit zero values override it
|
||||||
|
6. verify required `api_key_env` environment variable:
|
||||||
|
- missing/empty env value returns an invalid request with `ErrAPIKeyEnvMissing`
|
||||||
|
- only the environment-variable name is retained; secret value is never returned
|
||||||
7. resolve output contract and structured-output schema payload when `json_schema` mode is active.
|
7. resolve output contract and structured-output schema payload when `json_schema` mode is active.
|
||||||
8. read input artifacts.
|
8. read input artifacts.
|
||||||
9. render prompt messages.
|
9. render prompt messages, including any normalized message cache-control metadata.
|
||||||
10. compute prompt/input/render hashes and return `PreparedRun`.
|
10. compute prompt/input/render hashes and return `PreparedRun`.
|
||||||
|
|
||||||
|
`rendered_prompt_hash` includes cache-control metadata when present because it affects the outbound provider request. Prompts without cache control keep the role/content hash behavior.
|
||||||
|
|
||||||
`Prepare` does not call the LLM.
|
`Prepare` does not call the LLM.
|
||||||
|
|
||||||
|
Runtime target notes:
|
||||||
|
|
||||||
|
- Profile `extra_params` and request `extra_params` carry JSON-compatible values through prepared output, run metadata, and `domain.GenerateRequest.Target`.
|
||||||
|
- The OpenAI-compatible client serializes non-empty `reasoning_effort` as a top-level provider request field.
|
||||||
|
- The OpenAI-compatible client flattens `extra_params` into provider-specific top-level JSON request fields.
|
||||||
|
- Empty `extra_params` keys, reserved outbound field names, and values that cannot be JSON-encoded fail before the provider request.
|
||||||
|
- Resolved API-key values are never stored in `PreparedRun`, `RunResult`, logs, or HTTP responses.
|
||||||
|
|
||||||
## Run Flow
|
## Run Flow
|
||||||
|
|
||||||
`Run` performs:
|
`Run` performs:
|
||||||
@@ -114,7 +134,7 @@ Validation content failures are not run errors:
|
|||||||
4. build output artifact content type from output format.
|
4. build output artifact content type from output format.
|
||||||
5. validate output.
|
5. validate output.
|
||||||
6. optionally attempt bounded repair when repairer is injected and contract allows it.
|
6. optionally attempt bounded repair when repairer is injected and contract allows it.
|
||||||
7. return `RunResult` with artifact, raw output, validation, hashes, profile/model metadata, usage, and timestamps.
|
7. return `RunResult` with artifact, raw output, validation, hashes, profile/model metadata, token/cache usage, and timestamps.
|
||||||
|
|
||||||
## Repair Hook Boundary
|
## Repair Hook Boundary
|
||||||
|
|
||||||
|
|||||||
@@ -1,519 +0,0 @@
|
|||||||
# Code Quality And Deduplication Audit
|
|
||||||
|
|
||||||
## Executive Summary
|
|
||||||
|
|
||||||
Overall code quality is solid for the current project size. The repository follows the documented shape: thin CLI/HTTP adapters, a central `Runner`, strict config/prompt/profile loading, narrow filesystem and LLM adapters, and package tests around the important public behaviors.
|
|
||||||
|
|
||||||
The top three refactoring targets are:
|
|
||||||
|
|
||||||
1. centralize execution-target field mapping across profiles, CLI overrides, HTTP DTOs, prepared output, and LLM request serialization;
|
|
||||||
2. share filesystem repository scanning helpers between prompt and profile repositories;
|
|
||||||
3. reduce CLI command wiring duplication for app settings, runner construction, and common preflight behavior.
|
|
||||||
|
|
||||||
The codebase appears ready for a limited cleanup pass. No major architectural rewrite is warranted. The main risk before a release is maintenance drift when adding fields or policy to execution settings, repositories, or command setup.
|
|
||||||
|
|
||||||
## Repository Map Reviewed
|
|
||||||
|
|
||||||
Main areas inspected:
|
|
||||||
|
|
||||||
- `cmd/scriptorium`: process entrypoint path, via repository layout and command docs.
|
|
||||||
- `internal/adapter/cli`: `run`, `render`, `serve` parsing, config loading, app wiring, stdout/stderr behavior, summary output, exit-code policy.
|
|
||||||
- `internal/adapter/http`: request DTOs, strict JSON decoding, domain mapping, response metadata mapping, error mapping.
|
|
||||||
- `internal/config`: app config loading, defaults, strict YAML decoding, CLI override precedence.
|
|
||||||
- `internal/defaults`: built-in runtime/config/content-type defaults.
|
|
||||||
- `internal/domain`: domain request/result/profile/target/artifact/validation types.
|
|
||||||
- `internal/usecase`: `Runner.Prepare`, `Runner.Run`, execution target merge, schema structured-output setup, validation and repair policy.
|
|
||||||
- `internal/promptdef`: recursive prompt YAML loading, prompt normalization, content-file resolution, prompt output-contract validation.
|
|
||||||
- `internal/profile`: recursive profile YAML loading, strict decoding, duplicate ID handling, profile validation, raw API-key rejection.
|
|
||||||
- `internal/artifact`: `inline` and `file` artifact resolution.
|
|
||||||
- `internal/prompt`: Go template rendering and required input checks.
|
|
||||||
- `internal/llm`: OpenAI-compatible request construction, timeout/API-key handling, response parsing.
|
|
||||||
- `internal/validate`: basic/JSON/JSON Schema validation and schema document loading.
|
|
||||||
- `internal/format`: prepared-run text/JSON output.
|
|
||||||
- `examples/`: maintained example config, prompts, profiles, schemas, fixtures, HTTP/render examples.
|
|
||||||
- Package tests under `internal/**`.
|
|
||||||
|
|
||||||
Major execution paths reviewed:
|
|
||||||
|
|
||||||
- CLI `run`: parse flags/config, build `RunRequest`, construct runner, call LLM, write artifact, print summary, choose exit code.
|
|
||||||
- CLI `render`: parse flags/config, build `RunRequest`, construct runner without LLM, prepare only, format prepared run.
|
|
||||||
- CLI `serve`: parse flags/config, construct runner, expose HTTP `POST /v1/runs`.
|
|
||||||
- HTTP `POST /v1/runs`: strict decode DTO, map to `RunRequest`, map `RunResult` to JSON response.
|
|
||||||
- Runner `Prepare` and `Run`: prompt/profile/artifact/schema resolution, render, LLM generate, validation, repair hook.
|
|
||||||
|
|
||||||
Areas not deeply inspected:
|
|
||||||
|
|
||||||
- No `internal/app`, `internal/stage`, `internal/modules`, `internal/storage`, `internal/manifest`, `pkg`, or external test-suite directories exist in the current repository.
|
|
||||||
- Full test execution was not run because this is a report-only pass; inspection was static.
|
|
||||||
|
|
||||||
## High-Confidence Deduplication Opportunities
|
|
||||||
|
|
||||||
### 1. Execution Target Field Mapping Is Repeated Across Boundaries
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/domain/domain.go`
|
|
||||||
- `internal/usecase/runner.go`
|
|
||||||
- `internal/adapter/cli/run.go`
|
|
||||||
- `internal/adapter/http/dto.go`
|
|
||||||
- `internal/adapter/http/handler.go`
|
|
||||||
- `internal/llm/openai_compatible_client.go`
|
|
||||||
- `internal/format/prepared_run.go`
|
|
||||||
- tests in `internal/usecase`, `internal/adapter/http`, `internal/adapter/cli`, `internal/llm`, and `internal/format`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Execution settings fields are listed and copied in several places: profile-to-target conversion, target merge, CLI runtime override construction, HTTP request mapping, HTTP metadata mapping, prepared-run text output, and OpenAI-compatible request construction.
|
|
||||||
- Adding `service_tier` required coordinated edits across many of these sites, which is a strong signal that field-level mapping is too scattered.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- A future profile/runtime field can easily be parsed but not sent, displayed but not merged, accepted over HTTP but not included in metadata, or tested in one command but not another.
|
|
||||||
- This is public-interface drift risk because CLI, HTTP, render output, and provider requests all expose different slices of the same effective execution target.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Keep transport DTOs local to adapters, but add small mapping helpers at each boundary:
|
|
||||||
- a use-case/domain helper for profile-to-target copy and execution-target merge;
|
|
||||||
- an HTTP helper such as `executionTargetFromModelOverrideDTO` and `modelParamsDTOFromExecutionTarget`;
|
|
||||||
- an LLM helper such as `openAIChatRequestFromGenerateRequest` for outbound serialization.
|
|
||||||
- Add a table-driven test that constructs an `ExecutionTarget` with every supported field and verifies the HTTP metadata DTO and LLM request payload retain the intended fields.
|
|
||||||
- Avoid reflection-based generic copying; explicit mapping is still clearer here.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Expand runner merge tests to use an all-fields target.
|
|
||||||
- Add HTTP DTO mapping tests that fail when a domain execution field is omitted from request or metadata mapping.
|
|
||||||
- Add LLM request serialization tests for all serialized fields and explicit omission of non-serialized fields.
|
|
||||||
|
|
||||||
Risk level: high. The refactor itself is low-to-medium implementation risk, but the duplicated behavior has high drift risk.
|
|
||||||
|
|
||||||
### 2. Prompt And Profile Filesystem Repository Scanning Is Duplicated
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/promptdef/filesystem_repository.go`
|
|
||||||
- `internal/profile/filesystem_repository.go`
|
|
||||||
- repository tests in `internal/promptdef` and `internal/profile`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Both repositories recursively walk configured directories, filter `.yaml`/`.yml`, sort paths, compute relative paths, derive likely IDs from filenames, use strict YAML decoding, partially decode IDs to decide whether to surface malformed likely-target files, detect duplicate IDs, and include relative paths in errors.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Prompt/profile repository policy should remain aligned: nested scanning, stable ordering, duplicate ID errors, likely-target malformed file behavior, and relative-path diagnostics.
|
|
||||||
- Future changes to repository scanning or extension rules would need to be made in two packages.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Introduce a narrow internal helper for filesystem catalog behavior, for example `internal/filecatalog` or another small package whose scope is only:
|
|
||||||
- recursive YAML file discovery with context cancellation;
|
|
||||||
- stable sorting;
|
|
||||||
- relative path formatting;
|
|
||||||
- YAML extension checks;
|
|
||||||
- filename stem extraction.
|
|
||||||
- Keep prompt/profile-specific normalization and validation in their current packages.
|
|
||||||
- Do not make a generic repository framework.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Add helper-level tests for recursive YAML discovery, ordering, extension filtering, and relative path output.
|
|
||||||
- Keep existing prompt/profile behavior tests unchanged to confirm public error behavior survives.
|
|
||||||
|
|
||||||
Risk level: high for drift prevention; low implementation risk if the helper stays small.
|
|
||||||
|
|
||||||
### 3. CLI Command Setup And Runner Wiring Are Repeated
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/adapter/cli/run.go`
|
|
||||||
- `internal/adapter/cli/run_test.go`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- `run`, `render`, and `serve` each construct similar runner dependencies.
|
|
||||||
- `run` and `serve` both create an OpenAI-compatible client with default timeout.
|
|
||||||
- `run`, `render`, and `serve` all resolve app settings, clean dirs, validate required prompt/profile dirs, and create filesystem repositories/validator/renderer/readers.
|
|
||||||
- `run` and `render` share most execution request flags and request construction, with command-specific differences around `--schema-dir`, `--format`, and LLM use.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Adding or changing a dependency, default, or preflight check can drift between commands.
|
|
||||||
- The current duplication is still readable, but it is large enough that future command additions or option changes will be error-prone.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Add a small CLI-local wiring helper, such as `newRunnerFromDirs(promptDir, profileDir, schemaDir string, llmClient llm.Client) *usecase.Runner`.
|
|
||||||
- Add a small CLI-local resolved settings struct for common `prompt_dir`, `profile_dir`, `schema_dir`, and render-format handling.
|
|
||||||
- Preserve the existing command-specific parse functions and flag surfaces; do not hide command behavior behind a generic command framework.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Keep parser tests for each command.
|
|
||||||
- Add one test that verifies `run`, `render`, and `serve` use the same configured prompt/profile/schema dirs by exercising config-derived dirs.
|
|
||||||
- Keep command-level success tests for `run` and `render`.
|
|
||||||
|
|
||||||
Risk level: medium. Behavior is public, but a small CLI-local helper can be behavior-preserving.
|
|
||||||
|
|
||||||
### 4. HTTP Error Mapping Depends On Error Message Substrings
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/usecase/runner.go`
|
|
||||||
- `internal/adapter/http/handler.go`
|
|
||||||
- `internal/adapter/http/handler_test.go`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Runner creates `ErrInvalidRequest` with human-readable details for cases such as missing profile selection and missing API-key env.
|
|
||||||
- HTTP maps some specific invalid-request cases by checking `strings.Contains(err.Error(), ...)`.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- User-facing HTTP error codes can drift if a runner error message is clarified.
|
|
||||||
- This crosses package boundaries in a brittle way: HTTP should depend on stable error identity, not exact prose from `internal/usecase`.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Add narrow sentinel errors or typed invalid-request reasons in `internal/usecase`, for example profile-required and API-key-env-missing.
|
|
||||||
- Keep HTTP response messages stable and adapter-owned.
|
|
||||||
- Do not expose HTTP-specific error codes from the runner.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- HTTP error mapping tests should assert `profile_required` and `api_key_env_missing` via sentinel wrapping, not via message matching.
|
|
||||||
- Runner tests should assert `errors.Is` for the new reason errors.
|
|
||||||
|
|
||||||
Risk level: high for public API stability; low-to-medium implementation risk.
|
|
||||||
|
|
||||||
## Medium-Confidence Opportunities
|
|
||||||
|
|
||||||
### 1. Strict YAML Decode Setup Is Repeated
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/config/config.go`
|
|
||||||
- `internal/promptdef/filesystem_repository.go`
|
|
||||||
- `internal/profile/filesystem_repository.go`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Each package constructs a YAML decoder and enables `KnownFields(true)`.
|
|
||||||
|
|
||||||
Semantic differences that may be intentional:
|
|
||||||
|
|
||||||
- Prompt/profile loaders need likely-target behavior and raw `api_key` handling.
|
|
||||||
- Config loading has its own explicit/implicit file search policy.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Strict decoding is an architectural invariant. A future YAML loader could forget to enable it.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Consider a tiny helper for strict YAML decode from bytes.
|
|
||||||
- Keep package-specific error wrapping and partial ID decode logic local.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Existing unknown-field tests in config, promptdef, and profile should remain.
|
|
||||||
- Add any new YAML-consuming package with an unknown-field test.
|
|
||||||
|
|
||||||
Risk level: medium.
|
|
||||||
|
|
||||||
### 2. Schema Path Resolution Is Centralized, But Schema Loading Happens Twice
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/usecase/runner.go`
|
|
||||||
- `internal/validate/standard_validator.go`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- For `json_schema`, `Runner.Prepare` asks the validator to load the schema document for provider-level structured output.
|
|
||||||
- Later validation resolves and compiles the same schema path again.
|
|
||||||
|
|
||||||
Semantic differences that may be intentional:
|
|
||||||
|
|
||||||
- Provider structured-output payload needs the raw JSON schema document.
|
|
||||||
- Runtime validation needs a compiled schema.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- The current behavior is correct, but schema file errors can surface at different phases and the same file is read more than once during `Run`.
|
|
||||||
- Future caching or schema behavior changes should have one clear owner.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Keep path resolution inside `internal/validate`.
|
|
||||||
- Consider a schema service/loader interface that can return both raw document and compiled schema from one path, only if schema-related work grows.
|
|
||||||
- Do not add caching unless repeated schema loads become a measured cost.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Keep existing nested schema path tests.
|
|
||||||
- Add a test that `Prepare` fails before LLM call when structured-output schema cannot load.
|
|
||||||
|
|
||||||
Risk level: medium.
|
|
||||||
|
|
||||||
### 3. Artifact Hashing And Output Artifact Construction Are Split
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/artifact/reader.go`
|
|
||||||
- `internal/usecase/runner.go`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Input artifacts and generated output artifacts both compute SHA-256 hashes and fill size/body/content-type fields.
|
|
||||||
|
|
||||||
Semantic differences that may be intentional:
|
|
||||||
|
|
||||||
- Input artifacts derive content type from source extension or inline default.
|
|
||||||
- Output artifacts derive content type from prompt output format and always use the default output artifact name.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Hash algorithm and artifact metadata policy should remain consistent.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Consider an `artifact.Build` or `artifact.HashBody` helper only for shared hash/size construction.
|
|
||||||
- Keep source-specific content type and naming decisions local.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Existing artifact reader tests and runner output content-type tests should cover behavior.
|
|
||||||
- Add a small helper test if a shared hash function is introduced.
|
|
||||||
|
|
||||||
Risk level: medium-low.
|
|
||||||
|
|
||||||
### 4. Domain Contains An Unsupported `s3` Artifact Reference Constant
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/domain/domain.go`
|
|
||||||
- `internal/artifact/reader.go`
|
|
||||||
- docs that state only `inline` and `file` are implemented
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Not a duplication issue. This is a cleanup issue: `domain.ArtifactRefS3` exists, but no reader supports it and docs do not document it as implemented.
|
|
||||||
|
|
||||||
Semantic differences that may be intentional:
|
|
||||||
|
|
||||||
- The constant may be a placeholder for a future adapter.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Future-facing code outside `docs/roadmap/` can confuse maintainers and tests. It also weakens the otherwise clean "implemented behavior only" policy.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Remove the constant if no near-term S3 implementation is planned.
|
|
||||||
- If kept, add a code comment that it is intentionally unsupported today and ensure docs continue to state only `inline` and `file` are supported.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Existing unsupported artifact type tests should continue to pass.
|
|
||||||
|
|
||||||
Risk level: medium-low.
|
|
||||||
|
|
||||||
### 5. Test Fixture Setup Is Repeated In CLI And Use-Case Tests
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/adapter/cli/run_test.go`
|
|
||||||
- `internal/usecase/runner_test.go`
|
|
||||||
- `internal/usecase/integration_test.go`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Tests repeatedly construct temp prompt/profile dirs, write prompt/profile YAML, set up fake LLMs/readers/renderers/validators, and build runners.
|
|
||||||
|
|
||||||
Semantic differences that may be intentional:
|
|
||||||
|
|
||||||
- Package-local tests avoid exporting test helpers and keep each package independent.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Repeated setup makes behavior-preserving refactors noisier and can obscure the exact behavior under test.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Add package-local helper builders where duplication is highest, especially in CLI command tests.
|
|
||||||
- Avoid a cross-package test utility package unless multiple packages need the same public fixture contract.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- This is test cleanup only; existing test assertions should remain equivalent.
|
|
||||||
|
|
||||||
Risk level: low.
|
|
||||||
|
|
||||||
## Boundary And Responsibility Concerns
|
|
||||||
|
|
||||||
- HTTP error mapping currently depends on runner error message text. Stable reason identity should live in `internal/usecase`; HTTP status/code/message mapping should remain in `internal/adapter/http`.
|
|
||||||
- Execution-target merge policy lives in `internal/usecase`, which fits the architecture. The concern is not placement but incomplete centralization of field mapping around that policy.
|
|
||||||
- CLI app wiring currently constructs concrete repositories/readers/renderers/validators inline in each command. That is acceptable adapter responsibility, but repeated wiring should be centralized within the CLI adapter package.
|
|
||||||
- Prompt/profile filesystem traversal is duplicated in two repository packages. A narrow filesystem catalog helper would fit the architecture because it would not own prompt/profile policy.
|
|
||||||
- LLM provider request shape is properly isolated in `internal/llm`. It should remain there; do not move OpenAI/OpenRouter request-field policy into runner or prompt definitions.
|
|
||||||
|
|
||||||
## Path, Key, And Naming Construction Review
|
|
||||||
|
|
||||||
Path and naming construction is mostly explicit and low-risk:
|
|
||||||
|
|
||||||
- Config paths are cleaned in `internal/config` and again in CLI finalization.
|
|
||||||
- Prompt `content_file` paths are resolved relative to the prompt YAML file in `internal/promptdef`.
|
|
||||||
- Schema paths are resolved through `internal/validate.StandardValidator`.
|
|
||||||
- OpenAI-compatible endpoint paths use `defaults.OpenAIChatCompletionsPath`.
|
|
||||||
- Output artifact name comes from `defaults.OutputArtifactName`.
|
|
||||||
|
|
||||||
Areas needing cleanup:
|
|
||||||
|
|
||||||
- Prompt/profile recursive YAML discovery and relative-path formatting should share one helper.
|
|
||||||
- CLI path cleaning after config resolution is repeated and should be consolidated with common command settings finalization.
|
|
||||||
- Structured schema names are derived in `internal/usecase`; this is currently one place and should stay there unless structured-output support expands.
|
|
||||||
|
|
||||||
No remote keys, cache paths, manifest paths, lock files, or generated report paths exist in the current implementation.
|
|
||||||
|
|
||||||
## Resolution And Catalog Review
|
|
||||||
|
|
||||||
Named concept resolution is mostly consistent:
|
|
||||||
|
|
||||||
- Prompt resolution uses YAML `id` and optional `version`, not file path.
|
|
||||||
- Profile resolution uses YAML `id`, not file path.
|
|
||||||
- Prompt/profile subdirectories are organizational only.
|
|
||||||
- Duplicate prompt/profile IDs fail instead of choosing first match.
|
|
||||||
- Schema resolution is explicit path-based relative to `schema_dir`; no basename search.
|
|
||||||
- Input artifacts are resolved by `artifact.Reader`, with `inline` and `file` as implemented types.
|
|
||||||
- Prompt required-input checks happen in the renderer.
|
|
||||||
|
|
||||||
Recommended centralization:
|
|
||||||
|
|
||||||
- Share only filesystem catalog mechanics between prompt/profile repositories.
|
|
||||||
- Keep prompt ID/version, profile ID, schema path, and input resolution policies in their current owning packages.
|
|
||||||
|
|
||||||
## Config And Command-Loading Review
|
|
||||||
|
|
||||||
Config precedence is consistent with policy:
|
|
||||||
|
|
||||||
1. built-in defaults;
|
|
||||||
2. config file values;
|
|
||||||
3. CLI overrides.
|
|
||||||
|
|
||||||
Intentional differences:
|
|
||||||
|
|
||||||
- `render` supports `--format`; `run` does not.
|
|
||||||
- `serve` supports `--addr`; `run`/`render` do not.
|
|
||||||
- `serve` rejects runtime model override flags.
|
|
||||||
- `render` does not expose `--schema-dir`, even though it can prepare `json_schema` prompts through config/default schema settings.
|
|
||||||
- CLI runtime model override flags are narrower than HTTP model override fields; this is currently documented by omission in CLI docs.
|
|
||||||
|
|
||||||
Likely cleanup areas:
|
|
||||||
|
|
||||||
- Common command settings finalization can be made smaller and less repetitive.
|
|
||||||
- Runner dependency construction should be a CLI-local helper.
|
|
||||||
- The `runConfig` flag-set booleans work, but adding more runtime override fields will continue to require touching several fields and the override condition.
|
|
||||||
|
|
||||||
## State, Manifest, Or Progress Handling Review
|
|
||||||
|
|
||||||
Scriptorium has no durable state, manifests, checkpoints, progress records, cache state, or resume behavior. This matches `docs/policy/architecture.md` and `docs/operations.md`.
|
|
||||||
|
|
||||||
There is no drift affecting resume, retry, force, dry-run, or audit behavior because those features do not exist. Validation repair hooks exist in the runner but are not wired by CLI/HTTP today; that boundary is documented.
|
|
||||||
|
|
||||||
## Refactors To Avoid
|
|
||||||
|
|
||||||
- Do not introduce a generic workflow/stage engine. Scriptorium intentionally executes one prompt request.
|
|
||||||
- Do not build a plugin architecture for prompt/profile/schema/artifact backends before another backend exists.
|
|
||||||
- Do not replace explicit CLI parse functions with a broad command framework.
|
|
||||||
- Do not merge CLI and HTTP adapters. Their public interfaces and failure semantics differ.
|
|
||||||
- Do not create a generic reflection-based mapper for domain/DTO/request structs.
|
|
||||||
- Do not centralize prompt/profile normalization into one generic YAML repository; their validation and error semantics are different.
|
|
||||||
- Do not add schema caching or a manifest/state store as part of deduplication.
|
|
||||||
- Do not document or implement unsupported artifact backends as cleanup.
|
|
||||||
|
|
||||||
## Recommended Implementation Sequence
|
|
||||||
|
|
||||||
1. Execution-target mapping cleanup
|
|
||||||
- Goal: reduce drift when adding profile/runtime/provider fields.
|
|
||||||
- Files to update: `internal/usecase`, `internal/adapter/http`, `internal/llm`, targeted tests.
|
|
||||||
- Acceptance criteria: all existing field mapping tests pass; a new all-fields test fails if a mapped execution field is omitted.
|
|
||||||
- Suggested validation: `go test ./internal/usecase ./internal/adapter/http ./internal/llm ./internal/format`.
|
|
||||||
- Small enough for one implementation prompt: yes.
|
|
||||||
|
|
||||||
2. Prompt/profile filesystem catalog helper
|
|
||||||
- Goal: centralize recursive YAML discovery, path sorting, extension filtering, and relative path formatting.
|
|
||||||
- Files to update: prompt/profile repositories plus new narrow helper package.
|
|
||||||
- Acceptance criteria: existing repository tests pass unchanged; new helper tests cover nested discovery and stable ordering.
|
|
||||||
- Suggested validation: `go test ./internal/promptdef ./internal/profile`.
|
|
||||||
- Small enough for one implementation prompt: yes.
|
|
||||||
|
|
||||||
3. CLI wiring and config finalization helper
|
|
||||||
- Goal: centralize common command settings resolution and runner construction.
|
|
||||||
- Files to update: `internal/adapter/cli/run.go` and CLI tests.
|
|
||||||
- Acceptance criteria: no flag behavior changes; run/render/serve config precedence tests pass.
|
|
||||||
- Suggested validation: `go test ./internal/adapter/cli`.
|
|
||||||
- Small enough for one implementation prompt: yes.
|
|
||||||
|
|
||||||
4. Stable use-case invalid-request reasons
|
|
||||||
- Goal: remove HTTP dependency on runner error message substrings.
|
|
||||||
- Files to update: `internal/usecase/runner.go`, `internal/adapter/http/handler.go`, tests.
|
|
||||||
- Acceptance criteria: HTTP error codes/messages remain unchanged; runner exposes stable `errors.Is` reason errors.
|
|
||||||
- Suggested validation: `go test ./internal/usecase ./internal/adapter/http`.
|
|
||||||
- Small enough for one implementation prompt: yes.
|
|
||||||
|
|
||||||
5. Test helper cleanup
|
|
||||||
- Goal: reduce repeated fixture construction after behavior-preserving refactors are complete.
|
|
||||||
- Files to update: primarily `internal/adapter/cli/run_test.go`, optionally `internal/usecase/runner_test.go`.
|
|
||||||
- Acceptance criteria: test intent remains clear; no cross-package helper package unless strongly justified.
|
|
||||||
- Suggested validation: `go test ./internal/adapter/cli ./internal/usecase`.
|
|
||||||
- Small enough for one implementation prompt: yes.
|
|
||||||
|
|
||||||
6. Dead-code/legacy sweep
|
|
||||||
- Goal: remove or clearly mark unsupported placeholders such as `ArtifactRefS3`.
|
|
||||||
- Files to update: domain/artifact tests/docs only if needed.
|
|
||||||
- Acceptance criteria: docs continue to describe only implemented behavior outside roadmap.
|
|
||||||
- Suggested validation: `go test ./internal/domain ./internal/artifact`.
|
|
||||||
- Small enough for one implementation prompt: yes.
|
|
||||||
|
|
||||||
## Test Strategy
|
|
||||||
|
|
||||||
Tests to add before or during cleanup:
|
|
||||||
|
|
||||||
- Execution target all-fields mapping tests:
|
|
||||||
- runner profile/default/request merge;
|
|
||||||
- HTTP request DTO to domain target;
|
|
||||||
- HTTP domain result to metadata DTO;
|
|
||||||
- LLM domain target to outbound JSON payload.
|
|
||||||
- Repository catalog helper tests:
|
|
||||||
- recursive scan;
|
|
||||||
- `.yaml` and `.yml` filtering;
|
|
||||||
- stable sorted paths;
|
|
||||||
- relative clean path formatting;
|
|
||||||
- context cancellation if the helper preserves current behavior.
|
|
||||||
- HTTP error mapping tests:
|
|
||||||
- profile-required and API-key-env-missing should rely on sentinel errors, not message text.
|
|
||||||
- CLI wiring regression tests:
|
|
||||||
- `run`, `render`, and `serve` preserve config precedence and required directory checks.
|
|
||||||
- Schema behavior tests:
|
|
||||||
- `Prepare` fails before LLM generation when `json_schema` structured-output schema cannot be loaded.
|
|
||||||
|
|
||||||
Validation commands for cleanup work:
|
|
||||||
|
|
||||||
- `go test ./internal/usecase ./internal/adapter/http ./internal/llm ./internal/format`
|
|
||||||
- `go test ./internal/promptdef ./internal/profile`
|
|
||||||
- `go test ./internal/adapter/cli`
|
|
||||||
- `go test ./...` before merging broader cleanup.
|
|
||||||
|
|
||||||
No automated docs or link checker is currently present.
|
|
||||||
|
|
||||||
## Appendix: Findings Not Worth Acting On
|
|
||||||
|
|
||||||
- Repeated `select { case <-ctx.Done(): ... }` checks are acceptable. They are local, simple, and appear at IO/loop boundaries where behavior is easy to read.
|
|
||||||
- CLI and HTTP request validation should remain separate. Their external contracts differ, and centralizing all request validation would blur adapter responsibilities.
|
|
||||||
- Prompt and profile validation should not be merged. They both use YAML and IDs, but their schemas, normalization rules, and error policies differ.
|
|
||||||
- Prepared-run text formatting is verbose but intentionally presentation-specific. Avoid abstracting it until another formatter needs the same layout policy.
|
|
||||||
- Hashing appears in artifact loading and runner metadata, but not all hashes represent the same thing. A tiny body-hash helper may be useful later; a generic hashing subsystem is not justified.
|
|
||||||
- HTTP DTOs duplicate domain field names by design. They should remain transport-owned so JSON compatibility can evolve deliberately.
|
|
||||||
- Config `applyConfig` and `ApplyCLIOverrides` look similar, but they apply different source labels and validation contexts. A broad merge abstraction would likely reduce clarity.
|
|
||||||
@@ -1,504 +0,0 @@
|
|||||||
# Cleanup Implementation Roadmap
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
This roadmap turns the findings in [audit.md](audit.md) into a staged, decision-complete cleanup plan for Scriptorium.
|
|
||||||
|
|
||||||
Audience: LLM coding agents implementing the cleanup in order.
|
|
||||||
|
|
||||||
Controlling policies:
|
|
||||||
|
|
||||||
- [Documentation policy](../policy/documentation.md)
|
|
||||||
- [Architecture policy](../policy/architecture.md)
|
|
||||||
- [Development guide](../policy/development.md)
|
|
||||||
|
|
||||||
## Global Implementation Rules
|
|
||||||
|
|
||||||
- Implement stages in order.
|
|
||||||
- Keep public CLI flags, HTTP request/response shapes, config precedence, prompt/profile ID semantics, and validation behavior stable unless a stage explicitly says otherwise.
|
|
||||||
- Keep adapters thin and use-case policy in `internal/usecase`.
|
|
||||||
- Prefer explicit helpers over reflection, generic workflow abstractions, or broad framework-style rewrites.
|
|
||||||
- Update non-roadmap docs only when implemented behavior changes or when a stale implemented-behavior statement is found during a stage.
|
|
||||||
- Do not document future behavior outside `docs/roadmap/`.
|
|
||||||
- Run the stage-specific tests before moving to the next stage.
|
|
||||||
- Run `go test ./...` after the final stage.
|
|
||||||
|
|
||||||
## Stage 1: Execution Target Mapping Cleanup
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Reduce drift when adding or changing execution/runtime fields such as `service_tier`, `api_key_env`, `reasoning_effort`, or future provider request keys.
|
|
||||||
|
|
||||||
### Scope
|
|
||||||
|
|
||||||
Update only explicit execution-target mapping and serialization paths. Do not add new CLI flags or new provider features.
|
|
||||||
|
|
||||||
### Implementation
|
|
||||||
|
|
||||||
In `internal/usecase`:
|
|
||||||
|
|
||||||
- Keep execution-target merge policy in `internal/usecase`.
|
|
||||||
- Add focused helper coverage around `resolveExecutionTarget`, `mergeExecutionTarget`, and profile-to-target conversion.
|
|
||||||
- Keep the existing semantics:
|
|
||||||
- built-in defaults first;
|
|
||||||
- profile values override defaults;
|
|
||||||
- request overrides override profile values;
|
|
||||||
- zero numeric values do not override;
|
|
||||||
- empty/whitespace string values do not override;
|
|
||||||
- non-empty `ExtraParams` replaces the previous map with a copy.
|
|
||||||
|
|
||||||
In `internal/adapter/http`:
|
|
||||||
|
|
||||||
- Add local helper functions:
|
|
||||||
- `executionTargetFromModelOverrideDTO(*modelOverrideRequestDTO) *domain.ExecutionTarget`
|
|
||||||
- `modelParamsDTOFromExecutionTarget(domain.ExecutionTarget) modelParamsDTO`
|
|
||||||
- Use those helpers in `handler.go`.
|
|
||||||
- Keep DTO types unexported and transport-owned.
|
|
||||||
- Keep HTTP response field names and omission behavior unchanged.
|
|
||||||
|
|
||||||
In `internal/llm`:
|
|
||||||
|
|
||||||
- Add a local helper such as `openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error)` or an equivalent small function.
|
|
||||||
- Keep endpoint construction, HTTP client timeout handling, API-key environment lookup, and response parsing in `Generate`.
|
|
||||||
- Keep outbound serialization behavior unchanged:
|
|
||||||
- send `model` and `messages`;
|
|
||||||
- send `temperature`, `max_tokens`, `top_p`, and `service_tier` only when currently sent;
|
|
||||||
- send `response_format` only when structured output is present;
|
|
||||||
- do not serialize `reasoning_effort` or `extra_params`.
|
|
||||||
|
|
||||||
Do not:
|
|
||||||
|
|
||||||
- use reflection to copy fields;
|
|
||||||
- move HTTP DTOs into `internal/domain`;
|
|
||||||
- add generic mapper packages;
|
|
||||||
- change prepared-run JSON tags.
|
|
||||||
|
|
||||||
### Tests
|
|
||||||
|
|
||||||
Add or update tests so an all-fields `domain.ExecutionTarget` catches omissions.
|
|
||||||
|
|
||||||
Required tests:
|
|
||||||
|
|
||||||
- Runner merge/profile conversion:
|
|
||||||
- profile values populate all supported execution fields;
|
|
||||||
- runtime overrides beat profile values for all overrideable fields;
|
|
||||||
- empty string overrides do not erase profile values;
|
|
||||||
- empty `ExtraParams` does not erase profile values.
|
|
||||||
- HTTP adapter:
|
|
||||||
- request `model` object maps every supported field into `RunRequest.Execution`;
|
|
||||||
- response `metadata.model_params` includes every supported field according to current DTO tags.
|
|
||||||
- LLM client:
|
|
||||||
- outbound JSON includes every serialized execution field;
|
|
||||||
- outbound JSON omits `service_tier` when empty;
|
|
||||||
- outbound JSON still omits `reasoning_effort` and `extra_params`.
|
|
||||||
|
|
||||||
### Validation
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go test ./internal/usecase ./internal/adapter/http ./internal/llm ./internal/format
|
|
||||||
```
|
|
||||||
|
|
||||||
### Acceptance Criteria
|
|
||||||
|
|
||||||
- No public behavior changes.
|
|
||||||
- Adding a new execution target field later has obvious mapping/test locations.
|
|
||||||
- Existing HTTP and LLM behavior remains stable.
|
|
||||||
- Stage is small enough for one implementation prompt.
|
|
||||||
|
|
||||||
## Stage 2: Prompt/Profile Filesystem Catalog Helper
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Centralize shared recursive YAML discovery mechanics while preserving prompt/profile-specific validation and error behavior.
|
|
||||||
|
|
||||||
### Scope
|
|
||||||
|
|
||||||
Create a narrow helper package for filesystem catalog mechanics only.
|
|
||||||
|
|
||||||
Recommended package:
|
|
||||||
|
|
||||||
- `internal/filecatalog`
|
|
||||||
|
|
||||||
### Implementation
|
|
||||||
|
|
||||||
Add helper functions with explicit, small responsibilities:
|
|
||||||
|
|
||||||
- recursively find YAML files under a root directory;
|
|
||||||
- honor context cancellation during walking;
|
|
||||||
- accept `.yaml` and `.yml`;
|
|
||||||
- return stable sorted full paths;
|
|
||||||
- compute clean relative paths from a root;
|
|
||||||
- return filename stems with `.yaml`/`.yml` stripped.
|
|
||||||
|
|
||||||
Use the helper in:
|
|
||||||
|
|
||||||
- `internal/promptdef/filesystem_repository.go`
|
|
||||||
- `internal/profile/filesystem_repository.go`
|
|
||||||
|
|
||||||
Preserve existing behavior:
|
|
||||||
|
|
||||||
- prompt/profile lookup uses YAML `id`, not file path;
|
|
||||||
- subdirectories are organizational only;
|
|
||||||
- duplicate prompt/profile IDs are invalid;
|
|
||||||
- malformed likely-target files still surface errors;
|
|
||||||
- relative nested paths still appear in errors;
|
|
||||||
- prompt `content_file` resolution remains relative to the prompt YAML file;
|
|
||||||
- prompt/profile strict YAML and validation stay in their existing packages.
|
|
||||||
|
|
||||||
Do not:
|
|
||||||
|
|
||||||
- create a generic repository framework;
|
|
||||||
- merge prompt and profile normalization;
|
|
||||||
- move prompt/profile domain policy into the helper;
|
|
||||||
- change error messages except for unavoidable wording caused by helper extraction.
|
|
||||||
|
|
||||||
### Tests
|
|
||||||
|
|
||||||
Add tests for `internal/filecatalog`:
|
|
||||||
|
|
||||||
- nested YAML discovery;
|
|
||||||
- `.yaml` and `.yml` accepted;
|
|
||||||
- non-YAML files ignored;
|
|
||||||
- returned paths sorted deterministically;
|
|
||||||
- relative path formatting works for nested files;
|
|
||||||
- filename stem stripping handles both extensions.
|
|
||||||
|
|
||||||
Keep existing prompt/profile repository tests passing.
|
|
||||||
|
|
||||||
### Validation
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go test ./internal/filecatalog ./internal/promptdef ./internal/profile
|
|
||||||
```
|
|
||||||
|
|
||||||
### Acceptance Criteria
|
|
||||||
|
|
||||||
- Prompt/profile repository tests pass without behavior expectation changes.
|
|
||||||
- Shared filesystem scanning logic exists in one place.
|
|
||||||
- Prompt/profile packages still own their own validation and normalization.
|
|
||||||
- Stage is small enough for one implementation prompt.
|
|
||||||
|
|
||||||
## Stage 3: CLI Wiring And Settings Finalization Cleanup
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Reduce duplicated command setup while preserving each command's public flag surface and behavior.
|
|
||||||
|
|
||||||
### Scope
|
|
||||||
|
|
||||||
Clean up `internal/adapter/cli` only, except for tests.
|
|
||||||
|
|
||||||
### Implementation
|
|
||||||
|
|
||||||
Add CLI-local helpers. Recommended helpers:
|
|
||||||
|
|
||||||
- `commonCommandSettings` or similar struct containing resolved `promptDir`, `profileDir`, `schemaDir`, `serverAddr`, and `defaultRenderFormat` where applicable.
|
|
||||||
- `resolveCommonSettings(fs *flag.FlagSet, configPath string, overrides appconfig.CLIOverrides) (commonCommandSettings, error)`.
|
|
||||||
- `validateRequiredLibraryDirs(promptDir, profileDir string) error`.
|
|
||||||
- `newRunner(promptDir, profileDir, schemaDir string, llmClient llm.Client) *usecase.Runner`.
|
|
||||||
- Optionally `newOpenAIClient() (*llm.OpenAICompatibleClient, error)` if it removes exact duplication without obscuring command behavior.
|
|
||||||
|
|
||||||
Preserve command differences:
|
|
||||||
|
|
||||||
- `run` exposes runtime model override flags and `--schema-dir`;
|
|
||||||
- `render` exposes runtime model override flags and `--format`, but not `--schema-dir`;
|
|
||||||
- `serve` exposes `--addr` and `--schema-dir`, but no runtime model override flags;
|
|
||||||
- `render` default format comes from `defaults.render_format` unless `--format` is set;
|
|
||||||
- deprecated `--prompt-id` and `--profile-id` aliases remain accepted.
|
|
||||||
|
|
||||||
Keep existing parse functions:
|
|
||||||
|
|
||||||
- `parseRunArgs`
|
|
||||||
- `parseRenderArgs`
|
|
||||||
- `parseServeArgs`
|
|
||||||
|
|
||||||
Do not:
|
|
||||||
|
|
||||||
- replace the standard library `flag` package;
|
|
||||||
- introduce a command framework;
|
|
||||||
- make `serve` accept runtime model override flags;
|
|
||||||
- change error prefixes such as `run parse error`, `render parse error`, or `serve parse error`;
|
|
||||||
- change CLI output behavior.
|
|
||||||
|
|
||||||
### Tests
|
|
||||||
|
|
||||||
Required regression tests:
|
|
||||||
|
|
||||||
- `run`, `render`, and `serve` still apply config precedence correctly.
|
|
||||||
- Missing effective `prompt_dir` and `profile_dir` still return the same guidance.
|
|
||||||
- `render` still uses config default render format and explicit `--format` override.
|
|
||||||
- `serve` still rejects runtime model override flags.
|
|
||||||
- `run` and `render` still build equivalent runtime override requests for shared flags.
|
|
||||||
|
|
||||||
### Validation
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go test ./internal/adapter/cli
|
|
||||||
```
|
|
||||||
|
|
||||||
### Acceptance Criteria
|
|
||||||
|
|
||||||
- No CLI flag, output, exit-code, or precedence changes.
|
|
||||||
- Runner dependency construction is centralized inside the CLI adapter.
|
|
||||||
- Command-specific behavior remains easy to read.
|
|
||||||
- Stage is small enough for one implementation prompt.
|
|
||||||
|
|
||||||
## Stage 4: Stable Use-Case Error Reasons For HTTP Mapping
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Remove HTTP error mapping's dependency on runner error message substrings.
|
|
||||||
|
|
||||||
### Scope
|
|
||||||
|
|
||||||
Change error identity, not public HTTP error responses.
|
|
||||||
|
|
||||||
### Implementation
|
|
||||||
|
|
||||||
In `internal/usecase`:
|
|
||||||
|
|
||||||
- Add stable sentinel errors for invalid-request reasons that HTTP currently distinguishes by message text.
|
|
||||||
- Required sentinels:
|
|
||||||
- missing profile selection, for the case where neither request profile nor prompt `default_profile` is available;
|
|
||||||
- missing API-key environment value, for the case where `api_key_env` is set but the named environment variable is unset or empty.
|
|
||||||
- Wrap these sentinels with `ErrInvalidRequest` so existing broad invalid-request checks keep working.
|
|
||||||
- Preserve clear human-readable runner errors.
|
|
||||||
|
|
||||||
In `internal/adapter/http`:
|
|
||||||
|
|
||||||
- Replace `strings.Contains(err.Error(), ...)` checks for these cases with `errors.Is`.
|
|
||||||
- Keep current HTTP status codes, error codes, and response messages:
|
|
||||||
- `400 profile_required`;
|
|
||||||
- `400 api_key_env_missing`.
|
|
||||||
|
|
||||||
Do not:
|
|
||||||
|
|
||||||
- expose HTTP-specific error codes from `internal/usecase`;
|
|
||||||
- change the HTTP JSON error body shape;
|
|
||||||
- remove broad fallback handling for `usecase.ErrInvalidRequest`.
|
|
||||||
|
|
||||||
### Tests
|
|
||||||
|
|
||||||
Required tests:
|
|
||||||
|
|
||||||
- Runner tests assert `errors.Is(err, usecase.ErrProfileRequired)` or the chosen sentinel name for missing profile selection.
|
|
||||||
- Runner tests assert `errors.Is(err, usecase.ErrAPIKeyEnvMissing)` or the chosen sentinel name for missing API-key environment value.
|
|
||||||
- HTTP handler tests still assert unchanged status/code/message for both cases.
|
|
||||||
- HTTP handler tests should not construct errors by relying on exact runner prose for these two cases.
|
|
||||||
|
|
||||||
### Validation
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go test ./internal/usecase ./internal/adapter/http
|
|
||||||
```
|
|
||||||
|
|
||||||
### Acceptance Criteria
|
|
||||||
|
|
||||||
- HTTP mapping no longer depends on runner message substrings for the two distinguished invalid-request cases.
|
|
||||||
- Public HTTP behavior is unchanged.
|
|
||||||
- Runner errors remain clear in CLI output.
|
|
||||||
- Stage is small enough for one implementation prompt.
|
|
||||||
|
|
||||||
## Stage 5: Schema Failure Regression Coverage
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Protect the current structured-output invariant before future schema cleanup: `Prepare` must load a `json_schema` document before any LLM call.
|
|
||||||
|
|
||||||
### Scope
|
|
||||||
|
|
||||||
Add regression coverage only. Do not add schema caching or change schema loading architecture in this stage.
|
|
||||||
|
|
||||||
### Implementation
|
|
||||||
|
|
||||||
In `internal/usecase/runner_test.go` or an appropriate package test:
|
|
||||||
|
|
||||||
- Add a test where a prompt uses `validation_mode: json_schema` with a missing or failing schema document.
|
|
||||||
- Assert `Runner.Prepare` fails with `ErrValidation`.
|
|
||||||
- Assert no LLM call is made for `Runner.Run` when structured-output schema loading fails.
|
|
||||||
|
|
||||||
If existing tests already cover part of this behavior, consolidate assertions without making the test suite harder to read.
|
|
||||||
|
|
||||||
Do not:
|
|
||||||
|
|
||||||
- cache compiled schemas;
|
|
||||||
- change `validate.StandardValidator` behavior;
|
|
||||||
- introduce a schema service abstraction.
|
|
||||||
|
|
||||||
### Validation
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go test ./internal/usecase ./internal/validate
|
|
||||||
```
|
|
||||||
|
|
||||||
### Acceptance Criteria
|
|
||||||
|
|
||||||
- Missing structured-output schema fails before generation.
|
|
||||||
- Existing JSON Schema validation behavior is unchanged.
|
|
||||||
- Stage is small enough for one implementation prompt.
|
|
||||||
|
|
||||||
## Stage 6: Test Fixture Cleanup
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Reduce repeated test setup after behavior-preserving production refactors are complete.
|
|
||||||
|
|
||||||
### Scope
|
|
||||||
|
|
||||||
Prefer package-local test helpers. Avoid cross-package test utility packages unless a helper is needed by more than two packages and represents a stable public fixture contract.
|
|
||||||
|
|
||||||
### Implementation
|
|
||||||
|
|
||||||
In `internal/adapter/cli/run_test.go`:
|
|
||||||
|
|
||||||
- Consolidate repeated temp prompt/profile/input setup into local helper functions.
|
|
||||||
- Keep helper names behavior-focused, for example:
|
|
||||||
- `newCLITestLibrary`
|
|
||||||
- `writePromptFileWithDefaultProfile`
|
|
||||||
- `writeProfileFile`
|
|
||||||
- `runCLICommand`
|
|
||||||
- Do not hide assertions inside helpers unless the assertion is truly setup validation.
|
|
||||||
|
|
||||||
In `internal/usecase/runner_test.go`:
|
|
||||||
|
|
||||||
- Keep existing fake interfaces package-local.
|
|
||||||
- Remove only high-volume duplication that obscures test intent.
|
|
||||||
|
|
||||||
Do not:
|
|
||||||
|
|
||||||
- move package-private fake types into production code;
|
|
||||||
- create a broad `internal/testutil` package unless a later cleanup stage proves it necessary;
|
|
||||||
- rewrite tests into table-driven form when cases have meaningfully different setup.
|
|
||||||
|
|
||||||
### Validation
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go test ./internal/adapter/cli ./internal/usecase
|
|
||||||
```
|
|
||||||
|
|
||||||
### Acceptance Criteria
|
|
||||||
|
|
||||||
- Test intent is at least as clear as before.
|
|
||||||
- No production behavior changes.
|
|
||||||
- Test fixture setup has less repeated boilerplate in CLI tests.
|
|
||||||
- Stage is small enough for one implementation prompt.
|
|
||||||
|
|
||||||
## Stage 7: Unsupported Placeholder Sweep
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Remove code that suggests unimplemented artifact behavior outside roadmap documentation.
|
|
||||||
|
|
||||||
### Scope
|
|
||||||
|
|
||||||
Remove unsupported placeholders only when they are not needed by current tests or public docs.
|
|
||||||
|
|
||||||
### Implementation
|
|
||||||
|
|
||||||
Remove `domain.ArtifactRefS3` from `internal/domain/domain.go` unless new evidence shows it is intentionally needed by implemented code.
|
|
||||||
|
|
||||||
Preserve current behavior:
|
|
||||||
|
|
||||||
- supported artifact reference types remain `inline` and `file`;
|
|
||||||
- unsupported artifact reference types still return `artifact.ErrUnsupportedRefType`;
|
|
||||||
- docs continue to describe only `inline` and `file` outside roadmap files.
|
|
||||||
|
|
||||||
Update tests only if they reference the removed constant. Prefer testing unsupported artifact behavior with a literal custom type such as `domain.ArtifactRefType("s3")` or `domain.ArtifactRefType("unsupported")`.
|
|
||||||
|
|
||||||
Do not:
|
|
||||||
|
|
||||||
- add S3 support;
|
|
||||||
- document S3 as implemented;
|
|
||||||
- add future-backend placeholders elsewhere.
|
|
||||||
|
|
||||||
### Validation
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go test ./internal/domain ./internal/artifact
|
|
||||||
```
|
|
||||||
|
|
||||||
### Acceptance Criteria
|
|
||||||
|
|
||||||
- Unsupported placeholder constant is removed.
|
|
||||||
- Unsupported artifact-type behavior remains covered.
|
|
||||||
- No non-roadmap doc claims unimplemented artifact support.
|
|
||||||
- Stage is small enough for one implementation prompt.
|
|
||||||
|
|
||||||
## Stage 8: Final Verification And Documentation Alignment
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Confirm the cleanup sequence preserved behavior and documentation accuracy.
|
|
||||||
|
|
||||||
### Implementation
|
|
||||||
|
|
||||||
Run the full test suite:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go test ./...
|
|
||||||
```
|
|
||||||
|
|
||||||
Run the maintained render smoke command:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go run ./cmd/scriptorium render --config ./examples/config.yml --prompt generic.markdown_summary --input transcript=./examples/fixtures/transcript.md --input glossary=./examples/fixtures/glossary.yml --format json
|
|
||||||
```
|
|
||||||
|
|
||||||
Search for stale or unsupported terms:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
rg -n "ArtifactRefS3|s3|strings\\.Contains\\(err\\.Error\\(\\)|TODO|future|planned" internal docs README.md examples
|
|
||||||
```
|
|
||||||
|
|
||||||
Review results manually:
|
|
||||||
|
|
||||||
- `s3` should not appear as an implemented artifact type.
|
|
||||||
- `strings.Contains(err.Error())` should not be used for stable use-case reason mapping.
|
|
||||||
- Any `future` or `planned` wording outside `docs/roadmap/` must describe current boundaries, not aspirational behavior.
|
|
||||||
|
|
||||||
Update docs only if cleanup changed implemented behavior or if the search reveals stale implemented-behavior docs.
|
|
||||||
|
|
||||||
### Acceptance Criteria
|
|
||||||
|
|
||||||
- Full test suite passes.
|
|
||||||
- Maintained render smoke command succeeds.
|
|
||||||
- No stale unsupported artifact placeholder remains.
|
|
||||||
- Non-roadmap docs describe implemented behavior only.
|
|
||||||
- Working tree contains only intentional cleanup changes.
|
|
||||||
|
|
||||||
## Deferred Work
|
|
||||||
|
|
||||||
Do not implement these during the staged cleanup unless a later audit makes them high-confidence:
|
|
||||||
|
|
||||||
- schema caching or a combined raw/compiled schema service;
|
|
||||||
- artifact hash/build helper beyond a small helper introduced opportunistically during touched code;
|
|
||||||
- generic YAML repository framework;
|
|
||||||
- generic CLI command framework;
|
|
||||||
- plugin architecture for future prompt/profile/schema/artifact backends;
|
|
||||||
- durable state, manifests, checkpoints, or resume behavior.
|
|
||||||
|
|
||||||
## Completion Criteria
|
|
||||||
|
|
||||||
The cleanup roadmap is complete when all stages have been implemented in order, the final verification passes, and the resulting code still satisfies:
|
|
||||||
|
|
||||||
- `Runner.Run` reuses `Runner.Prepare`;
|
|
||||||
- CLI and HTTP adapters instantiate `Runner` without a repairer;
|
|
||||||
- unknown config/prompt/profile YAML and HTTP JSON fields are rejected;
|
|
||||||
- raw API key values are not accepted or emitted;
|
|
||||||
- prompt/profile subdirectories remain organizational only;
|
|
||||||
- schema paths remain explicit and relative to `schema_dir` when not absolute;
|
|
||||||
- public CLI and HTTP behavior remains stable.
|
|
||||||
262
docs/roadmap/implementation.md
Normal file
262
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,262 @@
|
|||||||
|
# Runtime Parameter Implementation Plan
|
||||||
|
|
||||||
|
This plan implements the target state in `docs/roadmap/params.md`.
|
||||||
|
|
||||||
|
Audience: LLM coding agents implementing the feature in order. Follow `docs/policy/architecture.md`, `docs/policy/development.md`, and `docs/policy/documentation.md` before changing code.
|
||||||
|
|
||||||
|
## Constraints
|
||||||
|
|
||||||
|
- Keep adapters thin. CLI and HTTP should capture caller intent and map it into domain request types; merge decisions belong in `internal/usecase`.
|
||||||
|
- Keep external decoding strict. Unknown YAML/JSON fields must continue to fail.
|
||||||
|
- Do not accept or emit raw API key values.
|
||||||
|
- Do not add dependencies unless there is a clear need. This feature should use the standard library plus existing dependencies.
|
||||||
|
- Do not expand the HTTP API surface beyond `POST /v1/runs`.
|
||||||
|
- Do not add provider-specific adapter packages.
|
||||||
|
- Keep each stage passing `go test ./...` before moving to the next stage.
|
||||||
|
|
||||||
|
## Stage 1: Presence-Aware Request Overrides
|
||||||
|
|
||||||
|
Goal: make per-request numeric execution overrides presence-aware while keeping resolved execution settings concrete.
|
||||||
|
|
||||||
|
### Domain Changes
|
||||||
|
|
||||||
|
1. In `internal/domain/domain.go`, add a request-only type:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type ExecutionTargetOverride struct {
|
||||||
|
Endpoint string `json:"endpoint,omitempty"`
|
||||||
|
Model string `json:"model,omitempty"`
|
||||||
|
Temperature *float64 `json:"temperature,omitempty"`
|
||||||
|
MaxTokens *int `json:"max_tokens,omitempty"`
|
||||||
|
TopP *float64 `json:"top_p,omitempty"`
|
||||||
|
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||||
|
ServiceTier string `json:"service_tier,omitempty"`
|
||||||
|
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||||
|
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||||
|
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Change `domain.RunRequest.Execution` from `*ExecutionTarget` to `*ExecutionTargetOverride`.
|
||||||
|
3. Change `ExecutionProfile.ExtraParams` and `ExecutionTarget.ExtraParams` from `map[string]string` to `map[string]any`.
|
||||||
|
4. Keep `ExecutionTarget` concrete. It represents the resolved effective runtime target after defaults, profile, and request overrides are merged.
|
||||||
|
|
||||||
|
### Runner Changes
|
||||||
|
|
||||||
|
1. Update `internal/usecase/runner.go` so profile values still merge over built-in defaults and request overrides merge over that result.
|
||||||
|
2. Keep the existing concrete profile merge semantics for profile numeric fields.
|
||||||
|
3. Add a separate request override merge path that uses pointer presence:
|
||||||
|
- `nil` numeric pointer means omitted; preserve the current value.
|
||||||
|
- non-nil numeric pointer means explicit override, even when the value is `0`.
|
||||||
|
4. Validate request override numeric values before or during merge:
|
||||||
|
- `temperature`: `0 <= value <= 2`
|
||||||
|
- `max_tokens`: `value >= 0`
|
||||||
|
- `top_p`: `0 <= value <= 1`
|
||||||
|
- `timeout_seconds`: `value >= 0`
|
||||||
|
5. Preserve existing validation after merge:
|
||||||
|
- effective endpoint required
|
||||||
|
- effective model required
|
||||||
|
- `api_key_env`, when set, must name a non-empty environment variable
|
||||||
|
6. Preserve secret handling. The resolved API key value must never be stored in `PreparedRun`, `RunResult`, logs, or HTTP responses.
|
||||||
|
|
||||||
|
### CLI Changes
|
||||||
|
|
||||||
|
1. Update `internal/adapter/cli/run.go` request construction to build `domain.ExecutionTargetOverride`.
|
||||||
|
2. Use the existing `flagWasSet` booleans to populate numeric pointers only when the user provided the flag.
|
||||||
|
3. Required behavior:
|
||||||
|
- omitted `--temperature` preserves profile/default temperature;
|
||||||
|
- `--temperature 0` explicitly sets temperature to zero;
|
||||||
|
- omitted `--top-p` preserves profile/default top-p;
|
||||||
|
- `--top-p 0` explicitly sets top-p to zero;
|
||||||
|
- omitted `--max-tokens` preserves profile/default max tokens;
|
||||||
|
- `--max-tokens 0` explicitly sets max tokens to zero;
|
||||||
|
- omitted `--timeout` preserves profile/default timeout;
|
||||||
|
- `--timeout 0s` explicitly sets timeout seconds to zero.
|
||||||
|
4. Do not add new CLI flags in this stage.
|
||||||
|
|
||||||
|
### HTTP Changes
|
||||||
|
|
||||||
|
1. Update `internal/adapter/http/dto.go` so numeric model override fields are pointers:
|
||||||
|
- `Temperature *float64`
|
||||||
|
- `MaxTokens *int`
|
||||||
|
- `TopP *float64`
|
||||||
|
- `TimeoutSeconds *int`
|
||||||
|
2. Update DTO mapping in `internal/adapter/http/handler.go` to build `domain.ExecutionTargetOverride`.
|
||||||
|
3. Preserve strict JSON decoding and existing error mapping.
|
||||||
|
4. Required behavior:
|
||||||
|
- omitted numeric JSON fields preserve profile/default values;
|
||||||
|
- explicit numeric zero JSON fields override profile/default values.
|
||||||
|
|
||||||
|
### Tests
|
||||||
|
|
||||||
|
Add or update tests in:
|
||||||
|
|
||||||
|
- `internal/usecase/runner_test.go`
|
||||||
|
- `internal/adapter/cli/run_test.go`
|
||||||
|
- `internal/adapter/http/handler_test.go`
|
||||||
|
|
||||||
|
Required test coverage:
|
||||||
|
|
||||||
|
- Runner preserves profile value when request numeric override is omitted.
|
||||||
|
- Runner applies explicit zero request override for `temperature`.
|
||||||
|
- Runner applies explicit zero request override for `top_p`.
|
||||||
|
- Runner applies explicit zero request override for `max_tokens`.
|
||||||
|
- Runner applies explicit zero request override for `timeout_seconds`.
|
||||||
|
- Invalid request override ranges fail as invalid request errors.
|
||||||
|
- CLI `--temperature 0` reaches effective settings as zero.
|
||||||
|
- HTTP `"temperature": 0` reaches effective settings as zero.
|
||||||
|
- HTTP omitted `temperature` preserves profile/default value.
|
||||||
|
|
||||||
|
### Verification
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go test ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
## Stage 2: JSON-Compatible `extra_params`
|
||||||
|
|
||||||
|
Goal: allow provider-specific parameters to carry JSON-compatible values throughout profile, HTTP, prepared output, metadata, and LLM request construction.
|
||||||
|
|
||||||
|
### Domain And Loader Changes
|
||||||
|
|
||||||
|
1. Complete all compile fixes from changing `ExtraParams` to `map[string]any`.
|
||||||
|
2. Ensure `internal/profile/filesystem_repository.go` continues to decode profiles strictly while allowing nested JSON-compatible values under `extra_params`.
|
||||||
|
3. Add profile repository tests for `extra_params` containing:
|
||||||
|
- string
|
||||||
|
- number
|
||||||
|
- boolean
|
||||||
|
- nested object or array
|
||||||
|
4. Ensure formatter output remains deterministic:
|
||||||
|
- keep sorting `extra_params` keys in `internal/format/prepared_run.go`;
|
||||||
|
- render non-string values with stable JSON encoding in text output.
|
||||||
|
5. Preserve JSON formatter behavior through normal `encoding/json` output.
|
||||||
|
|
||||||
|
### HTTP Changes
|
||||||
|
|
||||||
|
1. Change HTTP model override `ExtraParams` to `map[string]any`.
|
||||||
|
2. Add handler tests proving HTTP accepts JSON-compatible `extra_params` values.
|
||||||
|
3. Preserve strict rejection of unknown fields and raw API-key payload fields.
|
||||||
|
|
||||||
|
### Verification
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go test ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
## Stage 3: Outbound Serialization
|
||||||
|
|
||||||
|
Goal: serialize `reasoning_effort` and `extra_params` to the OpenAI-compatible chat-completions request.
|
||||||
|
|
||||||
|
### LLM Adapter Changes
|
||||||
|
|
||||||
|
1. In `internal/llm/openai_compatible_client.go`, add first-class outbound support for `reasoning_effort`.
|
||||||
|
2. Add `extra_params` support by flattening `domain.ExecutionTarget.ExtraParams` into additional top-level JSON request fields.
|
||||||
|
3. Implement reserved-field collision checks before the HTTP request is made.
|
||||||
|
4. Reserved keys must include:
|
||||||
|
- `model`
|
||||||
|
- `session_id`
|
||||||
|
- `messages`
|
||||||
|
- `temperature`
|
||||||
|
- `max_tokens`
|
||||||
|
- `top_p`
|
||||||
|
- `service_tier`
|
||||||
|
- `reasoning_effort`
|
||||||
|
- `response_format`
|
||||||
|
5. Reject empty `extra_params` keys.
|
||||||
|
6. Ensure each `extra_params` value can be marshaled as JSON. If marshaling fails, return `ErrInvalidRequest` with context.
|
||||||
|
7. Keep existing request behavior unchanged when `reasoning_effort` and `extra_params` are unset.
|
||||||
|
|
||||||
|
### Recommended Implementation Shape
|
||||||
|
|
||||||
|
Use a custom marshal path for the outbound chat request rather than string manipulation.
|
||||||
|
|
||||||
|
One acceptable shape:
|
||||||
|
|
||||||
|
- Add `ReasoningEffort string` and `ExtraParams map[string]any` to the internal `openAIChatRequest`.
|
||||||
|
- Add a helper that converts `openAIChatRequest` into `map[string]any`, inserts first-class fields when set, then inserts `ExtraParams` after collision validation.
|
||||||
|
- Marshal that map with `encoding/json`.
|
||||||
|
|
||||||
|
Do not construct outbound JSON with manual string concatenation.
|
||||||
|
|
||||||
|
### Tests
|
||||||
|
|
||||||
|
Update `internal/llm/openai_compatible_client_test.go`.
|
||||||
|
|
||||||
|
Required test coverage:
|
||||||
|
|
||||||
|
- outbound JSON includes `reasoning_effort` when set;
|
||||||
|
- outbound JSON omits `reasoning_effort` when unset;
|
||||||
|
- outbound JSON includes string, number, boolean, object, and array `extra_params`;
|
||||||
|
- reserved `extra_params` keys fail before provider call;
|
||||||
|
- empty `extra_params` keys fail before provider call;
|
||||||
|
- existing message, cache-control, service-tier, response-format, and usage parsing tests continue to pass.
|
||||||
|
|
||||||
|
### Verification
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go test ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
## Stage 4: Documentation And Examples
|
||||||
|
|
||||||
|
Goal: move implemented behavior from roadmap to canonical docs after code is complete.
|
||||||
|
|
||||||
|
Update only after Stages 1 through 3 are implemented.
|
||||||
|
|
||||||
|
### Required Docs
|
||||||
|
|
||||||
|
Update:
|
||||||
|
|
||||||
|
- `docs/config.md`
|
||||||
|
- `docs/cli.md`
|
||||||
|
- `docs/integrations/http-api.md`
|
||||||
|
- `docs/integrations/openai-compatible-chat.md`
|
||||||
|
- `docs/internal/runner.md`
|
||||||
|
- `docs/internal/adapters.md`
|
||||||
|
|
||||||
|
Required documentation content:
|
||||||
|
|
||||||
|
- `reasoning_effort` is serialized outbound when set.
|
||||||
|
- `extra_params` serializes as provider-specific top-level outbound JSON fields.
|
||||||
|
- `extra_params` supports JSON-compatible values.
|
||||||
|
- reserved `extra_params` fields are rejected.
|
||||||
|
- per-request numeric overrides distinguish omitted values from explicit zero values.
|
||||||
|
- CLI explicit zero behavior for existing numeric flags.
|
||||||
|
- HTTP explicit zero behavior for model override numeric fields.
|
||||||
|
- no raw API-key values are accepted or emitted.
|
||||||
|
|
||||||
|
### Examples
|
||||||
|
|
||||||
|
Update examples only if needed to keep them accurate and runnable.
|
||||||
|
|
||||||
|
If adding an `extra_params` example, keep it secret-free and simple. Prefer a harmless provider-routing example over a vendor-specific feature that requires special credentials.
|
||||||
|
|
||||||
|
### Verification
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go test ./...
|
||||||
|
go run ./cmd/scriptorium render \
|
||||||
|
--config ./examples/config.yml \
|
||||||
|
--prompt generic.markdown_summary \
|
||||||
|
--input transcript=./examples/fixtures/transcript.md \
|
||||||
|
--input glossary=./examples/fixtures/glossary.yml \
|
||||||
|
--format json
|
||||||
|
```
|
||||||
|
|
||||||
|
## Final Checks
|
||||||
|
|
||||||
|
Before considering the feature complete:
|
||||||
|
|
||||||
|
1. Confirm `git diff` contains only intended code, test, doc, and example changes.
|
||||||
|
2. Confirm all non-roadmap docs describe implemented behavior only.
|
||||||
|
3. Confirm no output path exposes raw API key values.
|
||||||
|
4. Confirm `go test ./...` passes.
|
||||||
|
5. Confirm the render smoke command passes.
|
||||||
96
docs/roadmap/params.md
Normal file
96
docs/roadmap/params.md
Normal file
@@ -0,0 +1,96 @@
|
|||||||
|
# Runtime Parameter Feature Roadmap
|
||||||
|
|
||||||
|
This roadmap defines the target behavior for runtime model parameters.
|
||||||
|
|
||||||
|
Current behavior has two limitations:
|
||||||
|
|
||||||
|
- `reasoning_effort` and `extra_params` are parsed into effective execution settings but are not serialized into outbound OpenAI-compatible chat-completions requests.
|
||||||
|
- Per-request numeric execution overrides use zero-value merge semantics, so callers cannot reliably override a profile value with an explicit zero such as `temperature: 0`.
|
||||||
|
|
||||||
|
The implementation plan for this feature lives in `docs/roadmap/implementation.md`.
|
||||||
|
|
||||||
|
## Target State
|
||||||
|
|
||||||
|
Scriptorium should preserve the existing separation between prompt definitions, execution profiles, and per-request execution overrides while making runtime parameter behavior explicit and predictable.
|
||||||
|
|
||||||
|
Expected end state:
|
||||||
|
|
||||||
|
- Effective execution settings remain visible in prepared-run output, run metadata, and HTTP metadata without exposing raw secret values.
|
||||||
|
- `reasoning_effort` is treated as a first-class effective execution setting and is serialized to the outbound OpenAI-compatible request when set.
|
||||||
|
- `extra_params` supports provider-specific OpenAI-compatible request fields.
|
||||||
|
- `extra_params` is serialized as additional top-level outbound JSON fields.
|
||||||
|
- `extra_params` values support JSON-compatible scalar, object, and array values.
|
||||||
|
- `extra_params` cannot override first-class outbound request fields.
|
||||||
|
- Per-request numeric overrides preserve caller intent, including explicit zero values.
|
||||||
|
- Omitted per-request numeric overrides continue to inherit the selected profile and built-in defaults.
|
||||||
|
- External decoding remains strict for config, prompt, profile, and HTTP request payloads.
|
||||||
|
|
||||||
|
## Policy Decisions
|
||||||
|
|
||||||
|
### `extra_params`
|
||||||
|
|
||||||
|
`extra_params` should serialize as additional top-level outbound JSON fields in the OpenAI-compatible chat-completions request.
|
||||||
|
|
||||||
|
Reasoning:
|
||||||
|
|
||||||
|
Most OpenAI-compatible providers expose vendor-specific chat-completions parameters as top-level fields. This keeps Scriptorium's adapter compatible with that ecosystem without adding first-class fields for every provider option.
|
||||||
|
|
||||||
|
`extra_params` must not silently override Scriptorium-owned fields. Reserved outbound fields include at least:
|
||||||
|
|
||||||
|
- `model`
|
||||||
|
- `session_id`
|
||||||
|
- `messages`
|
||||||
|
- `temperature`
|
||||||
|
- `max_tokens`
|
||||||
|
- `top_p`
|
||||||
|
- `service_tier`
|
||||||
|
- `reasoning_effort`
|
||||||
|
- `response_format`
|
||||||
|
|
||||||
|
If a caller supplies a reserved key through `extra_params`, Scriptorium should fail before making the outbound HTTP request.
|
||||||
|
|
||||||
|
`extra_params` should use JSON-compatible values rather than only strings.
|
||||||
|
|
||||||
|
Reasoning:
|
||||||
|
|
||||||
|
Provider-specific parameters commonly need booleans, numbers, objects, or arrays. String-only values would force awkward encoding and would likely require a later compatibility break.
|
||||||
|
|
||||||
|
### Presence-Aware Overrides
|
||||||
|
|
||||||
|
Per-request execution overrides should use a presence-aware type with pointer fields for optional numeric values.
|
||||||
|
|
||||||
|
Reasoning:
|
||||||
|
|
||||||
|
The resolved execution target should remain a concrete value used by prepared runs, generated requests, and metadata. Optionality matters at the request boundary, not after the runner has resolved the effective target.
|
||||||
|
|
||||||
|
This keeps adapter and merge logic precise while avoiding nil checks in formatter, metadata, and LLM serialization paths.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
In scope:
|
||||||
|
|
||||||
|
- Runtime merge behavior for per-request execution overrides.
|
||||||
|
- HTTP model override decoding for explicit zero numeric values.
|
||||||
|
- CLI execution override handling for explicit zero numeric flags.
|
||||||
|
- Outbound serialization of `reasoning_effort`.
|
||||||
|
- Outbound serialization of JSON-compatible `extra_params`.
|
||||||
|
- Tests and documentation for the changed implemented behavior.
|
||||||
|
|
||||||
|
Out of scope:
|
||||||
|
|
||||||
|
- Expanding the HTTP API beyond `POST /v1/runs`.
|
||||||
|
- Adding built-in HTTP authentication or authorization.
|
||||||
|
- Adding durable run state, run history, or multi-step orchestration.
|
||||||
|
- Adding broad provider-specific adapter packages.
|
||||||
|
- Adding new CLI flags for every provider-specific parameter.
|
||||||
|
|
||||||
|
## Acceptance Criteria
|
||||||
|
|
||||||
|
- A profile containing `reasoning_effort: medium` produces an outbound request with `reasoning_effort`.
|
||||||
|
- HTTP callers can pass `reasoning_effort` through the existing `model` override object and have it appear outbound.
|
||||||
|
- A profile or HTTP request containing JSON-compatible `extra_params` produces outbound top-level JSON fields according to the reserved-field policy.
|
||||||
|
- Reserved `extra_params` collisions fail before the outbound provider call.
|
||||||
|
- CLI callers can pass `--temperature 0` and observe `temperature: 0` in rendered/effective settings and outbound requests.
|
||||||
|
- HTTP callers can send `"temperature": 0` and observe the same behavior.
|
||||||
|
- Omitting `temperature` continues to preserve the selected profile/default value.
|
||||||
|
- Raw API key values remain unsupported in config, profiles, CLI flags, HTTP payloads, logs, and rendered output.
|
||||||
@@ -252,6 +252,39 @@ Relevant links:
|
|||||||
- [Configuration reference](config.md)
|
- [Configuration reference](config.md)
|
||||||
- [Operations guide](operations.md)
|
- [Operations guide](operations.md)
|
||||||
|
|
||||||
|
## Prompt Cache Misses Or No Cache Usage
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- CLI run summary omits `cached_tokens` / `cache_write_tokens`.
|
||||||
|
- HTTP `metadata.usage.cached_tokens` and `metadata.usage.cache_write_tokens` are both `0`.
|
||||||
|
- Provider cost or latency does not improve after repeated similar runs.
|
||||||
|
|
||||||
|
Likely cause:
|
||||||
|
|
||||||
|
- The selected prompt has no `messages[].cache_control`.
|
||||||
|
- Dynamic per-run input appears before the cache-controlled message and changes the provider cache key.
|
||||||
|
- The provider does not support the serialized cache-control shape for the selected model.
|
||||||
|
- The provider imposes minimum token thresholds or cache-breakpoint limits.
|
||||||
|
|
||||||
|
Diagnostic step:
|
||||||
|
|
||||||
|
- Run `render --format json` and verify the intended rendered message includes `cache_control`.
|
||||||
|
- Confirm stable reusable context appears before the cache-controlled message, with dynamic input after it.
|
||||||
|
- Check provider docs/logs for model support, minimum token thresholds, and breakpoint limits.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- Move stable reusable context before the cache-controlled message.
|
||||||
|
- Move highly dynamic input after the cache breakpoint.
|
||||||
|
- Keep `cache_control.type: ephemeral` and, when using `ttl`, set `ttl: 1h`.
|
||||||
|
- Use CLI cache counters or HTTP cache usage fields to verify cache reads/writes after rerunning.
|
||||||
|
|
||||||
|
Relevant links:
|
||||||
|
|
||||||
|
- [Configuration reference](config.md)
|
||||||
|
- [OpenAI-compatible chat integration](integrations/openai-compatible-chat.md)
|
||||||
|
|
||||||
## Validation Status Failed (`run` Exit 2 Or HTTP 200 With Failed Status)
|
## Validation Status Failed (`run` Exit 2 Or HTTP 200 With Failed Status)
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
|
|||||||
@@ -81,6 +81,14 @@ type serveConfig struct {
|
|||||||
schemaDir string
|
schemaDir string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type commonCommandSettings struct {
|
||||||
|
promptDir string
|
||||||
|
profileDir string
|
||||||
|
schemaDir string
|
||||||
|
serverAddr string
|
||||||
|
defaultRenderFormat renderformat.PreparedRunOutputFormat
|
||||||
|
}
|
||||||
|
|
||||||
type listFlag []string
|
type listFlag []string
|
||||||
|
|
||||||
func (l *listFlag) String() string {
|
func (l *listFlag) String() string {
|
||||||
@@ -125,22 +133,13 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
|
|||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
|
|
||||||
llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
|
llmClient, err := newOpenAIClient()
|
||||||
Timeout: defaults.LLMRequestTimeoutDefault,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(stderr, "llm client error: %v\n", err)
|
fmt.Fprintf(stderr, "llm client error: %v\n", err)
|
||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
|
|
||||||
runner := usecase.NewRunner(
|
runner := newRunner(cfg.promptDir, cfg.profileDir, cfg.schemaDir, llmClient)
|
||||||
promptdef.NewFilesystemRepository(cfg.promptDir),
|
|
||||||
profile.NewFilesystemRepository(cfg.profileDir),
|
|
||||||
artifactadapter.NewCompositeReader(),
|
|
||||||
prompt.NewGoRenderer(),
|
|
||||||
llmClient,
|
|
||||||
validate.NewStandardValidator(cfg.schemaDir),
|
|
||||||
)
|
|
||||||
|
|
||||||
res, runErr := runner.Run(context.Background(), req)
|
res, runErr := runner.Run(context.Background(), req)
|
||||||
if runErr != nil {
|
if runErr != nil {
|
||||||
@@ -170,14 +169,7 @@ func renderCommand(args []string, stdout, stderr io.Writer) int {
|
|||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
|
|
||||||
runner := usecase.NewRunner(
|
runner := newRunner(cfg.promptDir, cfg.profileDir, cfg.schemaDir, nil)
|
||||||
promptdef.NewFilesystemRepository(cfg.promptDir),
|
|
||||||
profile.NewFilesystemRepository(cfg.profileDir),
|
|
||||||
artifactadapter.NewCompositeReader(),
|
|
||||||
prompt.NewGoRenderer(),
|
|
||||||
nil,
|
|
||||||
validate.NewStandardValidator(cfg.schemaDir),
|
|
||||||
)
|
|
||||||
|
|
||||||
prepared, prepErr := runner.Prepare(context.Background(), req)
|
prepared, prepErr := runner.Prepare(context.Background(), req)
|
||||||
if prepErr != nil {
|
if prepErr != nil {
|
||||||
@@ -205,22 +197,13 @@ func serveCommand(args []string, stderr io.Writer) int {
|
|||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
|
|
||||||
llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
|
llmClient, err := newOpenAIClient()
|
||||||
Timeout: defaults.LLMRequestTimeoutDefault,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(stderr, "llm client error: %v\n", err)
|
fmt.Fprintf(stderr, "llm client error: %v\n", err)
|
||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
|
|
||||||
runner := usecase.NewRunner(
|
runner := newRunner(cfg.promptDir, cfg.profileDir, cfg.schemaDir, llmClient)
|
||||||
promptdef.NewFilesystemRepository(cfg.promptDir),
|
|
||||||
profile.NewFilesystemRepository(cfg.profileDir),
|
|
||||||
artifactadapter.NewCompositeReader(),
|
|
||||||
prompt.NewGoRenderer(),
|
|
||||||
llmClient,
|
|
||||||
validate.NewStandardValidator(cfg.schemaDir),
|
|
||||||
)
|
|
||||||
|
|
||||||
h := httpadapter.NewHandler(runner)
|
h := httpadapter.NewHandler(runner)
|
||||||
srv := &http.Server{
|
srv := &http.Server{
|
||||||
@@ -307,7 +290,7 @@ func parseServeArgs(args []string) (*serveConfig, error) {
|
|||||||
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
|
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
|
||||||
}
|
}
|
||||||
|
|
||||||
settings, err := resolveAppSettings(fs, cfg.configPath, appconfig.CLIOverrides{
|
settings, err := resolveCommonSettings(fs, cfg.configPath, appconfig.CLIOverrides{
|
||||||
PromptDir: cfg.promptDirIfSet(fs),
|
PromptDir: cfg.promptDirIfSet(fs),
|
||||||
ProfileDir: cfg.profileDirIfSet(fs),
|
ProfileDir: cfg.profileDirIfSet(fs),
|
||||||
SchemaDir: cfg.schemaDirIfSet(fs),
|
SchemaDir: cfg.schemaDirIfSet(fs),
|
||||||
@@ -317,16 +300,13 @@ func parseServeArgs(args []string) (*serveConfig, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg.promptDir = settings.PromptDir
|
cfg.promptDir = settings.promptDir
|
||||||
cfg.profileDir = settings.ProfileDir
|
cfg.profileDir = settings.profileDir
|
||||||
cfg.schemaDir = settings.SchemaDir
|
cfg.schemaDir = settings.schemaDir
|
||||||
cfg.addr = settings.ServerAddr
|
cfg.addr = settings.serverAddr
|
||||||
|
|
||||||
if strings.TrimSpace(cfg.promptDir) == "" {
|
if err := validateRequiredLibraryDirs(cfg.promptDir, cfg.profileDir); err != nil {
|
||||||
return nil, errors.New(errPromptDirRequired)
|
return nil, err
|
||||||
}
|
|
||||||
if strings.TrimSpace(cfg.profileDir) == "" {
|
|
||||||
return nil, errors.New(errProfileDirRequired)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg.promptDir = filepath.Clean(cfg.promptDir)
|
cfg.promptDir = filepath.Clean(cfg.promptDir)
|
||||||
@@ -359,7 +339,7 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
|
|||||||
return fmt.Errorf("unexpected positional args: %v", fs.Args())
|
return fmt.Errorf("unexpected positional args: %v", fs.Args())
|
||||||
}
|
}
|
||||||
|
|
||||||
settings, err := resolveAppSettings(fs, cfg.configPath, appconfig.CLIOverrides{
|
settings, err := resolveCommonSettings(fs, cfg.configPath, appconfig.CLIOverrides{
|
||||||
PromptDir: cfg.promptDirIfSet(fs),
|
PromptDir: cfg.promptDirIfSet(fs),
|
||||||
ProfileDir: cfg.profileDirIfSet(fs),
|
ProfileDir: cfg.profileDirIfSet(fs),
|
||||||
SchemaDir: cfg.schemaDirIfSet(fs),
|
SchemaDir: cfg.schemaDirIfSet(fs),
|
||||||
@@ -368,16 +348,13 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg.promptDir = settings.PromptDir
|
cfg.promptDir = settings.promptDir
|
||||||
cfg.profileDir = settings.ProfileDir
|
cfg.profileDir = settings.profileDir
|
||||||
cfg.schemaDir = settings.SchemaDir
|
cfg.schemaDir = settings.schemaDir
|
||||||
cfg.defaultRenderFormat = settings.DefaultRenderFormat
|
cfg.defaultRenderFormat = settings.defaultRenderFormat
|
||||||
|
|
||||||
if strings.TrimSpace(cfg.promptDir) == "" {
|
if err := validateRequiredLibraryDirs(cfg.promptDir, cfg.profileDir); err != nil {
|
||||||
return errors.New(errPromptDirRequired)
|
return err
|
||||||
}
|
|
||||||
if strings.TrimSpace(cfg.profileDir) == "" {
|
|
||||||
return errors.New(errProfileDirRequired)
|
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(cfg.promptID) == "" {
|
if strings.TrimSpace(cfg.promptID) == "" {
|
||||||
return errors.New("--prompt is required")
|
return errors.New("--prompt is required")
|
||||||
@@ -476,6 +453,47 @@ func resolveAppSettings(fs *flag.FlagSet, configPath string, overrides appconfig
|
|||||||
return merged, nil
|
return merged, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolveCommonSettings(fs *flag.FlagSet, configPath string, overrides appconfig.CLIOverrides) (commonCommandSettings, error) {
|
||||||
|
settings, err := resolveAppSettings(fs, configPath, overrides)
|
||||||
|
if err != nil {
|
||||||
|
return commonCommandSettings{}, err
|
||||||
|
}
|
||||||
|
return commonCommandSettings{
|
||||||
|
promptDir: settings.PromptDir,
|
||||||
|
profileDir: settings.ProfileDir,
|
||||||
|
schemaDir: settings.SchemaDir,
|
||||||
|
serverAddr: settings.ServerAddr,
|
||||||
|
defaultRenderFormat: settings.DefaultRenderFormat,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateRequiredLibraryDirs(promptDir, profileDir string) error {
|
||||||
|
if strings.TrimSpace(promptDir) == "" {
|
||||||
|
return errors.New(errPromptDirRequired)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(profileDir) == "" {
|
||||||
|
return errors.New(errProfileDirRequired)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRunner(promptDir, profileDir, schemaDir string, llmClient llm.Client) *usecase.Runner {
|
||||||
|
return usecase.NewRunner(
|
||||||
|
promptdef.NewFilesystemRepository(promptDir),
|
||||||
|
profile.NewFilesystemRepository(profileDir),
|
||||||
|
artifactadapter.NewCompositeReader(),
|
||||||
|
prompt.NewGoRenderer(),
|
||||||
|
llmClient,
|
||||||
|
validate.NewStandardValidator(schemaDir),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newOpenAIClient() (*llm.OpenAICompatibleClient, error) {
|
||||||
|
return llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
|
||||||
|
Timeout: defaults.LLMRequestTimeoutDefault,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func buildRunRequestFromConfig(cfg *runConfig) (domain.RunRequest, error) {
|
func buildRunRequestFromConfig(cfg *runConfig) (domain.RunRequest, error) {
|
||||||
inputMappings, err := parseMappings(cfg.inputRaw, false)
|
inputMappings, err := parseMappings(cfg.inputRaw, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -495,18 +513,25 @@ func buildRunRequestFromConfig(cfg *runConfig) (domain.RunRequest, error) {
|
|||||||
inputs[name] = domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path}
|
inputs[name] = domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path}
|
||||||
}
|
}
|
||||||
|
|
||||||
var modelOverride *domain.ExecutionTarget
|
var modelOverride *domain.ExecutionTargetOverride
|
||||||
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.apiKeyEnvSet || cfg.timeoutSet {
|
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.apiKeyEnvSet || cfg.timeoutSet {
|
||||||
modelOverride = &domain.ExecutionTarget{
|
modelOverride = &domain.ExecutionTargetOverride{
|
||||||
Endpoint: cfg.llmBaseURL,
|
Endpoint: cfg.llmBaseURL,
|
||||||
Model: cfg.model,
|
Model: cfg.model,
|
||||||
Temperature: cfg.temperature,
|
|
||||||
MaxTokens: cfg.maxTokens,
|
|
||||||
TopP: cfg.topP,
|
|
||||||
APIKeyEnv: cfg.apiKeyEnv,
|
APIKeyEnv: cfg.apiKeyEnv,
|
||||||
}
|
}
|
||||||
|
if cfg.temperatureSet {
|
||||||
|
modelOverride.Temperature = &cfg.temperature
|
||||||
|
}
|
||||||
|
if cfg.maxTokensSet {
|
||||||
|
modelOverride.MaxTokens = &cfg.maxTokens
|
||||||
|
}
|
||||||
|
if cfg.topPSet {
|
||||||
|
modelOverride.TopP = &cfg.topP
|
||||||
|
}
|
||||||
if cfg.timeoutSet {
|
if cfg.timeoutSet {
|
||||||
modelOverride.TimeoutSeconds = int(cfg.timeout.Seconds())
|
timeoutSeconds := int(cfg.timeout.Seconds())
|
||||||
|
modelOverride.TimeoutSeconds = &timeoutSeconds
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -588,7 +613,7 @@ func printSummary(stderr io.Writer, res *domain.RunResult) {
|
|||||||
if res == nil {
|
if res == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d\n",
|
fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d",
|
||||||
res.PromptID,
|
res.PromptID,
|
||||||
res.PromptVersion,
|
res.PromptVersion,
|
||||||
res.SelectedProfileID,
|
res.SelectedProfileID,
|
||||||
@@ -602,6 +627,10 @@ func printSummary(stderr io.Writer, res *domain.RunResult) {
|
|||||||
res.Usage.CompletionTokens,
|
res.Usage.CompletionTokens,
|
||||||
res.Usage.TotalTokens,
|
res.Usage.TotalTokens,
|
||||||
)
|
)
|
||||||
|
if res.Usage.CachedTokens != 0 || res.Usage.CacheWriteTokens != 0 {
|
||||||
|
fmt.Fprintf(stderr, " cached_tokens=%d cache_write_tokens=%d", res.Usage.CachedTokens, res.Usage.CacheWriteTokens)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
func printUsage(w io.Writer) {
|
func printUsage(w io.Writer) {
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -410,6 +411,28 @@ defaults:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseRenderArgsExplicitFormatOverridesConfigDefaultFormat(t *testing.T) {
|
||||||
|
configPath := writeAppConfigFile(t, `
|
||||||
|
prompt_dir: ./from-config/prompts
|
||||||
|
profile_dir: ./from-config/profiles
|
||||||
|
defaults:
|
||||||
|
render_format: json
|
||||||
|
`)
|
||||||
|
|
||||||
|
cfg, err := parseRenderArgs([]string{
|
||||||
|
"--config", configPath,
|
||||||
|
"--prompt", "p",
|
||||||
|
"--input", "a=b",
|
||||||
|
"--format", "text",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected valid args, got %v", err)
|
||||||
|
}
|
||||||
|
if cfg.outputFormat != renderformat.PreparedRunFormatText {
|
||||||
|
t.Fatalf("expected explicit --format text to override config default, got %q", cfg.outputFormat)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseServeArgsWithExplicitConfigLoadsSettingsAndCLIAddrOverrides(t *testing.T) {
|
func TestParseServeArgsWithExplicitConfigLoadsSettingsAndCLIAddrOverrides(t *testing.T) {
|
||||||
configPath := writeAppConfigFile(t, `
|
configPath := writeAppConfigFile(t, `
|
||||||
prompt_dir: ./from-config/prompts
|
prompt_dir: ./from-config/prompts
|
||||||
@@ -471,6 +494,59 @@ server:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *testing.T) {
|
||||||
|
runCfg, err := parseRunArgs([]string{
|
||||||
|
"--prompt-dir", "./prompts",
|
||||||
|
"--profile-dir", "./profiles",
|
||||||
|
"--prompt", "prompt-1",
|
||||||
|
"--profile", "profile-1",
|
||||||
|
"--input", "transcript=./transcript.md",
|
||||||
|
"--var", "session_date=2026-05-01",
|
||||||
|
"--llm-base-url", "http://localhost:8000/v1",
|
||||||
|
"--model", "model-x",
|
||||||
|
"--temperature", "0.8",
|
||||||
|
"--max-tokens", "123",
|
||||||
|
"--top-p", "0.6",
|
||||||
|
"--timeout", "90s",
|
||||||
|
"--api-key-env", "SCRIPTORIUM_API_KEY",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected valid run args, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
renderCfg, err := parseRenderArgs([]string{
|
||||||
|
"--prompt-dir", "./prompts",
|
||||||
|
"--profile-dir", "./profiles",
|
||||||
|
"--prompt", "prompt-1",
|
||||||
|
"--profile", "profile-1",
|
||||||
|
"--input", "transcript=./transcript.md",
|
||||||
|
"--var", "session_date=2026-05-01",
|
||||||
|
"--llm-base-url", "http://localhost:8000/v1",
|
||||||
|
"--model", "model-x",
|
||||||
|
"--temperature", "0.8",
|
||||||
|
"--max-tokens", "123",
|
||||||
|
"--top-p", "0.6",
|
||||||
|
"--timeout", "90s",
|
||||||
|
"--api-key-env", "SCRIPTORIUM_API_KEY",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected valid render args, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
runReq, err := buildRunRequestFromConfig(runCfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected run request build success, got %v", err)
|
||||||
|
}
|
||||||
|
renderReq, err := buildRunRequestFromConfig(&renderCfg.runConfig)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected render request build success, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !reflect.DeepEqual(runReq, renderReq) {
|
||||||
|
t.Fatalf("expected run/render shared flag requests to match.\nrun=%#v\nrender=%#v", runReq, renderReq)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseRunArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) {
|
func TestParseRunArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) {
|
||||||
configPath := writeAppConfigFile(t, `
|
configPath := writeAppConfigFile(t, `
|
||||||
profile_dir: ./profiles
|
profile_dir: ./profiles
|
||||||
@@ -586,42 +662,29 @@ func TestRunCommandVarsOptional(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
|
func TestRunCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
|
||||||
tmp := t.TempDir()
|
lib := newCLITestLibrary(t)
|
||||||
promptDir := filepath.Join(tmp, "prompts")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||||
profileDir := filepath.Join(tmp, "profiles")
|
|
||||||
if err := os.MkdirAll(promptDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(profileDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
inputPath := filepath.Join(tmp, "transcript.md")
|
|
||||||
if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ts := newTestLLMServer("from-config-dirs", nil)
|
ts := newTestLLMServer("from-config-dirs", nil)
|
||||||
defer ts.Close()
|
defer ts.Close()
|
||||||
|
|
||||||
writePromptFile(t, promptDir, "prompt.default", "local-default")
|
writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
|
||||||
writeProfileFile(t, profileDir, "local-default", ts.URL+"/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model")
|
||||||
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
||||||
prompt_dir: %s
|
prompt_dir: %s
|
||||||
profile_dir: %s
|
profile_dir: %s
|
||||||
`, promptDir, profileDir))
|
`, lib.promptDir, lib.profileDir))
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
||||||
var stderr bytes.Buffer
|
|
||||||
code := runCommand([]string{
|
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
"--prompt", "prompt.default",
|
"--prompt", "prompt.default",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
}, &stdout, &stderr)
|
})
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String())
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
if stdout.String() != "from-config-dirs" {
|
if stdout != "from-config-dirs" {
|
||||||
t.Fatalf("unexpected stdout output: %q", stdout.String())
|
t.Fatalf("unexpected stdout output: %q", stdout)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -630,28 +693,15 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te
|
|||||||
const secret = "super-secret-render-key"
|
const secret = "super-secret-render-key"
|
||||||
t.Setenv(envName, secret)
|
t.Setenv(envName, secret)
|
||||||
|
|
||||||
tmp := t.TempDir()
|
lib := newCLITestLibrary(t)
|
||||||
promptDir := filepath.Join(tmp, "prompts")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
||||||
profileDir := filepath.Join(tmp, "profiles")
|
|
||||||
if err := os.MkdirAll(promptDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(profileDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
inputPath := filepath.Join(tmp, "transcript.md")
|
|
||||||
if err := os.WriteFile(inputPath, []byte("hello transcript"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
writePromptFileWithTemplate(t, promptDir, "prompt.render", "local-default", "Date {{.session_date}} - Summarize: {{input \"transcript\"}}")
|
writePromptFileWithTemplate(t, lib.promptDir, "prompt.render", "local-default", "Date {{.session_date}} - Summarize: {{input \"transcript\"}}")
|
||||||
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
var stderr bytes.Buffer
|
"--prompt-dir", lib.promptDir,
|
||||||
code := renderCommand([]string{
|
"--profile-dir", lib.profileDir,
|
||||||
"--prompt-dir", promptDir,
|
|
||||||
"--profile-dir", profileDir,
|
|
||||||
"--prompt", "prompt.render",
|
"--prompt", "prompt.render",
|
||||||
"--profile", "local-default",
|
"--profile", "local-default",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
@@ -663,15 +713,15 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te
|
|||||||
"--top-p", "0.2",
|
"--top-p", "0.2",
|
||||||
"--timeout", "20s",
|
"--timeout", "20s",
|
||||||
"--api-key-env", envName,
|
"--api-key-env", envName,
|
||||||
}, &stdout, &stderr)
|
})
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String())
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
if stderr.Len() != 0 {
|
if stderr != "" {
|
||||||
t.Fatalf("expected empty stderr on success, got %q", stderr.String())
|
t.Fatalf("expected empty stderr on success, got %q", stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
out := stdout.String()
|
out := stdout
|
||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
"prompt: prompt.render",
|
"prompt: prompt.render",
|
||||||
"selected_profile_id: local-default",
|
"selected_profile_id: local-default",
|
||||||
@@ -697,112 +747,102 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRenderCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
|
func TestRenderCommandExplicitZeroTemperatureReachesEffectiveSettings(t *testing.T) {
|
||||||
tmp := t.TempDir()
|
lib := newCLITestLibrary(t)
|
||||||
promptDir := filepath.Join(tmp, "prompts")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
||||||
profileDir := filepath.Join(tmp, "profiles")
|
|
||||||
if err := os.MkdirAll(promptDir, 0o755); err != nil {
|
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
||||||
t.Fatal(err)
|
profile := `id: local-default
|
||||||
}
|
endpoint: http://127.0.0.1:1/v1
|
||||||
if err := os.MkdirAll(profileDir, 0o755); err != nil {
|
model: profile-model
|
||||||
t.Fatal(err)
|
temperature: 0.7
|
||||||
}
|
`
|
||||||
inputPath := filepath.Join(tmp, "transcript.md")
|
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-default.yaml"), []byte(profile), 0o644); err != nil {
|
||||||
if err := os.WriteFile(inputPath, []byte("hello transcript"), 0o644); err != nil {
|
t.Fatalf("failed to write profile fixture: %v", err)
|
||||||
t.Fatal(err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
writePromptFile(t, promptDir, "prompt.render", "local-default")
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
"--prompt-dir", lib.promptDir,
|
||||||
|
"--profile-dir", lib.profileDir,
|
||||||
|
"--prompt", "prompt.render",
|
||||||
|
"--input", "transcript=" + inputPath,
|
||||||
|
"--temperature", "0",
|
||||||
|
})
|
||||||
|
if code != ExitOK {
|
||||||
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout, "\n temperature: 0\n") {
|
||||||
|
t.Fatalf("expected explicit zero temperature in effective settings, got:\n%s", stdout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
||||||
|
|
||||||
|
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
||||||
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
||||||
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
||||||
prompt_dir: %s
|
prompt_dir: %s
|
||||||
profile_dir: %s
|
profile_dir: %s
|
||||||
`, promptDir, profileDir))
|
`, lib.promptDir, lib.profileDir))
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
var stderr bytes.Buffer
|
|
||||||
code := renderCommand([]string{
|
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
"--prompt", "prompt.render",
|
"--prompt", "prompt.render",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
}, &stdout, &stderr)
|
})
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String())
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
if !strings.Contains(stdout.String(), "prompt: prompt.render") {
|
if !strings.Contains(stdout, "prompt: prompt.render") {
|
||||||
t.Fatalf("expected rendered output, got %q", stdout.String())
|
t.Fatalf("expected rendered output, got %q", stdout)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRenderCommandExplicitTextFormatWorks(t *testing.T) {
|
func TestRenderCommandExplicitTextFormatWorks(t *testing.T) {
|
||||||
tmp := t.TempDir()
|
lib := newCLITestLibrary(t)
|
||||||
promptDir := filepath.Join(tmp, "prompts")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
||||||
profileDir := filepath.Join(tmp, "profiles")
|
|
||||||
if err := os.MkdirAll(promptDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(profileDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
inputPath := filepath.Join(tmp, "transcript.md")
|
|
||||||
if err := os.WriteFile(inputPath, []byte("hello transcript"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
writePromptFile(t, promptDir, "prompt.render", "local-default")
|
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
||||||
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
var stderr bytes.Buffer
|
"--prompt-dir", lib.promptDir,
|
||||||
code := renderCommand([]string{
|
"--profile-dir", lib.profileDir,
|
||||||
"--prompt-dir", promptDir,
|
|
||||||
"--profile-dir", profileDir,
|
|
||||||
"--prompt", "prompt.render",
|
"--prompt", "prompt.render",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
"--format", "text",
|
"--format", "text",
|
||||||
}, &stdout, &stderr)
|
})
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String())
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
if !strings.Contains(stdout.String(), "prompt: prompt.render") {
|
if !strings.Contains(stdout, "prompt: prompt.render") {
|
||||||
t.Fatalf("expected text output for explicit --format text, got %q", stdout.String())
|
t.Fatalf("expected text output for explicit --format text, got %q", stdout)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRenderCommandExplicitJSONFormatOutputsValidJSON(t *testing.T) {
|
func TestRenderCommandExplicitJSONFormatOutputsValidJSON(t *testing.T) {
|
||||||
tmp := t.TempDir()
|
lib := newCLITestLibrary(t)
|
||||||
promptDir := filepath.Join(tmp, "prompts")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
||||||
profileDir := filepath.Join(tmp, "profiles")
|
|
||||||
if err := os.MkdirAll(promptDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(profileDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
inputPath := filepath.Join(tmp, "transcript.md")
|
|
||||||
if err := os.WriteFile(inputPath, []byte("hello transcript"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
writePromptFile(t, promptDir, "prompt.render", "local-default")
|
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
||||||
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
var stderr bytes.Buffer
|
"--prompt-dir", lib.promptDir,
|
||||||
code := renderCommand([]string{
|
"--profile-dir", lib.profileDir,
|
||||||
"--prompt-dir", promptDir,
|
|
||||||
"--profile-dir", profileDir,
|
|
||||||
"--prompt", "prompt.render",
|
"--prompt", "prompt.render",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
"--format", "json",
|
"--format", "json",
|
||||||
}, &stdout, &stderr)
|
})
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String())
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
var payload map[string]any
|
var payload map[string]any
|
||||||
if err := json.Unmarshal(stdout.Bytes(), &payload); err != nil {
|
if err := json.Unmarshal([]byte(stdout), &payload); err != nil {
|
||||||
t.Fatalf("expected valid json output, got %v\nbody=%s", err, stdout.String())
|
t.Fatalf("expected valid json output, got %v\nbody=%s", err, stdout)
|
||||||
}
|
}
|
||||||
if payload["prompt_id"] != "prompt.render" {
|
if payload["prompt_id"] != "prompt.render" {
|
||||||
t.Fatalf("expected prompt_id, got %#v", payload["prompt_id"])
|
t.Fatalf("expected prompt_id, got %#v", payload["prompt_id"])
|
||||||
@@ -834,38 +874,25 @@ func TestRenderCommandUnknownFormatFailsClearly(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRenderCommandOutWritesToFile(t *testing.T) {
|
func TestRenderCommandOutWritesToFile(t *testing.T) {
|
||||||
tmp := t.TempDir()
|
lib := newCLITestLibrary(t)
|
||||||
promptDir := filepath.Join(tmp, "prompts")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
||||||
profileDir := filepath.Join(tmp, "profiles")
|
outPath := filepath.Join(lib.rootDir, "render.txt")
|
||||||
if err := os.MkdirAll(promptDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(profileDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
inputPath := filepath.Join(tmp, "transcript.md")
|
|
||||||
if err := os.WriteFile(inputPath, []byte("hello transcript"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
outPath := filepath.Join(tmp, "render.txt")
|
|
||||||
|
|
||||||
writePromptFile(t, promptDir, "prompt.render", "local-default")
|
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
||||||
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
var stderr bytes.Buffer
|
"--prompt-dir", lib.promptDir,
|
||||||
code := renderCommand([]string{
|
"--profile-dir", lib.profileDir,
|
||||||
"--prompt-dir", promptDir,
|
|
||||||
"--profile-dir", profileDir,
|
|
||||||
"--prompt", "prompt.render",
|
"--prompt", "prompt.render",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
"--out", outPath,
|
"--out", outPath,
|
||||||
}, &stdout, &stderr)
|
})
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String())
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
if stdout.Len() != 0 {
|
if stdout != "" {
|
||||||
t.Fatalf("expected empty stdout when --out is set, got %q", stdout.String())
|
t.Fatalf("expected empty stdout when --out is set, got %q", stdout)
|
||||||
}
|
}
|
||||||
out, err := os.ReadFile(outPath)
|
out, err := os.ReadFile(outPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -877,36 +904,23 @@ func TestRenderCommandOutWritesToFile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
|
func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
|
||||||
tmp := t.TempDir()
|
lib := newCLITestLibrary(t)
|
||||||
promptDir := filepath.Join(tmp, "prompts")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||||
profileDir := filepath.Join(tmp, "profiles")
|
|
||||||
if err := os.MkdirAll(promptDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(profileDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
inputPath := filepath.Join(tmp, "transcript.md")
|
|
||||||
if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
writePromptFile(t, promptDir, "prompt.default", "local-default")
|
writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
|
||||||
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model")
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model")
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
var stderr bytes.Buffer
|
"--prompt-dir", lib.promptDir,
|
||||||
code := renderCommand([]string{
|
"--profile-dir", lib.profileDir,
|
||||||
"--prompt-dir", promptDir,
|
|
||||||
"--profile-dir", profileDir,
|
|
||||||
"--prompt", "prompt.default",
|
"--prompt", "prompt.default",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
}, &stdout, &stderr)
|
})
|
||||||
|
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String())
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
out := stdout.String()
|
out := stdout
|
||||||
if !strings.Contains(out, "selected_profile_id: local-default") {
|
if !strings.Contains(out, "selected_profile_id: local-default") {
|
||||||
t.Fatalf("expected prompt default profile in output, got %q", out)
|
t.Fatalf("expected prompt default profile in output, got %q", out)
|
||||||
}
|
}
|
||||||
@@ -916,38 +930,25 @@ func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
||||||
tmp := t.TempDir()
|
lib := newCLITestLibrary(t)
|
||||||
promptDir := filepath.Join(tmp, "prompts")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||||
profileDir := filepath.Join(tmp, "profiles")
|
|
||||||
if err := os.MkdirAll(promptDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(profileDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
inputPath := filepath.Join(tmp, "transcript.md")
|
|
||||||
if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
writePromptFile(t, promptDir, "prompt.default", "local-default")
|
writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
|
||||||
writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model")
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model")
|
||||||
writeProfileFile(t, profileDir, "quality", "http://127.0.0.1:1/v1", "quality-model")
|
writeProfileFile(t, lib.profileDir, "quality", "http://127.0.0.1:1/v1", "quality-model")
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
var stderr bytes.Buffer
|
"--prompt-dir", lib.promptDir,
|
||||||
code := renderCommand([]string{
|
"--profile-dir", lib.profileDir,
|
||||||
"--prompt-dir", promptDir,
|
|
||||||
"--profile-dir", profileDir,
|
|
||||||
"--prompt", "prompt.default",
|
"--prompt", "prompt.default",
|
||||||
"--profile", "quality",
|
"--profile", "quality",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
}, &stdout, &stderr)
|
})
|
||||||
|
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String())
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
out := stdout.String()
|
out := stdout
|
||||||
if !strings.Contains(out, "selected_profile_id: quality") {
|
if !strings.Contains(out, "selected_profile_id: quality") {
|
||||||
t.Fatalf("expected explicit profile in output, got %q", out)
|
t.Fatalf("expected explicit profile in output, got %q", out)
|
||||||
}
|
}
|
||||||
@@ -957,104 +958,67 @@ func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
|
func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
|
||||||
tmp := t.TempDir()
|
lib := newCLITestLibrary(t)
|
||||||
promptDir := filepath.Join(tmp, "prompts")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||||
profileDir := filepath.Join(tmp, "profiles")
|
|
||||||
if err := os.MkdirAll(promptDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(profileDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
inputPath := filepath.Join(tmp, "transcript.md")
|
|
||||||
if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ts := newTestLLMServer("default-output", nil)
|
ts := newTestLLMServer("default-output", nil)
|
||||||
defer ts.Close()
|
defer ts.Close()
|
||||||
|
|
||||||
writePromptFile(t, promptDir, "prompt.default", "local-default")
|
writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
|
||||||
writeProfileFile(t, profileDir, "local-default", ts.URL+"/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model")
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
||||||
var stderr bytes.Buffer
|
"--prompt-dir", lib.promptDir,
|
||||||
code := runCommand([]string{
|
"--profile-dir", lib.profileDir,
|
||||||
"--prompt-dir", promptDir,
|
|
||||||
"--profile-dir", profileDir,
|
|
||||||
"--prompt", "prompt.default",
|
"--prompt", "prompt.default",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
}, &stdout, &stderr)
|
})
|
||||||
|
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String())
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
if stdout.String() != "default-output" {
|
if stdout != "default-output" {
|
||||||
t.Fatalf("unexpected stdout output: %q", stdout.String())
|
t.Fatalf("unexpected stdout output: %q", stdout)
|
||||||
}
|
}
|
||||||
if !strings.Contains(stderr.String(), "selected_profile=local-default") {
|
if !strings.Contains(stderr, "selected_profile=local-default") {
|
||||||
t.Fatalf("expected selected profile in summary, got %q", stderr.String())
|
t.Fatalf("expected selected profile in summary, got %q", stderr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
func TestRunCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
||||||
tmp := t.TempDir()
|
lib := newCLITestLibrary(t)
|
||||||
promptDir := filepath.Join(tmp, "prompts")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||||
profileDir := filepath.Join(tmp, "profiles")
|
|
||||||
if err := os.MkdirAll(promptDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(profileDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
inputPath := filepath.Join(tmp, "transcript.md")
|
|
||||||
if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defaultServer := newTestLLMServer("from-default", nil)
|
defaultServer := newTestLLMServer("from-default", nil)
|
||||||
defer defaultServer.Close()
|
defer defaultServer.Close()
|
||||||
overrideServer := newTestLLMServer("from-override", nil)
|
overrideServer := newTestLLMServer("from-override", nil)
|
||||||
defer overrideServer.Close()
|
defer overrideServer.Close()
|
||||||
|
|
||||||
writePromptFile(t, promptDir, "prompt.default", "local-default")
|
writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
|
||||||
writeProfileFile(t, profileDir, "local-default", defaultServer.URL+"/v1", "default-model")
|
writeProfileFile(t, lib.profileDir, "local-default", defaultServer.URL+"/v1", "default-model")
|
||||||
writeProfileFile(t, profileDir, "quality", overrideServer.URL+"/v1", "quality-model")
|
writeProfileFile(t, lib.profileDir, "quality", overrideServer.URL+"/v1", "quality-model")
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
||||||
var stderr bytes.Buffer
|
"--prompt-dir", lib.promptDir,
|
||||||
code := runCommand([]string{
|
"--profile-dir", lib.profileDir,
|
||||||
"--prompt-dir", promptDir,
|
|
||||||
"--profile-dir", profileDir,
|
|
||||||
"--prompt", "prompt.default",
|
"--prompt", "prompt.default",
|
||||||
"--profile", "quality",
|
"--profile", "quality",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
}, &stdout, &stderr)
|
})
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String())
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
if stdout.String() != "from-override" {
|
if stdout != "from-override" {
|
||||||
t.Fatalf("expected explicit profile output, got %q", stdout.String())
|
t.Fatalf("expected explicit profile output, got %q", stdout)
|
||||||
}
|
}
|
||||||
if !strings.Contains(stderr.String(), "selected_profile=quality") {
|
if !strings.Contains(stderr, "selected_profile=quality") {
|
||||||
t.Fatalf("expected selected profile quality, got %q", stderr.String())
|
t.Fatalf("expected selected profile quality, got %q", stderr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) {
|
func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) {
|
||||||
tmp := t.TempDir()
|
lib := newCLITestLibrary(t)
|
||||||
promptDir := filepath.Join(tmp, "prompts")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||||
profileDir := filepath.Join(tmp, "profiles")
|
|
||||||
if err := os.MkdirAll(promptDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(profileDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
inputPath := filepath.Join(tmp, "transcript.md")
|
|
||||||
if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var baseHits int32
|
var baseHits int32
|
||||||
baseServer := newTestLLMServer("base", &baseHits)
|
baseServer := newTestLLMServer("base", &baseHits)
|
||||||
@@ -1071,14 +1035,12 @@ func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
defer overrideServer.Close()
|
defer overrideServer.Close()
|
||||||
|
|
||||||
writePromptFile(t, promptDir, "prompt.default", "local-default")
|
writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
|
||||||
writeProfileFile(t, profileDir, "local-default", baseServer.URL+"/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", baseServer.URL+"/v1", "profile-model")
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
||||||
var stderr bytes.Buffer
|
"--prompt-dir", lib.promptDir,
|
||||||
code := runCommand([]string{
|
"--profile-dir", lib.profileDir,
|
||||||
"--prompt-dir", promptDir,
|
|
||||||
"--profile-dir", profileDir,
|
|
||||||
"--prompt", "prompt.default",
|
"--prompt", "prompt.default",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
"--llm-base-url", overrideServer.URL + "/v1",
|
"--llm-base-url", overrideServer.URL + "/v1",
|
||||||
@@ -1087,9 +1049,9 @@ func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) {
|
|||||||
"--max-tokens", "55",
|
"--max-tokens", "55",
|
||||||
"--top-p", "0.2",
|
"--top-p", "0.2",
|
||||||
"--timeout", "20s",
|
"--timeout", "20s",
|
||||||
}, &stdout, &stderr)
|
})
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String())
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
if atomic.LoadInt32(&baseHits) != 0 {
|
if atomic.LoadInt32(&baseHits) != 0 {
|
||||||
t.Fatalf("expected base profile endpoint not to be hit, got %d", baseHits)
|
t.Fatalf("expected base profile endpoint not to be hit, got %d", baseHits)
|
||||||
@@ -1097,8 +1059,8 @@ func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) {
|
|||||||
if atomic.LoadInt32(&overrideHits) != 1 {
|
if atomic.LoadInt32(&overrideHits) != 1 {
|
||||||
t.Fatalf("expected override endpoint to be hit once, got %d", overrideHits)
|
t.Fatalf("expected override endpoint to be hit once, got %d", overrideHits)
|
||||||
}
|
}
|
||||||
if stdout.String() != "override" {
|
if stdout != "override" {
|
||||||
t.Fatalf("unexpected stdout output: %q", stdout.String())
|
t.Fatalf("unexpected stdout output: %q", stdout)
|
||||||
}
|
}
|
||||||
if !strings.Contains(observedBody, `"model":"override-model"`) {
|
if !strings.Contains(observedBody, `"model":"override-model"`) {
|
||||||
t.Fatalf("expected override model in request body, got %s", observedBody)
|
t.Fatalf("expected override model in request body, got %s", observedBody)
|
||||||
@@ -1131,6 +1093,81 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
|
|||||||
if !strings.Contains(stderr.String(), "prompt=p@1") {
|
if !strings.Contains(stderr.String(), "prompt=p@1") {
|
||||||
t.Fatalf("expected summary on stderr, got %q", stderr.String())
|
t.Fatalf("expected summary on stderr, got %q", stderr.String())
|
||||||
}
|
}
|
||||||
|
if strings.Contains(stderr.String(), "cached_tokens=") || strings.Contains(stderr.String(), "cache_write_tokens=") {
|
||||||
|
t.Fatalf("expected zero cache usage to be omitted from summary, got %q", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) {
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
printSummary(&stderr, &domain.RunResult{
|
||||||
|
PromptID: "p",
|
||||||
|
PromptVersion: "1",
|
||||||
|
SelectedProfileID: "exec",
|
||||||
|
ModelName: "m",
|
||||||
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic},
|
||||||
|
RenderedPromptHash: "h",
|
||||||
|
InputHashes: map[string]string{"in": "x"},
|
||||||
|
Usage: domain.TokenUsage{
|
||||||
|
PromptTokens: 10,
|
||||||
|
CompletionTokens: 5,
|
||||||
|
TotalTokens: 15,
|
||||||
|
CachedTokens: 0,
|
||||||
|
CacheWriteTokens: 3,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
summary := stderr.String()
|
||||||
|
if !strings.Contains(summary, "usage=10/5/15") {
|
||||||
|
t.Fatalf("expected base usage summary, got %q", summary)
|
||||||
|
}
|
||||||
|
if !strings.Contains(summary, "cached_tokens=0 cache_write_tokens=3") {
|
||||||
|
t.Fatalf("expected cache usage in summary, got %q", summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type cliTestLibrary struct {
|
||||||
|
rootDir string
|
||||||
|
promptDir string
|
||||||
|
profileDir string
|
||||||
|
}
|
||||||
|
|
||||||
|
func newCLITestLibrary(t *testing.T) *cliTestLibrary {
|
||||||
|
t.Helper()
|
||||||
|
root := t.TempDir()
|
||||||
|
lib := &cliTestLibrary{
|
||||||
|
rootDir: root,
|
||||||
|
promptDir: filepath.Join(root, "prompts"),
|
||||||
|
profileDir: filepath.Join(root, "profiles"),
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(lib.promptDir, 0o755); err != nil {
|
||||||
|
t.Fatalf("failed to create prompt fixture directory: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(lib.profileDir, 0o755); err != nil {
|
||||||
|
t.Fatalf("failed to create profile fixture directory: %v", err)
|
||||||
|
}
|
||||||
|
return lib
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *cliTestLibrary) writeInputFile(t *testing.T, name, body string) string {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(l.rootDir, name)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
t.Fatalf("failed to create input fixture directory: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||||
|
t.Fatalf("failed to write input fixture: %v", err)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
func runCLICommand(t *testing.T, command func([]string, io.Writer, io.Writer) int, args []string) (int, string, string) {
|
||||||
|
t.Helper()
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := command(args, &stdout, &stderr)
|
||||||
|
return code, stdout.String(), stderr.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
func writePromptFile(t *testing.T, dir, id, defaultProfile string) {
|
func writePromptFile(t *testing.T, dir, id, defaultProfile string) {
|
||||||
|
|||||||
@@ -23,14 +23,14 @@ type inputRefDTO struct {
|
|||||||
type modelOverrideRequestDTO struct {
|
type modelOverrideRequestDTO struct {
|
||||||
Endpoint string `json:"endpoint,omitempty"`
|
Endpoint string `json:"endpoint,omitempty"`
|
||||||
Model string `json:"model,omitempty"`
|
Model string `json:"model,omitempty"`
|
||||||
Temperature float64 `json:"temperature,omitempty"`
|
Temperature *float64 `json:"temperature,omitempty"`
|
||||||
MaxTokens int `json:"max_tokens,omitempty"`
|
MaxTokens *int `json:"max_tokens,omitempty"`
|
||||||
TopP float64 `json:"top_p,omitempty"`
|
TopP *float64 `json:"top_p,omitempty"`
|
||||||
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
|
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||||
ServiceTier string `json:"service_tier,omitempty"`
|
ServiceTier string `json:"service_tier,omitempty"`
|
||||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||||
ExtraParams map[string]string `json:"extra_params,omitempty"`
|
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type runResponseDTO struct {
|
type runResponseDTO struct {
|
||||||
@@ -79,13 +79,15 @@ type modelParamsDTO struct {
|
|||||||
ServiceTier string `json:"service_tier,omitempty"`
|
ServiceTier string `json:"service_tier,omitempty"`
|
||||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||||
ExtraParams map[string]string `json:"extra_params,omitempty"`
|
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type tokenUsageDTO struct {
|
type tokenUsageDTO struct {
|
||||||
PromptTokens int `json:"prompt_tokens"`
|
PromptTokens int `json:"prompt_tokens"`
|
||||||
CompletionTokens int `json:"completion_tokens"`
|
CompletionTokens int `json:"completion_tokens"`
|
||||||
TotalTokens int `json:"total_tokens"`
|
TotalTokens int `json:"total_tokens"`
|
||||||
|
CachedTokens int `json:"cached_tokens"`
|
||||||
|
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type validationDTO struct {
|
type validationDTO struct {
|
||||||
|
|||||||
@@ -61,20 +61,9 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var model *domain.ExecutionTarget
|
var model *domain.ExecutionTargetOverride
|
||||||
if req.Model != nil {
|
if req.Model != nil {
|
||||||
model = &domain.ExecutionTarget{
|
model = executionTargetOverrideFromModelOverrideDTO(req.Model)
|
||||||
Endpoint: req.Model.Endpoint,
|
|
||||||
Model: req.Model.Model,
|
|
||||||
Temperature: req.Model.Temperature,
|
|
||||||
MaxTokens: req.Model.MaxTokens,
|
|
||||||
TopP: req.Model.TopP,
|
|
||||||
TimeoutSeconds: req.Model.TimeoutSeconds,
|
|
||||||
ServiceTier: req.Model.ServiceTier,
|
|
||||||
ReasoningEffort: req.Model.ReasoningEffort,
|
|
||||||
APIKeyEnv: req.Model.APIKeyEnv,
|
|
||||||
ExtraParams: req.Model.ExtraParams,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := h.runner.Run(r.Context(), domain.RunRequest{
|
res, err := h.runner.Run(r.Context(), domain.RunRequest{
|
||||||
@@ -110,23 +99,14 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
SelectedProfileID: res.SelectedProfileID,
|
SelectedProfileID: res.SelectedProfileID,
|
||||||
ModelName: res.ModelName,
|
ModelName: res.ModelName,
|
||||||
Endpoint: res.Endpoint,
|
Endpoint: res.Endpoint,
|
||||||
ModelParams: modelParamsDTO{
|
ModelParams: modelParamsDTOFromExecutionTarget(res.EffectiveModelParams),
|
||||||
Endpoint: res.EffectiveModelParams.Endpoint,
|
|
||||||
Model: res.EffectiveModelParams.Model,
|
|
||||||
Temperature: res.EffectiveModelParams.Temperature,
|
|
||||||
MaxTokens: res.EffectiveModelParams.MaxTokens,
|
|
||||||
TopP: res.EffectiveModelParams.TopP,
|
|
||||||
TimeoutSeconds: res.EffectiveModelParams.TimeoutSeconds,
|
|
||||||
ServiceTier: res.EffectiveModelParams.ServiceTier,
|
|
||||||
ReasoningEffort: res.EffectiveModelParams.ReasoningEffort,
|
|
||||||
APIKeyEnv: res.EffectiveModelParams.APIKeyEnv,
|
|
||||||
ExtraParams: res.EffectiveModelParams.ExtraParams,
|
|
||||||
},
|
|
||||||
InputHashes: res.InputHashes,
|
InputHashes: res.InputHashes,
|
||||||
Usage: tokenUsageDTO{
|
Usage: tokenUsageDTO{
|
||||||
PromptTokens: res.Usage.PromptTokens,
|
PromptTokens: res.Usage.PromptTokens,
|
||||||
CompletionTokens: res.Usage.CompletionTokens,
|
CompletionTokens: res.Usage.CompletionTokens,
|
||||||
TotalTokens: res.Usage.TotalTokens,
|
TotalTokens: res.Usage.TotalTokens,
|
||||||
|
CachedTokens: res.Usage.CachedTokens,
|
||||||
|
CacheWriteTokens: res.Usage.CacheWriteTokens,
|
||||||
},
|
},
|
||||||
StartTime: res.StartTime,
|
StartTime: res.StartTime,
|
||||||
EndTime: res.EndTime,
|
EndTime: res.EndTime,
|
||||||
@@ -143,6 +123,39 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeJSON(w, http.StatusOK, resp)
|
writeJSON(w, http.StatusOK, resp)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func executionTargetOverrideFromModelOverrideDTO(dto *modelOverrideRequestDTO) *domain.ExecutionTargetOverride {
|
||||||
|
if dto == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &domain.ExecutionTargetOverride{
|
||||||
|
Endpoint: dto.Endpoint,
|
||||||
|
Model: dto.Model,
|
||||||
|
Temperature: dto.Temperature,
|
||||||
|
MaxTokens: dto.MaxTokens,
|
||||||
|
TopP: dto.TopP,
|
||||||
|
TimeoutSeconds: dto.TimeoutSeconds,
|
||||||
|
ServiceTier: dto.ServiceTier,
|
||||||
|
ReasoningEffort: dto.ReasoningEffort,
|
||||||
|
APIKeyEnv: dto.APIKeyEnv,
|
||||||
|
ExtraParams: dto.ExtraParams,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func modelParamsDTOFromExecutionTarget(target domain.ExecutionTarget) modelParamsDTO {
|
||||||
|
return modelParamsDTO{
|
||||||
|
Endpoint: target.Endpoint,
|
||||||
|
Model: target.Model,
|
||||||
|
Temperature: target.Temperature,
|
||||||
|
MaxTokens: target.MaxTokens,
|
||||||
|
TopP: target.TopP,
|
||||||
|
TimeoutSeconds: target.TimeoutSeconds,
|
||||||
|
ServiceTier: target.ServiceTier,
|
||||||
|
ReasoningEffort: target.ReasoningEffort,
|
||||||
|
APIKeyEnv: target.APIKeyEnv,
|
||||||
|
ExtraParams: target.ExtraParams,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func mapValidation(v domain.ValidationResult) validationDTO {
|
func mapValidation(v domain.ValidationResult) validationDTO {
|
||||||
return validationDTO{
|
return validationDTO{
|
||||||
Status: string(v.Status),
|
Status: string(v.Status),
|
||||||
@@ -164,9 +177,9 @@ func mapRunError(err error) (int, string, string) {
|
|||||||
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
|
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
|
||||||
case errors.Is(err, profile.ErrInvalidYAML), errors.Is(err, profile.ErrInvalidProfile):
|
case errors.Is(err, profile.ErrInvalidYAML), errors.Is(err, profile.ErrInvalidProfile):
|
||||||
return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile"
|
return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile"
|
||||||
case errors.Is(err, usecase.ErrInvalidRequest) && strings.Contains(err.Error(), "profile id is required either in request or prompt default_profile"):
|
case errors.Is(err, usecase.ErrProfileRequired):
|
||||||
return http.StatusBadRequest, "profile_required", "profile_id is required when prompt default_profile is not set"
|
return http.StatusBadRequest, "profile_required", "profile_id is required when prompt default_profile is not set"
|
||||||
case errors.Is(err, usecase.ErrInvalidRequest) && strings.Contains(err.Error(), "api key environment variable"):
|
case errors.Is(err, usecase.ErrAPIKeyEnvMissing):
|
||||||
return http.StatusBadRequest, "api_key_env_missing", "api_key_env is set but the environment variable is missing"
|
return http.StatusBadRequest, "api_key_env_missing", "api_key_env is set but the environment variable is missing"
|
||||||
case errors.Is(err, usecase.ErrInvalidRequest):
|
case errors.Is(err, usecase.ErrInvalidRequest):
|
||||||
return http.StatusBadRequest, "invalid_request", "invalid run request"
|
return http.StatusBadRequest, "invalid_request", "invalid run request"
|
||||||
|
|||||||
@@ -4,15 +4,16 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||||
@@ -32,6 +33,34 @@ func (f *fakeRunner) Run(ctx context.Context, req domain.RunRequest) (*domain.Ru
|
|||||||
return f.result, nil
|
return f.result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type handlerPromptRepo struct {
|
||||||
|
def *domain.PromptDefinition
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r handlerPromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
||||||
|
return r.def, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type handlerProfileRepo struct {
|
||||||
|
profile *domain.ExecutionProfile
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r handlerProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
return r.profile, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type handlerArtifactReader struct{}
|
||||||
|
|
||||||
|
func (handlerArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
||||||
|
return &domain.Artifact{Name: "input", Body: []byte("input"), Hash: "hash"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type handlerRenderer struct{}
|
||||||
|
|
||||||
|
func (handlerRenderer) Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
|
||||||
|
return &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
||||||
start := time.Now().UTC()
|
start := time.Now().UTC()
|
||||||
end := start.Add(2 * time.Second)
|
end := start.Add(2 * time.Second)
|
||||||
@@ -66,7 +95,13 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
APIKeyEnv: envName,
|
APIKeyEnv: envName,
|
||||||
},
|
},
|
||||||
InputHashes: map[string]string{"transcript": "h1"},
|
InputHashes: map[string]string{"transcript": "h1"},
|
||||||
Usage: domain.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3},
|
Usage: domain.TokenUsage{
|
||||||
|
PromptTokens: 1,
|
||||||
|
CompletionTokens: 2,
|
||||||
|
TotalTokens: 3,
|
||||||
|
CachedTokens: 4,
|
||||||
|
CacheWriteTokens: 5,
|
||||||
|
},
|
||||||
StartTime: start,
|
StartTime: start,
|
||||||
EndTime: end,
|
EndTime: end,
|
||||||
Duration: 2 * time.Second,
|
Duration: 2 * time.Second,
|
||||||
@@ -111,6 +146,13 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" {
|
if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" {
|
||||||
t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"])
|
t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"])
|
||||||
}
|
}
|
||||||
|
usage := metadata["usage"].(map[string]any)
|
||||||
|
if usage["prompt_tokens"] != float64(1) || usage["completion_tokens"] != float64(2) || usage["total_tokens"] != float64(3) {
|
||||||
|
t.Fatalf("unexpected base usage metadata: %#v", usage)
|
||||||
|
}
|
||||||
|
if usage["cached_tokens"] != float64(4) || usage["cache_write_tokens"] != float64(5) {
|
||||||
|
t.Fatalf("unexpected cache usage metadata: %#v", usage)
|
||||||
|
}
|
||||||
modelParams := metadata["model_params"].(map[string]any)
|
modelParams := metadata["model_params"].(map[string]any)
|
||||||
if modelParams["api_key_env"] != envName {
|
if modelParams["api_key_env"] != envName {
|
||||||
t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"])
|
t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"])
|
||||||
@@ -134,7 +176,7 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
if r.last.Execution == nil || r.last.Execution.Model != "gpt-x" {
|
if r.last.Execution == nil || r.last.Execution.Model != "gpt-x" {
|
||||||
t.Fatalf("expected model override, got %#v", r.last.Execution)
|
t.Fatalf("expected model override, got %#v", r.last.Execution)
|
||||||
}
|
}
|
||||||
if r.last.Execution.TimeoutSeconds != 120 {
|
if r.last.Execution.TimeoutSeconds == nil || *r.last.Execution.TimeoutSeconds != 120 {
|
||||||
t.Fatalf("expected timeout_seconds override 120, got %#v", r.last.Execution)
|
t.Fatalf("expected timeout_seconds override 120, got %#v", r.last.Execution)
|
||||||
}
|
}
|
||||||
if r.last.Execution.ServiceTier != "flex" {
|
if r.last.Execution.ServiceTier != "flex" {
|
||||||
@@ -171,6 +213,265 @@ func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
|
|||||||
if metadata["selected_profile_id"] != "prompt-default" {
|
if metadata["selected_profile_id"] != "prompt-default" {
|
||||||
t.Fatalf("expected selected_profile_id from result, got %#v", metadata["selected_profile_id"])
|
t.Fatalf("expected selected_profile_id from result, got %#v", metadata["selected_profile_id"])
|
||||||
}
|
}
|
||||||
|
usage := metadata["usage"].(map[string]any)
|
||||||
|
if usage["cached_tokens"] != float64(0) || usage["cache_write_tokens"] != float64(0) {
|
||||||
|
t.Fatalf("expected zero cache usage fields to be included, got %#v", usage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
|
||||||
|
r := &fakeRunner{result: &domain.RunResult{
|
||||||
|
Artifact: domain.Artifact{Body: []byte("ok")},
|
||||||
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
|
}}
|
||||||
|
h := NewHandler(r)
|
||||||
|
|
||||||
|
reqBody := `{
|
||||||
|
"prompt_id": "prompt-1",
|
||||||
|
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
|
||||||
|
"model": {
|
||||||
|
"endpoint": "http://override/v1",
|
||||||
|
"model": "override-model",
|
||||||
|
"temperature": 0.6,
|
||||||
|
"max_tokens": 250,
|
||||||
|
"top_p": 0.85,
|
||||||
|
"timeout_seconds": 33,
|
||||||
|
"service_tier": "flex",
|
||||||
|
"reasoning_effort": "medium",
|
||||||
|
"api_key_env": "SCRIPTORIUM_API_KEY",
|
||||||
|
"extra_params": {"provider_option":"on"}
|
||||||
|
}
|
||||||
|
}`
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(reqBody))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if r.last.Execution == nil {
|
||||||
|
t.Fatalf("expected execution override in run request")
|
||||||
|
}
|
||||||
|
got := r.last.Execution
|
||||||
|
if got.Endpoint != "http://override/v1" ||
|
||||||
|
got.Model != "override-model" ||
|
||||||
|
got.ServiceTier != "flex" ||
|
||||||
|
got.ReasoningEffort != "medium" ||
|
||||||
|
got.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
|
||||||
|
t.Fatalf("unexpected mapped execution target: %+v", got)
|
||||||
|
}
|
||||||
|
if got.Temperature == nil || *got.Temperature != 0.6 {
|
||||||
|
t.Fatalf("unexpected mapped temperature: %#v", got.Temperature)
|
||||||
|
}
|
||||||
|
if got.MaxTokens == nil || *got.MaxTokens != 250 {
|
||||||
|
t.Fatalf("unexpected mapped max_tokens: %#v", got.MaxTokens)
|
||||||
|
}
|
||||||
|
if got.TopP == nil || *got.TopP != 0.85 {
|
||||||
|
t.Fatalf("unexpected mapped top_p: %#v", got.TopP)
|
||||||
|
}
|
||||||
|
if got.TimeoutSeconds == nil || *got.TimeoutSeconds != 33 {
|
||||||
|
t.Fatalf("unexpected mapped timeout_seconds: %#v", got.TimeoutSeconds)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got.ExtraParams, map[string]any{"provider_option": "on"}) {
|
||||||
|
t.Fatalf("unexpected mapped extra_params: %#v", got.ExtraParams)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerModelOverrideAcceptsJSONCompatibleExtraParams(t *testing.T) {
|
||||||
|
r := &fakeRunner{result: &domain.RunResult{
|
||||||
|
Artifact: domain.Artifact{Body: []byte("ok")},
|
||||||
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
|
}}
|
||||||
|
h := NewHandler(r)
|
||||||
|
|
||||||
|
reqBody := `{
|
||||||
|
"prompt_id": "prompt-1",
|
||||||
|
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
|
||||||
|
"model": {
|
||||||
|
"extra_params": {
|
||||||
|
"string_value": "enabled",
|
||||||
|
"number_value": 42,
|
||||||
|
"boolean_value": true,
|
||||||
|
"object_value": {"nested": "value", "count": 2},
|
||||||
|
"array_value": ["first", 3, false]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}`
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(reqBody))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if r.last.Execution == nil {
|
||||||
|
t.Fatal("expected execution override in run request")
|
||||||
|
}
|
||||||
|
want := map[string]any{
|
||||||
|
"string_value": "enabled",
|
||||||
|
"number_value": float64(42),
|
||||||
|
"boolean_value": true,
|
||||||
|
"object_value": map[string]any{"nested": "value", "count": float64(2)},
|
||||||
|
"array_value": []any{"first", float64(3), false},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(r.last.Execution.ExtraParams, want) {
|
||||||
|
t.Fatalf("unexpected mapped extra_params:\ngot=%#v\nwant=%#v", r.last.Execution.ExtraParams, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerModelOverrideExplicitZeroTemperatureMapsAsPresent(t *testing.T) {
|
||||||
|
r := &fakeRunner{result: &domain.RunResult{
|
||||||
|
Artifact: domain.Artifact{Body: []byte("ok")},
|
||||||
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0},
|
||||||
|
}}
|
||||||
|
h := NewHandler(r)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id": "prompt-1",
|
||||||
|
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
|
||||||
|
"model": {"temperature": 0}
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if r.last.Execution == nil || r.last.Execution.Temperature == nil {
|
||||||
|
t.Fatalf("expected temperature override to be present, got %#v", r.last.Execution)
|
||||||
|
}
|
||||||
|
if *r.last.Execution.Temperature != 0 {
|
||||||
|
t.Fatalf("expected zero temperature override, got %v", *r.last.Execution.Temperature)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerModelOverrideOmittedTemperatureMapsAsAbsent(t *testing.T) {
|
||||||
|
r := &fakeRunner{result: &domain.RunResult{
|
||||||
|
Artifact: domain.Artifact{Body: []byte("ok")},
|
||||||
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0.7},
|
||||||
|
}}
|
||||||
|
h := NewHandler(r)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id": "prompt-1",
|
||||||
|
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
|
||||||
|
"model": {"model": "override-model"}
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if r.last.Execution == nil {
|
||||||
|
t.Fatal("expected model override")
|
||||||
|
}
|
||||||
|
if r.last.Execution.Temperature != nil {
|
||||||
|
t.Fatalf("expected omitted temperature to remain absent, got %#v", r.last.Execution.Temperature)
|
||||||
|
}
|
||||||
|
|
||||||
|
var resp map[string]any
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||||
|
t.Fatalf("invalid JSON response: %v", err)
|
||||||
|
}
|
||||||
|
metadata := resp["metadata"].(map[string]any)
|
||||||
|
params := metadata["model_params"].(map[string]any)
|
||||||
|
if params["temperature"] != 0.7 {
|
||||||
|
t.Fatalf("expected effective profile/default temperature in response, got %#v", params["temperature"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing.T) {
|
||||||
|
r := &fakeRunner{result: &domain.RunResult{
|
||||||
|
Artifact: domain.Artifact{
|
||||||
|
Name: "output",
|
||||||
|
ContentType: "text/plain",
|
||||||
|
Body: []byte("ok"),
|
||||||
|
Size: 2,
|
||||||
|
Hash: "abc",
|
||||||
|
},
|
||||||
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: domain.ExecutionTarget{
|
||||||
|
Endpoint: "http://llm/v1",
|
||||||
|
Model: "gpt-test",
|
||||||
|
Temperature: 0.4,
|
||||||
|
MaxTokens: 321,
|
||||||
|
TopP: 0.7,
|
||||||
|
TimeoutSeconds: 45,
|
||||||
|
ServiceTier: "priority",
|
||||||
|
ReasoningEffort: "high",
|
||||||
|
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
||||||
|
ExtraParams: map[string]any{
|
||||||
|
"provider_option": "on",
|
||||||
|
"number_value": 42,
|
||||||
|
"object_value": map[string]any{"nested": "value"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
h := NewHandler(r)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
var resp map[string]any
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||||
|
t.Fatalf("invalid JSON response: %v", err)
|
||||||
|
}
|
||||||
|
metadata := resp["metadata"].(map[string]any)
|
||||||
|
params := metadata["model_params"].(map[string]any)
|
||||||
|
|
||||||
|
if params["endpoint"] != "http://llm/v1" {
|
||||||
|
t.Fatalf("unexpected endpoint: %#v", params["endpoint"])
|
||||||
|
}
|
||||||
|
if params["model"] != "gpt-test" {
|
||||||
|
t.Fatalf("unexpected model: %#v", params["model"])
|
||||||
|
}
|
||||||
|
if params["temperature"] != 0.4 {
|
||||||
|
t.Fatalf("unexpected temperature: %#v", params["temperature"])
|
||||||
|
}
|
||||||
|
if params["max_tokens"] != float64(321) {
|
||||||
|
t.Fatalf("unexpected max_tokens: %#v", params["max_tokens"])
|
||||||
|
}
|
||||||
|
if params["top_p"] != 0.7 {
|
||||||
|
t.Fatalf("unexpected top_p: %#v", params["top_p"])
|
||||||
|
}
|
||||||
|
if params["timeout_seconds"] != float64(45) {
|
||||||
|
t.Fatalf("unexpected timeout_seconds: %#v", params["timeout_seconds"])
|
||||||
|
}
|
||||||
|
if params["service_tier"] != "priority" {
|
||||||
|
t.Fatalf("unexpected service_tier: %#v", params["service_tier"])
|
||||||
|
}
|
||||||
|
if params["reasoning_effort"] != "high" {
|
||||||
|
t.Fatalf("unexpected reasoning_effort: %#v", params["reasoning_effort"])
|
||||||
|
}
|
||||||
|
if params["api_key_env"] != "SCRIPTORIUM_API_KEY" {
|
||||||
|
t.Fatalf("unexpected api_key_env: %#v", params["api_key_env"])
|
||||||
|
}
|
||||||
|
extraParams, ok := params["extra_params"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected extra_params object, got %#v", params["extra_params"])
|
||||||
|
}
|
||||||
|
if extraParams["provider_option"] != "on" {
|
||||||
|
t.Fatalf("unexpected extra_params.provider_option: %#v", extraParams["provider_option"])
|
||||||
|
}
|
||||||
|
if extraParams["number_value"] != float64(42) {
|
||||||
|
t.Fatalf("unexpected extra_params.number_value: %#v", extraParams["number_value"])
|
||||||
|
}
|
||||||
|
objectValue, ok := extraParams["object_value"].(map[string]any)
|
||||||
|
if !ok || objectValue["nested"] != "value" {
|
||||||
|
t.Fatalf("unexpected extra_params.object_value: %#v", extraParams["object_value"])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerInvalidJSON(t *testing.T) {
|
func TestHandlerInvalidJSON(t *testing.T) {
|
||||||
@@ -205,6 +506,54 @@ func TestHandlerMissingPromptID(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandlerReservedExtraParamsThroughRunnerMapsToInvalidRequest(t *testing.T) {
|
||||||
|
llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
runner := usecase.NewRunner(
|
||||||
|
handlerPromptRepo{def: &domain.PromptDefinition{
|
||||||
|
ID: "p",
|
||||||
|
Version: "1",
|
||||||
|
DefaultProfile: "exec",
|
||||||
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "hi"}},
|
||||||
|
OutputFormat: domain.FormatText,
|
||||||
|
Validation: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone},
|
||||||
|
}},
|
||||||
|
handlerProfileRepo{profile: &domain.ExecutionProfile{
|
||||||
|
ID: "exec",
|
||||||
|
Endpoint: "http://example.invalid/v1",
|
||||||
|
Model: "model",
|
||||||
|
}},
|
||||||
|
handlerArtifactReader{},
|
||||||
|
handlerRenderer{},
|
||||||
|
llmClient,
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
h := NewHandler(runner)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id":"p",
|
||||||
|
"inputs":{"x":{"type":"file","uri":"a"}},
|
||||||
|
"model":{"extra_params":{"model":"collision"}}
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("expected 400, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
var resp map[string]any
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||||
|
t.Fatalf("invalid JSON response: %v", err)
|
||||||
|
}
|
||||||
|
errBody := resp["error"].(map[string]any)
|
||||||
|
if errBody["code"] != "invalid_request" {
|
||||||
|
t.Fatalf("expected invalid_request code, got %#v", errBody["code"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandlerUsecaseErrorMapping(t *testing.T) {
|
func TestHandlerUsecaseErrorMapping(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -216,10 +565,10 @@ func TestHandlerUsecaseErrorMapping(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{name: "prompt not found", err: wrap(usecase.ErrProfileLoad, promptdef.ErrPromptDefinitionNotFound), status: http.StatusNotFound, code: "prompt_not_found", message: "prompt definition not found"},
|
{name: "prompt not found", err: wrap(usecase.ErrProfileLoad, promptdef.ErrPromptDefinitionNotFound), status: http.StatusNotFound, code: "prompt_not_found", message: "prompt definition not found"},
|
||||||
{name: "prompt load invalid", err: wrap(usecase.ErrProfileLoad, promptdef.ErrInvalidPromptDefinition), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition"},
|
{name: "prompt load invalid", err: wrap(usecase.ErrProfileLoad, promptdef.ErrInvalidPromptDefinition), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition"},
|
||||||
{name: "missing profile/default", err: wrap(usecase.ErrInvalidRequest, errors.New("profile id is required either in request or prompt default_profile")), status: http.StatusBadRequest, code: "profile_required", message: "profile_id is required when prompt default_profile is not set"},
|
{name: "missing profile/default", err: wrap(usecase.ErrInvalidRequest, usecase.ErrProfileRequired), status: http.StatusBadRequest, code: "profile_required", message: "profile_id is required when prompt default_profile is not set"},
|
||||||
{name: "profile not found", err: wrap(usecase.ErrProfileLoad, profile.ErrProfileNotFound), status: http.StatusNotFound, code: "profile_not_found", message: "execution profile not found"},
|
{name: "profile not found", err: wrap(usecase.ErrProfileLoad, profile.ErrProfileNotFound), status: http.StatusNotFound, code: "profile_not_found", message: "execution profile not found"},
|
||||||
{name: "profile invalid", err: wrap(usecase.ErrProfileLoad, profile.ErrInvalidProfile), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile"},
|
{name: "profile invalid", err: wrap(usecase.ErrProfileLoad, profile.ErrInvalidProfile), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile"},
|
||||||
{name: "api key env missing", err: wrap(usecase.ErrInvalidRequest, errors.New(`api key environment variable "SCRIPTORIUM_API_KEY" is not set`)), status: http.StatusBadRequest, code: "api_key_env_missing", message: "api_key_env is set but the environment variable is missing"},
|
{name: "api key env missing", err: wrap(usecase.ErrInvalidRequest, usecase.ErrAPIKeyEnvMissing), status: http.StatusBadRequest, code: "api_key_env_missing", message: "api_key_env is set but the environment variable is missing"},
|
||||||
{name: "artifact", err: wrap(usecase.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"},
|
{name: "artifact", err: wrap(usecase.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"},
|
||||||
{name: "prompt render", err: wrap(usecase.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"},
|
{name: "prompt render", err: wrap(usecase.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"},
|
||||||
{name: "llm", err: wrap(usecase.ErrLLMGenerate, fmt.Errorf("llm failed")), status: http.StatusBadGateway, code: "llm_failed", message: "model generation request failed", avoidCause: "llm failed"},
|
{name: "llm", err: wrap(usecase.ErrLLMGenerate, fmt.Errorf("llm failed")), status: http.StatusBadGateway, code: "llm_failed", message: "model generation request failed", avoidCause: "llm failed"},
|
||||||
|
|||||||
@@ -46,8 +46,8 @@ func TestCompositeReader_Read(t *testing.T) {
|
|||||||
|
|
||||||
t.Run("unsupported ref type", func(t *testing.T) {
|
t.Run("unsupported ref type", func(t *testing.T) {
|
||||||
ref := domain.ArtifactRef{
|
ref := domain.ArtifactRef{
|
||||||
Type: domain.ArtifactRefS3,
|
Type: domain.ArtifactRefType("unsupported"),
|
||||||
URI: "s3://bucket/key",
|
URI: "unsupported://bucket/key",
|
||||||
}
|
}
|
||||||
_, err := reader.Read(ctx, ref)
|
_, err := reader.Read(ctx, ref)
|
||||||
if !errors.Is(err, ErrUnsupportedRefType) {
|
if !errors.Is(err, ErrUnsupportedRefType) {
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ type ArtifactRefType string
|
|||||||
const (
|
const (
|
||||||
ArtifactRefInline ArtifactRefType = "inline"
|
ArtifactRefInline ArtifactRefType = "inline"
|
||||||
ArtifactRefFile ArtifactRefType = "file"
|
ArtifactRefFile ArtifactRefType = "file"
|
||||||
ArtifactRefS3 ArtifactRefType = "s3"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// OutputFormat defines the desired format of the generated artifact.
|
// OutputFormat defines the desired format of the generated artifact.
|
||||||
@@ -41,6 +40,24 @@ const (
|
|||||||
ValidationSkipped ValidationStatus = "skipped"
|
ValidationSkipped ValidationStatus = "skipped"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// CacheControlType defines provider cache behavior for prompt content.
|
||||||
|
type CacheControlType string
|
||||||
|
|
||||||
|
const (
|
||||||
|
CacheControlEphemeral CacheControlType = "ephemeral"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// SessionIDMaxLength is OpenRouter's documented maximum session_id length.
|
||||||
|
SessionIDMaxLength = 256
|
||||||
|
)
|
||||||
|
|
||||||
|
// CacheControl describes provider cache metadata attached to prompt content.
|
||||||
|
type CacheControl struct {
|
||||||
|
Type CacheControlType `yaml:"type" json:"type"`
|
||||||
|
TTL string `yaml:"ttl,omitempty" json:"ttl,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
// RunRequest represents a request to generate a single artifact.
|
// RunRequest represents a request to generate a single artifact.
|
||||||
type RunRequest struct {
|
type RunRequest struct {
|
||||||
PromptID string
|
PromptID string
|
||||||
@@ -48,7 +65,7 @@ type RunRequest struct {
|
|||||||
ProfileID string
|
ProfileID string
|
||||||
Inputs map[string]ArtifactRef
|
Inputs map[string]ArtifactRef
|
||||||
Vars map[string]string
|
Vars map[string]string
|
||||||
Execution *ExecutionTarget
|
Execution *ExecutionTargetOverride
|
||||||
Validation *OutputContract
|
Validation *OutputContract
|
||||||
Metadata map[string]string
|
Metadata map[string]string
|
||||||
}
|
}
|
||||||
@@ -83,9 +100,11 @@ type PreparedRun struct {
|
|||||||
PromptHash string `json:"prompt_hash,omitempty"`
|
PromptHash string `json:"prompt_hash,omitempty"`
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||||
|
TargetPresence ExecutionTargetPresence `json:"-"`
|
||||||
OutputContract OutputContract `json:"output_contract"`
|
OutputContract OutputContract `json:"output_contract"`
|
||||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||||
|
SessionID string `json:"session_id,omitempty"`
|
||||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||||
Messages []RenderedMessage `json:"messages"`
|
Messages []RenderedMessage `json:"messages"`
|
||||||
StartTime time.Time `json:"start_time,omitempty"`
|
StartTime time.Time `json:"start_time,omitempty"`
|
||||||
@@ -116,6 +135,7 @@ type PromptDefinition struct {
|
|||||||
Version string `yaml:"version"`
|
Version string `yaml:"version"`
|
||||||
DefaultProfile string `yaml:"default_profile"`
|
DefaultProfile string `yaml:"default_profile"`
|
||||||
Description string `yaml:"description"`
|
Description string `yaml:"description"`
|
||||||
|
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
|
||||||
Inputs []PromptInput `yaml:"inputs"`
|
Inputs []PromptInput `yaml:"inputs"`
|
||||||
Templates []PromptMessageTemplate `yaml:"templates"`
|
Templates []PromptMessageTemplate `yaml:"templates"`
|
||||||
OutputFormat OutputFormat `yaml:"output_format"`
|
OutputFormat OutputFormat `yaml:"output_format"`
|
||||||
@@ -135,6 +155,7 @@ type PromptMessageTemplate struct {
|
|||||||
Role string `yaml:"role"`
|
Role string `yaml:"role"`
|
||||||
Content string `yaml:"content"`
|
Content string `yaml:"content"`
|
||||||
ContentFile string `yaml:"content_file"`
|
ContentFile string `yaml:"content_file"`
|
||||||
|
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecutionProfile describes how and where to execute a model.
|
// ExecutionProfile describes how and where to execute a model.
|
||||||
@@ -149,7 +170,30 @@ type ExecutionProfile struct {
|
|||||||
ServiceTier string `yaml:"service_tier"`
|
ServiceTier string `yaml:"service_tier"`
|
||||||
ReasoningEffort string `yaml:"reasoning_effort"`
|
ReasoningEffort string `yaml:"reasoning_effort"`
|
||||||
APIKeyEnv string `yaml:"api_key_env"`
|
APIKeyEnv string `yaml:"api_key_env"`
|
||||||
ExtraParams map[string]string `yaml:"extra_params"`
|
ExtraParams map[string]any `yaml:"extra_params"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecutionTargetOverride represents per-request runtime setting overrides.
|
||||||
|
type ExecutionTargetOverride struct {
|
||||||
|
Endpoint string `json:"endpoint,omitempty"`
|
||||||
|
Model string `json:"model,omitempty"`
|
||||||
|
Temperature *float64 `json:"temperature,omitempty"`
|
||||||
|
MaxTokens *int `json:"max_tokens,omitempty"`
|
||||||
|
TopP *float64 `json:"top_p,omitempty"`
|
||||||
|
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||||
|
ServiceTier string `json:"service_tier,omitempty"`
|
||||||
|
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||||
|
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||||
|
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecutionTargetPresence tracks which effective runtime fields came from an
|
||||||
|
// explicit request override even when the resolved value is a zero value.
|
||||||
|
type ExecutionTargetPresence struct {
|
||||||
|
Temperature bool
|
||||||
|
MaxTokens bool
|
||||||
|
TopP bool
|
||||||
|
TimeoutSeconds bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecutionTarget represents effective model runtime settings for a run.
|
// ExecutionTarget represents effective model runtime settings for a run.
|
||||||
@@ -163,7 +207,7 @@ type ExecutionTarget struct {
|
|||||||
ServiceTier string `yaml:"service_tier" json:"service_tier"`
|
ServiceTier string `yaml:"service_tier" json:"service_tier"`
|
||||||
ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"`
|
ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"`
|
||||||
APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"`
|
APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"`
|
||||||
ExtraParams map[string]string `yaml:"extra_params" json:"extra_params"`
|
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// OutputContract defines the requirements for the output artifact.
|
// OutputContract defines the requirements for the output artifact.
|
||||||
@@ -176,6 +220,7 @@ type OutputContract struct {
|
|||||||
|
|
||||||
// RenderedPrompt represents the prompt after template application.
|
// RenderedPrompt represents the prompt after template application.
|
||||||
type RenderedPrompt struct {
|
type RenderedPrompt struct {
|
||||||
|
SessionID string `json:"session_id,omitempty"`
|
||||||
Messages []RenderedMessage `json:"messages"`
|
Messages []RenderedMessage `json:"messages"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -183,12 +228,14 @@ type RenderedPrompt struct {
|
|||||||
type RenderedMessage struct {
|
type RenderedMessage struct {
|
||||||
Role string `json:"role"`
|
Role string `json:"role"`
|
||||||
Content string `json:"content"`
|
Content string `json:"content"`
|
||||||
|
CacheControl *CacheControl `json:"cache_control,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// GenerateRequest is the internal request passed to the LLM client.
|
// GenerateRequest is the internal request passed to the LLM client.
|
||||||
type GenerateRequest struct {
|
type GenerateRequest struct {
|
||||||
Prompt RenderedPrompt
|
Prompt RenderedPrompt
|
||||||
Target ExecutionTarget
|
Target ExecutionTarget
|
||||||
|
TargetPresence ExecutionTargetPresence
|
||||||
StructuredOutput *StructuredOutputSpec
|
StructuredOutput *StructuredOutputSpec
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -223,6 +270,8 @@ type TokenUsage struct {
|
|||||||
PromptTokens int
|
PromptTokens int
|
||||||
CompletionTokens int
|
CompletionTokens int
|
||||||
TotalTokens int
|
TotalTokens int
|
||||||
|
CachedTokens int
|
||||||
|
CacheWriteTokens int
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidationResult represents the outcome of an output validation.
|
// ValidationResult represents the outcome of an output validation.
|
||||||
|
|||||||
@@ -53,3 +53,88 @@ func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPreparedRunJSONIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
||||||
|
prepared := PreparedRun{
|
||||||
|
PromptID: "prompt.id",
|
||||||
|
SelectedProfileID: "local-fast",
|
||||||
|
EffectiveModelParams: ExecutionTarget{
|
||||||
|
Endpoint: "http://llm/v1",
|
||||||
|
Model: "gpt-test",
|
||||||
|
},
|
||||||
|
RenderedPromptHash: "rendered-hash",
|
||||||
|
Messages: []RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "You are helpful.",
|
||||||
|
CacheControl: &CacheControl{
|
||||||
|
Type: CacheControlEphemeral,
|
||||||
|
TTL: "1h",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{Role: "user", Content: "Summarize this."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := json.Marshal(prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var decoded struct {
|
||||||
|
Messages []map[string]any `json:"messages"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(b, &decoded); err != nil {
|
||||||
|
t.Fatalf("unmarshal failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(decoded.Messages) != 2 {
|
||||||
|
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
|
||||||
|
}
|
||||||
|
|
||||||
|
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected cache_control on first message, got %#v", decoded.Messages[0])
|
||||||
|
}
|
||||||
|
if cacheControl["type"] != string(CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
||||||
|
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
||||||
|
}
|
||||||
|
if _, ok := decoded.Messages[1]["cache_control"]; ok {
|
||||||
|
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparedRunJSONIncludesSessionIDOnlyWhenPresent(t *testing.T) {
|
||||||
|
prepared := PreparedRun{
|
||||||
|
PromptID: "prompt.id",
|
||||||
|
SelectedProfileID: "local-fast",
|
||||||
|
EffectiveModelParams: ExecutionTarget{
|
||||||
|
Endpoint: "http://llm/v1",
|
||||||
|
Model: "gpt-test",
|
||||||
|
},
|
||||||
|
SessionID: "session-123",
|
||||||
|
RenderedPromptHash: "rendered-hash",
|
||||||
|
Messages: []RenderedMessage{{Role: "user", Content: "Summarize this."}},
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := json.Marshal(prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var decoded map[string]any
|
||||||
|
if err := json.Unmarshal(b, &decoded); err != nil {
|
||||||
|
t.Fatalf("unmarshal failed: %v", err)
|
||||||
|
}
|
||||||
|
if decoded["session_id"] != "session-123" {
|
||||||
|
t.Fatalf("expected session_id in prepared run JSON, got %#v", decoded["session_id"])
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared.SessionID = ""
|
||||||
|
b, err = json.Marshal(prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal failed: %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(b), "session_id") {
|
||||||
|
t.Fatalf("expected empty session_id to be omitted, got %s", b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
54
internal/filecatalog/catalog.go
Normal file
54
internal/filecatalog/catalog.go
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
package filecatalog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FindYAMLFiles returns sorted full paths for .yaml and .yml files under root.
|
||||||
|
func FindYAMLFiles(ctx context.Context, root string) ([]string, error) {
|
||||||
|
var files []string
|
||||||
|
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
if d.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !isYAMLFile(d.Name()) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
files = append(files, path)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
sort.Strings(files)
|
||||||
|
return files, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// RelativePath computes a clean relative path from root to path.
|
||||||
|
func RelativePath(root string, path string) string {
|
||||||
|
rel, err := filepath.Rel(root, path)
|
||||||
|
if err != nil {
|
||||||
|
return filepath.Clean(path)
|
||||||
|
}
|
||||||
|
return filepath.Clean(rel)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stem strips .yaml or .yml from a file name.
|
||||||
|
func Stem(name string) string {
|
||||||
|
name = strings.TrimSuffix(name, ".yaml")
|
||||||
|
name = strings.TrimSuffix(name, ".yml")
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
|
||||||
|
func isYAMLFile(name string) bool {
|
||||||
|
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
|
||||||
|
}
|
||||||
84
internal/filecatalog/catalog_test.go
Normal file
84
internal/filecatalog/catalog_test.go
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
package filecatalog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFindYAMLFilesNestedSortedAndFiltered(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
mustWriteFile(t, filepath.Join(root, "z", "prompt.yml"), "id: z")
|
||||||
|
mustWriteFile(t, filepath.Join(root, "a", "profile.yaml"), "id: a")
|
||||||
|
mustWriteFile(t, filepath.Join(root, "a", "ignore.txt"), "not yaml")
|
||||||
|
mustWriteFile(t, filepath.Join(root, "b", "ignore.yaml.bak"), "not yaml")
|
||||||
|
|
||||||
|
got, err := FindYAMLFiles(context.Background(), root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []string{
|
||||||
|
filepath.Join(root, "a", "profile.yaml"),
|
||||||
|
filepath.Join(root, "z", "prompt.yml"),
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("expected sorted YAML files %v, got %v", want, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFindYAMLFilesHonorsContextCancellation(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
mustWriteFile(t, filepath.Join(root, "one.yaml"), "id: one")
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
_, err := FindYAMLFiles(ctx, root)
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("expected context.Canceled, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRelativePathNested(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
path := filepath.Join(root, "nested", "profiles", "local.yaml")
|
||||||
|
got := RelativePath(root, path)
|
||||||
|
want := filepath.Join("nested", "profiles", "local.yaml")
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("expected relative path %q, got %q", want, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStemStripsYAMLExtensions(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
in string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "yaml", in: "prompt.yaml", want: "prompt"},
|
||||||
|
{name: "yml", in: "profile.yml", want: "profile"},
|
||||||
|
{name: "other", in: "file.txt", want: "file.txt"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if got := Stem(tc.in); got != tc.want {
|
||||||
|
t.Fatalf("expected %q, got %q", tc.want, got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustWriteFile(t *testing.T, path string, content string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
t.Fatalf("failed to create directory: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||||
|
t.Fatalf("failed to write file %q: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -96,6 +96,9 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
|||||||
if prepared.PromptHash != "" {
|
if prepared.PromptHash != "" {
|
||||||
fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash)
|
fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash)
|
||||||
}
|
}
|
||||||
|
if prepared.SessionID != "" {
|
||||||
|
fmt.Fprintf(&b, "session_id: %s\n", prepared.SessionID)
|
||||||
|
}
|
||||||
fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash)
|
fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash)
|
||||||
|
|
||||||
target := prepared.EffectiveModelParams
|
target := prepared.EffectiveModelParams
|
||||||
@@ -123,7 +126,11 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
|||||||
}
|
}
|
||||||
sort.Strings(keys)
|
sort.Strings(keys)
|
||||||
for _, k := range keys {
|
for _, k := range keys {
|
||||||
fmt.Fprintf(&b, " %s: %s\n", k, target.ExtraParams[k])
|
renderedValue, err := formatExtraParamTextValue(target.ExtraParams[k])
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to format extra_params.%s: %w", k, err)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, " %s: %s\n", k, renderedValue)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -151,6 +158,13 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
|||||||
messages := byRole[role]
|
messages := byRole[role]
|
||||||
for i, msg := range messages {
|
for i, msg := range messages {
|
||||||
fmt.Fprintf(&b, " - message: %d\n", i+1)
|
fmt.Fprintf(&b, " - message: %d\n", i+1)
|
||||||
|
if msg.CacheControl != nil {
|
||||||
|
fmt.Fprintf(&b, " cache_control: %s", msg.CacheControl.Type)
|
||||||
|
if msg.CacheControl.TTL != "" {
|
||||||
|
fmt.Fprintf(&b, " ttl=%s", msg.CacheControl.TTL)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(&b)
|
||||||
|
}
|
||||||
fmt.Fprintln(&b, " content: |")
|
fmt.Fprintln(&b, " content: |")
|
||||||
content := msg.Content
|
content := msg.Content
|
||||||
if content == "" {
|
if content == "" {
|
||||||
@@ -165,3 +179,15 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
|||||||
|
|
||||||
return b.Bytes(), nil
|
return b.Bytes(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func formatExtraParamTextValue(value any) (string, error) {
|
||||||
|
if s, ok := value.(string); ok {
|
||||||
|
return s, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return string(b), nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -49,6 +49,36 @@ func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTextFormatterRendersExtraParamsDeterministically(t *testing.T) {
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
prepared.EffectiveModelParams.ExtraParams = map[string]any{
|
||||||
|
"z_string": "enabled",
|
||||||
|
"b_number": 42,
|
||||||
|
"a_object": map[string]any{
|
||||||
|
"nested": "value",
|
||||||
|
"count": 2,
|
||||||
|
},
|
||||||
|
"c_array": []any{"first", 3, false},
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
s := string(out)
|
||||||
|
|
||||||
|
want := strings.Join([]string{
|
||||||
|
" extra_params:",
|
||||||
|
" a_object: {\"count\":2,\"nested\":\"value\"}",
|
||||||
|
" b_number: 42",
|
||||||
|
" c_array: [\"first\",3,false]",
|
||||||
|
" z_string: enabled",
|
||||||
|
}, "\n")
|
||||||
|
if !strings.Contains(s, want) {
|
||||||
|
t.Fatalf("expected deterministic extra_params block %q, got:\n%s", want, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
||||||
const secret = "super-secret-api-key"
|
const secret = "super-secret-api-key"
|
||||||
t.Setenv("SCRIPTORIUM_API_KEY", secret)
|
t.Setenv("SCRIPTORIUM_API_KEY", secret)
|
||||||
@@ -62,8 +92,80 @@ func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTextFormatterIncludesMessageCacheControlBeforeContent(t *testing.T) {
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
prepared.Messages = []domain.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "System guidance.",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
TTL: "1h",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{Role: "user", Content: "Summarize the transcript."},
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
s := string(out)
|
||||||
|
if !strings.Contains(s, " system:\n - message: 1\n cache_control: ephemeral ttl=1h\n content: |") {
|
||||||
|
t.Fatalf("expected system message cache control before content, got:\n%s", s)
|
||||||
|
}
|
||||||
|
if strings.Count(s, "cache_control:") != 1 {
|
||||||
|
t.Fatalf("expected exactly one cache_control line, got:\n%s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTextFormatterIncludesSessionIDWhenPresent(t *testing.T) {
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
prepared.SessionID = "session-123"
|
||||||
|
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(out), "session_id: session-123\n") {
|
||||||
|
t.Fatalf("expected session_id in text output, got:\n%s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTextFormatterOmitsEmptyCacheControlTTL(t *testing.T) {
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
prepared.Messages = []domain.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "System guidance.",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
s := string(out)
|
||||||
|
if !strings.Contains(s, " cache_control: ephemeral\n") {
|
||||||
|
t.Fatalf("expected cache_control line without ttl, got:\n%s", s)
|
||||||
|
}
|
||||||
|
if strings.Contains(s, "ttl=") {
|
||||||
|
t.Fatalf("expected empty ttl to be omitted, got:\n%s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
||||||
prepared := samplePreparedRun()
|
prepared := samplePreparedRun()
|
||||||
|
prepared.SessionID = "session-123"
|
||||||
|
prepared.EffectiveModelParams.ExtraParams = map[string]any{
|
||||||
|
"number": 42,
|
||||||
|
"nested": map[string]any{
|
||||||
|
"enabled": true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -87,9 +189,24 @@ func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
|||||||
if decoded["rendered_prompt_hash"] != "rendered-hash" {
|
if decoded["rendered_prompt_hash"] != "rendered-hash" {
|
||||||
t.Fatalf("expected rendered_prompt_hash in json output, got %#v", decoded["rendered_prompt_hash"])
|
t.Fatalf("expected rendered_prompt_hash in json output, got %#v", decoded["rendered_prompt_hash"])
|
||||||
}
|
}
|
||||||
if _, ok := decoded["effective_model_params"]; !ok {
|
if decoded["session_id"] != "session-123" {
|
||||||
|
t.Fatalf("expected session_id in json output, got %#v", decoded["session_id"])
|
||||||
|
}
|
||||||
|
modelParams, ok := decoded["effective_model_params"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
t.Fatalf("expected effective_model_params in json output, got %#v", decoded)
|
t.Fatalf("expected effective_model_params in json output, got %#v", decoded)
|
||||||
}
|
}
|
||||||
|
extraParams, ok := modelParams["extra_params"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected extra_params in json output, got %#v", modelParams["extra_params"])
|
||||||
|
}
|
||||||
|
if extraParams["number"] != float64(42) {
|
||||||
|
t.Fatalf("unexpected numeric extra param in json output: %#v", extraParams["number"])
|
||||||
|
}
|
||||||
|
nested, ok := extraParams["nested"].(map[string]any)
|
||||||
|
if !ok || nested["enabled"] != true {
|
||||||
|
t.Fatalf("unexpected nested extra param in json output: %#v", extraParams["nested"])
|
||||||
|
}
|
||||||
if _, ok := decoded["input_hashes"]; !ok {
|
if _, ok := decoded["input_hashes"]; !ok {
|
||||||
t.Fatalf("expected input_hashes in json output, got %#v", decoded)
|
t.Fatalf("expected input_hashes in json output, got %#v", decoded)
|
||||||
}
|
}
|
||||||
@@ -98,6 +215,47 @@ func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestJSONFormatterIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
prepared.Messages = []domain.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "System guidance.",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
TTL: "1h",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{Role: "user", Content: "Summarize the transcript."},
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var decoded struct {
|
||||||
|
Messages []map[string]any `json:"messages"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(out, &decoded); err != nil {
|
||||||
|
t.Fatalf("expected valid json output, got %v", err)
|
||||||
|
}
|
||||||
|
if len(decoded.Messages) != 2 {
|
||||||
|
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
|
||||||
|
}
|
||||||
|
|
||||||
|
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected first message cache_control, got %#v", decoded.Messages[0])
|
||||||
|
}
|
||||||
|
if cacheControl["type"] != string(domain.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
||||||
|
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
||||||
|
}
|
||||||
|
if _, ok := decoded.Messages[1]["cache_control"]; ok {
|
||||||
|
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
||||||
const secret = "super-secret-api-key"
|
const secret = "super-secret-api-key"
|
||||||
t.Setenv("SCRIPTORIUM_API_KEY", secret)
|
t.Setenv("SCRIPTORIUM_API_KEY", secret)
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
@@ -75,14 +76,6 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
return nil, fmt.Errorf("%w: timeout_seconds must be greater than or equal to 0", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: timeout_seconds must be greater than or equal to 0", ErrInvalidRequest)
|
||||||
}
|
}
|
||||||
|
|
||||||
model := strings.TrimSpace(req.Target.Model)
|
|
||||||
if model == "" {
|
|
||||||
model = strings.TrimSpace(c.defaultModel)
|
|
||||||
}
|
|
||||||
if model == "" {
|
|
||||||
return nil, fmt.Errorf("%w: model is required", ErrInvalidRequest)
|
|
||||||
}
|
|
||||||
|
|
||||||
endpoint := strings.TrimSpace(req.Target.Endpoint)
|
endpoint := strings.TrimSpace(req.Target.Endpoint)
|
||||||
if endpoint == "" {
|
if endpoint == "" {
|
||||||
endpoint = c.baseURL
|
endpoint = c.baseURL
|
||||||
@@ -92,39 +85,17 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
}
|
}
|
||||||
endpoint = strings.TrimRight(endpoint, "/") + defaults.OpenAIChatCompletionsPath
|
endpoint = strings.TrimRight(endpoint, "/") + defaults.OpenAIChatCompletionsPath
|
||||||
|
|
||||||
wireReq := openAIChatRequest{
|
wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel)
|
||||||
Model: model,
|
|
||||||
}
|
|
||||||
|
|
||||||
wireReq.Messages = make([]openAIChatMessage, 0, len(req.Prompt.Messages))
|
|
||||||
for _, msg := range req.Prompt.Messages {
|
|
||||||
wireReq.Messages = append(wireReq.Messages, openAIChatMessage{
|
|
||||||
Role: msg.Role,
|
|
||||||
Content: msg.Content,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if req.Target.Temperature != 0 {
|
|
||||||
wireReq.Temperature = &req.Target.Temperature
|
|
||||||
}
|
|
||||||
if req.Target.MaxTokens != 0 {
|
|
||||||
wireReq.MaxTokens = &req.Target.MaxTokens
|
|
||||||
}
|
|
||||||
if req.Target.TopP != 0 {
|
|
||||||
wireReq.TopP = &req.Target.TopP
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(req.Target.ServiceTier) != "" {
|
|
||||||
wireReq.ServiceTier = req.Target.ServiceTier
|
|
||||||
}
|
|
||||||
if req.StructuredOutput != nil {
|
|
||||||
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
wireReq.ResponseFormat = responseFormat
|
|
||||||
|
wirePayload, err := openAIChatRequestPayload(wireReq)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
payload, err := json.Marshal(wireReq)
|
payload, err := json.Marshal(wirePayload)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
|
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
|
||||||
}
|
}
|
||||||
@@ -145,6 +116,8 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
effectiveTimeout := c.timeout
|
effectiveTimeout := c.timeout
|
||||||
if req.Target.TimeoutSeconds > 0 {
|
if req.Target.TimeoutSeconds > 0 {
|
||||||
effectiveTimeout = time.Duration(req.Target.TimeoutSeconds) * time.Second
|
effectiveTimeout = time.Duration(req.Target.TimeoutSeconds) * time.Second
|
||||||
|
} else if req.TargetPresence.TimeoutSeconds {
|
||||||
|
effectiveTimeout = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
httpClient := c.httpClient
|
httpClient := c.httpClient
|
||||||
@@ -186,33 +159,166 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
PromptTokens: wireResp.Usage.PromptTokens,
|
PromptTokens: wireResp.Usage.PromptTokens,
|
||||||
CompletionTokens: wireResp.Usage.CompletionTokens,
|
CompletionTokens: wireResp.Usage.CompletionTokens,
|
||||||
TotalTokens: wireResp.Usage.TotalTokens,
|
TotalTokens: wireResp.Usage.TotalTokens,
|
||||||
|
CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens,
|
||||||
|
CacheWriteTokens: wireResp.Usage.CacheWriteTokens,
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) {
|
||||||
|
model := strings.TrimSpace(req.Target.Model)
|
||||||
|
if model == "" {
|
||||||
|
model = strings.TrimSpace(defaultModel)
|
||||||
|
}
|
||||||
|
if model == "" {
|
||||||
|
return openAIChatRequest{}, errors.New("model is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
wireReq := openAIChatRequest{
|
||||||
|
Model: model,
|
||||||
|
}
|
||||||
|
if sessionID := strings.TrimSpace(req.Prompt.SessionID); sessionID != "" {
|
||||||
|
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
|
||||||
|
return openAIChatRequest{}, fmt.Errorf("session_id length %d exceeds maximum %d", n, domain.SessionIDMaxLength)
|
||||||
|
}
|
||||||
|
wireReq.SessionID = sessionID
|
||||||
|
}
|
||||||
|
|
||||||
|
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
|
||||||
|
for _, msg := range req.Prompt.Messages {
|
||||||
|
wireReq.Messages = append(wireReq.Messages, openAIChatRequestMessageFromRenderedMessage(msg))
|
||||||
|
}
|
||||||
|
|
||||||
|
if req.Target.Temperature != 0 || req.TargetPresence.Temperature {
|
||||||
|
wireReq.Temperature = &req.Target.Temperature
|
||||||
|
}
|
||||||
|
if req.Target.MaxTokens != 0 || req.TargetPresence.MaxTokens {
|
||||||
|
wireReq.MaxTokens = &req.Target.MaxTokens
|
||||||
|
}
|
||||||
|
if req.Target.TopP != 0 || req.TargetPresence.TopP {
|
||||||
|
wireReq.TopP = &req.Target.TopP
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(req.Target.ServiceTier) != "" {
|
||||||
|
wireReq.ServiceTier = req.Target.ServiceTier
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(req.Target.ReasoningEffort) != "" {
|
||||||
|
wireReq.ReasoningEffort = req.Target.ReasoningEffort
|
||||||
|
}
|
||||||
|
if len(req.Target.ExtraParams) > 0 {
|
||||||
|
wireReq.ExtraParams = req.Target.ExtraParams
|
||||||
|
}
|
||||||
|
if req.StructuredOutput != nil {
|
||||||
|
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
|
||||||
|
if err != nil {
|
||||||
|
return openAIChatRequest{}, err
|
||||||
|
}
|
||||||
|
wireReq.ResponseFormat = responseFormat
|
||||||
|
}
|
||||||
|
|
||||||
|
return wireReq, nil
|
||||||
|
}
|
||||||
|
|
||||||
type openAIChatRequest struct {
|
type openAIChatRequest struct {
|
||||||
Model string `json:"model"`
|
Model string `json:"model"`
|
||||||
Messages []openAIChatMessage `json:"messages"`
|
SessionID string `json:"session_id,omitempty"`
|
||||||
|
Messages []openAIChatRequestMessage `json:"messages"`
|
||||||
Temperature *float64 `json:"temperature,omitempty"`
|
Temperature *float64 `json:"temperature,omitempty"`
|
||||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
MaxTokens *int `json:"max_tokens,omitempty"`
|
||||||
TopP *float64 `json:"top_p,omitempty"`
|
TopP *float64 `json:"top_p,omitempty"`
|
||||||
ServiceTier string `json:"service_tier,omitempty"`
|
ServiceTier string `json:"service_tier,omitempty"`
|
||||||
|
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||||
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"`
|
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"`
|
||||||
|
ExtraParams map[string]any `json:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type openAIChatMessage struct {
|
func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
||||||
|
out := map[string]any{
|
||||||
|
"model": req.Model,
|
||||||
|
"messages": req.Messages,
|
||||||
|
}
|
||||||
|
if req.SessionID != "" {
|
||||||
|
out["session_id"] = req.SessionID
|
||||||
|
}
|
||||||
|
if req.Temperature != nil {
|
||||||
|
out["temperature"] = *req.Temperature
|
||||||
|
}
|
||||||
|
if req.MaxTokens != nil {
|
||||||
|
out["max_tokens"] = *req.MaxTokens
|
||||||
|
}
|
||||||
|
if req.TopP != nil {
|
||||||
|
out["top_p"] = *req.TopP
|
||||||
|
}
|
||||||
|
if req.ServiceTier != "" {
|
||||||
|
out["service_tier"] = req.ServiceTier
|
||||||
|
}
|
||||||
|
if req.ReasoningEffort != "" {
|
||||||
|
out["reasoning_effort"] = req.ReasoningEffort
|
||||||
|
}
|
||||||
|
if req.ResponseFormat != nil {
|
||||||
|
out["response_format"] = req.ResponseFormat
|
||||||
|
}
|
||||||
|
|
||||||
|
for key, value := range req.ExtraParams {
|
||||||
|
if key == "" {
|
||||||
|
return nil, errors.New("extra_params key must not be empty")
|
||||||
|
}
|
||||||
|
if _, reserved := reservedOpenAIChatRequestFields[key]; reserved {
|
||||||
|
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
|
||||||
|
}
|
||||||
|
if _, err := json.Marshal(value); err != nil {
|
||||||
|
return nil, fmt.Errorf("extra_params.%s must be JSON-serializable: %w", key, err)
|
||||||
|
}
|
||||||
|
out[key] = value
|
||||||
|
}
|
||||||
|
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var reservedOpenAIChatRequestFields = map[string]struct{}{
|
||||||
|
"model": {},
|
||||||
|
"session_id": {},
|
||||||
|
"messages": {},
|
||||||
|
"temperature": {},
|
||||||
|
"max_tokens": {},
|
||||||
|
"top_p": {},
|
||||||
|
"service_tier": {},
|
||||||
|
"reasoning_effort": {},
|
||||||
|
"response_format": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
type openAIChatRequestMessage struct {
|
||||||
|
Role string `json:"role"`
|
||||||
|
Content any `json:"content"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type openAIChatTextContentBlock struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
Text string `json:"text"`
|
||||||
|
CacheControl *openAICacheControl `json:"cache_control,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type openAICacheControl struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
TTL string `json:"ttl,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type openAIChatResponseMessage struct {
|
||||||
Role string `json:"role"`
|
Role string `json:"role"`
|
||||||
Content string `json:"content"`
|
Content string `json:"content"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type openAIChatResponse struct {
|
type openAIChatResponse struct {
|
||||||
Choices []struct {
|
Choices []struct {
|
||||||
Message openAIChatMessage `json:"message"`
|
Message openAIChatResponseMessage `json:"message"`
|
||||||
} `json:"choices"`
|
} `json:"choices"`
|
||||||
Usage struct {
|
Usage struct {
|
||||||
PromptTokens int `json:"prompt_tokens"`
|
PromptTokens int `json:"prompt_tokens"`
|
||||||
CompletionTokens int `json:"completion_tokens"`
|
CompletionTokens int `json:"completion_tokens"`
|
||||||
TotalTokens int `json:"total_tokens"`
|
TotalTokens int `json:"total_tokens"`
|
||||||
|
PromptTokensDetails struct {
|
||||||
|
CachedTokens int `json:"cached_tokens"`
|
||||||
|
} `json:"prompt_tokens_details"`
|
||||||
|
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||||
} `json:"usage"`
|
} `json:"usage"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -227,6 +333,28 @@ type openAIJSONSchemaEnvelope struct {
|
|||||||
Schema any `json:"schema"`
|
Schema any `json:"schema"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func openAIChatRequestMessageFromRenderedMessage(msg domain.RenderedMessage) openAIChatRequestMessage {
|
||||||
|
wireMsg := openAIChatRequestMessage{
|
||||||
|
Role: msg.Role,
|
||||||
|
Content: msg.Content,
|
||||||
|
}
|
||||||
|
if msg.CacheControl == nil {
|
||||||
|
return wireMsg
|
||||||
|
}
|
||||||
|
|
||||||
|
wireMsg.Content = []openAIChatTextContentBlock{
|
||||||
|
{
|
||||||
|
Type: "text",
|
||||||
|
Text: msg.Content,
|
||||||
|
CacheControl: &openAICacheControl{
|
||||||
|
Type: string(msg.CacheControl.Type),
|
||||||
|
TTL: msg.CacheControl.TTL,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return wireMsg
|
||||||
|
}
|
||||||
|
|
||||||
func toOpenAIResponseFormat(spec *domain.StructuredOutputSpec) (*openAIResponseFormat, error) {
|
func toOpenAIResponseFormat(spec *domain.StructuredOutputSpec) (*openAIResponseFormat, error) {
|
||||||
if spec == nil {
|
if spec == nil {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"math"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -89,6 +90,9 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
|
|||||||
if resp.Usage.PromptTokens != 11 || resp.Usage.CompletionTokens != 22 || resp.Usage.TotalTokens != 33 {
|
if resp.Usage.PromptTokens != 11 || resp.Usage.CompletionTokens != 22 || resp.Usage.TotalTokens != 33 {
|
||||||
t.Fatalf("unexpected usage: %+v", resp.Usage)
|
t.Fatalf("unexpected usage: %+v", resp.Usage)
|
||||||
}
|
}
|
||||||
|
if resp.Usage.CachedTokens != 0 || resp.Usage.CacheWriteTokens != 0 {
|
||||||
|
t.Fatalf("expected absent cache usage fields to remain zero, got %+v", resp.Usage)
|
||||||
|
}
|
||||||
|
|
||||||
if obs.Authorization != "Bearer secret-key" {
|
if obs.Authorization != "Bearer secret-key" {
|
||||||
t.Fatalf("unexpected Authorization header: %q", obs.Authorization)
|
t.Fatalf("unexpected Authorization header: %q", obs.Authorization)
|
||||||
@@ -96,6 +100,15 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
|
|||||||
if got, ok := obs.Body["model"].(string); !ok || got != "gpt-test" {
|
if got, ok := obs.Body["model"].(string); !ok || got != "gpt-test" {
|
||||||
t.Fatalf("unexpected model payload: %#v", obs.Body["model"])
|
t.Fatalf("unexpected model payload: %#v", obs.Body["model"])
|
||||||
}
|
}
|
||||||
|
if got, ok := obs.Body["temperature"].(float64); !ok || got != 0.4 {
|
||||||
|
t.Fatalf("unexpected temperature payload: %#v", obs.Body["temperature"])
|
||||||
|
}
|
||||||
|
if got, ok := obs.Body["max_tokens"].(float64); !ok || got != 123 {
|
||||||
|
t.Fatalf("unexpected max_tokens payload: %#v", obs.Body["max_tokens"])
|
||||||
|
}
|
||||||
|
if got, ok := obs.Body["top_p"].(float64); !ok || got != 0.7 {
|
||||||
|
t.Fatalf("unexpected top_p payload: %#v", obs.Body["top_p"])
|
||||||
|
}
|
||||||
if got, ok := obs.Body["service_tier"].(string); !ok || got != "priority" {
|
if got, ok := obs.Body["service_tier"].(string); !ok || got != "priority" {
|
||||||
t.Fatalf("unexpected service_tier payload: %#v", obs.Body["service_tier"])
|
t.Fatalf("unexpected service_tier payload: %#v", obs.Body["service_tier"])
|
||||||
}
|
}
|
||||||
@@ -135,6 +148,245 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientSerializesCacheControlledMessageAsContentBlock(t *testing.T) {
|
||||||
|
var observedBody map[string]any
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
defer r.Body.Close()
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||||
|
t.Fatalf("failed to decode request body: %v", err)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "Stable instructions.",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
TTL: "1h",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{Role: "user", Content: "Dynamic request."},
|
||||||
|
}},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, forbidden := range []string{"cache_control", "extra_params"} {
|
||||||
|
if _, exists := observedBody[forbidden]; exists {
|
||||||
|
t.Fatalf("expected top-level %s to be omitted, got %#v", forbidden, observedBody[forbidden])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
msgs, ok := observedBody["messages"].([]any)
|
||||||
|
if !ok || len(msgs) != 2 {
|
||||||
|
t.Fatalf("unexpected messages payload: %#v", observedBody["messages"])
|
||||||
|
}
|
||||||
|
msg0 := msgs[0].(map[string]any)
|
||||||
|
if msg0["role"] != "system" {
|
||||||
|
t.Fatalf("unexpected first message role: %#v", msg0["role"])
|
||||||
|
}
|
||||||
|
contentBlocks, ok := msg0["content"].([]any)
|
||||||
|
if !ok || len(contentBlocks) != 1 {
|
||||||
|
t.Fatalf("expected first message content block array, got %#v", msg0["content"])
|
||||||
|
}
|
||||||
|
block := contentBlocks[0].(map[string]any)
|
||||||
|
if block["type"] != "text" || block["text"] != "Stable instructions." {
|
||||||
|
t.Fatalf("unexpected text content block: %#v", block)
|
||||||
|
}
|
||||||
|
cacheControl, ok := block["cache_control"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected cache_control on content block, got %#v", block)
|
||||||
|
}
|
||||||
|
if cacheControl["type"] != string(domain.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
||||||
|
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
||||||
|
}
|
||||||
|
|
||||||
|
msg1 := msgs[1].(map[string]any)
|
||||||
|
if msg1["role"] != "user" || msg1["content"] != "Dynamic request." {
|
||||||
|
t.Fatalf("expected uncached message to keep string content, got %#v", msg1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientOmitsEmptyCacheControlTTL(t *testing.T) {
|
||||||
|
var observedBody map[string]any
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
defer r.Body.Close()
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||||
|
t.Fatalf("failed to decode request body: %v", err)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "Stable instructions.",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
msgs := observedBody["messages"].([]any)
|
||||||
|
msg0 := msgs[0].(map[string]any)
|
||||||
|
contentBlocks := msg0["content"].([]any)
|
||||||
|
block := contentBlocks[0].(map[string]any)
|
||||||
|
cacheControl := block["cache_control"].(map[string]any)
|
||||||
|
if cacheControl["type"] != string(domain.CacheControlEphemeral) {
|
||||||
|
t.Fatalf("unexpected cache_control type: %#v", cacheControl)
|
||||||
|
}
|
||||||
|
if _, exists := cacheControl["ttl"]; exists {
|
||||||
|
t.Fatalf("expected empty ttl to be omitted, got %#v", cacheControl)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientSerializesSessionID(t *testing.T) {
|
||||||
|
var observedBody map[string]any
|
||||||
|
var observedSessionHeader string
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
observedSessionHeader = r.Header.Get("x-session-id")
|
||||||
|
defer r.Body.Close()
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||||
|
t.Fatalf("failed to decode request body: %v", err)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{
|
||||||
|
SessionID: " session-123 ",
|
||||||
|
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
|
||||||
|
},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if observedBody["session_id"] != "session-123" {
|
||||||
|
t.Fatalf("expected top-level session_id, got %#v", observedBody["session_id"])
|
||||||
|
}
|
||||||
|
if observedSessionHeader != "" {
|
||||||
|
t.Fatalf("did not expect x-session-id header, got %q", observedSessionHeader)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientOmitsEmptySessionID(t *testing.T) {
|
||||||
|
var observedBody map[string]any
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
defer r.Body.Close()
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||||
|
t.Fatalf("failed to decode request body: %v", err)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{
|
||||||
|
SessionID: " ",
|
||||||
|
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
|
||||||
|
},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if _, exists := observedBody["session_id"]; exists {
|
||||||
|
t.Fatalf("expected empty session_id to be omitted, got %#v", observedBody["session_id"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientRejectsTooLongSessionID(t *testing.T) {
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
|
BaseURL: "http://example.com/v1",
|
||||||
|
Model: "model",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{
|
||||||
|
SessionID: strings.Repeat("x", domain.SessionIDMaxLength+1),
|
||||||
|
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid request error")
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientParsesCacheUsage(t *testing.T) {
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = w.Write([]byte(`{
|
||||||
|
"choices": [{"message": {"role": "assistant", "content": "ok"}}],
|
||||||
|
"usage": {
|
||||||
|
"prompt_tokens": 100,
|
||||||
|
"completion_tokens": 20,
|
||||||
|
"total_tokens": 120,
|
||||||
|
"prompt_tokens_details": {"cached_tokens": 80},
|
||||||
|
"cache_write_tokens": 60
|
||||||
|
}
|
||||||
|
}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "model"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if resp.Usage.PromptTokens != 100 || resp.Usage.CompletionTokens != 20 || resp.Usage.TotalTokens != 120 {
|
||||||
|
t.Fatalf("unexpected base usage fields: %+v", resp.Usage)
|
||||||
|
}
|
||||||
|
if resp.Usage.CachedTokens != 80 || resp.Usage.CacheWriteTokens != 60 {
|
||||||
|
t.Fatalf("unexpected cache usage fields: %+v", resp.Usage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *testing.T) {
|
func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *testing.T) {
|
||||||
var observedBody map[string]any
|
var observedBody map[string]any
|
||||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -166,6 +418,281 @@ func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *test
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientSerializesReasoningEffortAndExtraParams(t *testing.T) {
|
||||||
|
var observedBody map[string]any
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
defer r.Body.Close()
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||||
|
t.Fatalf("failed to decode request body: %v", err)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{
|
||||||
|
Model: "model",
|
||||||
|
ReasoningEffort: "high",
|
||||||
|
ExtraParams: map[string]any{
|
||||||
|
"string_value": "on",
|
||||||
|
"number_value": 42,
|
||||||
|
"boolean_value": true,
|
||||||
|
"object_value": map[string]any{"nested": "value", "count": 2},
|
||||||
|
"array_value": []any{"first", 3, false},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if observedBody["reasoning_effort"] != "high" {
|
||||||
|
t.Fatalf("expected reasoning_effort high, got %#v", observedBody["reasoning_effort"])
|
||||||
|
}
|
||||||
|
if observedBody["string_value"] != "on" {
|
||||||
|
t.Fatalf("unexpected string extra param: %#v", observedBody["string_value"])
|
||||||
|
}
|
||||||
|
if observedBody["number_value"] != float64(42) {
|
||||||
|
t.Fatalf("unexpected number extra param: %#v", observedBody["number_value"])
|
||||||
|
}
|
||||||
|
if observedBody["boolean_value"] != true {
|
||||||
|
t.Fatalf("unexpected boolean extra param: %#v", observedBody["boolean_value"])
|
||||||
|
}
|
||||||
|
objectValue, ok := observedBody["object_value"].(map[string]any)
|
||||||
|
if !ok || objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
|
||||||
|
t.Fatalf("unexpected object extra param: %#v", observedBody["object_value"])
|
||||||
|
}
|
||||||
|
if _, exists := observedBody["extra_params"]; exists {
|
||||||
|
t.Fatalf("expected extra_params wrapper omitted, got %#v", observedBody["extra_params"])
|
||||||
|
}
|
||||||
|
arrayValue, ok := observedBody["array_value"].([]any)
|
||||||
|
if !ok || len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
|
||||||
|
t.Fatalf("unexpected array extra param: %#v", observedBody["array_value"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientOmitsReasoningEffortWhenUnset(t *testing.T) {
|
||||||
|
var observedBody map[string]any
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
defer r.Body.Close()
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||||
|
t.Fatalf("failed to decode request body: %v", err)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if _, exists := observedBody["reasoning_effort"]; exists {
|
||||||
|
t.Fatalf("expected reasoning_effort omitted, got %#v", observedBody["reasoning_effort"])
|
||||||
|
}
|
||||||
|
if _, exists := observedBody["extra_params"]; exists {
|
||||||
|
t.Fatalf("expected extra_params wrapper omitted, got %#v", observedBody["extra_params"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientSerializesExplicitZeroNumericOverrides(t *testing.T) {
|
||||||
|
var observedBody map[string]any
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
defer r.Body.Close()
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||||
|
t.Fatalf("failed to decode request body: %v", err)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model"},
|
||||||
|
TargetPresence: domain.ExecutionTargetPresence{
|
||||||
|
Temperature: true,
|
||||||
|
MaxTokens: true,
|
||||||
|
TopP: true,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if observedBody["temperature"] != float64(0) {
|
||||||
|
t.Fatalf("expected explicit zero temperature, got %#v", observedBody["temperature"])
|
||||||
|
}
|
||||||
|
if observedBody["max_tokens"] != float64(0) {
|
||||||
|
t.Fatalf("expected explicit zero max_tokens, got %#v", observedBody["max_tokens"])
|
||||||
|
}
|
||||||
|
if observedBody["top_p"] != float64(0) {
|
||||||
|
t.Fatalf("expected explicit zero top_p, got %#v", observedBody["top_p"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientOmitsImplicitZeroNumericFields(t *testing.T) {
|
||||||
|
var observedBody map[string]any
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
defer r.Body.Close()
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||||
|
t.Fatalf("failed to decode request body: %v", err)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
for _, field := range []string{"temperature", "max_tokens", "top_p"} {
|
||||||
|
if _, exists := observedBody[field]; exists {
|
||||||
|
t.Fatalf("expected implicit zero field %q to be omitted, got body %#v", field, observedBody)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientExplicitZeroTimeoutDisablesClientTimeout(t *testing.T) {
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
|
BaseURL: ts.URL + "/v1",
|
||||||
|
Timeout: time.Nanosecond,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model", TimeoutSeconds: 0},
|
||||||
|
TargetPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected explicit zero timeout to disable client timeout, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientOmittedTimeoutUsesClientTimeout(t *testing.T) {
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
|
BaseURL: ts.URL + "/v1",
|
||||||
|
Timeout: time.Nanosecond,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model", TimeoutSeconds: 0},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected omitted timeout to use client timeout")
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrRequestFailed) {
|
||||||
|
t.Fatalf("expected ErrRequestFailed, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
extraParams map[string]any
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "empty key", extraParams: map[string]any{"": "empty"}, want: "key must not be empty"},
|
||||||
|
{name: "unserializable value", extraParams: map[string]any{"bad": math.Inf(1)}, want: "JSON-serializable"},
|
||||||
|
}
|
||||||
|
for _, key := range []string{
|
||||||
|
"model",
|
||||||
|
"session_id",
|
||||||
|
"messages",
|
||||||
|
"temperature",
|
||||||
|
"max_tokens",
|
||||||
|
"top_p",
|
||||||
|
"service_tier",
|
||||||
|
"reasoning_effort",
|
||||||
|
"response_format",
|
||||||
|
} {
|
||||||
|
tests = append(tests, struct {
|
||||||
|
name string
|
||||||
|
extraParams map[string]any
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
name: "reserved key " + key,
|
||||||
|
extraParams: map[string]any{key: "collision"},
|
||||||
|
want: "reserved request field",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
called := false
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
called = true
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model", ExtraParams: tc.extraParams},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid request error")
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), tc.want) {
|
||||||
|
t.Fatalf("expected error to contain %q, got %v", tc.want, err)
|
||||||
|
}
|
||||||
|
if called {
|
||||||
|
t.Fatal("provider should not be called for invalid extra_params")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestOpenAICompatibleClientNoAuthorizationHeaderWhenNoAPIKey(t *testing.T) {
|
func TestOpenAICompatibleClientNoAuthorizationHeaderWhenNoAPIKey(t *testing.T) {
|
||||||
hadAuth := false
|
hadAuth := false
|
||||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|||||||
@@ -7,10 +7,10 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/filecatalog"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -34,7 +34,7 @@ func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*doma
|
|||||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
|
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
|
||||||
}
|
}
|
||||||
|
|
||||||
files, err := r.yamlFiles(ctx)
|
files, err := filecatalog.FindYAMLFiles(ctx, r.dir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read profile directory: %w", err)
|
return nil, fmt.Errorf("failed to read profile directory: %w", err)
|
||||||
}
|
}
|
||||||
@@ -47,24 +47,25 @@ func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*doma
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
relPath := r.relativePath(fullPath)
|
relPath := filecatalog.RelativePath(r.dir, fullPath)
|
||||||
fileMatch := profileIDFromFileName(filepath.Base(fullPath)) == id
|
fileMatch := filecatalog.Stem(filepath.Base(fullPath)) == id
|
||||||
data, err := os.ReadFile(fullPath)
|
data, err := os.ReadFile(fullPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
|
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
|
||||||
}
|
}
|
||||||
|
metadata := readProfileFileMetadata(data)
|
||||||
|
idMatch := fileMatch || metadata.id == id
|
||||||
|
if metadata.hasRawAPIKey {
|
||||||
|
if idMatch {
|
||||||
|
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
var prof domain.ExecutionProfile
|
var prof domain.ExecutionProfile
|
||||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||||
decoder.KnownFields(true)
|
decoder.KnownFields(true)
|
||||||
if err := decoder.Decode(&prof); err != nil {
|
if err := decoder.Decode(&prof); err != nil {
|
||||||
idMatch := fileMatch || profileFileHasID(data, id)
|
|
||||||
if strings.Contains(err.Error(), "field api_key not found") {
|
|
||||||
if idMatch {
|
|
||||||
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if idMatch {
|
if idMatch {
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
|
||||||
}
|
}
|
||||||
@@ -106,56 +107,36 @@ type profileMatch struct {
|
|||||||
path string
|
path string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *filesystemRepository) yamlFiles(ctx context.Context) ([]string, error) {
|
type profileFileMetadata struct {
|
||||||
var files []string
|
id string
|
||||||
err := filepath.WalkDir(r.dir, func(path string, d os.DirEntry, err error) error {
|
hasRawAPIKey bool
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
if d.IsDir() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if !isYAMLFile(d.Name()) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
files = append(files, path)
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
sort.Strings(files)
|
|
||||||
return files, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *filesystemRepository) relativePath(path string) string {
|
func readProfileFileMetadata(data []byte) profileFileMetadata {
|
||||||
rel, err := filepath.Rel(r.dir, path)
|
var node yaml.Node
|
||||||
if err != nil {
|
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&node); err != nil {
|
||||||
return filepath.Clean(path)
|
return profileFileMetadata{}
|
||||||
}
|
}
|
||||||
return filepath.Clean(rel)
|
if node.Kind != yaml.DocumentNode || len(node.Content) == 0 {
|
||||||
}
|
return profileFileMetadata{}
|
||||||
|
|
||||||
func isYAMLFile(name string) bool {
|
|
||||||
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
|
|
||||||
}
|
|
||||||
|
|
||||||
func profileIDFromFileName(name string) string {
|
|
||||||
name = strings.TrimSuffix(name, ".yaml")
|
|
||||||
name = strings.TrimSuffix(name, ".yml")
|
|
||||||
return name
|
|
||||||
}
|
|
||||||
|
|
||||||
func profileFileHasID(data []byte, id string) bool {
|
|
||||||
var raw struct {
|
|
||||||
ID string `yaml:"id"`
|
|
||||||
}
|
}
|
||||||
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&raw); err != nil {
|
mapping := node.Content[0]
|
||||||
return false
|
if mapping.Kind != yaml.MappingNode {
|
||||||
|
return profileFileMetadata{}
|
||||||
}
|
}
|
||||||
return strings.TrimSpace(raw.ID) == id
|
|
||||||
|
var metadata profileFileMetadata
|
||||||
|
for i := 0; i+1 < len(mapping.Content); i += 2 {
|
||||||
|
key := mapping.Content[i]
|
||||||
|
value := mapping.Content[i+1]
|
||||||
|
switch key.Value {
|
||||||
|
case "id":
|
||||||
|
metadata.id = strings.TrimSpace(value.Value)
|
||||||
|
case "api_key":
|
||||||
|
metadata.hasRawAPIKey = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return metadata
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateProfile(p *domain.ExecutionProfile) error {
|
func validateProfile(p *domain.ExecutionProfile) error {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package profile
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -85,6 +86,63 @@ temperature: 0.1
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("valid profile with JSON-compatible extra params", func(t *testing.T) {
|
||||||
|
writeProfileTestFile(t, filepath.Join(tmpDir, "json-extra-params.yaml"), `
|
||||||
|
id: json-extra-params
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: nested-model
|
||||||
|
extra_params:
|
||||||
|
string_value: enabled
|
||||||
|
number_value: 42
|
||||||
|
boolean_value: true
|
||||||
|
object_value:
|
||||||
|
nested: value
|
||||||
|
count: 2
|
||||||
|
array_value:
|
||||||
|
- first
|
||||||
|
- 3
|
||||||
|
- false
|
||||||
|
`)
|
||||||
|
|
||||||
|
p, err := repo.GetProfile(ctx, "json-extra-params")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var got map[string]any
|
||||||
|
encoded, err := json.Marshal(p.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected extra_params to marshal as JSON, got %v", err)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(encoded, &got); err != nil {
|
||||||
|
t.Fatalf("expected extra_params JSON to decode, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got["string_value"] != "enabled" {
|
||||||
|
t.Fatalf("unexpected string extra param: %#v", got["string_value"])
|
||||||
|
}
|
||||||
|
if got["number_value"] != float64(42) {
|
||||||
|
t.Fatalf("unexpected number extra param: %#v", got["number_value"])
|
||||||
|
}
|
||||||
|
if got["boolean_value"] != true {
|
||||||
|
t.Fatalf("unexpected boolean extra param: %#v", got["boolean_value"])
|
||||||
|
}
|
||||||
|
objectValue, ok := got["object_value"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected object extra param, got %#v", got["object_value"])
|
||||||
|
}
|
||||||
|
if objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
|
||||||
|
t.Fatalf("unexpected object extra param: %#v", objectValue)
|
||||||
|
}
|
||||||
|
arrayValue, ok := got["array_value"].([]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected array extra param, got %#v", got["array_value"])
|
||||||
|
}
|
||||||
|
if len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
|
||||||
|
t.Fatalf("unexpected array extra param: %#v", arrayValue)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
|
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
|
||||||
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
|
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
|
||||||
id: duplicate-profile
|
id: duplicate-profile
|
||||||
@@ -133,6 +191,20 @@ api_key: secret
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("raw api_key in non-target profile is ignored", func(t *testing.T) {
|
||||||
|
writeProfileTestFile(t, filepath.Join(tmpDir, "raw-api-key-non-target.yaml"), `
|
||||||
|
id: raw-api-key-non-target
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: m
|
||||||
|
api_key: secret
|
||||||
|
`)
|
||||||
|
|
||||||
|
_, err := repo.GetProfile(ctx, "does-not-exist-with-raw-key-nearby")
|
||||||
|
if !errors.Is(err, ErrProfileNotFound) {
|
||||||
|
t.Fatalf("expected ErrProfileNotFound for non-target raw api_key file, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("invalid yaml", func(t *testing.T) {
|
t.Run("invalid yaml", func(t *testing.T) {
|
||||||
_, err := repo.GetProfile(ctx, "invalid_yaml")
|
_, err := repo.GetProfile(ctx, "invalid_yaml")
|
||||||
if !errors.Is(err, ErrInvalidYAML) {
|
if !errors.Is(err, ErrInvalidYAML) {
|
||||||
|
|||||||
@@ -6,7 +6,9 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
"strings"
|
||||||
"text/template"
|
"text/template"
|
||||||
|
"unicode/utf8"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -50,6 +52,11 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
sessionID, err := renderSessionID(definition.SessionID, funcs, vars)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
var renderedMessages []domain.RenderedMessage
|
var renderedMessages []domain.RenderedMessage
|
||||||
|
|
||||||
for i, tmplMsg := range definition.Templates {
|
for i, tmplMsg := range definition.Templates {
|
||||||
@@ -77,10 +84,42 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
renderedMessages = append(renderedMessages, domain.RenderedMessage{
|
renderedMessages = append(renderedMessages, domain.RenderedMessage{
|
||||||
Role: tmplMsg.Role,
|
Role: tmplMsg.Role,
|
||||||
Content: buf.String(),
|
Content: buf.String(),
|
||||||
|
CacheControl: cloneCacheControl(tmplMsg.CacheControl),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
return &domain.RenderedPrompt{
|
return &domain.RenderedPrompt{
|
||||||
|
SessionID: sessionID,
|
||||||
Messages: renderedMessages,
|
Messages: renderedMessages,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
|
||||||
|
if strings.TrimSpace(raw) == "" {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
tmpl, err := template.New("session_id").Funcs(funcs).Option("missingkey=error").Parse(raw)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
if err := tmpl.Execute(&buf, vars); err != nil {
|
||||||
|
return "", fmt.Errorf("%w: session_id: %w", ErrRenderFailure, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionID := strings.TrimSpace(buf.String())
|
||||||
|
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
|
||||||
|
return "", fmt.Errorf("%w: session_id length %d exceeds maximum %d", ErrRenderFailure, n, domain.SessionIDMaxLength)
|
||||||
|
}
|
||||||
|
return sessionID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneCacheControl(in *domain.CacheControl) *domain.CacheControl {
|
||||||
|
if in == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := *in
|
||||||
|
return &out
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package prompt
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
@@ -78,6 +79,66 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("copying cache control to rendered messages", func(t *testing.T) {
|
||||||
|
def := &domain.PromptDefinition{
|
||||||
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "You are concise.",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
TTL: "1h",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{Role: "user", Content: "Analyze this: {{input \"transcript\"}}"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := renderer.Render(ctx, def, inputs, vars)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(res.Messages) != 2 {
|
||||||
|
t.Fatalf("expected 2 messages, got %d", len(res.Messages))
|
||||||
|
}
|
||||||
|
if res.Messages[0].CacheControl == nil {
|
||||||
|
t.Fatal("expected rendered cache control")
|
||||||
|
}
|
||||||
|
if res.Messages[0].CacheControl.Type != domain.CacheControlEphemeral {
|
||||||
|
t.Fatalf("unexpected cache control type: %q", res.Messages[0].CacheControl.Type)
|
||||||
|
}
|
||||||
|
if res.Messages[0].CacheControl.TTL != "1h" {
|
||||||
|
t.Fatalf("unexpected cache control ttl: %q", res.Messages[0].CacheControl.TTL)
|
||||||
|
}
|
||||||
|
if res.Messages[1].CacheControl != nil {
|
||||||
|
t.Fatalf("expected no cache control on second message, got %#v", res.Messages[1].CacheControl)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("rendered cache control does not alias source template", func(t *testing.T) {
|
||||||
|
source := &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}
|
||||||
|
def := &domain.PromptDefinition{
|
||||||
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{Role: "system", Content: "You are concise.", CacheControl: source},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := renderer.Render(ctx, def, inputs, vars)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if res.Messages[0].CacheControl == source {
|
||||||
|
t.Fatal("expected rendered cache control to be cloned")
|
||||||
|
}
|
||||||
|
|
||||||
|
res.Messages[0].CacheControl.TTL = ""
|
||||||
|
if source.TTL != "1h" {
|
||||||
|
t.Fatalf("source cache control was mutated, ttl=%q", source.TTL)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("accessing vars", func(t *testing.T) {
|
t.Run("accessing vars", func(t *testing.T) {
|
||||||
def := &domain.PromptDefinition{
|
def := &domain.PromptDefinition{
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
@@ -95,6 +156,78 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("rendering session id from vars", func(t *testing.T) {
|
||||||
|
def := &domain.PromptDefinition{
|
||||||
|
SessionID: " {{ .session_id }} ",
|
||||||
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{Role: "system", Content: "Speak in a {{.tone}} tone."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := renderer.Render(ctx, def, inputs, map[string]string{
|
||||||
|
"tone": "concise",
|
||||||
|
"session_id": "agent-session-123",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if res.SessionID != "agent-session-123" {
|
||||||
|
t.Fatalf("unexpected session id: %q", res.SessionID)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("empty rendered session id is omitted", func(t *testing.T) {
|
||||||
|
def := &domain.PromptDefinition{
|
||||||
|
SessionID: " ",
|
||||||
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{Role: "system", Content: "Speak in a {{.tone}} tone."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := renderer.Render(ctx, def, inputs, vars)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if res.SessionID != "" {
|
||||||
|
t.Fatalf("expected empty session id, got %q", res.SessionID)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("missing session id var fails rendering", func(t *testing.T) {
|
||||||
|
def := &domain.PromptDefinition{
|
||||||
|
SessionID: "{{ .session_id }}",
|
||||||
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{Role: "system", Content: "Speak in a {{.tone}} tone."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := renderer.Render(ctx, def, inputs, vars)
|
||||||
|
if !errors.Is(err, ErrRenderFailure) {
|
||||||
|
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("too long rendered session id fails rendering", func(t *testing.T) {
|
||||||
|
def := &domain.PromptDefinition{
|
||||||
|
SessionID: "{{ .session_id }}",
|
||||||
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{Role: "system", Content: "Speak in a {{.tone}} tone."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := renderer.Render(ctx, def, inputs, map[string]string{
|
||||||
|
"tone": "concise",
|
||||||
|
"session_id": strings.Repeat("x", domain.SessionIDMaxLength+1),
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrRenderFailure) {
|
||||||
|
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("inserting required input artifact", func(t *testing.T) {
|
t.Run("inserting required input artifact", func(t *testing.T) {
|
||||||
def := &domain.PromptDefinition{
|
def := &domain.PromptDefinition{
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
|
|||||||
@@ -7,10 +7,10 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/filecatalog"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -29,6 +29,7 @@ type promptDefinitionFile struct {
|
|||||||
Version string `yaml:"version"`
|
Version string `yaml:"version"`
|
||||||
DefaultProfile *string `yaml:"default_profile"`
|
DefaultProfile *string `yaml:"default_profile"`
|
||||||
Description string `yaml:"description"`
|
Description string `yaml:"description"`
|
||||||
|
SessionID string `yaml:"session_id"`
|
||||||
Inputs []promptInputFile `yaml:"inputs"`
|
Inputs []promptInputFile `yaml:"inputs"`
|
||||||
Messages []promptMessageFile `yaml:"messages"`
|
Messages []promptMessageFile `yaml:"messages"`
|
||||||
Output promptOutputContractFile `yaml:"output"`
|
Output promptOutputContractFile `yaml:"output"`
|
||||||
@@ -45,6 +46,12 @@ type promptMessageFile struct {
|
|||||||
Role string `yaml:"role"`
|
Role string `yaml:"role"`
|
||||||
Content string `yaml:"content"`
|
Content string `yaml:"content"`
|
||||||
ContentFile string `yaml:"content_file"`
|
ContentFile string `yaml:"content_file"`
|
||||||
|
CacheControl *cacheControlFile `yaml:"cache_control"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type cacheControlFile struct {
|
||||||
|
Type string `yaml:"type"`
|
||||||
|
TTL string `yaml:"ttl"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type promptOutputContractFile struct {
|
type promptOutputContractFile struct {
|
||||||
@@ -63,7 +70,7 @@ func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id strin
|
|||||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
|
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
|
||||||
}
|
}
|
||||||
|
|
||||||
files, err := r.yamlFiles(ctx)
|
files, err := filecatalog.FindYAMLFiles(ctx, r.dir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
|
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
|
||||||
}
|
}
|
||||||
@@ -76,8 +83,8 @@ func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id strin
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
relPath := r.relativePath(fullPath)
|
relPath := filecatalog.RelativePath(r.dir, fullPath)
|
||||||
fileMatch := promptIDFromFileName(filepath.Base(fullPath)) == id
|
fileMatch := filecatalog.Stem(filepath.Base(fullPath)) == id
|
||||||
|
|
||||||
raw, err := loadPromptDefinitionFile(fullPath)
|
raw, err := loadPromptDefinitionFile(fullPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -130,38 +137,6 @@ type promptDefinitionMatch struct {
|
|||||||
path string
|
path string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *filesystemRepository) yamlFiles(ctx context.Context) ([]string, error) {
|
|
||||||
var files []string
|
|
||||||
err := filepath.WalkDir(r.dir, func(path string, d os.DirEntry, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
if d.IsDir() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if !isYAMLFile(d.Name()) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
files = append(files, path)
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
sort.Strings(files)
|
|
||||||
return files, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *filesystemRepository) relativePath(path string) string {
|
|
||||||
rel, err := filepath.Rel(r.dir, path)
|
|
||||||
if err != nil {
|
|
||||||
return filepath.Clean(path)
|
|
||||||
}
|
|
||||||
return filepath.Clean(rel)
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadPromptDefinitionFile(path string) (*promptDefinitionFile, error) {
|
func loadPromptDefinitionFile(path string) (*promptDefinitionFile, error) {
|
||||||
data, err := os.ReadFile(path)
|
data, err := os.ReadFile(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -244,6 +219,11 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
|
|||||||
return nil, fmt.Errorf("message %d (%s) must set exactly one of content or content_file", i, role)
|
return nil, fmt.Errorf("message %d (%s) must set exactly one of content or content_file", i, role)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
cacheControl, err := normalizeCacheControl(msg.CacheControl)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("message %d (%s) cache_control: %w", i, role, err)
|
||||||
|
}
|
||||||
|
|
||||||
templateContent := msg.Content
|
templateContent := msg.Content
|
||||||
resolvedContentFile := ""
|
resolvedContentFile := ""
|
||||||
if hasContentFile {
|
if hasContentFile {
|
||||||
@@ -265,6 +245,7 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
|
|||||||
Role: role,
|
Role: role,
|
||||||
Content: templateContent,
|
Content: templateContent,
|
||||||
ContentFile: resolvedContentFile,
|
ContentFile: resolvedContentFile,
|
||||||
|
CacheControl: cacheControl,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -294,6 +275,7 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
|
|||||||
Version: version,
|
Version: version,
|
||||||
DefaultProfile: defaultProfile,
|
DefaultProfile: defaultProfile,
|
||||||
Description: strings.TrimSpace(raw.Description),
|
Description: strings.TrimSpace(raw.Description),
|
||||||
|
SessionID: strings.TrimSpace(raw.SessionID),
|
||||||
Inputs: inputs,
|
Inputs: inputs,
|
||||||
Templates: templates,
|
Templates: templates,
|
||||||
OutputFormat: raw.Output.Format,
|
OutputFormat: raw.Output.Format,
|
||||||
@@ -306,14 +288,28 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func isYAMLFile(name string) bool {
|
func normalizeCacheControl(raw *cacheControlFile) (*domain.CacheControl, error) {
|
||||||
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
|
if raw == nil {
|
||||||
}
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
func promptIDFromFileName(name string) string {
|
cacheType := strings.TrimSpace(raw.Type)
|
||||||
name = strings.TrimSuffix(name, ".yaml")
|
if cacheType == "" {
|
||||||
name = strings.TrimSuffix(name, ".yml")
|
return nil, errors.New("type is required")
|
||||||
return name
|
}
|
||||||
|
if domain.CacheControlType(cacheType) != domain.CacheControlEphemeral {
|
||||||
|
return nil, fmt.Errorf("unsupported type %q", cacheType)
|
||||||
|
}
|
||||||
|
|
||||||
|
ttl := strings.TrimSpace(raw.TTL)
|
||||||
|
if ttl != "" && ttl != "1h" {
|
||||||
|
return nil, fmt.Errorf("unsupported ttl %q", ttl)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlType(cacheType),
|
||||||
|
TTL: ttl,
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func isValidOutputFormat(f domain.OutputFormat) bool {
|
func isValidOutputFormat(f domain.OutputFormat) bool {
|
||||||
|
|||||||
@@ -68,6 +68,44 @@ func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("valid cache control with ttl", func(t *testing.T) {
|
||||||
|
p, err := repo.GetPromptDefinition(ctx, "valid-cache-control-ttl", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if len(p.Templates) != 2 {
|
||||||
|
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
|
||||||
|
}
|
||||||
|
assertCacheControl(t, p.Templates[0].CacheControl, domain.CacheControlEphemeral, "1h")
|
||||||
|
if p.Templates[1].CacheControl != nil {
|
||||||
|
t.Fatalf("expected second message cache control to be nil, got %#v", p.Templates[1].CacheControl)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("valid cache control without ttl", func(t *testing.T) {
|
||||||
|
p, err := repo.GetPromptDefinition(ctx, "valid-cache-control-without-ttl", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if len(p.Templates) != 2 {
|
||||||
|
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
|
||||||
|
}
|
||||||
|
assertCacheControl(t, p.Templates[0].CacheControl, domain.CacheControlEphemeral, "")
|
||||||
|
if p.Templates[1].CacheControl != nil {
|
||||||
|
t.Fatalf("expected second message cache control to be nil, got %#v", p.Templates[1].CacheControl)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("valid session id template", func(t *testing.T) {
|
||||||
|
p, err := repo.GetPromptDefinition(ctx, "valid-session-id", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if p.SessionID != "{{ .session_id }}" {
|
||||||
|
t.Fatalf("expected trimmed session_id template, got %q", p.SessionID)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("valid nested file-backed prompt resolves content file relative to nested YAML", func(t *testing.T) {
|
t.Run("valid nested file-backed prompt resolves content file relative to nested YAML", func(t *testing.T) {
|
||||||
nestedDir := filepath.Join(tmpDir, "dnd", "recap")
|
nestedDir := filepath.Join(tmpDir, "dnd", "recap")
|
||||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
||||||
@@ -258,6 +296,10 @@ output:
|
|||||||
{name: "invalid validation mode", id: "invalid_validation_mode", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"invalid validation mode"}},
|
{name: "invalid validation mode", id: "invalid_validation_mode", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"invalid validation mode"}},
|
||||||
{name: "json_schema without schema_path", id: "json_schema_without_schema_path", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"schema_path"}},
|
{name: "json_schema without schema_path", id: "json_schema_without_schema_path", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"schema_path"}},
|
||||||
{name: "unknown input field", id: "unknown_input_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unknown_input_setting not found"}},
|
{name: "unknown input field", id: "unknown_input_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unknown_input_setting not found"}},
|
||||||
|
{name: "empty cache control type", id: "empty_cache_control_type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "type is required"}},
|
||||||
|
{name: "unsupported cache control type", id: "unsupported_cache_control_type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported type"}},
|
||||||
|
{name: "unsupported cache control ttl", id: "unsupported_cache_control_ttl", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported ttl"}},
|
||||||
|
{name: "unknown cache control field", id: "unknown_cache_control_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unexpected not found"}},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
@@ -282,6 +324,19 @@ output:
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func assertCacheControl(t *testing.T, got *domain.CacheControl, wantType domain.CacheControlType, wantTTL string) {
|
||||||
|
t.Helper()
|
||||||
|
if got == nil {
|
||||||
|
t.Fatal("expected cache control, got nil")
|
||||||
|
}
|
||||||
|
if got.Type != wantType {
|
||||||
|
t.Fatalf("unexpected cache control type: got %q want %q", got.Type, wantType)
|
||||||
|
}
|
||||||
|
if got.TTL != wantTTL {
|
||||||
|
t.Fatalf("unexpected cache control ttl: got %q want %q", got.TTL, wantTTL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func writePromptTestFile(t *testing.T, path string, content string) {
|
func writePromptTestFile(t *testing.T, path string, content string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil {
|
if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil {
|
||||||
|
|||||||
10
internal/promptdef/testdata/empty_cache_control_type.yaml
vendored
Normal file
10
internal/promptdef/testdata/empty_cache_control_type.yaml
vendored
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
id: empty-cache-control-type
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: "Use cached instructions."
|
||||||
|
cache_control: {}
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
12
internal/promptdef/testdata/unknown_cache_control_field.yaml
vendored
Normal file
12
internal/promptdef/testdata/unknown_cache_control_field.yaml
vendored
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
id: unknown-cache-control-field
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: "Use cached instructions."
|
||||||
|
cache_control:
|
||||||
|
type: ephemeral
|
||||||
|
unexpected: true
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
12
internal/promptdef/testdata/unsupported_cache_control_ttl.yaml
vendored
Normal file
12
internal/promptdef/testdata/unsupported_cache_control_ttl.yaml
vendored
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
id: unsupported-cache-control-ttl
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: "Use cached instructions."
|
||||||
|
cache_control:
|
||||||
|
type: ephemeral
|
||||||
|
ttl: 5m
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
11
internal/promptdef/testdata/unsupported_cache_control_type.yaml
vendored
Normal file
11
internal/promptdef/testdata/unsupported_cache_control_type.yaml
vendored
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
id: unsupported-cache-control-type
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: "Use cached instructions."
|
||||||
|
cache_control:
|
||||||
|
type: persistent
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
14
internal/promptdef/testdata/valid_cache_control_ttl.yaml
vendored
Normal file
14
internal/promptdef/testdata/valid_cache_control_ttl.yaml
vendored
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
id: valid-cache-control-ttl
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: "Use cached instructions."
|
||||||
|
cache_control:
|
||||||
|
type: ephemeral
|
||||||
|
ttl: 1h
|
||||||
|
- role: user
|
||||||
|
content: "Summarize the input."
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
13
internal/promptdef/testdata/valid_cache_control_without_ttl.yaml
vendored
Normal file
13
internal/promptdef/testdata/valid_cache_control_without_ttl.yaml
vendored
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
id: valid-cache-control-without-ttl
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: "Use cached instructions."
|
||||||
|
cache_control:
|
||||||
|
type: ephemeral
|
||||||
|
- role: user
|
||||||
|
content: "Summarize the input."
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
10
internal/promptdef/testdata/valid_session_id.yaml
vendored
Normal file
10
internal/promptdef/testdata/valid_session_id.yaml
vendored
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
id: valid-session-id
|
||||||
|
version: "1.0.0"
|
||||||
|
session_id: " {{ .session_id }} "
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: Hello.
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
@@ -25,6 +25,8 @@ import (
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
ErrInvalidRequest = errors.New("invalid run request")
|
ErrInvalidRequest = errors.New("invalid run request")
|
||||||
|
ErrProfileRequired = errors.New("profile selection is required")
|
||||||
|
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
||||||
ErrProfileLoad = errors.New("failed to load prompt definition")
|
ErrProfileLoad = errors.New("failed to load prompt definition")
|
||||||
ErrArtifactLoad = errors.New("failed to load artifact")
|
ErrArtifactLoad = errors.New("failed to load artifact")
|
||||||
ErrPromptRender = errors.New("failed to render prompt")
|
ErrPromptRender = errors.New("failed to render prompt")
|
||||||
@@ -88,11 +90,15 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
}
|
}
|
||||||
|
|
||||||
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: prepared.Messages},
|
Prompt: domain.RenderedPrompt{SessionID: prepared.SessionID, Messages: prepared.Messages},
|
||||||
Target: prepared.EffectiveModelParams,
|
Target: prepared.EffectiveModelParams,
|
||||||
|
TargetPresence: prepared.TargetPresence,
|
||||||
StructuredOutput: prepared.StructuredOutput,
|
StructuredOutput: prepared.StructuredOutput,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if errors.Is(err, llm.ErrInvalidRequest) {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err)
|
return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -177,7 +183,7 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
|||||||
selectedProfileID = strings.TrimSpace(def.DefaultProfile)
|
selectedProfileID = strings.TrimSpace(def.DefaultProfile)
|
||||||
}
|
}
|
||||||
if selectedProfileID == "" {
|
if selectedProfileID == "" {
|
||||||
return nil, fmt.Errorf("%w: profile id is required either in request or prompt default_profile", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired)
|
||||||
}
|
}
|
||||||
|
|
||||||
execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID)
|
execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID)
|
||||||
@@ -185,7 +191,10 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
|||||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
effectiveModel := resolveExecutionTarget(execProfile, req.Execution)
|
effectiveModel, targetPresence, err := resolveExecutionTarget(execProfile, req.Execution)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
|
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
|
||||||
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
|
||||||
}
|
}
|
||||||
@@ -228,9 +237,11 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
|||||||
PromptHash: promptDefinitionHash,
|
PromptHash: promptDefinitionHash,
|
||||||
SelectedProfileID: selectedProfileID,
|
SelectedProfileID: selectedProfileID,
|
||||||
EffectiveModelParams: effectiveModel,
|
EffectiveModelParams: effectiveModel,
|
||||||
|
TargetPresence: targetPresence,
|
||||||
OutputContract: effectiveContract,
|
OutputContract: effectiveContract,
|
||||||
StructuredOutput: structuredOutput,
|
StructuredOutput: structuredOutput,
|
||||||
InputHashes: inputHashes,
|
InputHashes: inputHashes,
|
||||||
|
SessionID: renderedPrompt.SessionID,
|
||||||
RenderedPromptHash: hashRenderedPrompt(*renderedPrompt),
|
RenderedPromptHash: hashRenderedPrompt(*renderedPrompt),
|
||||||
Messages: renderedPrompt.Messages,
|
Messages: renderedPrompt.Messages,
|
||||||
StartTime: start,
|
StartTime: start,
|
||||||
@@ -352,22 +363,75 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution
|
|||||||
out.APIKeyEnv = override.APIKeyEnv
|
out.APIKeyEnv = override.APIKeyEnv
|
||||||
}
|
}
|
||||||
if len(override.ExtraParams) > 0 {
|
if len(override.ExtraParams) > 0 {
|
||||||
cp := make(map[string]string, len(override.ExtraParams))
|
out.ExtraParams = copyExtraParams(override.ExtraParams)
|
||||||
for k, v := range override.ExtraParams {
|
|
||||||
cp[k] = v
|
|
||||||
}
|
|
||||||
out.ExtraParams = cp
|
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTarget) domain.ExecutionTarget {
|
func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
|
||||||
|
out := base
|
||||||
|
var presence domain.ExecutionTargetPresence
|
||||||
|
if override.Endpoint != "" {
|
||||||
|
out.Endpoint = override.Endpoint
|
||||||
|
}
|
||||||
|
if override.Model != "" {
|
||||||
|
out.Model = override.Model
|
||||||
|
}
|
||||||
|
if override.Temperature != nil {
|
||||||
|
if *override.Temperature < 0 || *override.Temperature > 2 {
|
||||||
|
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("temperature must be between 0 and 2")
|
||||||
|
}
|
||||||
|
out.Temperature = *override.Temperature
|
||||||
|
presence.Temperature = true
|
||||||
|
}
|
||||||
|
if override.MaxTokens != nil {
|
||||||
|
if *override.MaxTokens < 0 {
|
||||||
|
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("max_tokens must be greater than or equal to 0")
|
||||||
|
}
|
||||||
|
out.MaxTokens = *override.MaxTokens
|
||||||
|
presence.MaxTokens = true
|
||||||
|
}
|
||||||
|
if override.TopP != nil {
|
||||||
|
if *override.TopP < 0 || *override.TopP > 1 {
|
||||||
|
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("top_p must be between 0 and 1")
|
||||||
|
}
|
||||||
|
out.TopP = *override.TopP
|
||||||
|
presence.TopP = true
|
||||||
|
}
|
||||||
|
if override.TimeoutSeconds != nil {
|
||||||
|
if *override.TimeoutSeconds < 0 {
|
||||||
|
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("timeout_seconds must be greater than or equal to 0")
|
||||||
|
}
|
||||||
|
out.TimeoutSeconds = *override.TimeoutSeconds
|
||||||
|
presence.TimeoutSeconds = true
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(override.ServiceTier) != "" {
|
||||||
|
out.ServiceTier = override.ServiceTier
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(override.ReasoningEffort) != "" {
|
||||||
|
out.ReasoningEffort = override.ReasoningEffort
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(override.APIKeyEnv) != "" {
|
||||||
|
out.APIKeyEnv = override.APIKeyEnv
|
||||||
|
}
|
||||||
|
if len(override.ExtraParams) > 0 {
|
||||||
|
out.ExtraParams = copyExtraParams(override.ExtraParams)
|
||||||
|
}
|
||||||
|
return out, presence, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
|
||||||
out := defaults.ExecutionTargetDefault()
|
out := defaults.ExecutionTargetDefault()
|
||||||
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
|
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
|
||||||
|
var presence domain.ExecutionTargetPresence
|
||||||
if override != nil {
|
if override != nil {
|
||||||
out = mergeExecutionTarget(out, *override)
|
var err error
|
||||||
|
out, presence, err = mergeExecutionTargetOverride(out, *override)
|
||||||
|
if err != nil {
|
||||||
|
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, err
|
||||||
}
|
}
|
||||||
return out
|
}
|
||||||
|
return out, presence, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateAPIKeyEnv(apiKeyEnv string) error {
|
func validateAPIKeyEnv(apiKeyEnv string) error {
|
||||||
@@ -376,7 +440,7 @@ func validateAPIKeyEnv(apiKeyEnv string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(os.Getenv(envName)) == "" {
|
if strings.TrimSpace(os.Getenv(envName)) == "" {
|
||||||
return fmt.Errorf("api key environment variable %q is not set", envName)
|
return fmt.Errorf("%w: api key environment variable %q is not set", ErrAPIKeyEnvMissing, envName)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -385,13 +449,6 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
|
|||||||
if p == nil {
|
if p == nil {
|
||||||
return domain.ExecutionTarget{}
|
return domain.ExecutionTarget{}
|
||||||
}
|
}
|
||||||
cp := map[string]string(nil)
|
|
||||||
if len(p.ExtraParams) > 0 {
|
|
||||||
cp = make(map[string]string, len(p.ExtraParams))
|
|
||||||
for k, v := range p.ExtraParams {
|
|
||||||
cp[k] = v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return domain.ExecutionTarget{
|
return domain.ExecutionTarget{
|
||||||
Endpoint: p.Endpoint,
|
Endpoint: p.Endpoint,
|
||||||
Model: p.Model,
|
Model: p.Model,
|
||||||
@@ -402,10 +459,21 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
|
|||||||
ServiceTier: p.ServiceTier,
|
ServiceTier: p.ServiceTier,
|
||||||
ReasoningEffort: p.ReasoningEffort,
|
ReasoningEffort: p.ReasoningEffort,
|
||||||
APIKeyEnv: p.APIKeyEnv,
|
APIKeyEnv: p.APIKeyEnv,
|
||||||
ExtraParams: cp,
|
ExtraParams: copyExtraParams(p.ExtraParams),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func copyExtraParams(src map[string]any) map[string]any {
|
||||||
|
if len(src) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cp := make(map[string]any, len(src))
|
||||||
|
for k, v := range src {
|
||||||
|
cp[k] = v
|
||||||
|
}
|
||||||
|
return cp
|
||||||
|
}
|
||||||
|
|
||||||
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract {
|
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract {
|
||||||
contract := def.Validation
|
contract := def.Validation
|
||||||
if contract.Format == "" {
|
if contract.Format == "" {
|
||||||
@@ -422,10 +490,23 @@ func resolveOutputContract(def *domain.PromptDefinition, override *domain.Output
|
|||||||
|
|
||||||
func hashRenderedPrompt(p domain.RenderedPrompt) string {
|
func hashRenderedPrompt(p domain.RenderedPrompt) string {
|
||||||
var b strings.Builder
|
var b strings.Builder
|
||||||
|
if p.SessionID != "" {
|
||||||
|
b.WriteString("session_id=")
|
||||||
|
b.WriteString(p.SessionID)
|
||||||
|
b.WriteString("\n---\n")
|
||||||
|
}
|
||||||
for _, msg := range p.Messages {
|
for _, msg := range p.Messages {
|
||||||
b.WriteString(msg.Role)
|
b.WriteString(msg.Role)
|
||||||
b.WriteByte('\n')
|
b.WriteByte('\n')
|
||||||
b.WriteString(msg.Content)
|
b.WriteString(msg.Content)
|
||||||
|
if msg.CacheControl != nil {
|
||||||
|
b.WriteString("\ncache_control.type=")
|
||||||
|
b.WriteString(string(msg.CacheControl.Type))
|
||||||
|
if msg.CacheControl.TTL != "" {
|
||||||
|
b.WriteString("\ncache_control.ttl=")
|
||||||
|
b.WriteString(msg.CacheControl.TTL)
|
||||||
|
}
|
||||||
|
}
|
||||||
b.WriteString("\n---\n")
|
b.WriteString("\n---\n")
|
||||||
}
|
}
|
||||||
h := sha256.Sum256([]byte(b.String()))
|
h := sha256.Sum256([]byte(b.String()))
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||||
@@ -160,7 +161,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
|
|||||||
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
|
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
|
||||||
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
|
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
|
||||||
}}
|
}}
|
||||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||||
llmClient := &fakeLLM{forbid: true}
|
llmClient := &fakeLLM{forbid: true}
|
||||||
|
|
||||||
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
||||||
@@ -172,7 +173,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
|
|||||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
||||||
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
|
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
|
||||||
},
|
},
|
||||||
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
|
Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -198,6 +199,9 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
|
|||||||
if len(prepared.Messages) != 2 {
|
if len(prepared.Messages) != 2 {
|
||||||
t.Fatalf("expected two messages, got %d", len(prepared.Messages))
|
t.Fatalf("expected two messages, got %d", len(prepared.Messages))
|
||||||
}
|
}
|
||||||
|
if prepared.SessionID != "session-123" {
|
||||||
|
t.Fatalf("expected prepared session id, got %q", prepared.SessionID)
|
||||||
|
}
|
||||||
if llmClient.calls != 0 {
|
if llmClient.calls != 0 {
|
||||||
t.Fatalf("prepare should not call llm, calls=%d", llmClient.calls)
|
t.Fatalf("prepare should not call llm, calls=%d", llmClient.calls)
|
||||||
}
|
}
|
||||||
@@ -234,6 +238,9 @@ func TestRunnerPrepareMissingExplicitProfileAndMissingDefaultProfileFails(t *tes
|
|||||||
if !errors.Is(err, ErrInvalidRequest) {
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
}
|
}
|
||||||
|
if !errors.Is(err, ErrProfileRequired) {
|
||||||
|
t.Fatalf("expected ErrProfileRequired, got %v", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) {
|
func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) {
|
||||||
@@ -266,11 +273,11 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
|
|||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Inputs: singleInputRef(),
|
Inputs: singleInputRef(),
|
||||||
Execution: &domain.ExecutionTarget{
|
Execution: &domain.ExecutionTargetOverride{
|
||||||
Endpoint: "http://override/v1",
|
Endpoint: "http://override/v1",
|
||||||
Model: "override-model",
|
Model: "override-model",
|
||||||
Temperature: 0.7,
|
Temperature: float64Ptr(0.7),
|
||||||
TimeoutSeconds: 30,
|
TimeoutSeconds: intPtr(30),
|
||||||
ServiceTier: "flex",
|
ServiceTier: "flex",
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -288,6 +295,143 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerPrepareRequestNumericOverridePresence(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
override *domain.ExecutionTargetOverride
|
||||||
|
wantTemperature float64
|
||||||
|
wantMaxTokens int
|
||||||
|
wantTopP float64
|
||||||
|
wantTimeoutSecs int
|
||||||
|
wantPresence domain.ExecutionTargetPresence
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "omitted preserves profile values",
|
||||||
|
override: &domain.ExecutionTargetOverride{},
|
||||||
|
wantTemperature: 0.7,
|
||||||
|
wantMaxTokens: 321,
|
||||||
|
wantTopP: 0.8,
|
||||||
|
wantTimeoutSecs: 45,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "explicit zero temperature",
|
||||||
|
override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(0)},
|
||||||
|
wantTemperature: 0,
|
||||||
|
wantMaxTokens: 321,
|
||||||
|
wantTopP: 0.8,
|
||||||
|
wantTimeoutSecs: 45,
|
||||||
|
wantPresence: domain.ExecutionTargetPresence{Temperature: true},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "explicit zero max tokens",
|
||||||
|
override: &domain.ExecutionTargetOverride{MaxTokens: intPtr(0)},
|
||||||
|
wantTemperature: 0.7,
|
||||||
|
wantMaxTokens: 0,
|
||||||
|
wantTopP: 0.8,
|
||||||
|
wantTimeoutSecs: 45,
|
||||||
|
wantPresence: domain.ExecutionTargetPresence{MaxTokens: true},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "explicit zero top p",
|
||||||
|
override: &domain.ExecutionTargetOverride{TopP: float64Ptr(0)},
|
||||||
|
wantTemperature: 0.7,
|
||||||
|
wantMaxTokens: 321,
|
||||||
|
wantTopP: 0,
|
||||||
|
wantTimeoutSecs: 45,
|
||||||
|
wantPresence: domain.ExecutionTargetPresence{TopP: true},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "explicit zero timeout",
|
||||||
|
override: &domain.ExecutionTargetOverride{TimeoutSeconds: intPtr(0)},
|
||||||
|
wantTemperature: 0.7,
|
||||||
|
wantMaxTokens: 321,
|
||||||
|
wantTopP: 0.8,
|
||||||
|
wantTimeoutSecs: 0,
|
||||||
|
wantPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"exec": {
|
||||||
|
ID: "exec",
|
||||||
|
Endpoint: "http://profile/v1",
|
||||||
|
Model: "profile-model",
|
||||||
|
Temperature: 0.7,
|
||||||
|
MaxTokens: 321,
|
||||||
|
TopP: 0.8,
|
||||||
|
TimeoutSeconds: 45,
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
&fakeLLM{forbid: true},
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
|
||||||
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
Execution: tc.override,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
got := prepared.EffectiveModelParams
|
||||||
|
if got.Temperature != tc.wantTemperature ||
|
||||||
|
got.MaxTokens != tc.wantMaxTokens ||
|
||||||
|
got.TopP != tc.wantTopP ||
|
||||||
|
got.TimeoutSeconds != tc.wantTimeoutSecs {
|
||||||
|
t.Fatalf("unexpected effective numeric settings: %+v", got)
|
||||||
|
}
|
||||||
|
if prepared.TargetPresence != tc.wantPresence {
|
||||||
|
t.Fatalf("unexpected target presence: got %+v want %+v", prepared.TargetPresence, tc.wantPresence)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPrepareInvalidRequestNumericOverridesFail(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
override *domain.ExecutionTargetOverride
|
||||||
|
}{
|
||||||
|
{name: "temperature below range", override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(-0.1)}},
|
||||||
|
{name: "temperature above range", override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(2.1)}},
|
||||||
|
{name: "max tokens below range", override: &domain.ExecutionTargetOverride{MaxTokens: intPtr(-1)}},
|
||||||
|
{name: "top p below range", override: &domain.ExecutionTargetOverride{TopP: float64Ptr(-0.1)}},
|
||||||
|
{name: "top p above range", override: &domain.ExecutionTargetOverride{TopP: float64Ptr(1.1)}},
|
||||||
|
{name: "timeout below range", override: &domain.ExecutionTargetOverride{TimeoutSeconds: intPtr(-1)}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
&fakeLLM{forbid: true},
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
|
||||||
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
Execution: tc.override,
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
|
func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
|
||||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
@@ -487,6 +631,35 @@ func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerPrepareJSONSchemaSchemaLoadFailureReturnsValidationError(t *testing.T) {
|
||||||
|
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||||
|
def.Validation.SchemaPath = "missing.schema.json"
|
||||||
|
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: def},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
&fakeLLM{forbid: true},
|
||||||
|
validator,
|
||||||
|
)
|
||||||
|
|
||||||
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrValidation) {
|
||||||
|
t.Fatalf("expected ErrValidation, got %v", err)
|
||||||
|
}
|
||||||
|
if validator.schemaLoads != 1 {
|
||||||
|
t.Fatalf("expected one schema load attempt, got %d", validator.schemaLoads)
|
||||||
|
}
|
||||||
|
if validator.schemaLoadPath != "missing.schema.json" {
|
||||||
|
t.Fatalf("expected schema path missing.schema.json, got %q", validator.schemaLoadPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) {
|
func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) {
|
||||||
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||||
def.Validation.SchemaPath = "missing.schema.json"
|
def.Validation.SchemaPath = "missing.schema.json"
|
||||||
@@ -545,6 +718,100 @@ func TestDeriveStructuredSchemaName(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHashRenderedPromptIncludesCacheControlWhenPresent(t *testing.T) {
|
||||||
|
uncached := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
|
{Role: "system", Content: "sys"},
|
||||||
|
{Role: "user", Content: "usr"},
|
||||||
|
}}
|
||||||
|
wantLegacyHash := hashString("system\nsys\n---\nuser\nusr\n---\n")
|
||||||
|
if got := hashRenderedPrompt(uncached); got != wantLegacyHash {
|
||||||
|
t.Fatalf("expected no-cache hash to preserve legacy input, got %q want %q", got, wantLegacyHash)
|
||||||
|
}
|
||||||
|
|
||||||
|
withCache := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "sys",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
TTL: "1h",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{Role: "user", Content: "usr"},
|
||||||
|
}}
|
||||||
|
alsoWithCache := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "sys",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
TTL: "1h",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{Role: "user", Content: "usr"},
|
||||||
|
}}
|
||||||
|
withoutTTL := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "sys",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{Role: "user", Content: "usr"},
|
||||||
|
}}
|
||||||
|
|
||||||
|
cachedHash := hashRenderedPrompt(withCache)
|
||||||
|
if cachedHash == hashRenderedPrompt(uncached) {
|
||||||
|
t.Fatal("expected cache control to change rendered prompt hash")
|
||||||
|
}
|
||||||
|
if cachedHash != hashRenderedPrompt(alsoWithCache) {
|
||||||
|
t.Fatal("expected identical cache control metadata to produce stable hash")
|
||||||
|
}
|
||||||
|
if cachedHash == hashRenderedPrompt(withoutTTL) {
|
||||||
|
t.Fatal("expected ttl changes to affect rendered prompt hash")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHashRenderedPromptIncludesSessionIDWhenPresent(t *testing.T) {
|
||||||
|
withoutSession := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
|
{Role: "system", Content: "sys"},
|
||||||
|
{Role: "user", Content: "usr"},
|
||||||
|
}}
|
||||||
|
withSession := domain.RenderedPrompt{
|
||||||
|
SessionID: "session-123",
|
||||||
|
Messages: []domain.RenderedMessage{
|
||||||
|
{Role: "system", Content: "sys"},
|
||||||
|
{Role: "user", Content: "usr"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
alsoWithSession := domain.RenderedPrompt{
|
||||||
|
SessionID: "session-123",
|
||||||
|
Messages: []domain.RenderedMessage{
|
||||||
|
{Role: "system", Content: "sys"},
|
||||||
|
{Role: "user", Content: "usr"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
otherSession := domain.RenderedPrompt{
|
||||||
|
SessionID: "session-456",
|
||||||
|
Messages: []domain.RenderedMessage{
|
||||||
|
{Role: "system", Content: "sys"},
|
||||||
|
{Role: "user", Content: "usr"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionHash := hashRenderedPrompt(withSession)
|
||||||
|
if sessionHash == hashRenderedPrompt(withoutSession) {
|
||||||
|
t.Fatal("expected session_id to change rendered prompt hash")
|
||||||
|
}
|
||||||
|
if sessionHash != hashRenderedPrompt(alsoWithSession) {
|
||||||
|
t.Fatal("expected identical session_id to produce stable hash")
|
||||||
|
}
|
||||||
|
if sessionHash == hashRenderedPrompt(otherSession) {
|
||||||
|
t.Fatal("expected session_id value changes to affect rendered prompt hash")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerRunSuccessful(t *testing.T) {
|
func TestRunnerRunSuccessful(t *testing.T) {
|
||||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
|
||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}
|
||||||
@@ -552,7 +819,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
|||||||
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
|
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
|
||||||
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
|
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
|
||||||
}}
|
}}
|
||||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
|
||||||
|
|
||||||
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
||||||
@@ -564,7 +831,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
|||||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
||||||
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
|
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
|
||||||
},
|
},
|
||||||
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
|
Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -599,6 +866,48 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
|||||||
if llmClient.lastReq.Target.TimeoutSeconds != 90 {
|
if llmClient.lastReq.Target.TimeoutSeconds != 90 {
|
||||||
t.Fatalf("expected timeout propagation, got %d", llmClient.lastReq.Target.TimeoutSeconds)
|
t.Fatalf("expected timeout propagation, got %d", llmClient.lastReq.Target.TimeoutSeconds)
|
||||||
}
|
}
|
||||||
|
if !llmClient.lastReq.TargetPresence.Temperature || !llmClient.lastReq.TargetPresence.TimeoutSeconds {
|
||||||
|
t.Fatalf("expected numeric override presence to be sent to llm, got %+v", llmClient.lastReq.TargetPresence)
|
||||||
|
}
|
||||||
|
if llmClient.lastReq.Prompt.SessionID != "session-123" {
|
||||||
|
t.Fatalf("expected session id to be sent to llm, got %q", llmClient.lastReq.Prompt.SessionID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunPassesExtraParamsToGenerateRequestTarget(t *testing.T) {
|
||||||
|
extraParams := map[string]any{
|
||||||
|
"string_value": "enabled",
|
||||||
|
"number_value": 42,
|
||||||
|
"boolean_value": true,
|
||||||
|
"object_value": map[string]any{"nested": "value"},
|
||||||
|
"array_value": []any{"first", 3, false},
|
||||||
|
}
|
||||||
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||||
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"exec": {
|
||||||
|
ID: "exec",
|
||||||
|
Endpoint: "http://profile/v1",
|
||||||
|
Model: "profile-model",
|
||||||
|
ExtraParams: extraParams,
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
|
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||||
|
|
||||||
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(res.EffectiveModelParams.ExtraParams, extraParams) {
|
||||||
|
t.Fatalf("expected run result extra_params to match profile values, got %#v", res.EffectiveModelParams.ExtraParams)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(llmClient.lastReq.Target.ExtraParams, extraParams) {
|
||||||
|
t.Fatalf("expected generate request extra_params to match profile values, got %#v", llmClient.lastReq.Target.ExtraParams)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T) {
|
func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T) {
|
||||||
@@ -617,7 +926,7 @@ func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T)
|
|||||||
Inputs: map[string]domain.ArtifactRef{
|
Inputs: map[string]domain.ArtifactRef{
|
||||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
||||||
},
|
},
|
||||||
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
|
Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)},
|
||||||
}
|
}
|
||||||
|
|
||||||
prepared, err := runner.Prepare(context.Background(), req)
|
prepared, err := runner.Prepare(context.Background(), req)
|
||||||
@@ -745,11 +1054,11 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T)
|
|||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Inputs: singleInputRef(),
|
Inputs: singleInputRef(),
|
||||||
Execution: &domain.ExecutionTarget{
|
Execution: &domain.ExecutionTargetOverride{
|
||||||
Endpoint: "http://override/v1",
|
Endpoint: "http://override/v1",
|
||||||
Model: "override-model",
|
Model: "override-model",
|
||||||
Temperature: 0.7,
|
Temperature: float64Ptr(0.7),
|
||||||
TimeoutSeconds: 30,
|
TimeoutSeconds: intPtr(30),
|
||||||
ServiceTier: "flex",
|
ServiceTier: "flex",
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -862,6 +1171,9 @@ func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) {
|
|||||||
if !errors.Is(err, ErrInvalidRequest) {
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
}
|
}
|
||||||
|
if !errors.Is(err, ErrAPIKeyEnvMissing) {
|
||||||
|
t.Fatalf("expected ErrAPIKeyEnvMissing, got %v", err)
|
||||||
|
}
|
||||||
if !strings.Contains(err.Error(), "SCRIPTORIUM_MISSING_KEY") {
|
if !strings.Contains(err.Error(), "SCRIPTORIUM_MISSING_KEY") {
|
||||||
t.Fatalf("expected missing env name in error, got %v", err)
|
t.Fatalf("expected missing env name in error, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -881,7 +1193,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
|
|||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Inputs: singleInputRef(),
|
Inputs: singleInputRef(),
|
||||||
Execution: &domain.ExecutionTarget{APIKeyEnv: envName},
|
Execution: &domain.ExecutionTargetOverride{APIKeyEnv: envName},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -906,7 +1218,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) {
|
|||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Inputs: singleInputRef(),
|
Inputs: singleInputRef(),
|
||||||
Execution: &domain.ExecutionTarget{APIKeyEnv: runtimeEnv},
|
Execution: &domain.ExecutionTargetOverride{APIKeyEnv: runtimeEnv},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -1005,6 +1317,28 @@ func TestRunnerRunLLMFailure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
&fakeLLM{err: llm.ErrInvalidRequest},
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
|
}
|
||||||
|
if errors.Is(err, ErrLLMGenerate) {
|
||||||
|
t.Fatalf("did not expect ErrLLMGenerate, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerRunValidationStillWorks(t *testing.T) {
|
func TestRunnerRunValidationStillWorks(t *testing.T) {
|
||||||
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
|
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
@@ -1047,7 +1381,7 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
|
|||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Inputs: singleInputRef(),
|
Inputs: singleInputRef(),
|
||||||
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "override-model", TimeoutSeconds: 22},
|
Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "override-model", TimeoutSeconds: intPtr(22)},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -1122,6 +1456,186 @@ func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecutionProfileToTargetPopulatesAllFieldsAndCopiesExtraParams(t *testing.T) {
|
||||||
|
src := &domain.ExecutionProfile{
|
||||||
|
ID: "exec",
|
||||||
|
Endpoint: "http://profile/v1",
|
||||||
|
Model: "profile-model",
|
||||||
|
Temperature: 0.2,
|
||||||
|
MaxTokens: 123,
|
||||||
|
TopP: 0.75,
|
||||||
|
TimeoutSeconds: 90,
|
||||||
|
ServiceTier: "priority",
|
||||||
|
ReasoningEffort: "medium",
|
||||||
|
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
||||||
|
ExtraParams: map[string]any{
|
||||||
|
"provider_option": "on",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
target := executionProfileToTarget(src)
|
||||||
|
if target.Endpoint != src.Endpoint ||
|
||||||
|
target.Model != src.Model ||
|
||||||
|
target.Temperature != src.Temperature ||
|
||||||
|
target.MaxTokens != src.MaxTokens ||
|
||||||
|
target.TopP != src.TopP ||
|
||||||
|
target.TimeoutSeconds != src.TimeoutSeconds ||
|
||||||
|
target.ServiceTier != src.ServiceTier ||
|
||||||
|
target.ReasoningEffort != src.ReasoningEffort ||
|
||||||
|
target.APIKeyEnv != src.APIKeyEnv {
|
||||||
|
t.Fatalf("expected all profile fields to populate target, got %+v", target)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(target.ExtraParams, src.ExtraParams) {
|
||||||
|
t.Fatalf("expected extra_params to match, got %#v", target.ExtraParams)
|
||||||
|
}
|
||||||
|
|
||||||
|
src.ExtraParams["provider_option"] = "changed"
|
||||||
|
if target.ExtraParams["provider_option"] != "on" {
|
||||||
|
t.Fatalf("expected extra_params copy to be independent, got %#v", target.ExtraParams)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testing.T) {
|
||||||
|
profileValue := &domain.ExecutionProfile{
|
||||||
|
ID: "exec",
|
||||||
|
Endpoint: "http://profile/v1",
|
||||||
|
Model: "profile-model",
|
||||||
|
Temperature: 0.3,
|
||||||
|
MaxTokens: 222,
|
||||||
|
TopP: 0.6,
|
||||||
|
TimeoutSeconds: 77,
|
||||||
|
ServiceTier: "priority",
|
||||||
|
ReasoningEffort: "low",
|
||||||
|
APIKeyEnv: "PROFILE_KEY",
|
||||||
|
ExtraParams: map[string]any{
|
||||||
|
"profile_option": "enabled",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
target, presence, err := resolveExecutionTarget(profileValue, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if presence != (domain.ExecutionTargetPresence{}) {
|
||||||
|
t.Fatalf("expected no request override presence, got %+v", presence)
|
||||||
|
}
|
||||||
|
if target.Endpoint != profileValue.Endpoint ||
|
||||||
|
target.Model != profileValue.Model ||
|
||||||
|
target.Temperature != profileValue.Temperature ||
|
||||||
|
target.MaxTokens != profileValue.MaxTokens ||
|
||||||
|
target.TopP != profileValue.TopP ||
|
||||||
|
target.TimeoutSeconds != profileValue.TimeoutSeconds ||
|
||||||
|
target.ServiceTier != profileValue.ServiceTier ||
|
||||||
|
target.ReasoningEffort != profileValue.ReasoningEffort ||
|
||||||
|
target.APIKeyEnv != profileValue.APIKeyEnv {
|
||||||
|
t.Fatalf("expected profile values to populate target, got %+v", target)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(target.ExtraParams, profileValue.ExtraParams) {
|
||||||
|
t.Fatalf("expected profile extra_params in target, got %#v", target.ExtraParams)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFields(t *testing.T) {
|
||||||
|
profileValue := &domain.ExecutionProfile{
|
||||||
|
ID: "exec",
|
||||||
|
Endpoint: "http://profile/v1",
|
||||||
|
Model: "profile-model",
|
||||||
|
Temperature: 0.2,
|
||||||
|
MaxTokens: 200,
|
||||||
|
TopP: 0.8,
|
||||||
|
TimeoutSeconds: 90,
|
||||||
|
ServiceTier: "priority",
|
||||||
|
ReasoningEffort: "medium",
|
||||||
|
APIKeyEnv: "PROFILE_KEY",
|
||||||
|
ExtraParams: map[string]any{
|
||||||
|
"profile_only": "yes",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
override := &domain.ExecutionTargetOverride{
|
||||||
|
Endpoint: "http://override/v1",
|
||||||
|
Model: "override-model",
|
||||||
|
Temperature: float64Ptr(0.9),
|
||||||
|
MaxTokens: intPtr(111),
|
||||||
|
TopP: float64Ptr(0.5),
|
||||||
|
TimeoutSeconds: intPtr(30),
|
||||||
|
ServiceTier: "flex",
|
||||||
|
ReasoningEffort: "high",
|
||||||
|
APIKeyEnv: "RUNTIME_KEY",
|
||||||
|
ExtraParams: map[string]any{
|
||||||
|
"runtime_only": "yes",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
target, presence, err := resolveExecutionTarget(profileValue, override)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if presence != (domain.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}) {
|
||||||
|
t.Fatalf("unexpected override presence: %+v", presence)
|
||||||
|
}
|
||||||
|
if target.Endpoint != override.Endpoint ||
|
||||||
|
target.Model != override.Model ||
|
||||||
|
target.Temperature != *override.Temperature ||
|
||||||
|
target.MaxTokens != *override.MaxTokens ||
|
||||||
|
target.TopP != *override.TopP ||
|
||||||
|
target.TimeoutSeconds != *override.TimeoutSeconds ||
|
||||||
|
target.ServiceTier != override.ServiceTier ||
|
||||||
|
target.ReasoningEffort != override.ReasoningEffort ||
|
||||||
|
target.APIKeyEnv != override.APIKeyEnv {
|
||||||
|
t.Fatalf("expected runtime overrides to win for all fields, got %+v", target)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(target.ExtraParams, override.ExtraParams) {
|
||||||
|
t.Fatalf("expected runtime extra_params to replace profile extra_params, got %#v", target.ExtraParams)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeExecutionTargetEmptyStringOverridesDoNotErase(t *testing.T) {
|
||||||
|
base := domain.ExecutionTarget{
|
||||||
|
Endpoint: "http://base/v1",
|
||||||
|
Model: "base-model",
|
||||||
|
ServiceTier: "priority",
|
||||||
|
ReasoningEffort: "medium",
|
||||||
|
APIKeyEnv: "BASE_KEY",
|
||||||
|
}
|
||||||
|
override := domain.ExecutionTarget{
|
||||||
|
Endpoint: "http://override/v1",
|
||||||
|
Model: "override-model",
|
||||||
|
ServiceTier: " ",
|
||||||
|
ReasoningEffort: " ",
|
||||||
|
APIKeyEnv: "",
|
||||||
|
}
|
||||||
|
|
||||||
|
merged := mergeExecutionTarget(base, override)
|
||||||
|
if merged.Endpoint != "http://override/v1" || merged.Model != "override-model" {
|
||||||
|
t.Fatalf("expected endpoint/model to override, got %+v", merged)
|
||||||
|
}
|
||||||
|
if merged.ServiceTier != "priority" {
|
||||||
|
t.Fatalf("expected empty service_tier override to be ignored, got %q", merged.ServiceTier)
|
||||||
|
}
|
||||||
|
if merged.ReasoningEffort != "medium" {
|
||||||
|
t.Fatalf("expected empty reasoning_effort override to be ignored, got %q", merged.ReasoningEffort)
|
||||||
|
}
|
||||||
|
if merged.APIKeyEnv != "BASE_KEY" {
|
||||||
|
t.Fatalf("expected empty api_key_env override to be ignored, got %q", merged.APIKeyEnv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeExecutionTargetEmptyExtraParamsDoesNotErase(t *testing.T) {
|
||||||
|
base := domain.ExecutionTarget{
|
||||||
|
ExtraParams: map[string]any{
|
||||||
|
"keep": "value",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
override := domain.ExecutionTarget{
|
||||||
|
ExtraParams: map[string]any{},
|
||||||
|
}
|
||||||
|
|
||||||
|
merged := mergeExecutionTarget(base, override)
|
||||||
|
if !reflect.DeepEqual(merged.ExtraParams, base.ExtraParams) {
|
||||||
|
t.Fatalf("expected empty extra_params override not to erase base values, got %#v", merged.ExtraParams)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestBuildOutputArtifactDefaults(t *testing.T) {
|
func TestBuildOutputArtifactDefaults(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -1189,6 +1703,14 @@ func singleInputRef() map[string]domain.ArtifactRef {
|
|||||||
return map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}}
|
return map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func float64Ptr(v float64) *float64 {
|
||||||
|
return &v
|
||||||
|
}
|
||||||
|
|
||||||
|
func intPtr(v int) *int {
|
||||||
|
return &v
|
||||||
|
}
|
||||||
|
|
||||||
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
|
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
|
||||||
return NewRunner(
|
return NewRunner(
|
||||||
promptRepo,
|
promptRepo,
|
||||||
|
|||||||
Reference in New Issue
Block a user