Compare commits
28 Commits
v0.9.0
...
e1e5351c5d
| Author | SHA1 | Date | |
|---|---|---|---|
| e1e5351c5d | |||
| d60ef66f53 | |||
| 4669b73d38 | |||
| 6f91603168 | |||
| 3ad247039b | |||
| 32e2433628 | |||
| 712c6b92b8 | |||
| 89cafcefec | |||
| 1d7fac0a47 | |||
| 03d4f27d2b | |||
| 4ac2038331 | |||
| 14a7e7e04c | |||
| 5e522bad8b | |||
| 23872dd742 | |||
| 7ffbf5f6ca | |||
| d0dc30fcc9 | |||
| b38f7b4dc3 | |||
| 0512995931 | |||
| 049a5feadb | |||
| 1798e9c575 | |||
| 5d4bc8c2b9 | |||
| 63fb8fc132 | |||
| 4d4bb7a121 | |||
| 5dcb3cd4fc | |||
| efe346893c | |||
| c95d6fcfec | |||
| 0badb4364d | |||
| 1f63f8afbb |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1,6 +1,5 @@
|
||||
# ---> Codex
|
||||
.codex
|
||||
AGENTS.md
|
||||
|
||||
# ---> Go
|
||||
# If you prefer the allow list template instead of the deny list, see community template:
|
||||
|
||||
@@ -28,10 +28,6 @@ steps:
|
||||
|
||||
build_binary linux amd64 ""
|
||||
build_binary linux arm64 ""
|
||||
build_binary darwin amd64 ""
|
||||
build_binary darwin arm64 ""
|
||||
build_binary windows amd64 ".exe"
|
||||
build_binary windows arm64 ".exe"
|
||||
|
||||
- name: publish-release
|
||||
image: woodpeckerci/plugin-release
|
||||
|
||||
4
AGENTS.md
Normal file
4
AGENTS.md
Normal file
@@ -0,0 +1,4 @@
|
||||
Please carefully review the relevant documents in `docs/policy` before making any changes to this repository.
|
||||
- `development.md` defines the contributor workflow for this application.
|
||||
- `architecture.md` provides the canonical high-level architecture policy for this repository, and should be reviewed before writing or changing any code.
|
||||
- `documentation.md` provides the canonical documentation policy for this repository, and should be reviewed before writing or changing any documentation.
|
||||
@@ -25,6 +25,7 @@ This command renders the prepared prompt and effective runtime settings without
|
||||
- [Configuration reference](docs/config.md)
|
||||
- [Operations guide](docs/operations.md)
|
||||
- [Troubleshooting](docs/troubleshooting.md)
|
||||
- [Go library package](docs/consumers/pkg-scriptorium.md)
|
||||
- [HTTP API integration](docs/integrations/http-api.md)
|
||||
- [OpenAI-compatible chat integration](docs/integrations/openai-compatible-chat.md)
|
||||
- [Narratio subprocess integration](docs/integrations/narratio.md)
|
||||
@@ -34,3 +35,4 @@ This command renders the prepared prompt and effective runtime settings without
|
||||
|
||||
- `examples/render-markdown-summary.sh`
|
||||
- `examples/http-run.json`
|
||||
- `examples/go-library/prepare`
|
||||
|
||||
398
convert.go
Normal file
398
convert.go
Normal file
@@ -0,0 +1,398 @@
|
||||
package scriptorium
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
)
|
||||
|
||||
func toDomainRunRequest(req RunRequest) domain.RunRequest {
|
||||
return domain.RunRequest{
|
||||
PromptID: req.PromptID,
|
||||
PromptVersion: req.PromptVersion,
|
||||
ProfileID: req.ProfileID,
|
||||
APIKey: req.APIKey,
|
||||
Inputs: toDomainArtifactRefMap(req.Inputs),
|
||||
Vars: copyStringMap(req.Vars),
|
||||
Execution: toDomainExecutionTargetOverride(req.Execution),
|
||||
Validation: toDomainOutputContractPtr(req.Validation),
|
||||
Metadata: copyStringMap(req.Metadata),
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
||||
if prepared == nil {
|
||||
return nil
|
||||
}
|
||||
return &PreparedRun{
|
||||
PromptID: prepared.PromptID,
|
||||
PromptVersion: prepared.PromptVersion,
|
||||
PromptHash: prepared.PromptHash,
|
||||
SelectedProfileID: prepared.SelectedProfileID,
|
||||
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
|
||||
OutputContract: fromDomainOutputContract(prepared.OutputContract),
|
||||
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
|
||||
InputHashes: copyStringMap(prepared.InputHashes),
|
||||
SessionID: prepared.SessionID,
|
||||
RenderedPromptHash: prepared.RenderedPromptHash,
|
||||
Messages: fromDomainRenderedMessages(prepared.Messages),
|
||||
StartTime: prepared.StartTime,
|
||||
EndTime: prepared.EndTime,
|
||||
DurationMS: prepared.DurationMS,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
return &RunResult{
|
||||
RunID: result.RunID,
|
||||
Artifact: fromDomainArtifact(result.Artifact),
|
||||
RawOutput: result.RawOutput,
|
||||
Validation: fromDomainValidationResult(result.Validation),
|
||||
PromptID: result.PromptID,
|
||||
PromptVersion: result.PromptVersion,
|
||||
PromptHash: result.PromptHash,
|
||||
RenderedPromptHash: result.RenderedPromptHash,
|
||||
SelectedProfileID: result.SelectedProfileID,
|
||||
ModelName: result.ModelName,
|
||||
Endpoint: result.Endpoint,
|
||||
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
|
||||
InputHashes: copyStringMap(result.InputHashes),
|
||||
Usage: fromDomainTokenUsage(result.Usage),
|
||||
StartTime: result.StartTime,
|
||||
EndTime: result.EndTime,
|
||||
Duration: result.Duration,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainGenerateRequest(req domain.GenerateRequest) GenerateRequest {
|
||||
return GenerateRequest{
|
||||
Prompt: fromDomainRenderedPrompt(req.Prompt),
|
||||
Target: fromDomainExecutionTarget(req.Target),
|
||||
TargetPresence: fromDomainExecutionTargetPresence(req.TargetPresence),
|
||||
StructuredOutput: fromDomainStructuredOutputSpec(req.StructuredOutput),
|
||||
APIKey: req.Target.APIKey,
|
||||
}
|
||||
}
|
||||
|
||||
func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse {
|
||||
if resp == nil {
|
||||
return nil
|
||||
}
|
||||
return &domain.GenerateResponse{
|
||||
Content: resp.Content,
|
||||
Usage: toDomainTokenUsage(resp.Usage),
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainRenderedPrompt(prompt domain.RenderedPrompt) RenderedPrompt {
|
||||
return RenderedPrompt{
|
||||
SessionID: prompt.SessionID,
|
||||
Messages: fromDomainRenderedMessages(prompt.Messages),
|
||||
}
|
||||
}
|
||||
|
||||
func toDomainArtifactRefMap(src map[string]ArtifactRef) map[string]domain.ArtifactRef {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]domain.ArtifactRef, len(src))
|
||||
for k, v := range src {
|
||||
out[k] = toDomainArtifactRef(v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func toDomainArtifactRef(ref ArtifactRef) domain.ArtifactRef {
|
||||
return domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefType(ref.Type),
|
||||
URI: ref.URI,
|
||||
Body: ref.Body,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainArtifact(artifact domain.Artifact) Artifact {
|
||||
return Artifact{
|
||||
Name: artifact.Name,
|
||||
ContentType: artifact.ContentType,
|
||||
Body: copyBytes(artifact.Body),
|
||||
URI: artifact.URI,
|
||||
Size: artifact.Size,
|
||||
Hash: artifact.Hash,
|
||||
}
|
||||
}
|
||||
|
||||
func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) *domain.ExecutionTargetOverride {
|
||||
if override == nil {
|
||||
return nil
|
||||
}
|
||||
return &domain.ExecutionTargetOverride{
|
||||
Endpoint: override.Endpoint,
|
||||
Model: override.Model,
|
||||
Temperature: copyFloat64Ptr(override.Temperature),
|
||||
MaxTokens: copyIntPtr(override.MaxTokens),
|
||||
TopP: copyFloat64Ptr(override.TopP),
|
||||
TimeoutSeconds: copyIntPtr(override.TimeoutSeconds),
|
||||
ServiceTier: override.ServiceTier,
|
||||
ReasoningEffort: override.ReasoningEffort,
|
||||
APIKeyEnv: override.APIKeyEnv,
|
||||
ExtraParams: copyAnyMap(override.ExtraParams),
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
||||
return ExecutionTarget{
|
||||
Endpoint: target.Endpoint,
|
||||
Model: target.Model,
|
||||
Temperature: target.Temperature,
|
||||
MaxTokens: target.MaxTokens,
|
||||
TopP: target.TopP,
|
||||
TimeoutSeconds: target.TimeoutSeconds,
|
||||
ServiceTier: target.ServiceTier,
|
||||
ReasoningEffort: target.ReasoningEffort,
|
||||
APIKeyEnv: target.APIKeyEnv,
|
||||
ExtraParams: copyAnyMap(target.ExtraParams),
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
||||
return ExecutionTargetPresence{
|
||||
Temperature: presence.Temperature,
|
||||
MaxTokens: presence.MaxTokens,
|
||||
TopP: presence.TopP,
|
||||
TimeoutSeconds: presence.TimeoutSeconds,
|
||||
}
|
||||
}
|
||||
|
||||
func toDomainOutputContractPtr(contract *OutputContract) *domain.OutputContract {
|
||||
if contract == nil {
|
||||
return nil
|
||||
}
|
||||
out := toDomainOutputContract(*contract)
|
||||
return &out
|
||||
}
|
||||
|
||||
func toDomainOutputContract(contract OutputContract) domain.OutputContract {
|
||||
return domain.OutputContract{
|
||||
Format: domain.OutputFormat(contract.Format),
|
||||
ValidationMode: domain.ValidationMode(contract.ValidationMode),
|
||||
SchemaPath: contract.SchemaPath,
|
||||
RepairAttempts: contract.RepairAttempts,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainOutputContract(contract domain.OutputContract) OutputContract {
|
||||
return OutputContract{
|
||||
Format: OutputFormat(contract.Format),
|
||||
ValidationMode: ValidationMode(contract.ValidationMode),
|
||||
SchemaPath: contract.SchemaPath,
|
||||
RepairAttempts: contract.RepairAttempts,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainValidationResult(result domain.ValidationResult) ValidationResult {
|
||||
return ValidationResult{
|
||||
Status: ValidationStatus(result.Status),
|
||||
Mode: ValidationMode(result.Mode),
|
||||
Errors: copyStringSlice(result.Errors),
|
||||
SchemaPath: result.SchemaPath,
|
||||
RepairAttempts: result.RepairAttempts,
|
||||
IsValid: result.IsValid,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainTokenUsage(usage domain.TokenUsage) TokenUsage {
|
||||
return TokenUsage{
|
||||
PromptTokens: usage.PromptTokens,
|
||||
CompletionTokens: usage.CompletionTokens,
|
||||
TotalTokens: usage.TotalTokens,
|
||||
CachedTokens: usage.CachedTokens,
|
||||
CacheWriteTokens: usage.CacheWriteTokens,
|
||||
}
|
||||
}
|
||||
|
||||
func toDomainTokenUsage(usage TokenUsage) domain.TokenUsage {
|
||||
return domain.TokenUsage{
|
||||
PromptTokens: usage.PromptTokens,
|
||||
CompletionTokens: usage.CompletionTokens,
|
||||
TotalTokens: usage.TotalTokens,
|
||||
CachedTokens: usage.CachedTokens,
|
||||
CacheWriteTokens: usage.CacheWriteTokens,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainRenderedMessages(messages []domain.RenderedMessage) []RenderedMessage {
|
||||
if messages == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]RenderedMessage, len(messages))
|
||||
for i, msg := range messages {
|
||||
out[i] = RenderedMessage{
|
||||
Role: msg.Role,
|
||||
Content: msg.Content,
|
||||
CacheControl: fromDomainCacheControl(msg.CacheControl),
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl {
|
||||
if cacheControl == nil {
|
||||
return nil
|
||||
}
|
||||
return &CacheControl{
|
||||
Type: CacheControlType(cacheControl.Type),
|
||||
TTL: cacheControl.TTL,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainStructuredOutputSpec(spec *domain.StructuredOutputSpec) *StructuredOutputSpec {
|
||||
if spec == nil {
|
||||
return nil
|
||||
}
|
||||
out := &StructuredOutputSpec{
|
||||
Type: StructuredOutputType(spec.Type),
|
||||
}
|
||||
if spec.JSONSchema != nil {
|
||||
out.JSONSchema = &StructuredOutputJSONSpec{
|
||||
Name: spec.JSONSchema.Name,
|
||||
Strict: spec.JSONSchema.Strict,
|
||||
Schema: copyAny(spec.JSONSchema.Schema),
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func copyStringMap(src map[string]string) map[string]string {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(src))
|
||||
for k, v := range src {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func copyAnyMap(src map[string]any) map[string]any {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(src))
|
||||
for k, v := range src {
|
||||
out[k] = copyAny(v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func copyAny(value any) any {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
switch v := value.(type) {
|
||||
case map[string]any:
|
||||
return copyAnyMap(v)
|
||||
case []any:
|
||||
out := make([]any, len(v))
|
||||
for i, item := range v {
|
||||
out[i] = copyAny(item)
|
||||
}
|
||||
return out
|
||||
case []string:
|
||||
return copyStringSlice(v)
|
||||
case []byte:
|
||||
return copyBytes(v)
|
||||
default:
|
||||
return copyReflectValue(reflect.ValueOf(value)).Interface()
|
||||
}
|
||||
}
|
||||
|
||||
func copyReflectValue(value reflect.Value) reflect.Value {
|
||||
if !value.IsValid() {
|
||||
return value
|
||||
}
|
||||
|
||||
switch value.Kind() {
|
||||
case reflect.Interface:
|
||||
if value.IsNil() {
|
||||
return reflect.Zero(value.Type())
|
||||
}
|
||||
copied := copyReflectValue(value.Elem())
|
||||
if copied.IsValid() && copied.Type().AssignableTo(value.Type()) {
|
||||
return copied
|
||||
}
|
||||
out := reflect.New(value.Type()).Elem()
|
||||
out.Set(copied)
|
||||
return out
|
||||
case reflect.Pointer:
|
||||
if value.IsNil() {
|
||||
return reflect.Zero(value.Type())
|
||||
}
|
||||
out := reflect.New(value.Type().Elem())
|
||||
out.Elem().Set(copyReflectValue(value.Elem()))
|
||||
return out
|
||||
case reflect.Map:
|
||||
if value.IsNil() {
|
||||
return reflect.Zero(value.Type())
|
||||
}
|
||||
out := reflect.MakeMapWithSize(value.Type(), value.Len())
|
||||
iter := value.MapRange()
|
||||
for iter.Next() {
|
||||
out.SetMapIndex(copyReflectValue(iter.Key()), copyReflectValue(iter.Value()))
|
||||
}
|
||||
return out
|
||||
case reflect.Slice:
|
||||
if value.IsNil() {
|
||||
return reflect.Zero(value.Type())
|
||||
}
|
||||
out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap())
|
||||
for i := 0; i < value.Len(); i++ {
|
||||
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
||||
}
|
||||
return out
|
||||
case reflect.Array:
|
||||
out := reflect.New(value.Type()).Elem()
|
||||
for i := 0; i < value.Len(); i++ {
|
||||
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
||||
}
|
||||
return out
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
func copyStringSlice(src []string) []string {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, len(src))
|
||||
copy(out, src)
|
||||
return out
|
||||
}
|
||||
|
||||
func copyBytes(src []byte) []byte {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]byte, len(src))
|
||||
copy(out, src)
|
||||
return out
|
||||
}
|
||||
|
||||
func copyFloat64Ptr(src *float64) *float64 {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
v := *src
|
||||
return &v
|
||||
}
|
||||
|
||||
func copyIntPtr(src *int) *int {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
v := *src
|
||||
return &v
|
||||
}
|
||||
19
docs/cli.md
19
docs/cli.md
@@ -29,9 +29,13 @@ Integration references:
|
||||
- `run` and `render` require:
|
||||
- `--prompt`
|
||||
- at least one `--input`
|
||||
- an effective `prompt_dir` and `profile_dir` (from flags or config)
|
||||
- `serve` requires an effective `prompt_dir` and `profile_dir` (from flags or config).
|
||||
- an effective `prompt_dir` from flags or config
|
||||
- `serve` requires an effective `prompt_dir` from flags or config.
|
||||
- `profile_dir` is optional. If omitted, only built-in profiles are available; if provided, custom profiles override built-ins with the same ID.
|
||||
- Built-in profile IDs are listed in the [configuration reference](config.md#profile-definition-files).
|
||||
- Positional arguments are rejected.
|
||||
- Prompt cache control is configured in prompt YAML (`messages[].cache_control`), not with CLI flags.
|
||||
- Provider-specific `reasoning_effort` and `extra_params` are configured in profile YAML or HTTP model overrides, not with CLI flags.
|
||||
|
||||
## Flag Reference
|
||||
|
||||
@@ -39,7 +43,7 @@ Integration references:
|
||||
|
||||
- `--config <path>`: app config file path.
|
||||
- `--prompt-dir <dir>`: prompt definition directory.
|
||||
- `--profile-dir <dir>`: profile definition directory.
|
||||
- `--profile-dir <dir>`: custom profile definition directory.
|
||||
- `--schema-dir <dir>`: schema base directory for `json_schema` validation.
|
||||
- `--prompt <id>`: prompt ID to execute. Required.
|
||||
- `--prompt-id <id>`: deprecated alias for `--prompt`.
|
||||
@@ -56,6 +60,11 @@ Integration references:
|
||||
- `--top-p <float>`: runtime top-p override.
|
||||
- `--timeout <duration>`: runtime timeout override (Go duration syntax, for example `30s`, `2m`).
|
||||
|
||||
Numeric runtime override flags are presence-aware:
|
||||
|
||||
- omitted numeric flags preserve the selected profile/default value
|
||||
- explicit zero values override the selected profile/default value (`--temperature 0`, `--max-tokens 0`, `--top-p 0`, `--timeout 0s`)
|
||||
|
||||
### `scriptorium render`
|
||||
|
||||
- Supports the same flags as `run`, except:
|
||||
@@ -72,7 +81,7 @@ Notes:
|
||||
- `--config <path>`: app config file path.
|
||||
- `--addr <listen-address>`: HTTP listen address.
|
||||
- `--prompt-dir <dir>`: prompt definition directory.
|
||||
- `--profile-dir <dir>`: profile definition directory.
|
||||
- `--profile-dir <dir>`: custom profile definition directory.
|
||||
- `--schema-dir <dir>`: schema base directory for `json_schema` validation.
|
||||
|
||||
Notes:
|
||||
@@ -82,6 +91,7 @@ Notes:
|
||||
|
||||
- `--input name=path` maps prompt input names to local file paths.
|
||||
- `--var name=value` maps template variable names to values.
|
||||
- If a prompt defines `session_id: "{{ .session_id }}"`, pass the OpenRouter sticky-routing value with `--var session_id=<value>`.
|
||||
- Both flags can be repeated.
|
||||
- Both flags also support comma-separated batches, for example:
|
||||
- `--input transcript=./t.md,glossary=./g.yml`
|
||||
@@ -93,6 +103,7 @@ Notes:
|
||||
- Writes generated artifact content to stdout by default.
|
||||
- Writes generated artifact content to `--out` when provided.
|
||||
- Prints run summary metadata to stderr on success.
|
||||
- Appends `cached_tokens=<n> cache_write_tokens=<n>` to the summary only when the provider reports non-zero cache usage.
|
||||
- Prints errors to stderr on failure.
|
||||
|
||||
`render`:
|
||||
|
||||
@@ -21,10 +21,9 @@ When `--config <path>` is provided, that file is required.
|
||||
|
||||
```yaml
|
||||
prompt_dir: ./examples/prompts
|
||||
profile_dir: ./examples/profiles
|
||||
```
|
||||
|
||||
This is enough to use `run` and `render` when prompt/profile files are valid.
|
||||
This is enough to use `run` and `render` when prompts select built-in profiles.
|
||||
|
||||
## Production-Oriented App Config
|
||||
|
||||
@@ -45,7 +44,7 @@ defaults:
|
||||
Top-level fields:
|
||||
|
||||
- `prompt_dir` (optional): default prompt definition directory.
|
||||
- `profile_dir` (optional): default profile definition directory.
|
||||
- `profile_dir` (optional): default custom profile definition directory.
|
||||
- `schema_dir` (optional): base directory for schema files used by `json_schema` validation.
|
||||
- `server.addr` (optional): default listen address for `serve`.
|
||||
- `defaults.render_format` (optional): default `render` output format (`text` or `json`).
|
||||
@@ -104,6 +103,7 @@ Field reference:
|
||||
- `version` (required): prompt version.
|
||||
- `default_profile` (optional): profile ID used when request does not provide `profile_id`.
|
||||
- `description` (optional): prompt description.
|
||||
- `session_id` (optional): Go-template string for OpenRouter sticky-routing `session_id`; rendered from request vars.
|
||||
- `inputs` (optional list): expected named inputs.
|
||||
- `messages` (required list): prompt message templates.
|
||||
- `output` (required object): output contract.
|
||||
@@ -119,6 +119,7 @@ Field reference:
|
||||
|
||||
- `role` (required)
|
||||
- `content` or `content_file` (exactly one is required)
|
||||
- `cache_control` (optional object): provider prompt-cache metadata for this message
|
||||
|
||||
Message rules:
|
||||
|
||||
@@ -128,6 +129,35 @@ Message rules:
|
||||
- Prompt decoding is strict; unknown YAML fields are rejected.
|
||||
- Duplicate prompt IDs are invalid. If multiple files declare the requested prompt ID, Scriptorium fails instead of choosing one.
|
||||
|
||||
`messages[].cache_control` fields:
|
||||
|
||||
- `type` (required when `cache_control` is present): currently only `ephemeral`.
|
||||
- `ttl` (optional): currently only `1h`; omitted from outbound requests when unset.
|
||||
|
||||
Example cache-controlled message:
|
||||
|
||||
```yaml
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./stable_context.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
ttl: 1h
|
||||
- role: user
|
||||
content: |
|
||||
{{input "transcript"}}
|
||||
```
|
||||
|
||||
Use cache control on stable reusable prompt content. Dynamic per-run inputs before the cache-controlled message change the provider cache key.
|
||||
|
||||
Example prompt-level session ID:
|
||||
|
||||
```yaml
|
||||
session_id: "{{ .session_id }}"
|
||||
```
|
||||
|
||||
When configured, `session_id` is rendered with the same variable context as messages. The rendered value is trimmed, omitted when empty, and rejected if longer than 256 characters. CLI callers pass the value through `--var session_id=<value>`; HTTP callers pass it through `"vars": {"session_id": "<value>"}`.
|
||||
|
||||
`output` fields:
|
||||
|
||||
- `format` (required): `text`, `markdown`, or `json`.
|
||||
@@ -142,7 +172,7 @@ Repair behavior boundary:
|
||||
|
||||
## Profile Definition Files
|
||||
|
||||
Execution profiles are YAML files anywhere under `profile_dir`, including nested subdirectories.
|
||||
Scriptorium includes built-in execution profiles. Custom execution profiles are YAML files anywhere under `profile_dir`, including nested subdirectories.
|
||||
|
||||
Subdirectories are organizational only. Callers still select profiles by the YAML `id`, not by file path. For example, `profiles/local/local-quality.yaml` may still declare `id: local-quality`, and callers use `--profile local-quality`.
|
||||
|
||||
@@ -158,6 +188,11 @@ top_p: 1.0
|
||||
timeout_seconds: 90
|
||||
api_key_env: SCRIPTORIUM_API_KEY
|
||||
service_tier: priority
|
||||
reasoning_effort: medium
|
||||
extra_params:
|
||||
provider_route: primary
|
||||
provider_options:
|
||||
retry_budget: 2
|
||||
```
|
||||
|
||||
Field reference:
|
||||
@@ -170,21 +205,55 @@ Field reference:
|
||||
- `top_p` (optional): range `0..1`
|
||||
- `timeout_seconds` (optional): `>= 0`
|
||||
- `service_tier` (optional): provider-specific request tier such as OpenRouter `flex` or `priority`
|
||||
- `reasoning_effort` (optional)
|
||||
- `reasoning_effort` (optional): serialized as top-level `reasoning_effort` in outbound chat-completions requests
|
||||
- `api_key_env` (optional)
|
||||
- `extra_params` (optional map of strings)
|
||||
- `extra_params` (optional map): JSON-compatible provider-specific parameters. Values may be strings, numbers, booleans, objects, or arrays.
|
||||
|
||||
Profile rules:
|
||||
|
||||
- `profile_dir` is optional. If omitted, only built-in profiles are available.
|
||||
- If `profile_dir` is set, custom profiles from that directory override built-in profiles with the same `id`.
|
||||
- Duplicate IDs within the custom profile directory are invalid. Matching IDs across custom and built-in profiles are valid override behavior.
|
||||
- Profile decoding is strict; unknown YAML fields are rejected.
|
||||
- Raw `api_key` is rejected; use `api_key_env`.
|
||||
- If `api_key_env` is set, that environment variable must be set when preparing/running.
|
||||
- Duplicate profile IDs are invalid. If multiple files declare the requested profile ID, Scriptorium fails instead of choosing one.
|
||||
- `extra_params` keys must not be empty and must not collide with reserved outbound request fields: `model`, `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, or `response_format`.
|
||||
|
||||
Built-in profile IDs:
|
||||
|
||||
| Provider | ID | Model | API key env |
|
||||
| --- | --- | --- | --- |
|
||||
| aion-labs | `aion-2` | `aion-labs/aion-2.0` | `OPENROUTER_API_KEY` |
|
||||
| anthropic | `claude-fable-latest` | `~anthropic/claude-fable-latest` | `OPENROUTER_API_KEY` |
|
||||
| anthropic | `claude-haiku-latest` | `~anthropic/claude-haiku-latest` | `OPENROUTER_API_KEY` |
|
||||
| anthropic | `claude-opus-latest` | `~anthropic/claude-opus-latest` | `OPENROUTER_API_KEY` |
|
||||
| anthropic | `claude-sonnet-latest` | `~anthropic/claude-sonnet-latest` | `OPENROUTER_API_KEY` |
|
||||
| deepseek | `deepseek-3-2` | `deepseek/deepseek-v3.2` | `OPENROUTER_API_KEY` |
|
||||
| deepseek | `deepseek-4-pro` | `deepseek/deepseek-v4-pro` | `OPENROUTER_API_KEY` |
|
||||
| google | `gemini-2-flash` | `google/gemini-2.5-flash` | `OPENROUTER_API_KEY` |
|
||||
| google | `gemini-2-flash-lite` | `google/gemini-2.5-flash-lite` | `OPENROUTER_API_KEY` |
|
||||
| google | `gemini-2-pro` | `google/gemini-2.5-pro` | `OPENROUTER_API_KEY` |
|
||||
| google | `gemini-3-flash-lite` | `google/gemini-3.1-flash-lite` | `OPENROUTER_API_KEY` |
|
||||
| google | `gemini-flash-latest` | `~google/gemini-flash-latest` | `OPENROUTER_API_KEY` |
|
||||
| google | `gemini-pro-latest` | `~google/gemini-pro-latest` | `OPENROUTER_API_KEY` |
|
||||
| google | `gemma-4-31b` | `google/gemma-4-31b-it:exacto` | `OPENROUTER_API_KEY` |
|
||||
| minimax | `minimax-m2` | `minimax/minimax-m2.5` | `OPENROUTER_API_KEY` |
|
||||
| minimax | `minimax-m3` | `minimax/minimax-m3` | `OPENROUTER_API_KEY` |
|
||||
| mistral | `mistral-large-2512` | `mistralai/mistral-large-2512` | `OPENROUTER_API_KEY` |
|
||||
| mistral | `mistral-medium-3-5` | `mistralai/mistral-medium-3-5` | `OPENROUTER_API_KEY` |
|
||||
| mistral | `mistral-small-3` | `mistralai/mistral-small-3.2-24b-instruct` | `OPENROUTER_API_KEY` |
|
||||
| mistral | `mistral-small-4` | `mistralai/mistral-small-2603` | `OPENROUTER_API_KEY` |
|
||||
| nvidia | `nemotron-3-ultra` | `nvidia/nemotron-3-ultra-550b-a55b` | `OPENROUTER_API_KEY` |
|
||||
| openai | `gpt-5-mini` | `openai/gpt-5.4-mini` | `OPENROUTER_API_KEY` |
|
||||
| openai | `gpt-5-nano` | `openai/gpt-5.4-nano` | `OPENROUTER_API_KEY` |
|
||||
|
||||
Current outbound request behavior:
|
||||
|
||||
- The OpenAI-compatible client currently serializes: `model`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, and optional `response_format` for `json_schema` prompts.
|
||||
- `reasoning_effort` and `extra_params` are parsed and carried in effective settings, but are not currently serialized into outbound chat-completions requests.
|
||||
- The OpenAI-compatible client currently serializes: `model`, optional `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, optional `response_format` for `json_schema` prompts, and `extra_params`.
|
||||
- `extra_params` are flattened into provider-specific top-level JSON request fields. They are not wrapped in an `extra_params` object on the outbound provider request.
|
||||
- Messages without `cache_control` serialize with string `content`.
|
||||
- Messages with `cache_control` serialize as a single text content-block array containing `cache_control`.
|
||||
|
||||
## Schema Behavior
|
||||
|
||||
@@ -211,7 +280,7 @@ Supported artifact reference types for request inputs are `file` and `inline`.
|
||||
|
||||
- App config: `examples/config.yml`
|
||||
- Prompt examples: `examples/prompts/`
|
||||
- Profile examples: `examples/profiles/`
|
||||
- Custom profile examples: `examples/profiles/`
|
||||
- Schema examples: `examples/schemas/`
|
||||
- Input fixtures: `examples/fixtures/`
|
||||
- Render example script: `examples/render-markdown-summary.sh`
|
||||
|
||||
13
docs/consumers/api.md
Normal file
13
docs/consumers/api.md
Normal file
@@ -0,0 +1,13 @@
|
||||
# Consumer API Overview
|
||||
|
||||
Scriptorium can be used by consumers through three implemented surfaces:
|
||||
|
||||
- CLI commands, documented in [CLI reference](../cli.md).
|
||||
- HTTP `POST /v1/runs`, documented in [HTTP API integration](../integrations/http-api.md).
|
||||
- Go package `gitea.maximumdirect.net/eric/scriptorium`, documented in [pkg-scriptorium](pkg-scriptorium.md).
|
||||
|
||||
The Go package is the typed in-process API. It prepares prompts, runs prompts, accepts file or inline artifacts, supports per-request execution overrides, and exposes stable public errors for `errors.Is`.
|
||||
|
||||
Use the Go package when the caller is a Go program that wants typed requests/results, context cancellation, repeated calls without subprocess overhead, or fake LLM injection for tests. Use the CLI or HTTP surfaces when process isolation, language neutrality, or an HTTP boundary is preferred.
|
||||
|
||||
Raw API key values are not accepted in public payloads and are not returned in prepared or run results. Execution profiles may reference an environment variable name through `api_key_env`.
|
||||
161
docs/consumers/pkg-scriptorium.md
Normal file
161
docs/consumers/pkg-scriptorium.md
Normal file
@@ -0,0 +1,161 @@
|
||||
# Package scriptorium
|
||||
|
||||
Import path:
|
||||
|
||||
```go
|
||||
import "gitea.maximumdirect.net/eric/scriptorium"
|
||||
```
|
||||
|
||||
The root package is a public facade over Scriptorium's prompt execution use case. It keeps `internal/*` packages private while exposing typed construction, preparation, execution, inputs, results, and errors.
|
||||
|
||||
## Construct An Engine
|
||||
|
||||
```go
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||
PromptDir: "./examples/prompts",
|
||||
ProfileDir: "./examples/profiles",
|
||||
SchemaDir: "./examples/schemas",
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
`PromptDir` is required unless an explicit prompt source option is supplied. `ProfileDir` is optional; omit it to use built-in profiles only, or set it to overlay custom profiles above built-ins. `SchemaDir` defaults to the built-in schema directory. `Timeout` and `HTTPClient` configure the default OpenAI-compatible client used by `Run` when no custom LLM client is supplied.
|
||||
|
||||
## Asset Sources
|
||||
|
||||
Directory fields on `Config` remain the compatibility path. Explicit source options override the matching directory field:
|
||||
|
||||
- `WithPromptFS(fsys, root)` and `WithPromptFile(path)`
|
||||
- `WithProfileFS(fsys, root)` and `WithProfileFile(path)`
|
||||
- `WithSchemaFS(fsys, root)` and `WithSchemaFile(path)`
|
||||
|
||||
Prompt and profile sources load standard Scriptorium YAML with the same strict validation as directory sources. Prompt `content_file` paths resolve relative to the prompt file in the same source. Profile options overlay custom profiles above built-ins. Schema `fs.FS` sources preserve prompt `schema_path` semantics; schema file options expose the file by its base name.
|
||||
|
||||
## In-Memory Profiles
|
||||
|
||||
Use `WithProfiles` when the consuming application already has profile settings in typed Go configuration:
|
||||
|
||||
```go
|
||||
profile := scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
ID: "app.default",
|
||||
Endpoint: "https://openrouter.ai/api/v1",
|
||||
Model: "mistralai/mistral-small-3.2-24b-instruct",
|
||||
APIKeyRequired: true,
|
||||
})
|
||||
|
||||
engine, err := scriptorium.NewEngine(cfg, scriptorium.WithProfiles(profile))
|
||||
```
|
||||
|
||||
In-memory profiles have highest precedence, followed by configured profile file/FS/directory sources, then built-in profiles. Duplicate IDs in one `WithProfiles` call return `ErrInvalidConfig`.
|
||||
|
||||
`Profile` and `OpenAICompatibleProfileConfig` include endpoint, model, numeric defaults, service tier, reasoning effort, `APIKeyRequired`, and JSON-compatible `ExtraParams`. They do not accept raw API-key fields. When `APIKeyRequired` is true, pass the secret with `RunRequest.APIKey`.
|
||||
|
||||
## Prepare A Prompt
|
||||
|
||||
`Prepare` resolves the prompt definition, profile, inputs, variables, output contract, structured-output metadata, and rendered messages without calling an LLM.
|
||||
|
||||
```go
|
||||
prepared, err := engine.Prepare(ctx, scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = prepared.Messages
|
||||
```
|
||||
|
||||
Input helpers:
|
||||
|
||||
- `scriptorium.File(path)` loads an input artifact from a file.
|
||||
- `scriptorium.Inline(body)` passes inline input content.
|
||||
- `scriptorium.InlineWithURI(uri, body)` passes inline content with URI metadata.
|
||||
|
||||
## Run A Prompt
|
||||
|
||||
`Run` prepares the prompt, calls the configured LLM client, builds the output artifact, and validates the output.
|
||||
|
||||
```go
|
||||
result, err := engine.Run(ctx, scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
APIKey: apiKey,
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = result.Artifact
|
||||
```
|
||||
|
||||
`RunResult` includes the run ID, output artifact, raw output, validation result, prompt/profile/model metadata, effective model parameters, input hashes, token/cache usage, and timing fields. Validation content failures return a successful `RunResult` with failed validation status. Runtime validation errors return `ErrValidation`.
|
||||
|
||||
For the public Go API, pass provider credentials with `RunRequest.APIKey`. The value is request-scoped, uses `json:"-"`, is preferred over profile `api_key_env` by the default OpenAI-compatible client, and is not included in `PreparedRun` or `RunResult` JSON. Do not store raw keys in config, prompt files, or profile YAML.
|
||||
|
||||
## Inject An LLM Client
|
||||
|
||||
Use `WithLLMClient` for tests or custom model integrations:
|
||||
|
||||
```go
|
||||
type fakeLLM struct{}
|
||||
|
||||
func (fakeLLM) Generate(ctx context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
|
||||
return &scriptorium.GenerateResponse{
|
||||
Content: "generated text",
|
||||
Usage: scriptorium.TokenUsage{TotalTokens: 12},
|
||||
}, nil
|
||||
}
|
||||
|
||||
engine, err := scriptorium.NewEngine(cfg, scriptorium.WithLLMClient(fakeLLM{}))
|
||||
```
|
||||
|
||||
The injected client receives the rendered prompt, effective execution target, target presence metadata for explicit numeric overrides, structured-output spec, and request API key when provided. `GenerateRequest.APIKey` also uses `json:"-"`; custom and fake clients should avoid logging or serializing it. `WithLLMClient(nil)` returns `ErrInvalidConfig`.
|
||||
|
||||
## Request Overrides
|
||||
|
||||
`RunRequest.Execution` accepts per-request overrides. Numeric override fields are pointers so explicit zero values are preserved:
|
||||
|
||||
```go
|
||||
zero := 0
|
||||
req.Execution = &scriptorium.ExecutionTargetOverride{
|
||||
MaxTokens: &zero,
|
||||
}
|
||||
```
|
||||
|
||||
## Errors
|
||||
|
||||
Public methods wrap context while preserving stable sentinel checks with `errors.Is`:
|
||||
|
||||
- `ErrInvalidConfig`
|
||||
- `ErrInvalidRequest`
|
||||
- `ErrPromptNotFound`
|
||||
- `ErrProfileNotFound`
|
||||
- `ErrPromptLoad`
|
||||
- `ErrProfileLoad`
|
||||
- `ErrArtifactLoad`
|
||||
- `ErrPromptRender`
|
||||
- `ErrLLMGenerate`
|
||||
- `ErrValidation`
|
||||
|
||||
Example:
|
||||
|
||||
```go
|
||||
if errors.Is(err, scriptorium.ErrPromptNotFound) {
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
Run the prepare-only example from the repository root:
|
||||
|
||||
```bash
|
||||
go run ./examples/go-library/prepare
|
||||
```
|
||||
@@ -50,7 +50,10 @@ Copyable request example file:
|
||||
"reasoning_effort": "medium",
|
||||
"api_key_env": "SCRIPTORIUM_API_KEY",
|
||||
"extra_params": {
|
||||
"route": "primary"
|
||||
"route": "primary",
|
||||
"provider_options": {
|
||||
"retry_budget": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"include_raw_output": false
|
||||
@@ -67,6 +70,14 @@ Input reference types currently supported by runtime artifact loading:
|
||||
- `file`
|
||||
- `inline`
|
||||
|
||||
Model override notes:
|
||||
|
||||
- Numeric model override fields distinguish omitted values from explicit zero values. For example, omitting `temperature` preserves the selected profile/default value, while `"temperature": 0` explicitly sets the effective temperature to zero.
|
||||
- `extra_params` accepts JSON-compatible values: strings, numbers, booleans, objects, and arrays.
|
||||
- `extra_params` are passed through effective model metadata and flattened into top-level provider request fields by the OpenAI-compatible client.
|
||||
- `extra_params` keys must not be empty and must not collide with reserved outbound fields: `model`, `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, or `response_format`.
|
||||
- Raw API-key values are not accepted. Use `api_key_env` to name an environment variable.
|
||||
|
||||
## Strict JSON Rules
|
||||
|
||||
Request decoding uses strict JSON field checks:
|
||||
@@ -119,7 +130,10 @@ Response shape:
|
||||
"reasoning_effort": "medium",
|
||||
"api_key_env": "SCRIPTORIUM_API_KEY",
|
||||
"extra_params": {
|
||||
"route": "primary"
|
||||
"route": "primary",
|
||||
"provider_options": {
|
||||
"retry_budget": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"input_hashes": {
|
||||
@@ -128,7 +142,9 @@ Response shape:
|
||||
"usage": {
|
||||
"prompt_tokens": 11,
|
||||
"completion_tokens": 22,
|
||||
"total_tokens": 33
|
||||
"total_tokens": 33,
|
||||
"cached_tokens": 0,
|
||||
"cache_write_tokens": 0
|
||||
},
|
||||
"start_time": "2026-05-04T12:00:00Z",
|
||||
"end_time": "2026-05-04T12:00:01Z",
|
||||
@@ -142,6 +158,8 @@ Response shape:
|
||||
|
||||
`raw_model_output` is omitted by default.
|
||||
|
||||
`metadata.usage.cached_tokens` and `metadata.usage.cache_write_tokens` are always present as numbers. They are `0` when the provider omits compatible cache usage fields or reports no cache activity.
|
||||
|
||||
To include it, send:
|
||||
|
||||
- `"include_raw_output": true`
|
||||
|
||||
@@ -26,15 +26,85 @@ Example:
|
||||
Serialized JSON fields:
|
||||
|
||||
- `model` (required after fallback resolution)
|
||||
- `messages` (role/content pairs from rendered prompt)
|
||||
- `temperature` (only when non-zero)
|
||||
- `max_tokens` (only when non-zero)
|
||||
- `top_p` (only when non-zero)
|
||||
- `session_id` (only when the rendered prompt includes a non-empty session ID)
|
||||
- `messages` (rendered prompt messages)
|
||||
- `temperature` (when non-zero, or when explicitly overridden to zero)
|
||||
- `max_tokens` (when non-zero, or when explicitly overridden to zero)
|
||||
- `top_p` (when non-zero, or when explicitly overridden to zero)
|
||||
- `service_tier` (only when non-empty)
|
||||
- `reasoning_effort` (only when non-empty)
|
||||
- `response_format` (only when structured output is provided)
|
||||
- profile/request `extra_params` as additional provider-specific top-level fields
|
||||
|
||||
`service_tier` is provider-specific. OpenRouter currently documents request values such as `flex` and `priority`; Scriptorium forwards any non-empty configured value and lets the backend validate support.
|
||||
|
||||
`reasoning_effort` is provider-specific. Scriptorium forwards any non-empty configured value as top-level `reasoning_effort` and lets the backend validate support.
|
||||
|
||||
`extra_params` are flattened into the outbound JSON object. They are not wrapped in an `extra_params` object:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "gpt-4o-mini",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "rendered text"
|
||||
}
|
||||
],
|
||||
"provider_route": "primary",
|
||||
"provider_options": {
|
||||
"retry_budget": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`extra_params` values must be JSON-compatible. Supported value shapes include strings, numbers, booleans, objects, and arrays.
|
||||
|
||||
Reserved `extra_params` keys are rejected before the HTTP request is made:
|
||||
|
||||
- `model`
|
||||
- `session_id`
|
||||
- `messages`
|
||||
- `temperature`
|
||||
- `max_tokens`
|
||||
- `top_p`
|
||||
- `service_tier`
|
||||
- `reasoning_effort`
|
||||
- `response_format`
|
||||
|
||||
Empty `extra_params` keys and values that cannot be encoded as JSON are also rejected before the HTTP request is made.
|
||||
|
||||
`session_id` is rendered from prompt YAML using request variables and serialized as a top-level JSON request field. Scriptorium does not send an `x-session-id` header. Empty rendered session IDs are omitted, and values longer than 256 characters are rejected before the HTTP request.
|
||||
|
||||
Messages without prompt cache control serialize with string `content`:
|
||||
|
||||
```json
|
||||
{
|
||||
"role": "system",
|
||||
"content": "rendered text"
|
||||
}
|
||||
```
|
||||
|
||||
Messages with prompt cache control serialize as a single text content-block array:
|
||||
|
||||
```json
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "rendered text",
|
||||
"cache_control": {
|
||||
"type": "ephemeral",
|
||||
"ttl": "1h"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
When cache-control `ttl` is unset in the prompt definition, `ttl` is omitted from the outbound payload.
|
||||
|
||||
Structured output is currently `json_schema` only, serialized as:
|
||||
|
||||
```json
|
||||
@@ -52,7 +122,12 @@ Structured output is currently `json_schema` only, serialized as:
|
||||
|
||||
## Authentication Header
|
||||
|
||||
If `Target.APIKeyEnv` is set:
|
||||
If `Target.APIKey` is set:
|
||||
|
||||
- set `Authorization: Bearer <value>`
|
||||
- do not read `Target.APIKeyEnv`
|
||||
|
||||
If `Target.APIKey` is empty and `Target.APIKeyEnv` is set:
|
||||
|
||||
- resolve environment variable value at request time
|
||||
- set `Authorization: Bearer <value>`
|
||||
@@ -61,7 +136,7 @@ If the environment variable is unset/empty:
|
||||
|
||||
- request fails before HTTP call (`ErrInvalidRequest`)
|
||||
|
||||
If `Target.APIKeyEnv` is empty:
|
||||
If both `Target.APIKey` and `Target.APIKeyEnv` are empty:
|
||||
|
||||
- no `Authorization` header is sent
|
||||
|
||||
@@ -72,6 +147,7 @@ Base timeout comes from client configuration.
|
||||
Per-request override:
|
||||
|
||||
- if `Target.TimeoutSeconds > 0`, use that value for request timeout
|
||||
- if `Target.TimeoutSeconds == 0` and the value came from an explicit request override, disable the HTTP client timeout
|
||||
- if `Target.TimeoutSeconds < 0`, request is rejected (`ErrInvalidRequest`)
|
||||
|
||||
## Response Expectations
|
||||
@@ -82,6 +158,13 @@ Expected successful response shape (subset used):
|
||||
- `usage.prompt_tokens`
|
||||
- `usage.completion_tokens`
|
||||
- `usage.total_tokens`
|
||||
- `usage.prompt_tokens_details.cached_tokens` (optional)
|
||||
- `usage.cache_write_tokens` (optional)
|
||||
|
||||
Absent cache usage fields are treated as zero. Parsed cache usage is exposed through run results and adapter response surfaces as:
|
||||
|
||||
- `cached_tokens`
|
||||
- `cache_write_tokens`
|
||||
|
||||
Malformed response conditions include:
|
||||
|
||||
@@ -99,10 +182,7 @@ Malformed responses return `ErrMalformedResponse`.
|
||||
|
||||
## Unsupported Or Non-Serialized Fields
|
||||
|
||||
The following fields may exist in profile/effective settings but are not currently serialized into outbound chat-completions payloads:
|
||||
|
||||
- `reasoning_effort`
|
||||
- `extra_params`
|
||||
The client does not serialize top-level `cache_control`.
|
||||
|
||||
No built-in retries, tool-calls, or multi-request payload modes are implemented in this client.
|
||||
|
||||
|
||||
@@ -8,12 +8,14 @@ This document describes implemented adapter/repository boundaries and their curr
|
||||
|
||||
- `internal/adapter/cli`: CLI command parsing, app wiring, stdout/stderr handling, exit codes.
|
||||
- `internal/adapter/http`: HTTP request/response mapping for `POST /v1/runs`.
|
||||
- `internal/promptdef`: filesystem prompt-definition repository.
|
||||
- `internal/profile`: filesystem execution-profile repository.
|
||||
- root package `scriptorium`: public Go library facade for preparing and running prompt requests.
|
||||
- `internal/promptdef`: filesystem and `fs.FS` prompt-definition repositories.
|
||||
- `internal/profile`: filesystem, `fs.FS`, and overlay execution-profile repositories.
|
||||
- `internal/profile/builtin`: embedded built-in execution-profile repository.
|
||||
- `internal/artifact`: input artifact reader.
|
||||
- `internal/prompt`: Go-template renderer.
|
||||
- `internal/llm`: OpenAI-compatible LLM client implementation.
|
||||
- `internal/validate`: output validator.
|
||||
- `internal/validate`: filesystem and `fs.FS` output validators.
|
||||
- `internal/format`: prepared-run formatters for `render` output.
|
||||
|
||||
## Inputs And Outputs
|
||||
@@ -22,16 +24,35 @@ CLI adapter:
|
||||
|
||||
- Input: process args, filesystem config/assets, environment.
|
||||
- Output: exit code, stdout artifact/prepared output, stderr summaries/errors.
|
||||
- `run` summaries include cache usage counters only when either parsed cache counter is non-zero.
|
||||
|
||||
HTTP adapter:
|
||||
|
||||
- Input: JSON request body (`runRequestDTO`).
|
||||
- Output: JSON success/error body with mapped status codes.
|
||||
- Success metadata includes token usage plus cache usage counters.
|
||||
|
||||
Filesystem repositories:
|
||||
Public library facade:
|
||||
|
||||
- Input: prompt/profile YAML files under configured directories.
|
||||
- Input: typed `scriptorium.RunRequest` values.
|
||||
- Output: typed `PreparedRun` and `RunResult` values plus public sentinel errors.
|
||||
- Custom LLM behavior is injected with `WithLLMClient`; otherwise the default OpenAI-compatible client is used.
|
||||
- `RunRequest.APIKey` is a request-scoped Go value only; it is converted into internal execution state for LLM generation and stripped from public result types.
|
||||
- Prompt, profile, and schema source options can use directories, single files, or `fs.FS` roots. Explicit source options override the matching `Config` directory field.
|
||||
- Public types are facade types converted at the package boundary; internal domain types remain internal.
|
||||
|
||||
Prompt/profile repositories:
|
||||
|
||||
- Input: prompt/profile YAML files under configured directories or `fs.FS` roots.
|
||||
- Output: normalized domain definitions/profiles or typed errors.
|
||||
- Single-file public sources are represented as `fs.FS` roots containing one YAML file; lookup still uses YAML `id` values.
|
||||
|
||||
Profile repository composition:
|
||||
|
||||
- Built-in profiles are embedded and loaded through the same profile validation rules as filesystem profiles.
|
||||
- When no custom profile directory is configured, the runner receives the built-in profile repository.
|
||||
- When a custom profile directory/file/`fs.FS` source is configured, the runner receives an overlay repository with custom profiles as primary and built-ins as fallback.
|
||||
- Overlay lookup falls back only after custom profile-not-found errors; custom load/validation/raw-key errors are returned directly.
|
||||
|
||||
Artifact reader:
|
||||
|
||||
@@ -42,11 +63,13 @@ LLM adapter:
|
||||
|
||||
- Input: `domain.GenerateRequest`.
|
||||
- Output: `domain.GenerateResponse`.
|
||||
- Direct API-key values are preferred when present; otherwise `api_key_env` is resolved from the process environment.
|
||||
|
||||
Validator:
|
||||
|
||||
- Input: artifact body + output contract.
|
||||
- Output: validation result or runtime validation error.
|
||||
- Schema documents may be loaded from a directory, single file, or `fs.FS` root in the public package. CLI and HTTP continue to use directory-backed schema loading.
|
||||
|
||||
## Boundaries
|
||||
|
||||
@@ -59,7 +82,7 @@ Validator:
|
||||
Primary app settings consumed by adapters:
|
||||
|
||||
- `prompt_dir`
|
||||
- `profile_dir`
|
||||
- `profile_dir` (optional custom profile source)
|
||||
- `schema_dir`
|
||||
- `server.addr`
|
||||
- `defaults.render_format`
|
||||
@@ -67,6 +90,8 @@ Primary app settings consumed by adapters:
|
||||
Execution profile/request settings used through runner:
|
||||
|
||||
- `endpoint`, `model`, `temperature`, `max_tokens`, `top_p`, `timeout_seconds`, `service_tier`, `api_key_env`, `reasoning_effort`, `extra_params`
|
||||
- CLI and HTTP request adapters preserve caller intent for numeric runtime overrides. Omitted values remain absent; explicit zero values are mapped as explicit overrides.
|
||||
- HTTP `extra_params` accepts JSON-compatible values and maps them to domain request overrides without provider-specific adapter logic.
|
||||
|
||||
## External Dependencies
|
||||
|
||||
@@ -81,7 +106,9 @@ Strict decoding and input checks:
|
||||
- config/prompt/profile loaders reject unknown YAML fields.
|
||||
- prompt/profile repositories scan nested subdirectories recursively.
|
||||
- prompt/profile lookup uses YAML `id` values; subdirectory paths are organizational only.
|
||||
- prompt `content_file` paths resolve relative to the prompt YAML file within the same source.
|
||||
- duplicate prompt/profile IDs are invalid and fail instead of using first-match behavior.
|
||||
- duplicate profile IDs across custom and built-in sources are allowed; the custom source overrides the built-in profile.
|
||||
- HTTP DTO decoder rejects unknown JSON fields.
|
||||
- raw API key payload fields are rejected by strict decoding in profile/http paths.
|
||||
|
||||
@@ -93,8 +120,16 @@ Artifact refs:
|
||||
LLM adapter:
|
||||
|
||||
- endpoint appends `/chat/completions`.
|
||||
- rendered messages without cache control serialize with string `content`.
|
||||
- rendered messages with cache control serialize as one text content block with `cache_control`.
|
||||
- non-empty `reasoning_effort` serializes as a top-level provider request field.
|
||||
- `extra_params` flatten into provider-specific top-level JSON request fields.
|
||||
- reserved `extra_params` keys are rejected before the provider call: `model`, `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, and `response_format`.
|
||||
- empty `extra_params` keys and values that cannot be JSON-encoded are rejected before the provider call.
|
||||
- compatible cache usage response fields are parsed into domain token usage.
|
||||
- non-2xx responses map to request failure errors.
|
||||
- malformed responses (including missing/empty first choice content) are errors.
|
||||
- direct API-key values are never serialized in provider request bodies.
|
||||
|
||||
Validator:
|
||||
|
||||
@@ -141,4 +176,5 @@ Behavior highlights:
|
||||
- Adapter packages do not own runner decision logic.
|
||||
- External request/response strictness is part of contract stability.
|
||||
- Prepared-render output never includes resolved API key values.
|
||||
- Outbound OpenAI-compatible request includes only currently serialized fields (`model`, `messages`, optional `temperature`, `max_tokens`, `top_p`, optional `service_tier`, optional `response_format`).
|
||||
- Outbound OpenAI-compatible request includes currently serialized first-class fields (`model`, optional `session_id`, `messages`, optional `temperature`, `max_tokens`, `top_p`, optional `service_tier`, optional `reasoning_effort`, optional `response_format`) plus validated `extra_params` flattened as provider-specific top-level fields.
|
||||
- Outbound cache control is message-level only; no top-level cache-control field is serialized.
|
||||
|
||||
@@ -103,16 +103,29 @@ Validation content failures are not run errors:
|
||||
- built-in execution defaults
|
||||
- selected profile values
|
||||
- request overrides
|
||||
6. verify required `api_key_env` environment variable:
|
||||
- missing/empty env value returns an invalid request with `ErrAPIKeyEnvMissing`
|
||||
- only the environment-variable name is retained; secret value is never returned
|
||||
- request numeric overrides are presence-aware, so omitted values preserve the current effective value and explicit zero values override it
|
||||
6. verify credentials when the effective target names `api_key_env`:
|
||||
- a request-scoped direct API key satisfies the credential requirement
|
||||
- otherwise a missing/empty env value returns an invalid request with `ErrAPIKeyEnvMissing`
|
||||
- only the environment-variable name is returned in public output; secret values are never returned
|
||||
7. resolve output contract and structured-output schema payload when `json_schema` mode is active.
|
||||
8. read input artifacts.
|
||||
9. render prompt messages.
|
||||
9. render prompt messages, including any normalized message cache-control metadata.
|
||||
10. compute prompt/input/render hashes and return `PreparedRun`.
|
||||
|
||||
`rendered_prompt_hash` includes cache-control metadata when present because it affects the outbound provider request. Prompts without cache control keep the role/content hash behavior.
|
||||
|
||||
`Prepare` does not call the LLM.
|
||||
|
||||
Runtime target notes:
|
||||
|
||||
- Profile `extra_params` and request `extra_params` carry JSON-compatible values through prepared output, run metadata, and `domain.GenerateRequest.Target`.
|
||||
- The OpenAI-compatible client serializes non-empty `reasoning_effort` as a top-level provider request field.
|
||||
- The OpenAI-compatible client flattens `extra_params` into provider-specific top-level JSON request fields.
|
||||
- Empty `extra_params` keys, reserved outbound field names, and values that cannot be JSON-encoded fail before the provider request.
|
||||
- Resolved API-key values are never serialized in prepared/run output, public results, logs, or HTTP responses.
|
||||
- Public direct API-key values are carried only far enough to call the configured LLM client and are excluded from JSON/YAML serialization.
|
||||
|
||||
## Run Flow
|
||||
|
||||
`Run` performs:
|
||||
@@ -123,7 +136,7 @@ Validation content failures are not run errors:
|
||||
4. build output artifact content type from output format.
|
||||
5. validate output.
|
||||
6. optionally attempt bounded repair when repairer is injected and contract allows it.
|
||||
7. return `RunResult` with artifact, raw output, validation, hashes, profile/model metadata, usage, and timestamps.
|
||||
7. return `RunResult` with artifact, raw output, validation, hashes, profile/model metadata, token/cache usage, and timestamps.
|
||||
|
||||
## Repair Hook Boundary
|
||||
|
||||
|
||||
@@ -2,12 +2,13 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
Project documentation must help four audiences:
|
||||
Project documentation must help five audiences:
|
||||
|
||||
1. users who need to run the application;
|
||||
2. administrators/operators who need to configure and operate it;
|
||||
3. developers who need to understand and change it safely;
|
||||
4. LLM coding agents that need clear scope, boundaries, and invariants.
|
||||
4. LLM coding agents that need clear scope, boundaries, and invariants;
|
||||
5. developers and LLM coding agents integrating this project from another codebase.
|
||||
|
||||
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
||||
|
||||
@@ -42,11 +43,14 @@ Canonical homes:
|
||||
|
||||
- project purpose and quickstart: `README.md`
|
||||
- development principles: `docs/policy/architecture.md`
|
||||
- public HTTP API reference: `docs/api.md`
|
||||
- configuration reference: `docs/config.md`
|
||||
- CLI reference: `docs/cli.md`
|
||||
- operations and recovery: `docs/operations.md`
|
||||
- troubleshooting: `docs/troubleshooting.md`
|
||||
- public API/package consumer guidance: `docs/consumers/`
|
||||
- implemented internals: `docs/internal/`
|
||||
- external protocol, service, and file-format contracts: `docs/integrations/`
|
||||
- future work: `docs/roadmap/`
|
||||
- contributor workflow: `docs/policy/development.md`
|
||||
- copyable examples: `examples/`
|
||||
@@ -106,7 +110,7 @@ Recommended:
|
||||
- `examples/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
### Modular, staged, service-oriented, or orchestration application
|
||||
### Modular, service-oriented, or orchestration application
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`, if CLI-based
|
||||
@@ -119,6 +123,31 @@ Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- validated examples under `examples/`
|
||||
|
||||
### Public HTTP API service
|
||||
|
||||
Required:
|
||||
- `docs/api.md`
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
- `docs/internal/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- `docs/consumers/`, for task-oriented client integration guides
|
||||
- `docs/integrations/`, for upstream/downstream service contracts
|
||||
- validated examples under `examples/`
|
||||
|
||||
### Project with public packages or consumer APIs
|
||||
|
||||
Required:
|
||||
- `docs/consumers/api.md`
|
||||
- one `docs/consumers/pkg-<name>.md` file per public package, if public packages exist
|
||||
|
||||
Recommended:
|
||||
- copyable consumer examples under `examples/`, if practical
|
||||
|
||||
## Required Documents
|
||||
|
||||
### README.md
|
||||
@@ -159,7 +188,35 @@ It should include:
|
||||
- architectural invariants;
|
||||
- explicit non-goals, if useful.
|
||||
|
||||
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
|
||||
Notably, this file should prescribe a core development *policy* that should remain unchanged as the application evolves. It is not a place for details (e.g., CLI flags) that could change over time.
|
||||
|
||||
The contents of `architecture.md` should be trim and concise. LLMs may be directed to review it routinely via AGENTS.md, CLAUDE.md, or similar.
|
||||
|
||||
### docs/api.md
|
||||
|
||||
**Audience:** external HTTP API consumers, developers, LLM coding agents integrating by HTTP
|
||||
|
||||
Required for projects whose primary public interface is HTTP.
|
||||
|
||||
`docs/api.md` is the canonical public HTTP API contract. It should be normative for external consumers and should not be duplicated by README, operations docs, consumer guides, or integration docs.
|
||||
|
||||
It should include:
|
||||
|
||||
1. base URL conventions;
|
||||
2. authentication and authorization behavior, if implemented;
|
||||
3. response envelope;
|
||||
4. supported media types and content negotiation behavior;
|
||||
5. shared query parameters;
|
||||
6. endpoint reference grouped by route family;
|
||||
7. request parameters and validation rules;
|
||||
8. response fields, units, nullability, and optionality;
|
||||
9. error response shape and status codes;
|
||||
10. pagination, caching, rate-limit, idempotency, and retry behavior, if implemented;
|
||||
11. compact request and response examples.
|
||||
|
||||
It must document only implemented endpoints and behavior. Planned endpoints, proposed fields, future filters, and experimental response shapes belong only under `docs/roadmap/`.
|
||||
|
||||
For HTTP API projects, `docs/consumers/` may provide task-oriented client integration guides, but those guides should link to `docs/api.md` for the authoritative endpoint contract.
|
||||
|
||||
### docs/policy/development.md
|
||||
|
||||
@@ -175,7 +232,7 @@ It should include:
|
||||
- dependency policy;
|
||||
- how to add config fields;
|
||||
- how to add CLI flags;
|
||||
- how to add stages/modules/adapters, if applicable;
|
||||
- how to add modules or adapters, if applicable;
|
||||
- how to update examples;
|
||||
- documentation update expectations.
|
||||
|
||||
@@ -216,7 +273,7 @@ Explain when commands are useful, not just their syntax.
|
||||
|
||||
**Audience:** administrators, operators
|
||||
|
||||
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
|
||||
Required for applications that maintain state, support resume behavior, run multi-step workflows, write durable artifacts, use remote storage, or require recovery procedures.
|
||||
|
||||
It should cover:
|
||||
|
||||
@@ -244,11 +301,40 @@ Each entry should include:
|
||||
- safe fix;
|
||||
- relevant links.
|
||||
|
||||
### docs/consumers/
|
||||
|
||||
**Audience:** developers and LLM coding agents integrating this project from another codebase
|
||||
|
||||
Required for projects with public packages, SDKs, client APIs, plugin APIs, or other application-facing integration surfaces.
|
||||
|
||||
This directory describes how an external codebase should consume the project's public API. It should be task-oriented and copyable where useful. It is not the place for internal implementation details or operator procedures.
|
||||
|
||||
For projects whose public API is HTTP, `docs/consumers/` is not required, and it should not duplicate the endpoint reference in `docs/api.md`. If present, it may provide practical integration workflows, client-specific examples, or migration notes that link back to `docs/api.md`.
|
||||
|
||||
`docs/consumers/api.md` should provide the consumer-facing overview and primary implementation workflow. It should include:
|
||||
|
||||
1. intended consumer audience and use cases;
|
||||
2. required inputs supplied by operators or deployment configuration;
|
||||
3. recommended public package or API workflow;
|
||||
4. minimal copyable example;
|
||||
5. consumer responsibilities and boundaries;
|
||||
6. retry, idempotency, or status behavior, if applicable;
|
||||
7. links to package-specific docs and canonical integration contracts.
|
||||
|
||||
Package-specific docs should be named `pkg-<name>.md` and should include:
|
||||
|
||||
1. import path;
|
||||
2. intended use cases;
|
||||
3. primary types and functions needed by consumers;
|
||||
4. minimal examples;
|
||||
5. validation, error, retry, and boundary behavior;
|
||||
6. links to canonical file-format or wire-protocol contracts.
|
||||
|
||||
### docs/internal/
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for modular, staged, service-oriented, or orchestration projects.
|
||||
Required for modular, service-oriented, or orchestration projects.
|
||||
|
||||
This directory describes implemented internal components. It is not the roadmap.
|
||||
|
||||
@@ -289,7 +375,9 @@ Roadmap docs should not be confused with current behavior.
|
||||
|
||||
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
||||
|
||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
|
||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses or exposes.
|
||||
|
||||
For public HTTP API services, `docs/integrations/` should document upstream, downstream, storage, protocol, or runtime contracts that the service depends on or bridges. It should not become a second copy of the public HTTP endpoint reference; that belongs in `docs/api.md`.
|
||||
|
||||
Use one file per integration where useful.
|
||||
|
||||
@@ -346,8 +434,10 @@ Before merging documentation changes, verify:
|
||||
|
||||
- README is concise and orientation-focused.
|
||||
- `docs/policy/architecture.md` describes development principles.
|
||||
- `docs/api.md` is the canonical HTTP contract for HTTP API services.
|
||||
- Future work appears only under `docs/roadmap/`.
|
||||
- User-facing docs avoid unnecessary internals.
|
||||
- Consumer-facing docs explain public APIs without duplicating HTTP endpoint or integration contracts.
|
||||
- Developer-facing docs preserve boundaries and invariants.
|
||||
- Config examples match the schema.
|
||||
- CLI examples match real commands and flags.
|
||||
|
||||
@@ -33,23 +33,23 @@ Relevant links:
|
||||
- [Configuration reference](config.md)
|
||||
- [CLI reference](cli.md)
|
||||
|
||||
## Missing Prompt/Profile Directory Settings
|
||||
## Missing Prompt Directory Settings
|
||||
|
||||
Symptom:
|
||||
|
||||
- CLI parse errors saying prompt directory or profile directory is required.
|
||||
- CLI parse errors saying prompt directory is required.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- Neither CLI flags nor config provide effective `prompt_dir` / `profile_dir`.
|
||||
- Neither CLI flags nor config provide an effective `prompt_dir`.
|
||||
|
||||
Diagnostic step:
|
||||
|
||||
- Run the failing command with explicit `--prompt-dir` and `--profile-dir` once to verify.
|
||||
- Run the failing command with explicit `--prompt-dir` once to verify.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- Set `prompt_dir` and `profile_dir` in config, or always pass both flags.
|
||||
- Set `prompt_dir` in config, or always pass `--prompt-dir`.
|
||||
|
||||
Relevant links:
|
||||
|
||||
@@ -252,6 +252,39 @@ Relevant links:
|
||||
- [Configuration reference](config.md)
|
||||
- [Operations guide](operations.md)
|
||||
|
||||
## Prompt Cache Misses Or No Cache Usage
|
||||
|
||||
Symptom:
|
||||
|
||||
- CLI run summary omits `cached_tokens` / `cache_write_tokens`.
|
||||
- HTTP `metadata.usage.cached_tokens` and `metadata.usage.cache_write_tokens` are both `0`.
|
||||
- Provider cost or latency does not improve after repeated similar runs.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- The selected prompt has no `messages[].cache_control`.
|
||||
- Dynamic per-run input appears before the cache-controlled message and changes the provider cache key.
|
||||
- The provider does not support the serialized cache-control shape for the selected model.
|
||||
- The provider imposes minimum token thresholds or cache-breakpoint limits.
|
||||
|
||||
Diagnostic step:
|
||||
|
||||
- Run `render --format json` and verify the intended rendered message includes `cache_control`.
|
||||
- Confirm stable reusable context appears before the cache-controlled message, with dynamic input after it.
|
||||
- Check provider docs/logs for model support, minimum token thresholds, and breakpoint limits.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- Move stable reusable context before the cache-controlled message.
|
||||
- Move highly dynamic input after the cache breakpoint.
|
||||
- Keep `cache_control.type: ephemeral` and, when using `ttl`, set `ttl: 1h`.
|
||||
- Use CLI cache counters or HTTP cache usage fields to verify cache reads/writes after rerunning.
|
||||
|
||||
Relevant links:
|
||||
|
||||
- [Configuration reference](config.md)
|
||||
- [OpenAI-compatible chat integration](integrations/openai-compatible-chat.md)
|
||||
|
||||
## Validation Status Failed (`run` Exit 2 Or HTTP 200 With Failed Status)
|
||||
|
||||
Symptom:
|
||||
|
||||
278
engine.go
Normal file
278
engine.go
Normal file
@@ -0,0 +1,278 @@
|
||||
package scriptorium
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
artifactadapter "gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile/builtin"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
||||
)
|
||||
|
||||
// ErrInvalidConfig indicates invalid public engine configuration.
|
||||
var ErrInvalidConfig = errors.New("invalid engine configuration")
|
||||
|
||||
var (
|
||||
ErrInvalidRequest = errors.New("invalid run request")
|
||||
ErrPromptNotFound = errors.New("prompt not found")
|
||||
ErrProfileNotFound = errors.New("profile not found")
|
||||
ErrPromptLoad = errors.New("failed to load prompt definition")
|
||||
ErrProfileLoad = errors.New("failed to load execution profile")
|
||||
ErrArtifactLoad = errors.New("failed to load artifact")
|
||||
ErrPromptRender = errors.New("failed to render prompt")
|
||||
ErrLLMGenerate = errors.New("failed to generate output")
|
||||
ErrValidation = errors.New("failed to validate output")
|
||||
)
|
||||
|
||||
// Engine prepares and runs Scriptorium prompt requests.
|
||||
type Engine struct {
|
||||
runner *usecase.Runner
|
||||
}
|
||||
|
||||
// Config configures a public Scriptorium engine.
|
||||
type Config struct {
|
||||
PromptDir string
|
||||
ProfileDir string
|
||||
SchemaDir string
|
||||
Timeout time.Duration
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
// Option customizes engine construction.
|
||||
type Option func(*engineOptions) error
|
||||
|
||||
type engineOptions struct {
|
||||
llmClient llm.Client
|
||||
promptDefs promptdef.Repository
|
||||
profiles profile.Repository
|
||||
memoryProfiles profile.Repository
|
||||
validator validate.Validator
|
||||
promptSource bool
|
||||
profileSource bool
|
||||
memorySource bool
|
||||
validatorSource bool
|
||||
}
|
||||
|
||||
// WithLLMClient injects a custom LLM client for execution.
|
||||
func WithLLMClient(client LLMClient) Option {
|
||||
return func(options *engineOptions) error {
|
||||
if client == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.llmClient = publicLLMClientAdapter{client: client}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func WithPromptFS(fsys fs.FS, root string) Option {
|
||||
return func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.promptDefs = promptdef.NewFSRepository(fsys, root)
|
||||
options.promptSource = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func WithPromptFile(path string) Option {
|
||||
return func(options *engineOptions) error {
|
||||
fsys, root, err := fileSource(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
options.promptDefs = promptdef.NewFSRepository(fsys, root)
|
||||
options.promptSource = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func WithProfileFS(fsys fs.FS, root string) Option {
|
||||
return func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.profiles = profile.NewFSRepository(fsys, root)
|
||||
options.profileSource = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func WithProfileFile(path string) Option {
|
||||
return func(options *engineOptions) error {
|
||||
fsys, root, err := fileSource(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
options.profiles = profile.NewFSRepository(fsys, root)
|
||||
options.profileSource = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithProfiles configures in-memory profiles that take precedence over
|
||||
// configured profile files and built-in profiles.
|
||||
func WithProfiles(profiles ...Profile) Option {
|
||||
return func(options *engineOptions) error {
|
||||
repo, err := newMemoryProfileRepository(profiles)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
options.memoryProfiles = repo
|
||||
options.memorySource = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func WithSchemaFS(fsys fs.FS, root string) Option {
|
||||
return func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.validator = validate.NewFSValidator(fsys, root)
|
||||
options.validatorSource = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func WithSchemaFile(path string) Option {
|
||||
return func(options *engineOptions) error {
|
||||
fsys, root, err := fileSource(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
options.validator = validate.NewFSValidator(fsys, root)
|
||||
options.validatorSource = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// NewEngine constructs an Engine using the same default internal components as
|
||||
// the CLI and HTTP adapters.
|
||||
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
var options engineOptions
|
||||
for _, opt := range opts {
|
||||
if opt == nil {
|
||||
continue
|
||||
}
|
||||
if err := opt(&options); err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
}
|
||||
|
||||
promptDefs := options.promptDefs
|
||||
if !options.promptSource {
|
||||
if strings.TrimSpace(cfg.PromptDir) == "" {
|
||||
return nil, fmt.Errorf("%w: prompt directory is required", ErrInvalidConfig)
|
||||
}
|
||||
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
|
||||
}
|
||||
|
||||
profiles := builtin.NewRepositoryWithDirectory(cfg.ProfileDir)
|
||||
if options.profileSource {
|
||||
profiles = builtin.NewRepositoryWithPrimary(options.profiles)
|
||||
}
|
||||
if options.memorySource {
|
||||
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
|
||||
}
|
||||
|
||||
validator := options.validator
|
||||
if !options.validatorSource {
|
||||
schemaDir := cfg.SchemaDir
|
||||
if strings.TrimSpace(schemaDir) == "" {
|
||||
schemaDir = defaults.SchemaDirDefault
|
||||
}
|
||||
validator = validate.NewStandardValidator(schemaDir)
|
||||
}
|
||||
|
||||
llmClient := options.llmClient
|
||||
if llmClient == nil {
|
||||
var err error
|
||||
llmClient, err = llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
|
||||
Timeout: cfg.Timeout,
|
||||
HTTPClient: cfg.HTTPClient,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
}
|
||||
|
||||
return &Engine{
|
||||
runner: usecase.NewRunner(
|
||||
promptDefs,
|
||||
profiles,
|
||||
artifactadapter.NewCompositeReader(),
|
||||
prompt.NewGoRenderer(),
|
||||
llmClient,
|
||||
validator,
|
||||
),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func fileSource(name string) (fs.FS, string, error) {
|
||||
cleanName := strings.TrimSpace(name)
|
||||
if cleanName == "" {
|
||||
return nil, "", ErrInvalidConfig
|
||||
}
|
||||
dir := filepath.Dir(cleanName)
|
||||
base := filepath.Base(cleanName)
|
||||
if base == "." || base == string(filepath.Separator) || strings.TrimSpace(base) == "" {
|
||||
return nil, "", ErrInvalidConfig
|
||||
}
|
||||
info, err := os.Stat(cleanName)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, cleanName, err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, cleanName)
|
||||
}
|
||||
return os.DirFS(dir), filepath.ToSlash(base), nil
|
||||
}
|
||||
|
||||
// Prepare resolves a prompt request without calling an LLM.
|
||||
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
prepared, err := e.runner.Prepare(ctx, toDomainRunRequest(req))
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return fromDomainPreparedRun(prepared), nil
|
||||
}
|
||||
|
||||
// Run executes a prompt request and returns the generated artifact and metadata.
|
||||
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
result, err := e.runner.Run(ctx, toDomainRunRequest(req))
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return fromDomainRunResult(result), nil
|
||||
}
|
||||
1395
engine_test.go
Normal file
1395
engine_test.go
Normal file
File diff suppressed because it is too large
Load Diff
77
errors.go
Normal file
77
errors.go
Normal file
@@ -0,0 +1,77 @@
|
||||
package scriptorium
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||
)
|
||||
|
||||
func mapPublicError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if hasPublicError(err) {
|
||||
return err
|
||||
}
|
||||
publicErr := publicErrorFor(err)
|
||||
if publicErr == nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("%w: %w", publicErr, err)
|
||||
}
|
||||
|
||||
func hasPublicError(err error) bool {
|
||||
for _, publicErr := range []error{
|
||||
ErrInvalidConfig,
|
||||
ErrInvalidRequest,
|
||||
ErrPromptNotFound,
|
||||
ErrProfileNotFound,
|
||||
ErrPromptLoad,
|
||||
ErrProfileLoad,
|
||||
ErrArtifactLoad,
|
||||
ErrPromptRender,
|
||||
ErrLLMGenerate,
|
||||
ErrValidation,
|
||||
} {
|
||||
if errors.Is(err, publicErr) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func publicErrorFor(err error) error {
|
||||
switch {
|
||||
case errors.Is(err, promptdef.ErrPromptDefinitionNotFound):
|
||||
return ErrPromptNotFound
|
||||
case errors.Is(err, profile.ErrProfileNotFound):
|
||||
return ErrProfileNotFound
|
||||
case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition):
|
||||
return ErrPromptLoad
|
||||
case isProfileLoadCause(err):
|
||||
return ErrProfileLoad
|
||||
case errors.Is(err, usecase.ErrArtifactLoad):
|
||||
return ErrArtifactLoad
|
||||
case errors.Is(err, usecase.ErrPromptRender):
|
||||
return ErrPromptRender
|
||||
case errors.Is(err, usecase.ErrLLMGenerate):
|
||||
return ErrLLMGenerate
|
||||
case errors.Is(err, usecase.ErrValidation):
|
||||
return ErrValidation
|
||||
case errors.Is(err, usecase.ErrInvalidRequest):
|
||||
return ErrInvalidRequest
|
||||
case errors.Is(err, usecase.ErrProfileLoad):
|
||||
return ErrPromptLoad
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func isProfileLoadCause(err error) bool {
|
||||
return errors.Is(err, profile.ErrInvalidYAML) ||
|
||||
errors.Is(err, profile.ErrInvalidProfile) ||
|
||||
errors.Is(err, profile.ErrRawAPIKeyNotAllowed)
|
||||
}
|
||||
50
examples/go-library/prepare/main.go
Normal file
50
examples/go-library/prepare/main.go
Normal file
@@ -0,0 +1,50 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
func main() {
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||
PromptDir: "./examples/prompts",
|
||||
ProfileDir: "./examples/profiles",
|
||||
SchemaDir: "./examples/schemas",
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
summary := struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
Model string `json:"model"`
|
||||
MessageCount int `json:"message_count"`
|
||||
InputHashes map[string]string `json:"input_hashes"`
|
||||
}{
|
||||
PromptID: prepared.PromptID,
|
||||
SelectedProfileID: prepared.SelectedProfileID,
|
||||
Model: prepared.EffectiveModelParams.Model,
|
||||
MessageCount: len(prepared.Messages),
|
||||
InputHashes: prepared.InputHashes,
|
||||
}
|
||||
|
||||
if err := json.NewEncoder(os.Stdout).Encode(summary); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile/builtin"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||
@@ -33,8 +33,7 @@ const (
|
||||
)
|
||||
|
||||
const (
|
||||
errPromptDirRequired = "prompt directory is required; provide --prompt-dir or config.yml prompt_dir"
|
||||
errProfileDirRequired = "profile directory is required; provide --profile-dir or config.yml profile_dir"
|
||||
errPromptDirRequired = "prompt directory is required; provide --prompt-dir or config.yml prompt_dir"
|
||||
)
|
||||
|
||||
type runConfig struct {
|
||||
@@ -305,12 +304,14 @@ func parseServeArgs(args []string) (*serveConfig, error) {
|
||||
cfg.schemaDir = settings.schemaDir
|
||||
cfg.addr = settings.serverAddr
|
||||
|
||||
if err := validateRequiredLibraryDirs(cfg.promptDir, cfg.profileDir); err != nil {
|
||||
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg.promptDir = filepath.Clean(cfg.promptDir)
|
||||
cfg.profileDir = filepath.Clean(cfg.profileDir)
|
||||
if strings.TrimSpace(cfg.profileDir) != "" {
|
||||
cfg.profileDir = filepath.Clean(cfg.profileDir)
|
||||
}
|
||||
cfg.schemaDir = filepath.Clean(cfg.schemaDir)
|
||||
return cfg, nil
|
||||
}
|
||||
@@ -353,7 +354,7 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
|
||||
cfg.schemaDir = settings.schemaDir
|
||||
cfg.defaultRenderFormat = settings.defaultRenderFormat
|
||||
|
||||
if err := validateRequiredLibraryDirs(cfg.promptDir, cfg.profileDir); err != nil {
|
||||
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(cfg.promptID) == "" {
|
||||
@@ -363,7 +364,9 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
|
||||
return errors.New("at least one --input is required")
|
||||
}
|
||||
cfg.promptDir = filepath.Clean(cfg.promptDir)
|
||||
cfg.profileDir = filepath.Clean(cfg.profileDir)
|
||||
if strings.TrimSpace(cfg.profileDir) != "" {
|
||||
cfg.profileDir = filepath.Clean(cfg.profileDir)
|
||||
}
|
||||
if cfg.outputPath != "" {
|
||||
cfg.outputPath = filepath.Clean(cfg.outputPath)
|
||||
}
|
||||
@@ -467,20 +470,17 @@ func resolveCommonSettings(fs *flag.FlagSet, configPath string, overrides appcon
|
||||
}, nil
|
||||
}
|
||||
|
||||
func validateRequiredLibraryDirs(promptDir, profileDir string) error {
|
||||
func validateRequiredLibraryDirs(promptDir string) error {
|
||||
if strings.TrimSpace(promptDir) == "" {
|
||||
return errors.New(errPromptDirRequired)
|
||||
}
|
||||
if strings.TrimSpace(profileDir) == "" {
|
||||
return errors.New(errProfileDirRequired)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newRunner(promptDir, profileDir, schemaDir string, llmClient llm.Client) *usecase.Runner {
|
||||
return usecase.NewRunner(
|
||||
promptdef.NewFilesystemRepository(promptDir),
|
||||
profile.NewFilesystemRepository(profileDir),
|
||||
builtin.NewRepositoryWithDirectory(profileDir),
|
||||
artifactadapter.NewCompositeReader(),
|
||||
prompt.NewGoRenderer(),
|
||||
llmClient,
|
||||
@@ -513,18 +513,25 @@ func buildRunRequestFromConfig(cfg *runConfig) (domain.RunRequest, error) {
|
||||
inputs[name] = domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path}
|
||||
}
|
||||
|
||||
var modelOverride *domain.ExecutionTarget
|
||||
var modelOverride *domain.ExecutionTargetOverride
|
||||
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.apiKeyEnvSet || cfg.timeoutSet {
|
||||
modelOverride = &domain.ExecutionTarget{
|
||||
Endpoint: cfg.llmBaseURL,
|
||||
Model: cfg.model,
|
||||
Temperature: cfg.temperature,
|
||||
MaxTokens: cfg.maxTokens,
|
||||
TopP: cfg.topP,
|
||||
APIKeyEnv: cfg.apiKeyEnv,
|
||||
modelOverride = &domain.ExecutionTargetOverride{
|
||||
Endpoint: cfg.llmBaseURL,
|
||||
Model: cfg.model,
|
||||
APIKeyEnv: cfg.apiKeyEnv,
|
||||
}
|
||||
if cfg.temperatureSet {
|
||||
modelOverride.Temperature = &cfg.temperature
|
||||
}
|
||||
if cfg.maxTokensSet {
|
||||
modelOverride.MaxTokens = &cfg.maxTokens
|
||||
}
|
||||
if cfg.topPSet {
|
||||
modelOverride.TopP = &cfg.topP
|
||||
}
|
||||
if cfg.timeoutSet {
|
||||
modelOverride.TimeoutSeconds = int(cfg.timeout.Seconds())
|
||||
timeoutSeconds := int(cfg.timeout.Seconds())
|
||||
modelOverride.TimeoutSeconds = &timeoutSeconds
|
||||
}
|
||||
}
|
||||
|
||||
@@ -606,7 +613,7 @@ func printSummary(stderr io.Writer, res *domain.RunResult) {
|
||||
if res == nil {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d\n",
|
||||
fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d",
|
||||
res.PromptID,
|
||||
res.PromptVersion,
|
||||
res.SelectedProfileID,
|
||||
@@ -620,6 +627,10 @@ func printSummary(stderr io.Writer, res *domain.RunResult) {
|
||||
res.Usage.CompletionTokens,
|
||||
res.Usage.TotalTokens,
|
||||
)
|
||||
if res.Usage.CachedTokens != 0 || res.Usage.CacheWriteTokens != 0 {
|
||||
fmt.Fprintf(stderr, " cached_tokens=%d cache_write_tokens=%d", res.Usage.CachedTokens, res.Usage.CacheWriteTokens)
|
||||
}
|
||||
fmt.Fprintln(stderr)
|
||||
}
|
||||
|
||||
func printUsage(w io.Writer) {
|
||||
|
||||
@@ -74,12 +74,12 @@ func TestParseRunArgsRequiredFlags(t *testing.T) {
|
||||
t.Fatalf("expected clear prompt-dir guidance, got %v", err)
|
||||
}
|
||||
|
||||
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p", "--input", "a=b"})
|
||||
if err == nil {
|
||||
t.Fatal("expected missing --profile-dir error")
|
||||
cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p", "--input", "a=b"})
|
||||
if err != nil {
|
||||
t.Fatalf("expected missing --profile-dir to be accepted, got %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "profile directory is required") {
|
||||
t.Fatalf("expected clear profile-dir guidance, got %v", err)
|
||||
if cfg.profileDir != "" {
|
||||
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
||||
}
|
||||
|
||||
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--input", "a=b"})
|
||||
@@ -161,17 +161,12 @@ func TestParseServeArgsRequiredFlags(t *testing.T) {
|
||||
t.Fatalf("expected clear prompt-dir guidance, got %v", err)
|
||||
}
|
||||
|
||||
_, err = parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts"})
|
||||
if err == nil {
|
||||
t.Fatal("expected missing --profile-dir error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "profile directory is required") {
|
||||
t.Fatalf("expected clear profile-dir guidance, got %v", err)
|
||||
}
|
||||
|
||||
cfg, err := parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles"})
|
||||
cfg, err := parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts"})
|
||||
if err != nil {
|
||||
t.Fatalf("expected valid serve args, got %v", err)
|
||||
t.Fatalf("expected missing --profile-dir to be accepted, got %v", err)
|
||||
}
|
||||
if cfg.profileDir != "" {
|
||||
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
||||
}
|
||||
if cfg.addr != defaults.HTTPAddrDefault {
|
||||
t.Fatalf("expected default addr %s, got %q", defaults.HTTPAddrDefault, cfg.addr)
|
||||
@@ -565,21 +560,21 @@ profile_dir: ./profiles
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRunArgsFailsClearlyWhenNoEffectiveProfileDir(t *testing.T) {
|
||||
func TestParseRunArgsAcceptsMissingEffectiveProfileDir(t *testing.T) {
|
||||
configPath := writeAppConfigFile(t, `
|
||||
prompt_dir: ./prompts
|
||||
`)
|
||||
|
||||
_, err := parseRunArgs([]string{
|
||||
cfg, err := parseRunArgs([]string{
|
||||
"--config", configPath,
|
||||
"--prompt", "p",
|
||||
"--input", "a=b",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected missing profile_dir error")
|
||||
if err != nil {
|
||||
t.Fatalf("expected missing profile_dir to be accepted, got %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "profile directory is required") || !strings.Contains(err.Error(), "config.yml profile_dir") {
|
||||
t.Fatalf("expected clear profile_dir guidance, got %v", err)
|
||||
if cfg.profileDir != "" {
|
||||
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -601,21 +596,21 @@ profile_dir: ./profiles
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRenderArgsFailsClearlyWhenNoEffectiveProfileDir(t *testing.T) {
|
||||
func TestParseRenderArgsAcceptsMissingEffectiveProfileDir(t *testing.T) {
|
||||
configPath := writeAppConfigFile(t, `
|
||||
prompt_dir: ./prompts
|
||||
`)
|
||||
|
||||
_, err := parseRenderArgs([]string{
|
||||
cfg, err := parseRenderArgs([]string{
|
||||
"--config", configPath,
|
||||
"--prompt", "p",
|
||||
"--input", "a=b",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected missing profile_dir error")
|
||||
if err != nil {
|
||||
t.Fatalf("expected missing profile_dir to be accepted, got %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "profile directory is required") || !strings.Contains(err.Error(), "config.yml profile_dir") {
|
||||
t.Fatalf("expected clear profile_dir guidance, got %v", err)
|
||||
if cfg.profileDir != "" {
|
||||
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -747,6 +742,35 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderCommandExplicitZeroTemperatureReachesEffectiveSettings(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
||||
|
||||
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
||||
profile := `id: local-default
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
model: profile-model
|
||||
temperature: 0.7
|
||||
`
|
||||
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-default.yaml"), []byte(profile), 0o644); err != nil {
|
||||
t.Fatalf("failed to write profile fixture: %v", err)
|
||||
}
|
||||
|
||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||
"--prompt-dir", lib.promptDir,
|
||||
"--profile-dir", lib.profileDir,
|
||||
"--prompt", "prompt.render",
|
||||
"--input", "transcript=" + inputPath,
|
||||
"--temperature", "0",
|
||||
})
|
||||
if code != ExitOK {
|
||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||
}
|
||||
if !strings.Contains(stdout, "\n temperature: 0\n") {
|
||||
t.Fatalf("expected explicit zero temperature in effective settings, got:\n%s", stdout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
||||
@@ -900,6 +924,29 @@ func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderCommandUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
|
||||
t.Setenv("OPENROUTER_API_KEY", "test-key")
|
||||
lib := newCLITestLibrary(t)
|
||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||
|
||||
writePromptFile(t, lib.promptDir, "prompt.builtin", "mistral-small-3")
|
||||
|
||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||
"--prompt-dir", lib.promptDir,
|
||||
"--prompt", "prompt.builtin",
|
||||
"--input", "transcript=" + inputPath,
|
||||
})
|
||||
if code != ExitOK {
|
||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||
}
|
||||
if !strings.Contains(stdout, "selected_profile_id: mistral-small-3") {
|
||||
t.Fatalf("expected built-in selected profile, got %q", stdout)
|
||||
}
|
||||
if !strings.Contains(stdout, "model: mistralai/mistral-small-3.2-24b-instruct") {
|
||||
t.Fatalf("expected built-in model, got %q", stdout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||
@@ -1064,6 +1111,38 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
|
||||
if !strings.Contains(stderr.String(), "prompt=p@1") {
|
||||
t.Fatalf("expected summary on stderr, got %q", stderr.String())
|
||||
}
|
||||
if strings.Contains(stderr.String(), "cached_tokens=") || strings.Contains(stderr.String(), "cache_write_tokens=") {
|
||||
t.Fatalf("expected zero cache usage to be omitted from summary, got %q", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) {
|
||||
var stderr bytes.Buffer
|
||||
|
||||
printSummary(&stderr, &domain.RunResult{
|
||||
PromptID: "p",
|
||||
PromptVersion: "1",
|
||||
SelectedProfileID: "exec",
|
||||
ModelName: "m",
|
||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic},
|
||||
RenderedPromptHash: "h",
|
||||
InputHashes: map[string]string{"in": "x"},
|
||||
Usage: domain.TokenUsage{
|
||||
PromptTokens: 10,
|
||||
CompletionTokens: 5,
|
||||
TotalTokens: 15,
|
||||
CachedTokens: 0,
|
||||
CacheWriteTokens: 3,
|
||||
},
|
||||
})
|
||||
|
||||
summary := stderr.String()
|
||||
if !strings.Contains(summary, "usage=10/5/15") {
|
||||
t.Fatalf("expected base usage summary, got %q", summary)
|
||||
}
|
||||
if !strings.Contains(summary, "cached_tokens=0 cache_write_tokens=3") {
|
||||
t.Fatalf("expected cache usage in summary, got %q", summary)
|
||||
}
|
||||
}
|
||||
|
||||
type cliTestLibrary struct {
|
||||
|
||||
@@ -21,16 +21,16 @@ type inputRefDTO struct {
|
||||
}
|
||||
|
||||
type modelOverrideRequestDTO struct {
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Temperature float64 `json:"temperature,omitempty"`
|
||||
MaxTokens int `json:"max_tokens,omitempty"`
|
||||
TopP float64 `json:"top_p,omitempty"`
|
||||
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
|
||||
ServiceTier string `json:"service_tier,omitempty"`
|
||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||
ExtraParams map[string]string `json:"extra_params,omitempty"`
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||
ServiceTier string `json:"service_tier,omitempty"`
|
||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||
}
|
||||
|
||||
type runResponseDTO struct {
|
||||
@@ -70,22 +70,24 @@ type metadataDTO struct {
|
||||
}
|
||||
|
||||
type modelParamsDTO struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Model string `json:"model"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
TopP float64 `json:"top_p"`
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
ServiceTier string `json:"service_tier,omitempty"`
|
||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||
ExtraParams map[string]string `json:"extra_params,omitempty"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Model string `json:"model"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
TopP float64 `json:"top_p"`
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
ServiceTier string `json:"service_tier,omitempty"`
|
||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||
}
|
||||
|
||||
type tokenUsageDTO struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||
}
|
||||
|
||||
type validationDTO struct {
|
||||
|
||||
@@ -61,9 +61,9 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
var model *domain.ExecutionTarget
|
||||
var model *domain.ExecutionTargetOverride
|
||||
if req.Model != nil {
|
||||
model = executionTargetFromModelOverrideDTO(req.Model)
|
||||
model = executionTargetOverrideFromModelOverrideDTO(req.Model)
|
||||
}
|
||||
|
||||
res, err := h.runner.Run(r.Context(), domain.RunRequest{
|
||||
@@ -105,6 +105,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
PromptTokens: res.Usage.PromptTokens,
|
||||
CompletionTokens: res.Usage.CompletionTokens,
|
||||
TotalTokens: res.Usage.TotalTokens,
|
||||
CachedTokens: res.Usage.CachedTokens,
|
||||
CacheWriteTokens: res.Usage.CacheWriteTokens,
|
||||
},
|
||||
StartTime: res.StartTime,
|
||||
EndTime: res.EndTime,
|
||||
@@ -121,11 +123,11 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
func executionTargetFromModelOverrideDTO(dto *modelOverrideRequestDTO) *domain.ExecutionTarget {
|
||||
func executionTargetOverrideFromModelOverrideDTO(dto *modelOverrideRequestDTO) *domain.ExecutionTargetOverride {
|
||||
if dto == nil {
|
||||
return nil
|
||||
}
|
||||
return &domain.ExecutionTarget{
|
||||
return &domain.ExecutionTargetOverride{
|
||||
Endpoint: dto.Endpoint,
|
||||
Model: dto.Model,
|
||||
Temperature: dto.Temperature,
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||
@@ -32,6 +33,34 @@ func (f *fakeRunner) Run(ctx context.Context, req domain.RunRequest) (*domain.Ru
|
||||
return f.result, nil
|
||||
}
|
||||
|
||||
type handlerPromptRepo struct {
|
||||
def *domain.PromptDefinition
|
||||
}
|
||||
|
||||
func (r handlerPromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
||||
return r.def, nil
|
||||
}
|
||||
|
||||
type handlerProfileRepo struct {
|
||||
profile *domain.ExecutionProfile
|
||||
}
|
||||
|
||||
func (r handlerProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
return r.profile, nil
|
||||
}
|
||||
|
||||
type handlerArtifactReader struct{}
|
||||
|
||||
func (handlerArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
||||
return &domain.Artifact{Name: "input", Body: []byte("input"), Hash: "hash"}, nil
|
||||
}
|
||||
|
||||
type handlerRenderer struct{}
|
||||
|
||||
func (handlerRenderer) Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
|
||||
return &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, nil
|
||||
}
|
||||
|
||||
func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
||||
start := time.Now().UTC()
|
||||
end := start.Add(2 * time.Second)
|
||||
@@ -66,11 +95,17 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
||||
APIKeyEnv: envName,
|
||||
},
|
||||
InputHashes: map[string]string{"transcript": "h1"},
|
||||
Usage: domain.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3},
|
||||
StartTime: start,
|
||||
EndTime: end,
|
||||
Duration: 2 * time.Second,
|
||||
RawOutput: "hello",
|
||||
Usage: domain.TokenUsage{
|
||||
PromptTokens: 1,
|
||||
CompletionTokens: 2,
|
||||
TotalTokens: 3,
|
||||
CachedTokens: 4,
|
||||
CacheWriteTokens: 5,
|
||||
},
|
||||
StartTime: start,
|
||||
EndTime: end,
|
||||
Duration: 2 * time.Second,
|
||||
RawOutput: "hello",
|
||||
}}
|
||||
|
||||
h := NewHandler(r)
|
||||
@@ -111,6 +146,13 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
||||
if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" {
|
||||
t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"])
|
||||
}
|
||||
usage := metadata["usage"].(map[string]any)
|
||||
if usage["prompt_tokens"] != float64(1) || usage["completion_tokens"] != float64(2) || usage["total_tokens"] != float64(3) {
|
||||
t.Fatalf("unexpected base usage metadata: %#v", usage)
|
||||
}
|
||||
if usage["cached_tokens"] != float64(4) || usage["cache_write_tokens"] != float64(5) {
|
||||
t.Fatalf("unexpected cache usage metadata: %#v", usage)
|
||||
}
|
||||
modelParams := metadata["model_params"].(map[string]any)
|
||||
if modelParams["api_key_env"] != envName {
|
||||
t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"])
|
||||
@@ -134,7 +176,7 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
||||
if r.last.Execution == nil || r.last.Execution.Model != "gpt-x" {
|
||||
t.Fatalf("expected model override, got %#v", r.last.Execution)
|
||||
}
|
||||
if r.last.Execution.TimeoutSeconds != 120 {
|
||||
if r.last.Execution.TimeoutSeconds == nil || *r.last.Execution.TimeoutSeconds != 120 {
|
||||
t.Fatalf("expected timeout_seconds override 120, got %#v", r.last.Execution)
|
||||
}
|
||||
if r.last.Execution.ServiceTier != "flex" {
|
||||
@@ -171,6 +213,10 @@ func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
|
||||
if metadata["selected_profile_id"] != "prompt-default" {
|
||||
t.Fatalf("expected selected_profile_id from result, got %#v", metadata["selected_profile_id"])
|
||||
}
|
||||
usage := metadata["usage"].(map[string]any)
|
||||
if usage["cached_tokens"] != float64(0) || usage["cache_write_tokens"] != float64(0) {
|
||||
t.Fatalf("expected zero cache usage fields to be included, got %#v", usage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
|
||||
@@ -211,20 +257,136 @@ func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
|
||||
got := r.last.Execution
|
||||
if got.Endpoint != "http://override/v1" ||
|
||||
got.Model != "override-model" ||
|
||||
got.Temperature != 0.6 ||
|
||||
got.MaxTokens != 250 ||
|
||||
got.TopP != 0.85 ||
|
||||
got.TimeoutSeconds != 33 ||
|
||||
got.ServiceTier != "flex" ||
|
||||
got.ReasoningEffort != "medium" ||
|
||||
got.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
|
||||
t.Fatalf("unexpected mapped execution target: %+v", got)
|
||||
}
|
||||
if !reflect.DeepEqual(got.ExtraParams, map[string]string{"provider_option": "on"}) {
|
||||
if got.Temperature == nil || *got.Temperature != 0.6 {
|
||||
t.Fatalf("unexpected mapped temperature: %#v", got.Temperature)
|
||||
}
|
||||
if got.MaxTokens == nil || *got.MaxTokens != 250 {
|
||||
t.Fatalf("unexpected mapped max_tokens: %#v", got.MaxTokens)
|
||||
}
|
||||
if got.TopP == nil || *got.TopP != 0.85 {
|
||||
t.Fatalf("unexpected mapped top_p: %#v", got.TopP)
|
||||
}
|
||||
if got.TimeoutSeconds == nil || *got.TimeoutSeconds != 33 {
|
||||
t.Fatalf("unexpected mapped timeout_seconds: %#v", got.TimeoutSeconds)
|
||||
}
|
||||
if !reflect.DeepEqual(got.ExtraParams, map[string]any{"provider_option": "on"}) {
|
||||
t.Fatalf("unexpected mapped extra_params: %#v", got.ExtraParams)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerModelOverrideAcceptsJSONCompatibleExtraParams(t *testing.T) {
|
||||
r := &fakeRunner{result: &domain.RunResult{
|
||||
Artifact: domain.Artifact{Body: []byte("ok")},
|
||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||
}}
|
||||
h := NewHandler(r)
|
||||
|
||||
reqBody := `{
|
||||
"prompt_id": "prompt-1",
|
||||
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
|
||||
"model": {
|
||||
"extra_params": {
|
||||
"string_value": "enabled",
|
||||
"number_value": 42,
|
||||
"boolean_value": true,
|
||||
"object_value": {"nested": "value", "count": 2},
|
||||
"array_value": ["first", 3, false]
|
||||
}
|
||||
}
|
||||
}`
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(reqBody))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
h.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if r.last.Execution == nil {
|
||||
t.Fatal("expected execution override in run request")
|
||||
}
|
||||
want := map[string]any{
|
||||
"string_value": "enabled",
|
||||
"number_value": float64(42),
|
||||
"boolean_value": true,
|
||||
"object_value": map[string]any{"nested": "value", "count": float64(2)},
|
||||
"array_value": []any{"first", float64(3), false},
|
||||
}
|
||||
if !reflect.DeepEqual(r.last.Execution.ExtraParams, want) {
|
||||
t.Fatalf("unexpected mapped extra_params:\ngot=%#v\nwant=%#v", r.last.Execution.ExtraParams, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerModelOverrideExplicitZeroTemperatureMapsAsPresent(t *testing.T) {
|
||||
r := &fakeRunner{result: &domain.RunResult{
|
||||
Artifact: domain.Artifact{Body: []byte("ok")},
|
||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0},
|
||||
}}
|
||||
h := NewHandler(r)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||
"prompt_id": "prompt-1",
|
||||
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
|
||||
"model": {"temperature": 0}
|
||||
}`))
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if r.last.Execution == nil || r.last.Execution.Temperature == nil {
|
||||
t.Fatalf("expected temperature override to be present, got %#v", r.last.Execution)
|
||||
}
|
||||
if *r.last.Execution.Temperature != 0 {
|
||||
t.Fatalf("expected zero temperature override, got %v", *r.last.Execution.Temperature)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerModelOverrideOmittedTemperatureMapsAsAbsent(t *testing.T) {
|
||||
r := &fakeRunner{result: &domain.RunResult{
|
||||
Artifact: domain.Artifact{Body: []byte("ok")},
|
||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0.7},
|
||||
}}
|
||||
h := NewHandler(r)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||
"prompt_id": "prompt-1",
|
||||
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
|
||||
"model": {"model": "override-model"}
|
||||
}`))
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if r.last.Execution == nil {
|
||||
t.Fatal("expected model override")
|
||||
}
|
||||
if r.last.Execution.Temperature != nil {
|
||||
t.Fatalf("expected omitted temperature to remain absent, got %#v", r.last.Execution.Temperature)
|
||||
}
|
||||
|
||||
var resp map[string]any
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("invalid JSON response: %v", err)
|
||||
}
|
||||
metadata := resp["metadata"].(map[string]any)
|
||||
params := metadata["model_params"].(map[string]any)
|
||||
if params["temperature"] != 0.7 {
|
||||
t.Fatalf("expected effective profile/default temperature in response, got %#v", params["temperature"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing.T) {
|
||||
r := &fakeRunner{result: &domain.RunResult{
|
||||
Artifact: domain.Artifact{
|
||||
@@ -245,8 +407,10 @@ func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing
|
||||
ServiceTier: "priority",
|
||||
ReasoningEffort: "high",
|
||||
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
||||
ExtraParams: map[string]string{
|
||||
ExtraParams: map[string]any{
|
||||
"provider_option": "on",
|
||||
"number_value": 42,
|
||||
"object_value": map[string]any{"nested": "value"},
|
||||
},
|
||||
},
|
||||
}}
|
||||
@@ -301,6 +465,13 @@ func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing
|
||||
if extraParams["provider_option"] != "on" {
|
||||
t.Fatalf("unexpected extra_params.provider_option: %#v", extraParams["provider_option"])
|
||||
}
|
||||
if extraParams["number_value"] != float64(42) {
|
||||
t.Fatalf("unexpected extra_params.number_value: %#v", extraParams["number_value"])
|
||||
}
|
||||
objectValue, ok := extraParams["object_value"].(map[string]any)
|
||||
if !ok || objectValue["nested"] != "value" {
|
||||
t.Fatalf("unexpected extra_params.object_value: %#v", extraParams["object_value"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerInvalidJSON(t *testing.T) {
|
||||
@@ -335,6 +506,54 @@ func TestHandlerMissingPromptID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerReservedExtraParamsThroughRunnerMapsToInvalidRequest(t *testing.T) {
|
||||
llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner := usecase.NewRunner(
|
||||
handlerPromptRepo{def: &domain.PromptDefinition{
|
||||
ID: "p",
|
||||
Version: "1",
|
||||
DefaultProfile: "exec",
|
||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "hi"}},
|
||||
OutputFormat: domain.FormatText,
|
||||
Validation: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone},
|
||||
}},
|
||||
handlerProfileRepo{profile: &domain.ExecutionProfile{
|
||||
ID: "exec",
|
||||
Endpoint: "http://example.invalid/v1",
|
||||
Model: "model",
|
||||
}},
|
||||
handlerArtifactReader{},
|
||||
handlerRenderer{},
|
||||
llmClient,
|
||||
nil,
|
||||
)
|
||||
h := NewHandler(runner)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||
"prompt_id":"p",
|
||||
"inputs":{"x":{"type":"file","uri":"a"}},
|
||||
"model":{"extra_params":{"model":"collision"}}
|
||||
}`))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
h.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("expected 400, got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
var resp map[string]any
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("invalid JSON response: %v", err)
|
||||
}
|
||||
errBody := resp["error"].(map[string]any)
|
||||
if errBody["code"] != "invalid_request" {
|
||||
t.Fatalf("expected invalid_request code, got %#v", errBody["code"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerUsecaseErrorMapping(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -40,14 +40,33 @@ const (
|
||||
ValidationSkipped ValidationStatus = "skipped"
|
||||
)
|
||||
|
||||
// CacheControlType defines provider cache behavior for prompt content.
|
||||
type CacheControlType string
|
||||
|
||||
const (
|
||||
CacheControlEphemeral CacheControlType = "ephemeral"
|
||||
)
|
||||
|
||||
const (
|
||||
// SessionIDMaxLength is OpenRouter's documented maximum session_id length.
|
||||
SessionIDMaxLength = 256
|
||||
)
|
||||
|
||||
// CacheControl describes provider cache metadata attached to prompt content.
|
||||
type CacheControl struct {
|
||||
Type CacheControlType `yaml:"type" json:"type"`
|
||||
TTL string `yaml:"ttl,omitempty" json:"ttl,omitempty"`
|
||||
}
|
||||
|
||||
// RunRequest represents a request to generate a single artifact.
|
||||
type RunRequest struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
APIKey string `json:"-" yaml:"-"`
|
||||
Inputs map[string]ArtifactRef
|
||||
Vars map[string]string
|
||||
Execution *ExecutionTarget
|
||||
Execution *ExecutionTargetOverride
|
||||
Validation *OutputContract
|
||||
Metadata map[string]string
|
||||
}
|
||||
@@ -77,19 +96,21 @@ type RunResult struct {
|
||||
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
|
||||
// It must never include resolved API key values, model output, or validation data.
|
||||
type PreparedRun struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
EndTime time.Time `json:"end_time,omitempty"`
|
||||
DurationMS int64 `json:"duration_ms,omitempty"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
TargetPresence ExecutionTargetPresence `json:"-"`
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
EndTime time.Time `json:"end_time,omitempty"`
|
||||
DurationMS int64 `json:"duration_ms,omitempty"`
|
||||
}
|
||||
|
||||
// ArtifactRef represents a reference to an input artifact.
|
||||
@@ -115,6 +136,7 @@ type PromptDefinition struct {
|
||||
Version string `yaml:"version"`
|
||||
DefaultProfile string `yaml:"default_profile"`
|
||||
Description string `yaml:"description"`
|
||||
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
|
||||
Inputs []PromptInput `yaml:"inputs"`
|
||||
Templates []PromptMessageTemplate `yaml:"templates"`
|
||||
OutputFormat OutputFormat `yaml:"output_format"`
|
||||
@@ -131,38 +153,65 @@ type PromptInput struct {
|
||||
|
||||
// PromptMessageTemplate defines a template for a chat message.
|
||||
type PromptMessageTemplate struct {
|
||||
Role string `yaml:"role"`
|
||||
Content string `yaml:"content"`
|
||||
ContentFile string `yaml:"content_file"`
|
||||
Role string `yaml:"role"`
|
||||
Content string `yaml:"content"`
|
||||
ContentFile string `yaml:"content_file"`
|
||||
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
||||
}
|
||||
|
||||
// ExecutionProfile describes how and where to execute a model.
|
||||
type ExecutionProfile struct {
|
||||
ID string `yaml:"id"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Model string `yaml:"model"`
|
||||
Temperature float64 `yaml:"temperature"`
|
||||
MaxTokens int `yaml:"max_tokens"`
|
||||
TopP float64 `yaml:"top_p"`
|
||||
TimeoutSeconds int `yaml:"timeout_seconds"`
|
||||
ServiceTier string `yaml:"service_tier"`
|
||||
ReasoningEffort string `yaml:"reasoning_effort"`
|
||||
APIKeyEnv string `yaml:"api_key_env"`
|
||||
ExtraParams map[string]string `yaml:"extra_params"`
|
||||
ID string `yaml:"id"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Model string `yaml:"model"`
|
||||
Temperature float64 `yaml:"temperature"`
|
||||
MaxTokens int `yaml:"max_tokens"`
|
||||
TopP float64 `yaml:"top_p"`
|
||||
TimeoutSeconds int `yaml:"timeout_seconds"`
|
||||
ServiceTier string `yaml:"service_tier"`
|
||||
ReasoningEffort string `yaml:"reasoning_effort"`
|
||||
APIKeyEnv string `yaml:"api_key_env"`
|
||||
APIKeyRequired bool `yaml:"-" json:"-"`
|
||||
ExtraParams map[string]any `yaml:"extra_params"`
|
||||
}
|
||||
|
||||
// ExecutionTargetOverride represents per-request runtime setting overrides.
|
||||
type ExecutionTargetOverride struct {
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||
ServiceTier string `json:"service_tier,omitempty"`
|
||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||
}
|
||||
|
||||
// ExecutionTargetPresence tracks which effective runtime fields came from an
|
||||
// explicit request override even when the resolved value is a zero value.
|
||||
type ExecutionTargetPresence struct {
|
||||
Temperature bool
|
||||
MaxTokens bool
|
||||
TopP bool
|
||||
TimeoutSeconds bool
|
||||
}
|
||||
|
||||
// ExecutionTarget represents effective model runtime settings for a run.
|
||||
type ExecutionTarget struct {
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||
Model string `yaml:"model" json:"model"`
|
||||
Temperature float64 `yaml:"temperature" json:"temperature"`
|
||||
MaxTokens int `yaml:"max_tokens" json:"max_tokens"`
|
||||
TopP float64 `yaml:"top_p" json:"top_p"`
|
||||
TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"`
|
||||
ServiceTier string `yaml:"service_tier" json:"service_tier"`
|
||||
ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"`
|
||||
APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"`
|
||||
ExtraParams map[string]string `yaml:"extra_params" json:"extra_params"`
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||
Model string `yaml:"model" json:"model"`
|
||||
Temperature float64 `yaml:"temperature" json:"temperature"`
|
||||
MaxTokens int `yaml:"max_tokens" json:"max_tokens"`
|
||||
TopP float64 `yaml:"top_p" json:"top_p"`
|
||||
TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"`
|
||||
ServiceTier string `yaml:"service_tier" json:"service_tier"`
|
||||
ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"`
|
||||
APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"`
|
||||
APIKey string `yaml:"-" json:"-"`
|
||||
APIKeyRequired bool `yaml:"-" json:"-"`
|
||||
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
|
||||
}
|
||||
|
||||
// OutputContract defines the requirements for the output artifact.
|
||||
@@ -175,19 +224,22 @@ type OutputContract struct {
|
||||
|
||||
// RenderedPrompt represents the prompt after template application.
|
||||
type RenderedPrompt struct {
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
}
|
||||
|
||||
// RenderedMessage is a single message in a rendered prompt.
|
||||
type RenderedMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
CacheControl *CacheControl `json:"cache_control,omitempty"`
|
||||
}
|
||||
|
||||
// GenerateRequest is the internal request passed to the LLM client.
|
||||
type GenerateRequest struct {
|
||||
Prompt RenderedPrompt
|
||||
Target ExecutionTarget
|
||||
TargetPresence ExecutionTargetPresence
|
||||
StructuredOutput *StructuredOutputSpec
|
||||
}
|
||||
|
||||
@@ -222,6 +274,8 @@ type TokenUsage struct {
|
||||
PromptTokens int
|
||||
CompletionTokens int
|
||||
TotalTokens int
|
||||
CachedTokens int
|
||||
CacheWriteTokens int
|
||||
}
|
||||
|
||||
// ValidationResult represents the outcome of an output validation.
|
||||
|
||||
@@ -20,6 +20,7 @@ func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
|
||||
Endpoint: "http://llm/v1",
|
||||
Model: "gpt-test",
|
||||
APIKeyEnv: envName,
|
||||
APIKey: secret,
|
||||
},
|
||||
InputHashes: map[string]string{"transcript": "hash-1"},
|
||||
RenderedPromptHash: "rendered-hash",
|
||||
@@ -53,3 +54,88 @@ func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedRunJSONIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
||||
prepared := PreparedRun{
|
||||
PromptID: "prompt.id",
|
||||
SelectedProfileID: "local-fast",
|
||||
EffectiveModelParams: ExecutionTarget{
|
||||
Endpoint: "http://llm/v1",
|
||||
Model: "gpt-test",
|
||||
},
|
||||
RenderedPromptHash: "rendered-hash",
|
||||
Messages: []RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "You are helpful.",
|
||||
CacheControl: &CacheControl{
|
||||
Type: CacheControlEphemeral,
|
||||
TTL: "1h",
|
||||
},
|
||||
},
|
||||
{Role: "user", Content: "Summarize this."},
|
||||
},
|
||||
}
|
||||
|
||||
b, err := json.Marshal(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal failed: %v", err)
|
||||
}
|
||||
|
||||
var decoded struct {
|
||||
Messages []map[string]any `json:"messages"`
|
||||
}
|
||||
if err := json.Unmarshal(b, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal failed: %v", err)
|
||||
}
|
||||
if len(decoded.Messages) != 2 {
|
||||
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
|
||||
}
|
||||
|
||||
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected cache_control on first message, got %#v", decoded.Messages[0])
|
||||
}
|
||||
if cacheControl["type"] != string(CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
||||
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
||||
}
|
||||
if _, ok := decoded.Messages[1]["cache_control"]; ok {
|
||||
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedRunJSONIncludesSessionIDOnlyWhenPresent(t *testing.T) {
|
||||
prepared := PreparedRun{
|
||||
PromptID: "prompt.id",
|
||||
SelectedProfileID: "local-fast",
|
||||
EffectiveModelParams: ExecutionTarget{
|
||||
Endpoint: "http://llm/v1",
|
||||
Model: "gpt-test",
|
||||
},
|
||||
SessionID: "session-123",
|
||||
RenderedPromptHash: "rendered-hash",
|
||||
Messages: []RenderedMessage{{Role: "user", Content: "Summarize this."}},
|
||||
}
|
||||
|
||||
b, err := json.Marshal(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal failed: %v", err)
|
||||
}
|
||||
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(b, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal failed: %v", err)
|
||||
}
|
||||
if decoded["session_id"] != "session-123" {
|
||||
t.Fatalf("expected session_id in prepared run JSON, got %#v", decoded["session_id"])
|
||||
}
|
||||
|
||||
prepared.SessionID = ""
|
||||
b, err = json.Marshal(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal failed: %v", err)
|
||||
}
|
||||
if strings.Contains(string(b), "session_id") {
|
||||
t.Fatalf("expected empty session_id to be omitted, got %s", b)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,6 +96,9 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
||||
if prepared.PromptHash != "" {
|
||||
fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash)
|
||||
}
|
||||
if prepared.SessionID != "" {
|
||||
fmt.Fprintf(&b, "session_id: %s\n", prepared.SessionID)
|
||||
}
|
||||
fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash)
|
||||
|
||||
target := prepared.EffectiveModelParams
|
||||
@@ -123,7 +126,11 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
fmt.Fprintf(&b, " %s: %s\n", k, target.ExtraParams[k])
|
||||
renderedValue, err := formatExtraParamTextValue(target.ExtraParams[k])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to format extra_params.%s: %w", k, err)
|
||||
}
|
||||
fmt.Fprintf(&b, " %s: %s\n", k, renderedValue)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,6 +158,13 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
||||
messages := byRole[role]
|
||||
for i, msg := range messages {
|
||||
fmt.Fprintf(&b, " - message: %d\n", i+1)
|
||||
if msg.CacheControl != nil {
|
||||
fmt.Fprintf(&b, " cache_control: %s", msg.CacheControl.Type)
|
||||
if msg.CacheControl.TTL != "" {
|
||||
fmt.Fprintf(&b, " ttl=%s", msg.CacheControl.TTL)
|
||||
}
|
||||
fmt.Fprintln(&b)
|
||||
}
|
||||
fmt.Fprintln(&b, " content: |")
|
||||
content := msg.Content
|
||||
if content == "" {
|
||||
@@ -165,3 +179,15 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
||||
|
||||
return b.Bytes(), nil
|
||||
}
|
||||
|
||||
func formatExtraParamTextValue(value any) (string, error) {
|
||||
if s, ok := value.(string); ok {
|
||||
return s, nil
|
||||
}
|
||||
|
||||
b, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
@@ -49,6 +49,36 @@ func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTextFormatterRendersExtraParamsDeterministically(t *testing.T) {
|
||||
prepared := samplePreparedRun()
|
||||
prepared.EffectiveModelParams.ExtraParams = map[string]any{
|
||||
"z_string": "enabled",
|
||||
"b_number": 42,
|
||||
"a_object": map[string]any{
|
||||
"nested": "value",
|
||||
"count": 2,
|
||||
},
|
||||
"c_array": []any{"first", 3, false},
|
||||
}
|
||||
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
s := string(out)
|
||||
|
||||
want := strings.Join([]string{
|
||||
" extra_params:",
|
||||
" a_object: {\"count\":2,\"nested\":\"value\"}",
|
||||
" b_number: 42",
|
||||
" c_array: [\"first\",3,false]",
|
||||
" z_string: enabled",
|
||||
}, "\n")
|
||||
if !strings.Contains(s, want) {
|
||||
t.Fatalf("expected deterministic extra_params block %q, got:\n%s", want, s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
||||
const secret = "super-secret-api-key"
|
||||
t.Setenv("SCRIPTORIUM_API_KEY", secret)
|
||||
@@ -62,8 +92,94 @@ func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTextFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) {
|
||||
const directKey = "direct-format-key"
|
||||
prepared := samplePreparedRun()
|
||||
prepared.EffectiveModelParams.APIKey = directKey
|
||||
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if strings.Contains(string(out), directKey) {
|
||||
t.Fatalf("text output should not include direct api key value: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTextFormatterIncludesMessageCacheControlBeforeContent(t *testing.T) {
|
||||
prepared := samplePreparedRun()
|
||||
prepared.Messages = []domain.RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "System guidance.",
|
||||
CacheControl: &domain.CacheControl{
|
||||
Type: domain.CacheControlEphemeral,
|
||||
TTL: "1h",
|
||||
},
|
||||
},
|
||||
{Role: "user", Content: "Summarize the transcript."},
|
||||
}
|
||||
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
s := string(out)
|
||||
if !strings.Contains(s, " system:\n - message: 1\n cache_control: ephemeral ttl=1h\n content: |") {
|
||||
t.Fatalf("expected system message cache control before content, got:\n%s", s)
|
||||
}
|
||||
if strings.Count(s, "cache_control:") != 1 {
|
||||
t.Fatalf("expected exactly one cache_control line, got:\n%s", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTextFormatterIncludesSessionIDWhenPresent(t *testing.T) {
|
||||
prepared := samplePreparedRun()
|
||||
prepared.SessionID = "session-123"
|
||||
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if !strings.Contains(string(out), "session_id: session-123\n") {
|
||||
t.Fatalf("expected session_id in text output, got:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTextFormatterOmitsEmptyCacheControlTTL(t *testing.T) {
|
||||
prepared := samplePreparedRun()
|
||||
prepared.Messages = []domain.RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "System guidance.",
|
||||
CacheControl: &domain.CacheControl{
|
||||
Type: domain.CacheControlEphemeral,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
s := string(out)
|
||||
if !strings.Contains(s, " cache_control: ephemeral\n") {
|
||||
t.Fatalf("expected cache_control line without ttl, got:\n%s", s)
|
||||
}
|
||||
if strings.Contains(s, "ttl=") {
|
||||
t.Fatalf("expected empty ttl to be omitted, got:\n%s", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
||||
prepared := samplePreparedRun()
|
||||
prepared.SessionID = "session-123"
|
||||
prepared.EffectiveModelParams.ExtraParams = map[string]any{
|
||||
"number": 42,
|
||||
"nested": map[string]any{
|
||||
"enabled": true,
|
||||
},
|
||||
}
|
||||
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
||||
if err != nil {
|
||||
@@ -87,9 +203,24 @@ func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
||||
if decoded["rendered_prompt_hash"] != "rendered-hash" {
|
||||
t.Fatalf("expected rendered_prompt_hash in json output, got %#v", decoded["rendered_prompt_hash"])
|
||||
}
|
||||
if _, ok := decoded["effective_model_params"]; !ok {
|
||||
if decoded["session_id"] != "session-123" {
|
||||
t.Fatalf("expected session_id in json output, got %#v", decoded["session_id"])
|
||||
}
|
||||
modelParams, ok := decoded["effective_model_params"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected effective_model_params in json output, got %#v", decoded)
|
||||
}
|
||||
extraParams, ok := modelParams["extra_params"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected extra_params in json output, got %#v", modelParams["extra_params"])
|
||||
}
|
||||
if extraParams["number"] != float64(42) {
|
||||
t.Fatalf("unexpected numeric extra param in json output: %#v", extraParams["number"])
|
||||
}
|
||||
nested, ok := extraParams["nested"].(map[string]any)
|
||||
if !ok || nested["enabled"] != true {
|
||||
t.Fatalf("unexpected nested extra param in json output: %#v", extraParams["nested"])
|
||||
}
|
||||
if _, ok := decoded["input_hashes"]; !ok {
|
||||
t.Fatalf("expected input_hashes in json output, got %#v", decoded)
|
||||
}
|
||||
@@ -98,6 +229,47 @@ func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONFormatterIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
||||
prepared := samplePreparedRun()
|
||||
prepared.Messages = []domain.RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "System guidance.",
|
||||
CacheControl: &domain.CacheControl{
|
||||
Type: domain.CacheControlEphemeral,
|
||||
TTL: "1h",
|
||||
},
|
||||
},
|
||||
{Role: "user", Content: "Summarize the transcript."},
|
||||
}
|
||||
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
var decoded struct {
|
||||
Messages []map[string]any `json:"messages"`
|
||||
}
|
||||
if err := json.Unmarshal(out, &decoded); err != nil {
|
||||
t.Fatalf("expected valid json output, got %v", err)
|
||||
}
|
||||
if len(decoded.Messages) != 2 {
|
||||
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
|
||||
}
|
||||
|
||||
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected first message cache_control, got %#v", decoded.Messages[0])
|
||||
}
|
||||
if cacheControl["type"] != string(domain.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
||||
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
||||
}
|
||||
if _, ok := decoded.Messages[1]["cache_control"]; ok {
|
||||
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
||||
const secret = "super-secret-api-key"
|
||||
t.Setenv("SCRIPTORIUM_API_KEY", secret)
|
||||
@@ -111,6 +283,20 @@ func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) {
|
||||
const directKey = "direct-format-key"
|
||||
prepared := samplePreparedRun()
|
||||
prepared.EffectiveModelParams.APIKey = directKey
|
||||
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if strings.Contains(string(out), directKey) {
|
||||
t.Fatalf("json output should not include direct api key value: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePreparedRunOutputFormatRecognizesSupportedNames(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
@@ -89,7 +90,12 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(wireReq)
|
||||
wirePayload, err := openAIChatRequestPayload(wireReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(wirePayload)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
|
||||
}
|
||||
@@ -99,7 +105,9 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
||||
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" {
|
||||
if apiKey := strings.TrimSpace(req.Target.APIKey); apiKey != "" {
|
||||
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
} else if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" {
|
||||
apiKey := strings.TrimSpace(os.Getenv(envName))
|
||||
if apiKey == "" {
|
||||
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
|
||||
@@ -110,6 +118,8 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
||||
effectiveTimeout := c.timeout
|
||||
if req.Target.TimeoutSeconds > 0 {
|
||||
effectiveTimeout = time.Duration(req.Target.TimeoutSeconds) * time.Second
|
||||
} else if req.TargetPresence.TimeoutSeconds {
|
||||
effectiveTimeout = 0
|
||||
}
|
||||
|
||||
httpClient := c.httpClient
|
||||
@@ -151,6 +161,8 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
||||
PromptTokens: wireResp.Usage.PromptTokens,
|
||||
CompletionTokens: wireResp.Usage.CompletionTokens,
|
||||
TotalTokens: wireResp.Usage.TotalTokens,
|
||||
CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens,
|
||||
CacheWriteTokens: wireResp.Usage.CacheWriteTokens,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
@@ -167,27 +179,36 @@ func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultMod
|
||||
wireReq := openAIChatRequest{
|
||||
Model: model,
|
||||
}
|
||||
|
||||
wireReq.Messages = make([]openAIChatMessage, 0, len(req.Prompt.Messages))
|
||||
for _, msg := range req.Prompt.Messages {
|
||||
wireReq.Messages = append(wireReq.Messages, openAIChatMessage{
|
||||
Role: msg.Role,
|
||||
Content: msg.Content,
|
||||
})
|
||||
if sessionID := strings.TrimSpace(req.Prompt.SessionID); sessionID != "" {
|
||||
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
|
||||
return openAIChatRequest{}, fmt.Errorf("session_id length %d exceeds maximum %d", n, domain.SessionIDMaxLength)
|
||||
}
|
||||
wireReq.SessionID = sessionID
|
||||
}
|
||||
|
||||
if req.Target.Temperature != 0 {
|
||||
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
|
||||
for _, msg := range req.Prompt.Messages {
|
||||
wireReq.Messages = append(wireReq.Messages, openAIChatRequestMessageFromRenderedMessage(msg))
|
||||
}
|
||||
|
||||
if req.Target.Temperature != 0 || req.TargetPresence.Temperature {
|
||||
wireReq.Temperature = &req.Target.Temperature
|
||||
}
|
||||
if req.Target.MaxTokens != 0 {
|
||||
if req.Target.MaxTokens != 0 || req.TargetPresence.MaxTokens {
|
||||
wireReq.MaxTokens = &req.Target.MaxTokens
|
||||
}
|
||||
if req.Target.TopP != 0 {
|
||||
if req.Target.TopP != 0 || req.TargetPresence.TopP {
|
||||
wireReq.TopP = &req.Target.TopP
|
||||
}
|
||||
if strings.TrimSpace(req.Target.ServiceTier) != "" {
|
||||
wireReq.ServiceTier = req.Target.ServiceTier
|
||||
}
|
||||
if strings.TrimSpace(req.Target.ReasoningEffort) != "" {
|
||||
wireReq.ReasoningEffort = req.Target.ReasoningEffort
|
||||
}
|
||||
if len(req.Target.ExtraParams) > 0 {
|
||||
wireReq.ExtraParams = req.Target.ExtraParams
|
||||
}
|
||||
if req.StructuredOutput != nil {
|
||||
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
|
||||
if err != nil {
|
||||
@@ -200,28 +221,106 @@ func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultMod
|
||||
}
|
||||
|
||||
type openAIChatRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []openAIChatMessage `json:"messages"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
ServiceTier string `json:"service_tier,omitempty"`
|
||||
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"`
|
||||
Model string `json:"model"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Messages []openAIChatRequestMessage `json:"messages"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
ServiceTier string `json:"service_tier,omitempty"`
|
||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"`
|
||||
ExtraParams map[string]any `json:"-"`
|
||||
}
|
||||
|
||||
type openAIChatMessage struct {
|
||||
func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
||||
out := map[string]any{
|
||||
"model": req.Model,
|
||||
"messages": req.Messages,
|
||||
}
|
||||
if req.SessionID != "" {
|
||||
out["session_id"] = req.SessionID
|
||||
}
|
||||
if req.Temperature != nil {
|
||||
out["temperature"] = *req.Temperature
|
||||
}
|
||||
if req.MaxTokens != nil {
|
||||
out["max_tokens"] = *req.MaxTokens
|
||||
}
|
||||
if req.TopP != nil {
|
||||
out["top_p"] = *req.TopP
|
||||
}
|
||||
if req.ServiceTier != "" {
|
||||
out["service_tier"] = req.ServiceTier
|
||||
}
|
||||
if req.ReasoningEffort != "" {
|
||||
out["reasoning_effort"] = req.ReasoningEffort
|
||||
}
|
||||
if req.ResponseFormat != nil {
|
||||
out["response_format"] = req.ResponseFormat
|
||||
}
|
||||
|
||||
for key, value := range req.ExtraParams {
|
||||
if key == "" {
|
||||
return nil, errors.New("extra_params key must not be empty")
|
||||
}
|
||||
if _, reserved := reservedOpenAIChatRequestFields[key]; reserved {
|
||||
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
|
||||
}
|
||||
if _, err := json.Marshal(value); err != nil {
|
||||
return nil, fmt.Errorf("extra_params.%s must be JSON-serializable: %w", key, err)
|
||||
}
|
||||
out[key] = value
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
var reservedOpenAIChatRequestFields = map[string]struct{}{
|
||||
"model": {},
|
||||
"session_id": {},
|
||||
"messages": {},
|
||||
"temperature": {},
|
||||
"max_tokens": {},
|
||||
"top_p": {},
|
||||
"service_tier": {},
|
||||
"reasoning_effort": {},
|
||||
"response_format": {},
|
||||
}
|
||||
|
||||
type openAIChatRequestMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content any `json:"content"`
|
||||
}
|
||||
|
||||
type openAIChatTextContentBlock struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
CacheControl *openAICacheControl `json:"cache_control,omitempty"`
|
||||
}
|
||||
|
||||
type openAICacheControl struct {
|
||||
Type string `json:"type"`
|
||||
TTL string `json:"ttl,omitempty"`
|
||||
}
|
||||
|
||||
type openAIChatResponseMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
type openAIChatResponse struct {
|
||||
Choices []struct {
|
||||
Message openAIChatMessage `json:"message"`
|
||||
Message openAIChatResponseMessage `json:"message"`
|
||||
} `json:"choices"`
|
||||
Usage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
PromptTokensDetails struct {
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
} `json:"prompt_tokens_details"`
|
||||
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||
} `json:"usage"`
|
||||
}
|
||||
|
||||
@@ -236,6 +335,28 @@ type openAIJSONSchemaEnvelope struct {
|
||||
Schema any `json:"schema"`
|
||||
}
|
||||
|
||||
func openAIChatRequestMessageFromRenderedMessage(msg domain.RenderedMessage) openAIChatRequestMessage {
|
||||
wireMsg := openAIChatRequestMessage{
|
||||
Role: msg.Role,
|
||||
Content: msg.Content,
|
||||
}
|
||||
if msg.CacheControl == nil {
|
||||
return wireMsg
|
||||
}
|
||||
|
||||
wireMsg.Content = []openAIChatTextContentBlock{
|
||||
{
|
||||
Type: "text",
|
||||
Text: msg.Content,
|
||||
CacheControl: &openAICacheControl{
|
||||
Type: string(msg.CacheControl.Type),
|
||||
TTL: msg.CacheControl.TTL,
|
||||
},
|
||||
},
|
||||
}
|
||||
return wireMsg
|
||||
}
|
||||
|
||||
func toOpenAIResponseFormat(spec *domain.StructuredOutputSpec) (*openAIResponseFormat, error) {
|
||||
if spec == nil {
|
||||
return nil, nil
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
@@ -89,6 +90,9 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
|
||||
if resp.Usage.PromptTokens != 11 || resp.Usage.CompletionTokens != 22 || resp.Usage.TotalTokens != 33 {
|
||||
t.Fatalf("unexpected usage: %+v", resp.Usage)
|
||||
}
|
||||
if resp.Usage.CachedTokens != 0 || resp.Usage.CacheWriteTokens != 0 {
|
||||
t.Fatalf("expected absent cache usage fields to remain zero, got %+v", resp.Usage)
|
||||
}
|
||||
|
||||
if obs.Authorization != "Bearer secret-key" {
|
||||
t.Fatalf("unexpected Authorization header: %q", obs.Authorization)
|
||||
@@ -144,6 +148,277 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientDirectAPIKeyPreferredOverEnv(t *testing.T) {
|
||||
const directKey = "direct-llm-key"
|
||||
t.Setenv("SCRIPTORIUM_TEST_API_KEY", "env-key")
|
||||
|
||||
var gotAuth string
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotAuth = r.Header.Get("Authorization")
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||
Target: domain.ExecutionTarget{
|
||||
Model: "model",
|
||||
APIKeyEnv: "SCRIPTORIUM_TEST_API_KEY",
|
||||
APIKey: directKey,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if gotAuth != "Bearer "+directKey {
|
||||
t.Fatalf("unexpected Authorization header: %q", gotAuth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientSerializesCacheControlledMessageAsContentBlock(t *testing.T) {
|
||||
var observedBody map[string]any
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
defer r.Body.Close()
|
||||
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||
t.Fatalf("failed to decode request body: %v", err)
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "Stable instructions.",
|
||||
CacheControl: &domain.CacheControl{
|
||||
Type: domain.CacheControlEphemeral,
|
||||
TTL: "1h",
|
||||
},
|
||||
},
|
||||
{Role: "user", Content: "Dynamic request."},
|
||||
}},
|
||||
Target: domain.ExecutionTarget{Model: "model"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
for _, forbidden := range []string{"cache_control", "extra_params"} {
|
||||
if _, exists := observedBody[forbidden]; exists {
|
||||
t.Fatalf("expected top-level %s to be omitted, got %#v", forbidden, observedBody[forbidden])
|
||||
}
|
||||
}
|
||||
|
||||
msgs, ok := observedBody["messages"].([]any)
|
||||
if !ok || len(msgs) != 2 {
|
||||
t.Fatalf("unexpected messages payload: %#v", observedBody["messages"])
|
||||
}
|
||||
msg0 := msgs[0].(map[string]any)
|
||||
if msg0["role"] != "system" {
|
||||
t.Fatalf("unexpected first message role: %#v", msg0["role"])
|
||||
}
|
||||
contentBlocks, ok := msg0["content"].([]any)
|
||||
if !ok || len(contentBlocks) != 1 {
|
||||
t.Fatalf("expected first message content block array, got %#v", msg0["content"])
|
||||
}
|
||||
block := contentBlocks[0].(map[string]any)
|
||||
if block["type"] != "text" || block["text"] != "Stable instructions." {
|
||||
t.Fatalf("unexpected text content block: %#v", block)
|
||||
}
|
||||
cacheControl, ok := block["cache_control"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected cache_control on content block, got %#v", block)
|
||||
}
|
||||
if cacheControl["type"] != string(domain.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
||||
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
||||
}
|
||||
|
||||
msg1 := msgs[1].(map[string]any)
|
||||
if msg1["role"] != "user" || msg1["content"] != "Dynamic request." {
|
||||
t.Fatalf("expected uncached message to keep string content, got %#v", msg1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientOmitsEmptyCacheControlTTL(t *testing.T) {
|
||||
var observedBody map[string]any
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
defer r.Body.Close()
|
||||
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||
t.Fatalf("failed to decode request body: %v", err)
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "Stable instructions.",
|
||||
CacheControl: &domain.CacheControl{
|
||||
Type: domain.CacheControlEphemeral,
|
||||
},
|
||||
},
|
||||
}},
|
||||
Target: domain.ExecutionTarget{Model: "model"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
msgs := observedBody["messages"].([]any)
|
||||
msg0 := msgs[0].(map[string]any)
|
||||
contentBlocks := msg0["content"].([]any)
|
||||
block := contentBlocks[0].(map[string]any)
|
||||
cacheControl := block["cache_control"].(map[string]any)
|
||||
if cacheControl["type"] != string(domain.CacheControlEphemeral) {
|
||||
t.Fatalf("unexpected cache_control type: %#v", cacheControl)
|
||||
}
|
||||
if _, exists := cacheControl["ttl"]; exists {
|
||||
t.Fatalf("expected empty ttl to be omitted, got %#v", cacheControl)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientSerializesSessionID(t *testing.T) {
|
||||
var observedBody map[string]any
|
||||
var observedSessionHeader string
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
observedSessionHeader = r.Header.Get("x-session-id")
|
||||
defer r.Body.Close()
|
||||
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||
t.Fatalf("failed to decode request body: %v", err)
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{
|
||||
SessionID: " session-123 ",
|
||||
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
|
||||
},
|
||||
Target: domain.ExecutionTarget{Model: "model"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if observedBody["session_id"] != "session-123" {
|
||||
t.Fatalf("expected top-level session_id, got %#v", observedBody["session_id"])
|
||||
}
|
||||
if observedSessionHeader != "" {
|
||||
t.Fatalf("did not expect x-session-id header, got %q", observedSessionHeader)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientOmitsEmptySessionID(t *testing.T) {
|
||||
var observedBody map[string]any
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
defer r.Body.Close()
|
||||
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||
t.Fatalf("failed to decode request body: %v", err)
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{
|
||||
SessionID: " ",
|
||||
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
|
||||
},
|
||||
Target: domain.ExecutionTarget{Model: "model"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if _, exists := observedBody["session_id"]; exists {
|
||||
t.Fatalf("expected empty session_id to be omitted, got %#v", observedBody["session_id"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientRejectsTooLongSessionID(t *testing.T) {
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||
BaseURL: "http://example.com/v1",
|
||||
Model: "model",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{
|
||||
SessionID: strings.Repeat("x", domain.SessionIDMaxLength+1),
|
||||
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid request error")
|
||||
}
|
||||
if !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientParsesCacheUsage(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{
|
||||
"choices": [{"message": {"role": "assistant", "content": "ok"}}],
|
||||
"usage": {
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 20,
|
||||
"total_tokens": 120,
|
||||
"prompt_tokens_details": {"cached_tokens": 80},
|
||||
"cache_write_tokens": 60
|
||||
}
|
||||
}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "model"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if resp.Usage.PromptTokens != 100 || resp.Usage.CompletionTokens != 20 || resp.Usage.TotalTokens != 120 {
|
||||
t.Fatalf("unexpected base usage fields: %+v", resp.Usage)
|
||||
}
|
||||
if resp.Usage.CachedTokens != 80 || resp.Usage.CacheWriteTokens != 60 {
|
||||
t.Fatalf("unexpected cache usage fields: %+v", resp.Usage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *testing.T) {
|
||||
var observedBody map[string]any
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -175,7 +450,7 @@ func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *test
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientOmitsReasoningEffortAndExtraParams(t *testing.T) {
|
||||
func TestOpenAICompatibleClientSerializesReasoningEffortAndExtraParams(t *testing.T) {
|
||||
var observedBody map[string]any
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
defer r.Body.Close()
|
||||
@@ -196,19 +471,257 @@ func TestOpenAICompatibleClientOmitsReasoningEffortAndExtraParams(t *testing.T)
|
||||
Target: domain.ExecutionTarget{
|
||||
Model: "model",
|
||||
ReasoningEffort: "high",
|
||||
ExtraParams: map[string]string{
|
||||
"provider_option": "on",
|
||||
ExtraParams: map[string]any{
|
||||
"string_value": "on",
|
||||
"number_value": 42,
|
||||
"boolean_value": true,
|
||||
"object_value": map[string]any{"nested": "value", "count": 2},
|
||||
"array_value": []any{"first", 3, false},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if observedBody["reasoning_effort"] != "high" {
|
||||
t.Fatalf("expected reasoning_effort high, got %#v", observedBody["reasoning_effort"])
|
||||
}
|
||||
if observedBody["string_value"] != "on" {
|
||||
t.Fatalf("unexpected string extra param: %#v", observedBody["string_value"])
|
||||
}
|
||||
if observedBody["number_value"] != float64(42) {
|
||||
t.Fatalf("unexpected number extra param: %#v", observedBody["number_value"])
|
||||
}
|
||||
if observedBody["boolean_value"] != true {
|
||||
t.Fatalf("unexpected boolean extra param: %#v", observedBody["boolean_value"])
|
||||
}
|
||||
objectValue, ok := observedBody["object_value"].(map[string]any)
|
||||
if !ok || objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
|
||||
t.Fatalf("unexpected object extra param: %#v", observedBody["object_value"])
|
||||
}
|
||||
if _, exists := observedBody["extra_params"]; exists {
|
||||
t.Fatalf("expected extra_params wrapper omitted, got %#v", observedBody["extra_params"])
|
||||
}
|
||||
arrayValue, ok := observedBody["array_value"].([]any)
|
||||
if !ok || len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
|
||||
t.Fatalf("unexpected array extra param: %#v", observedBody["array_value"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientOmitsReasoningEffortWhenUnset(t *testing.T) {
|
||||
var observedBody map[string]any
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
defer r.Body.Close()
|
||||
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||
t.Fatalf("failed to decode request body: %v", err)
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||
Target: domain.ExecutionTarget{Model: "model"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if _, exists := observedBody["reasoning_effort"]; exists {
|
||||
t.Fatalf("expected reasoning_effort omitted, got %#v", observedBody["reasoning_effort"])
|
||||
}
|
||||
if _, exists := observedBody["extra_params"]; exists {
|
||||
t.Fatalf("expected extra_params omitted, got %#v", observedBody["extra_params"])
|
||||
t.Fatalf("expected extra_params wrapper omitted, got %#v", observedBody["extra_params"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientSerializesExplicitZeroNumericOverrides(t *testing.T) {
|
||||
var observedBody map[string]any
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
defer r.Body.Close()
|
||||
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||
t.Fatalf("failed to decode request body: %v", err)
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||
Target: domain.ExecutionTarget{Model: "model"},
|
||||
TargetPresence: domain.ExecutionTargetPresence{
|
||||
Temperature: true,
|
||||
MaxTokens: true,
|
||||
TopP: true,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if observedBody["temperature"] != float64(0) {
|
||||
t.Fatalf("expected explicit zero temperature, got %#v", observedBody["temperature"])
|
||||
}
|
||||
if observedBody["max_tokens"] != float64(0) {
|
||||
t.Fatalf("expected explicit zero max_tokens, got %#v", observedBody["max_tokens"])
|
||||
}
|
||||
if observedBody["top_p"] != float64(0) {
|
||||
t.Fatalf("expected explicit zero top_p, got %#v", observedBody["top_p"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientOmitsImplicitZeroNumericFields(t *testing.T) {
|
||||
var observedBody map[string]any
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
defer r.Body.Close()
|
||||
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||
t.Fatalf("failed to decode request body: %v", err)
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||
Target: domain.ExecutionTarget{Model: "model"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
for _, field := range []string{"temperature", "max_tokens", "top_p"} {
|
||||
if _, exists := observedBody[field]; exists {
|
||||
t.Fatalf("expected implicit zero field %q to be omitted, got body %#v", field, observedBody)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientExplicitZeroTimeoutDisablesClientTimeout(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||
BaseURL: ts.URL + "/v1",
|
||||
Timeout: time.Nanosecond,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||
Target: domain.ExecutionTarget{Model: "model", TimeoutSeconds: 0},
|
||||
TargetPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected explicit zero timeout to disable client timeout, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientOmittedTimeoutUsesClientTimeout(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||
BaseURL: ts.URL + "/v1",
|
||||
Timeout: time.Nanosecond,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||
Target: domain.ExecutionTarget{Model: "model", TimeoutSeconds: 0},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected omitted timeout to use client timeout")
|
||||
}
|
||||
if !errors.Is(err, ErrRequestFailed) {
|
||||
t.Fatalf("expected ErrRequestFailed, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
extraParams map[string]any
|
||||
want string
|
||||
}{
|
||||
{name: "empty key", extraParams: map[string]any{"": "empty"}, want: "key must not be empty"},
|
||||
{name: "unserializable value", extraParams: map[string]any{"bad": math.Inf(1)}, want: "JSON-serializable"},
|
||||
}
|
||||
for _, key := range []string{
|
||||
"model",
|
||||
"session_id",
|
||||
"messages",
|
||||
"temperature",
|
||||
"max_tokens",
|
||||
"top_p",
|
||||
"service_tier",
|
||||
"reasoning_effort",
|
||||
"response_format",
|
||||
} {
|
||||
tests = append(tests, struct {
|
||||
name string
|
||||
extraParams map[string]any
|
||||
want string
|
||||
}{
|
||||
name: "reserved key " + key,
|
||||
extraParams: map[string]any{key: "collision"},
|
||||
want: "reserved request field",
|
||||
})
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
called := false
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
called = true
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||
Target: domain.ExecutionTarget{Model: "model", ExtraParams: tc.extraParams},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid request error")
|
||||
}
|
||||
if !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("expected error to contain %q, got %v", tc.want, err)
|
||||
}
|
||||
if called {
|
||||
t.Fatal("provider should not be called for invalid extra_params")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
9
internal/profile/builtin/assets/aion-labs/aion-2.yml
Normal file
9
internal/profile/builtin/assets/aion-labs/aion-2.yml
Normal file
@@ -0,0 +1,9 @@
|
||||
id: aion-2
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: aion-labs/aion-2.0
|
||||
temperature: 0.72
|
||||
reasoning_effort: high
|
||||
top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -0,0 +1,7 @@
|
||||
id: claude-fable-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "~anthropic/claude-fable-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 600
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -0,0 +1,7 @@
|
||||
id: claude-haiku-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "~anthropic/claude-haiku-latest"
|
||||
reasoning_effort: medium
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -0,0 +1,7 @@
|
||||
id: claude-opus-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "~anthropic/claude-opus-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -0,0 +1,7 @@
|
||||
id: claude-sonnet-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "~anthropic/claude-sonnet-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -0,0 +1,7 @@
|
||||
id: deepseek-3-2
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: deepseek/deepseek-v3.2
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -0,0 +1,7 @@
|
||||
id: deepseek-4-pro
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: deepseek/deepseek-v4-pro
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -0,0 +1,9 @@
|
||||
id: gemini-2-flash-lite
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "google/gemini-2.5-flash-lite"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -0,0 +1,9 @@
|
||||
id: gemini-2-flash
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "google/gemini-2.5-flash"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
9
internal/profile/builtin/assets/google/gemini-2-pro.yml
Normal file
9
internal/profile/builtin/assets/google/gemini-2-pro.yml
Normal file
@@ -0,0 +1,9 @@
|
||||
id: gemini-2-pro
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "google/gemini-2.5-pro"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -0,0 +1,9 @@
|
||||
id: gemini-3-flash-lite
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "google/gemini-3.1-flash-lite"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -0,0 +1,9 @@
|
||||
id: gemini-flash-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "~google/gemini-flash-latest"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -0,0 +1,9 @@
|
||||
id: gemini-pro-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "~google/gemini-pro-latest"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
9
internal/profile/builtin/assets/google/gemma-4-31b.yml
Normal file
9
internal/profile/builtin/assets/google/gemma-4-31b.yml
Normal file
@@ -0,0 +1,9 @@
|
||||
id: gemma-4-31b
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: google/gemma-4-31b-it:exacto
|
||||
temperature: 0.15
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
9
internal/profile/builtin/assets/minimax/minimax-m2.yml
Normal file
9
internal/profile/builtin/assets/minimax/minimax-m2.yml
Normal file
@@ -0,0 +1,9 @@
|
||||
id: minimax-m2
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: minimax/minimax-m2.5
|
||||
temperature: 0.5
|
||||
reasoning_effort: high
|
||||
top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
9
internal/profile/builtin/assets/minimax/minimax-m3.yml
Normal file
9
internal/profile/builtin/assets/minimax/minimax-m3.yml
Normal file
@@ -0,0 +1,9 @@
|
||||
id: minimax-m3
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: minimax/minimax-m3
|
||||
#temperature: 0.5
|
||||
reasoning_effort: high
|
||||
#top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -0,0 +1,7 @@
|
||||
id: mistral-large-2512
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: mistralai/mistral-large-2512
|
||||
temperature: 0.15
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
@@ -0,0 +1,8 @@
|
||||
id: mistral-medium-3-5
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: mistralai/mistral-medium-3-5
|
||||
temperature: 0.15
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
@@ -0,0 +1,7 @@
|
||||
id: mistral-small-3
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: mistralai/mistral-small-3.2-24b-instruct
|
||||
temperature: 0.05
|
||||
top_p: 1.0
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
@@ -0,0 +1,8 @@
|
||||
id: mistral-small-4
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: mistralai/mistral-small-2603
|
||||
temperature: 0.1
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
@@ -0,0 +1,7 @@
|
||||
id: nemotron-3-ultra
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: nvidia/nemotron-3-ultra-550b-a55b
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
7
internal/profile/builtin/assets/openai/gpt-5-mini.yml
Normal file
7
internal/profile/builtin/assets/openai/gpt-5-mini.yml
Normal file
@@ -0,0 +1,7 @@
|
||||
id: gpt-5-mini
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "openai/gpt-5.4-mini"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
7
internal/profile/builtin/assets/openai/gpt-5-nano.yml
Normal file
7
internal/profile/builtin/assets/openai/gpt-5-nano.yml
Normal file
@@ -0,0 +1,7 @@
|
||||
id: gpt-5-nano
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "openai/gpt-5.4-nano"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
31
internal/profile/builtin/repository.go
Normal file
31
internal/profile/builtin/repository.go
Normal file
@@ -0,0 +1,31 @@
|
||||
package builtin
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
)
|
||||
|
||||
const assetRoot = "assets"
|
||||
|
||||
//go:embed assets/**/*.yml
|
||||
var assets embed.FS
|
||||
|
||||
func NewRepository() profile.Repository {
|
||||
return profile.NewFSRepository(assets, assetRoot)
|
||||
}
|
||||
|
||||
func NewRepositoryWithPrimary(primary profile.Repository) profile.Repository {
|
||||
if primary == nil {
|
||||
return NewRepository()
|
||||
}
|
||||
return profile.NewOverlayRepository(primary, NewRepository())
|
||||
}
|
||||
|
||||
func NewRepositoryWithDirectory(dir string) profile.Repository {
|
||||
if strings.TrimSpace(dir) == "" {
|
||||
return NewRepository()
|
||||
}
|
||||
return NewRepositoryWithPrimary(profile.NewFilesystemRepository(dir))
|
||||
}
|
||||
127
internal/profile/builtin/repository_test.go
Normal file
127
internal/profile/builtin/repository_test.go
Normal file
@@ -0,0 +1,127 @@
|
||||
package builtin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
|
||||
repo := NewRepository()
|
||||
ids := loadBuiltInProfileIDs(t)
|
||||
if len(ids) == 0 {
|
||||
t.Fatal("expected built-in profiles")
|
||||
}
|
||||
|
||||
for id := range ids {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
p, err := repo.GetProfile(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("expected built-in profile %q to load, got %v", id, err)
|
||||
}
|
||||
if p.ID != id {
|
||||
t.Fatalf("expected profile id %q, got %q", id, p.ID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltInProfilesDoNotContainDuplicateIDsOrRawAPIKeys(t *testing.T) {
|
||||
loadBuiltInProfileIDs(t)
|
||||
}
|
||||
|
||||
func loadBuiltInProfileIDs(t *testing.T) map[string]string {
|
||||
t.Helper()
|
||||
|
||||
ids := map[string]string{}
|
||||
err := fs.WalkDir(assets, assetRoot, func(name string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() || !strings.HasSuffix(name, ".yml") {
|
||||
return nil
|
||||
}
|
||||
|
||||
data, err := assets.ReadFile(name)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read built-in profile %s: %v", name, err)
|
||||
}
|
||||
|
||||
var raw map[string]any
|
||||
if err := yaml.Unmarshal(data, &raw); err != nil {
|
||||
t.Fatalf("failed to decode built-in profile %s: %v", name, err)
|
||||
}
|
||||
if _, ok := raw["api_key"]; ok {
|
||||
t.Fatalf("built-in profile %s contains raw api_key", name)
|
||||
}
|
||||
id, ok := raw["id"].(string)
|
||||
if !ok || strings.TrimSpace(id) == "" {
|
||||
t.Fatalf("built-in profile %s has missing id", name)
|
||||
}
|
||||
if previous, ok := ids[id]; ok {
|
||||
t.Fatalf("duplicate built-in profile id %q in %s and %s", id, previous, name)
|
||||
}
|
||||
ids[id] = name
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to walk built-in profiles: %v", err)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func TestRepositoryWithPrimaryUsesPrimaryBeforeBuiltIns(t *testing.T) {
|
||||
repo := NewRepositoryWithPrimary(staticProfileRepo{
|
||||
profiles: map[string]string{"mistral-small-3": "custom-model"},
|
||||
})
|
||||
|
||||
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
||||
if err != nil {
|
||||
t.Fatalf("expected profile to load, got %v", err)
|
||||
}
|
||||
if p.Model != "custom-model" {
|
||||
t.Fatalf("expected primary profile to override built-in, got %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryWithPrimaryFallsBackToBuiltIns(t *testing.T) {
|
||||
repo := NewRepositoryWithPrimary(staticProfileRepo{})
|
||||
|
||||
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
||||
if err != nil {
|
||||
t.Fatalf("expected built-in profile to load, got %v", err)
|
||||
}
|
||||
if p.ID != "mistral-small-3" {
|
||||
t.Fatalf("unexpected profile: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryWithPrimaryDoesNotFallBackAfterPrimaryError(t *testing.T) {
|
||||
repo := NewRepositoryWithPrimary(staticProfileRepo{err: profile.ErrInvalidProfile})
|
||||
|
||||
_, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
||||
if !errors.Is(err, profile.ErrInvalidProfile) {
|
||||
t.Fatalf("expected primary error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type staticProfileRepo struct {
|
||||
profiles map[string]string
|
||||
err error
|
||||
}
|
||||
|
||||
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
if r.err != nil {
|
||||
return nil, r.err
|
||||
}
|
||||
if model, ok := r.profiles[id]; ok {
|
||||
return &domain.ExecutionProfile{ID: id, Endpoint: "http://primary/v1", Model: model}, nil
|
||||
}
|
||||
return nil, profile.ErrProfileNotFound
|
||||
}
|
||||
@@ -5,12 +5,12 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/filecatalog"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
@@ -30,11 +30,56 @@ func NewFilesystemRepository(dir string) Repository {
|
||||
}
|
||||
|
||||
func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
return loadProfile(ctx, os.DirFS(r.dir), ".", id)
|
||||
}
|
||||
|
||||
type fsRepository struct {
|
||||
fsys fs.FS
|
||||
root string
|
||||
}
|
||||
|
||||
func NewFSRepository(fsys fs.FS, root string) Repository {
|
||||
return &fsRepository{fsys: fsys, root: root}
|
||||
}
|
||||
|
||||
func (r *fsRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
return loadProfile(ctx, r.fsys, r.root, id)
|
||||
}
|
||||
|
||||
type overlayRepository struct {
|
||||
primary Repository
|
||||
fallback Repository
|
||||
}
|
||||
|
||||
func NewOverlayRepository(primary, fallback Repository) Repository {
|
||||
return &overlayRepository{primary: primary, fallback: fallback}
|
||||
}
|
||||
|
||||
func (r *overlayRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
if r.primary != nil {
|
||||
prof, err := r.primary.GetProfile(ctx, id)
|
||||
if err == nil {
|
||||
return prof, nil
|
||||
}
|
||||
if !errors.Is(err, ErrProfileNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if r.fallback == nil {
|
||||
return nil, ErrProfileNotFound
|
||||
}
|
||||
return r.fallback.GetProfile(ctx, id)
|
||||
}
|
||||
|
||||
func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*domain.ExecutionProfile, error) {
|
||||
if strings.TrimSpace(id) == "" {
|
||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
|
||||
}
|
||||
if fsys == nil {
|
||||
return nil, fmt.Errorf("failed to read profile directory: filesystem is nil")
|
||||
}
|
||||
|
||||
files, err := filecatalog.FindYAMLFiles(ctx, r.dir)
|
||||
files, err := findProfileYAMLFiles(ctx, fsys, root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read profile directory: %w", err)
|
||||
}
|
||||
@@ -47,9 +92,9 @@ func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*doma
|
||||
default:
|
||||
}
|
||||
|
||||
relPath := filecatalog.RelativePath(r.dir, fullPath)
|
||||
fileMatch := filecatalog.Stem(filepath.Base(fullPath)) == id
|
||||
data, err := os.ReadFile(fullPath)
|
||||
relPath := displayPath(root, fullPath)
|
||||
fileMatch := profileFileStem(path.Base(fullPath)) == id
|
||||
data, err := fs.ReadFile(fsys, fullPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
|
||||
}
|
||||
@@ -102,6 +147,61 @@ func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*doma
|
||||
return nil, ErrProfileNotFound
|
||||
}
|
||||
|
||||
func findProfileYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, error) {
|
||||
cleanRoot := cleanFSRoot(root)
|
||||
var files []string
|
||||
err := fs.WalkDir(fsys, cleanRoot, func(name string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !isProfileYAMLFile(d.Name()) {
|
||||
return nil
|
||||
}
|
||||
files = append(files, name)
|
||||
return nil
|
||||
})
|
||||
return files, err
|
||||
}
|
||||
|
||||
func cleanFSRoot(root string) string {
|
||||
root = strings.TrimSpace(root)
|
||||
if root == "" || root == "." {
|
||||
return "."
|
||||
}
|
||||
return path.Clean(root)
|
||||
}
|
||||
|
||||
func displayPath(root string, name string) string {
|
||||
cleanRoot := cleanFSRoot(root)
|
||||
cleanName := path.Clean(name)
|
||||
if cleanRoot == "." {
|
||||
return cleanName
|
||||
}
|
||||
prefix := strings.TrimSuffix(cleanRoot, "/") + "/"
|
||||
if strings.HasPrefix(cleanName, prefix) {
|
||||
return strings.TrimPrefix(cleanName, prefix)
|
||||
}
|
||||
return cleanName
|
||||
}
|
||||
|
||||
func profileFileStem(name string) string {
|
||||
name = strings.TrimSuffix(name, ".yaml")
|
||||
name = strings.TrimSuffix(name, ".yml")
|
||||
return name
|
||||
}
|
||||
|
||||
func isProfileYAMLFile(name string) bool {
|
||||
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
|
||||
}
|
||||
|
||||
type profileMatch struct {
|
||||
profile *domain.ExecutionProfile
|
||||
path string
|
||||
|
||||
@@ -2,11 +2,15 @@ package profile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
)
|
||||
|
||||
func TestFilesystemRepository_GetProfile(t *testing.T) {
|
||||
@@ -85,6 +89,63 @@ temperature: 0.1
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid profile with JSON-compatible extra params", func(t *testing.T) {
|
||||
writeProfileTestFile(t, filepath.Join(tmpDir, "json-extra-params.yaml"), `
|
||||
id: json-extra-params
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: nested-model
|
||||
extra_params:
|
||||
string_value: enabled
|
||||
number_value: 42
|
||||
boolean_value: true
|
||||
object_value:
|
||||
nested: value
|
||||
count: 2
|
||||
array_value:
|
||||
- first
|
||||
- 3
|
||||
- false
|
||||
`)
|
||||
|
||||
p, err := repo.GetProfile(ctx, "json-extra-params")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
var got map[string]any
|
||||
encoded, err := json.Marshal(p.ExtraParams)
|
||||
if err != nil {
|
||||
t.Fatalf("expected extra_params to marshal as JSON, got %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(encoded, &got); err != nil {
|
||||
t.Fatalf("expected extra_params JSON to decode, got %v", err)
|
||||
}
|
||||
|
||||
if got["string_value"] != "enabled" {
|
||||
t.Fatalf("unexpected string extra param: %#v", got["string_value"])
|
||||
}
|
||||
if got["number_value"] != float64(42) {
|
||||
t.Fatalf("unexpected number extra param: %#v", got["number_value"])
|
||||
}
|
||||
if got["boolean_value"] != true {
|
||||
t.Fatalf("unexpected boolean extra param: %#v", got["boolean_value"])
|
||||
}
|
||||
objectValue, ok := got["object_value"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected object extra param, got %#v", got["object_value"])
|
||||
}
|
||||
if objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
|
||||
t.Fatalf("unexpected object extra param: %#v", objectValue)
|
||||
}
|
||||
arrayValue, ok := got["array_value"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected array extra param, got %#v", got["array_value"])
|
||||
}
|
||||
if len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
|
||||
t.Fatalf("unexpected array extra param: %#v", arrayValue)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
|
||||
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
|
||||
id: duplicate-profile
|
||||
@@ -203,3 +264,216 @@ func writeProfileTestFile(t *testing.T, path string, content string) {
|
||||
t.Fatalf("failed to write profile test file %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSRepository(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("loads valid profiles from nested directories", func(t *testing.T) {
|
||||
repo := NewFSRepository(fstest.MapFS{
|
||||
"profiles/provider/nested.yaml": profileMapFile(`
|
||||
id: nested-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: nested-model
|
||||
temperature: 0.1
|
||||
`),
|
||||
}, "profiles")
|
||||
|
||||
p, err := repo.GetProfile(ctx, "nested-profile")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if p.ID != "nested-profile" || p.Model != "nested-model" {
|
||||
t.Fatalf("unexpected profile: %+v", p)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects unknown YAML fields", func(t *testing.T) {
|
||||
repo := NewFSRepository(fstest.MapFS{
|
||||
"profiles/unknown.yaml": profileMapFile(`
|
||||
id: unknown-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: model
|
||||
unknown: value
|
||||
`),
|
||||
}, "profiles")
|
||||
|
||||
_, err := repo.GetProfile(ctx, "unknown-profile")
|
||||
if !errors.Is(err, ErrInvalidYAML) {
|
||||
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects raw api_key in selected profile", func(t *testing.T) {
|
||||
repo := NewFSRepository(fstest.MapFS{
|
||||
"profiles/raw.yaml": profileMapFile(`
|
||||
id: raw-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: model
|
||||
api_key: secret
|
||||
`),
|
||||
}, "profiles")
|
||||
|
||||
_, err := repo.GetProfile(ctx, "raw-profile")
|
||||
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ignores raw api_key in non-selected profiles", func(t *testing.T) {
|
||||
repo := NewFSRepository(fstest.MapFS{
|
||||
"profiles/raw.yaml": profileMapFile(`
|
||||
id: raw-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: model
|
||||
api_key: secret
|
||||
`),
|
||||
"profiles/valid.yaml": profileMapFile(`
|
||||
id: valid-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: model
|
||||
`),
|
||||
}, "profiles")
|
||||
|
||||
p, err := repo.GetProfile(ctx, "valid-profile")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if p.ID != "valid-profile" {
|
||||
t.Fatalf("unexpected profile: %+v", p)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects duplicate IDs within one source", func(t *testing.T) {
|
||||
repo := NewFSRepository(fstest.MapFS{
|
||||
"profiles/a.yaml": profileMapFile(`
|
||||
id: duplicate-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: first
|
||||
`),
|
||||
"profiles/nested/b.yaml": profileMapFile(`
|
||||
id: duplicate-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: second
|
||||
`),
|
||||
}, "profiles")
|
||||
|
||||
_, err := repo.GetProfile(ctx, "duplicate-profile")
|
||||
if !errors.Is(err, ErrInvalidProfile) {
|
||||
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||
}
|
||||
for _, want := range []string{"duplicate execution profile id", "a.yaml", "nested/b.yaml"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("expected error to contain %q, got %v", want, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestOverlayRepository(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"}
|
||||
fallbackProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://fallback", Model: "fallback"}
|
||||
|
||||
t.Run("returns primary matches before fallback matches", func(t *testing.T) {
|
||||
repo := NewOverlayRepository(
|
||||
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": primaryProfile}},
|
||||
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
|
||||
)
|
||||
|
||||
p, err := repo.GetProfile(ctx, "shared")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if p.Model != "primary" {
|
||||
t.Fatalf("expected primary profile, got %+v", p)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("falls back on primary not found", func(t *testing.T) {
|
||||
repo := NewOverlayRepository(
|
||||
staticProfileRepo{},
|
||||
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
|
||||
)
|
||||
|
||||
p, err := repo.GetProfile(ctx, "shared")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if p.Model != "fallback" {
|
||||
t.Fatalf("expected fallback profile, got %+v", p)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("does not fall back after primary load errors", func(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
err error
|
||||
}{
|
||||
{name: "invalid yaml", err: ErrInvalidYAML},
|
||||
{name: "invalid profile", err: ErrInvalidProfile},
|
||||
{name: "raw api key", err: ErrRawAPIKeyNotAllowed},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
repo := NewOverlayRepository(
|
||||
staticProfileRepo{err: tc.err},
|
||||
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
|
||||
)
|
||||
|
||||
_, err := repo.GetProfile(ctx, "shared")
|
||||
if !errors.Is(err, tc.err) {
|
||||
t.Fatalf("expected %v, got %v", tc.err, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("returns not found when both sources miss", func(t *testing.T) {
|
||||
repo := NewOverlayRepository(staticProfileRepo{}, staticProfileRepo{})
|
||||
|
||||
_, err := repo.GetProfile(ctx, "missing")
|
||||
if !errors.Is(err, ErrProfileNotFound) {
|
||||
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nil primary uses fallback", func(t *testing.T) {
|
||||
repo := NewOverlayRepository(nil, staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}})
|
||||
|
||||
p, err := repo.GetProfile(ctx, "shared")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if p.Model != "fallback" {
|
||||
t.Fatalf("expected fallback profile, got %+v", p)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nil fallback returns not found after primary miss", func(t *testing.T) {
|
||||
repo := NewOverlayRepository(staticProfileRepo{}, nil)
|
||||
|
||||
_, err := repo.GetProfile(ctx, "missing")
|
||||
if !errors.Is(err, ErrProfileNotFound) {
|
||||
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func profileMapFile(content string) *fstest.MapFile {
|
||||
return &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))}
|
||||
}
|
||||
|
||||
type staticProfileRepo struct {
|
||||
profiles map[string]*domain.ExecutionProfile
|
||||
err error
|
||||
}
|
||||
|
||||
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
if r.err != nil {
|
||||
return nil, r.err
|
||||
}
|
||||
if p, ok := r.profiles[id]; ok {
|
||||
cp := *p
|
||||
return &cp, nil
|
||||
}
|
||||
return nil, ErrProfileNotFound
|
||||
}
|
||||
|
||||
@@ -6,7 +6,9 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"strings"
|
||||
"text/template"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -50,6 +52,11 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
||||
},
|
||||
}
|
||||
|
||||
sessionID, err := renderSessionID(definition.SessionID, funcs, vars)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var renderedMessages []domain.RenderedMessage
|
||||
|
||||
for i, tmplMsg := range definition.Templates {
|
||||
@@ -75,12 +82,44 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
||||
}
|
||||
|
||||
renderedMessages = append(renderedMessages, domain.RenderedMessage{
|
||||
Role: tmplMsg.Role,
|
||||
Content: buf.String(),
|
||||
Role: tmplMsg.Role,
|
||||
Content: buf.String(),
|
||||
CacheControl: cloneCacheControl(tmplMsg.CacheControl),
|
||||
})
|
||||
}
|
||||
|
||||
return &domain.RenderedPrompt{
|
||||
Messages: renderedMessages,
|
||||
SessionID: sessionID,
|
||||
Messages: renderedMessages,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
tmpl, err := template.New("session_id").Funcs(funcs).Option("missingkey=error").Parse(raw)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, err)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := tmpl.Execute(&buf, vars); err != nil {
|
||||
return "", fmt.Errorf("%w: session_id: %w", ErrRenderFailure, err)
|
||||
}
|
||||
|
||||
sessionID := strings.TrimSpace(buf.String())
|
||||
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
|
||||
return "", fmt.Errorf("%w: session_id length %d exceeds maximum %d", ErrRenderFailure, n, domain.SessionIDMaxLength)
|
||||
}
|
||||
return sessionID, nil
|
||||
}
|
||||
|
||||
func cloneCacheControl(in *domain.CacheControl) *domain.CacheControl {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := *in
|
||||
return &out
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package prompt
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
@@ -78,6 +79,66 @@ func TestGoRenderer_Render(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("copying cache control to rendered messages", func(t *testing.T) {
|
||||
def := &domain.PromptDefinition{
|
||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||
Templates: []domain.PromptMessageTemplate{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "You are concise.",
|
||||
CacheControl: &domain.CacheControl{
|
||||
Type: domain.CacheControlEphemeral,
|
||||
TTL: "1h",
|
||||
},
|
||||
},
|
||||
{Role: "user", Content: "Analyze this: {{input \"transcript\"}}"},
|
||||
},
|
||||
}
|
||||
|
||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(res.Messages) != 2 {
|
||||
t.Fatalf("expected 2 messages, got %d", len(res.Messages))
|
||||
}
|
||||
if res.Messages[0].CacheControl == nil {
|
||||
t.Fatal("expected rendered cache control")
|
||||
}
|
||||
if res.Messages[0].CacheControl.Type != domain.CacheControlEphemeral {
|
||||
t.Fatalf("unexpected cache control type: %q", res.Messages[0].CacheControl.Type)
|
||||
}
|
||||
if res.Messages[0].CacheControl.TTL != "1h" {
|
||||
t.Fatalf("unexpected cache control ttl: %q", res.Messages[0].CacheControl.TTL)
|
||||
}
|
||||
if res.Messages[1].CacheControl != nil {
|
||||
t.Fatalf("expected no cache control on second message, got %#v", res.Messages[1].CacheControl)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rendered cache control does not alias source template", func(t *testing.T) {
|
||||
source := &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}
|
||||
def := &domain.PromptDefinition{
|
||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||
Templates: []domain.PromptMessageTemplate{
|
||||
{Role: "system", Content: "You are concise.", CacheControl: source},
|
||||
},
|
||||
}
|
||||
|
||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if res.Messages[0].CacheControl == source {
|
||||
t.Fatal("expected rendered cache control to be cloned")
|
||||
}
|
||||
|
||||
res.Messages[0].CacheControl.TTL = ""
|
||||
if source.TTL != "1h" {
|
||||
t.Fatalf("source cache control was mutated, ttl=%q", source.TTL)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("accessing vars", func(t *testing.T) {
|
||||
def := &domain.PromptDefinition{
|
||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||
@@ -95,6 +156,78 @@ func TestGoRenderer_Render(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rendering session id from vars", func(t *testing.T) {
|
||||
def := &domain.PromptDefinition{
|
||||
SessionID: " {{ .session_id }} ",
|
||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||
Templates: []domain.PromptMessageTemplate{
|
||||
{Role: "system", Content: "Speak in a {{.tone}} tone."},
|
||||
},
|
||||
}
|
||||
|
||||
res, err := renderer.Render(ctx, def, inputs, map[string]string{
|
||||
"tone": "concise",
|
||||
"session_id": "agent-session-123",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if res.SessionID != "agent-session-123" {
|
||||
t.Fatalf("unexpected session id: %q", res.SessionID)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty rendered session id is omitted", func(t *testing.T) {
|
||||
def := &domain.PromptDefinition{
|
||||
SessionID: " ",
|
||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||
Templates: []domain.PromptMessageTemplate{
|
||||
{Role: "system", Content: "Speak in a {{.tone}} tone."},
|
||||
},
|
||||
}
|
||||
|
||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if res.SessionID != "" {
|
||||
t.Fatalf("expected empty session id, got %q", res.SessionID)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing session id var fails rendering", func(t *testing.T) {
|
||||
def := &domain.PromptDefinition{
|
||||
SessionID: "{{ .session_id }}",
|
||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||
Templates: []domain.PromptMessageTemplate{
|
||||
{Role: "system", Content: "Speak in a {{.tone}} tone."},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
||||
if !errors.Is(err, ErrRenderFailure) {
|
||||
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("too long rendered session id fails rendering", func(t *testing.T) {
|
||||
def := &domain.PromptDefinition{
|
||||
SessionID: "{{ .session_id }}",
|
||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||
Templates: []domain.PromptMessageTemplate{
|
||||
{Role: "system", Content: "Speak in a {{.tone}} tone."},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := renderer.Render(ctx, def, inputs, map[string]string{
|
||||
"tone": "concise",
|
||||
"session_id": strings.Repeat("x", domain.SessionIDMaxLength+1),
|
||||
})
|
||||
if !errors.Is(err, ErrRenderFailure) {
|
||||
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("inserting required input artifact", func(t *testing.T) {
|
||||
def := &domain.PromptDefinition{
|
||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||
|
||||
@@ -5,7 +5,9 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
@@ -24,11 +26,17 @@ type filesystemRepository struct {
|
||||
dir string
|
||||
}
|
||||
|
||||
type fsRepository struct {
|
||||
fsys fs.FS
|
||||
root string
|
||||
}
|
||||
|
||||
type promptDefinitionFile struct {
|
||||
ID string `yaml:"id"`
|
||||
Version string `yaml:"version"`
|
||||
DefaultProfile *string `yaml:"default_profile"`
|
||||
Description string `yaml:"description"`
|
||||
SessionID string `yaml:"session_id"`
|
||||
Inputs []promptInputFile `yaml:"inputs"`
|
||||
Messages []promptMessageFile `yaml:"messages"`
|
||||
Output promptOutputContractFile `yaml:"output"`
|
||||
@@ -42,9 +50,15 @@ type promptInputFile struct {
|
||||
}
|
||||
|
||||
type promptMessageFile struct {
|
||||
Role string `yaml:"role"`
|
||||
Content string `yaml:"content"`
|
||||
ContentFile string `yaml:"content_file"`
|
||||
Role string `yaml:"role"`
|
||||
Content string `yaml:"content"`
|
||||
ContentFile string `yaml:"content_file"`
|
||||
CacheControl *cacheControlFile `yaml:"cache_control"`
|
||||
}
|
||||
|
||||
type cacheControlFile struct {
|
||||
Type string `yaml:"type"`
|
||||
TTL string `yaml:"ttl"`
|
||||
}
|
||||
|
||||
type promptOutputContractFile struct {
|
||||
@@ -58,6 +72,10 @@ func NewFilesystemRepository(dir string) Repository {
|
||||
return &filesystemRepository{dir: dir}
|
||||
}
|
||||
|
||||
func NewFSRepository(fsys fs.FS, root string) Repository {
|
||||
return &fsRepository{fsys: fsys, root: root}
|
||||
}
|
||||
|
||||
func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
||||
if strings.TrimSpace(id) == "" {
|
||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
|
||||
@@ -125,6 +143,10 @@ func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id strin
|
||||
return nil, ErrPromptDefinitionNotFound
|
||||
}
|
||||
|
||||
func (r *fsRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
||||
return loadPromptDefinition(ctx, r.fsys, r.root, id, version)
|
||||
}
|
||||
|
||||
type promptDefinitionMatch struct {
|
||||
def *domain.PromptDefinition
|
||||
path string
|
||||
@@ -159,7 +181,187 @@ func promptDefinitionFileHasID(path string, id string) bool {
|
||||
return strings.TrimSpace(raw.ID) == id
|
||||
}
|
||||
|
||||
func loadPromptDefinition(ctx context.Context, fsys fs.FS, root string, id string, version string) (*domain.PromptDefinition, error) {
|
||||
if strings.TrimSpace(id) == "" {
|
||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
|
||||
}
|
||||
if fsys == nil {
|
||||
return nil, fmt.Errorf("failed to read prompt definition directory: filesystem is nil")
|
||||
}
|
||||
|
||||
files, err := findPromptDefinitionYAMLFiles(ctx, fsys, root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
|
||||
}
|
||||
|
||||
var matches []promptDefinitionMatch
|
||||
for _, fullPath := range files {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
relPath := displayPath(root, fullPath)
|
||||
fileMatch := filecatalog.Stem(path.Base(fullPath)) == id
|
||||
data, err := fs.ReadFile(fsys, fullPath)
|
||||
if err != nil {
|
||||
if fileMatch {
|
||||
return nil, fmt.Errorf("%w: %s: failed to read prompt definition file: %v", ErrInvalidYAML, relPath, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
raw, err := decodePromptDefinition(data)
|
||||
if err != nil {
|
||||
if fileMatch || promptDefinitionDataHasID(data, id) {
|
||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
def, err := normalizePromptDefinitionFromFS(raw, fsys, fullPath)
|
||||
if err != nil {
|
||||
if fileMatch || strings.TrimSpace(raw.ID) == id {
|
||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, relPath, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if def.ID != id {
|
||||
continue
|
||||
}
|
||||
if version != "" && def.Version != version {
|
||||
continue
|
||||
}
|
||||
matches = append(matches, promptDefinitionMatch{
|
||||
def: def,
|
||||
path: relPath,
|
||||
})
|
||||
}
|
||||
|
||||
if len(matches) > 1 {
|
||||
paths := make([]string, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
paths = append(paths, match.path)
|
||||
}
|
||||
if version != "" {
|
||||
return nil, fmt.Errorf("%w: duplicate prompt definition id %q version %q found in: %s", ErrInvalidPromptDefinition, id, version, strings.Join(paths, ", "))
|
||||
}
|
||||
return nil, fmt.Errorf("%w: duplicate prompt definition id %q found in: %s", ErrInvalidPromptDefinition, id, strings.Join(paths, ", "))
|
||||
}
|
||||
|
||||
if len(matches) == 1 {
|
||||
return matches[0].def, nil
|
||||
}
|
||||
|
||||
return nil, ErrPromptDefinitionNotFound
|
||||
}
|
||||
|
||||
func findPromptDefinitionYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, error) {
|
||||
cleanRoot := cleanFSRoot(root)
|
||||
var files []string
|
||||
err := fs.WalkDir(fsys, cleanRoot, func(name string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !isPromptDefinitionYAMLFile(d.Name()) {
|
||||
return nil
|
||||
}
|
||||
files = append(files, name)
|
||||
return nil
|
||||
})
|
||||
return files, err
|
||||
}
|
||||
|
||||
func cleanFSRoot(root string) string {
|
||||
root = strings.TrimSpace(root)
|
||||
if root == "" || root == "." {
|
||||
return "."
|
||||
}
|
||||
return path.Clean(root)
|
||||
}
|
||||
|
||||
func displayPath(root string, name string) string {
|
||||
cleanRoot := cleanFSRoot(root)
|
||||
cleanName := path.Clean(name)
|
||||
if cleanRoot == "." {
|
||||
return cleanName
|
||||
}
|
||||
prefix := strings.TrimSuffix(cleanRoot, "/") + "/"
|
||||
if strings.HasPrefix(cleanName, prefix) {
|
||||
return strings.TrimPrefix(cleanName, prefix)
|
||||
}
|
||||
return cleanName
|
||||
}
|
||||
|
||||
func isPromptDefinitionYAMLFile(name string) bool {
|
||||
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
|
||||
}
|
||||
|
||||
func decodePromptDefinition(data []byte) (*promptDefinitionFile, error) {
|
||||
var raw promptDefinitionFile
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||
decoder.KnownFields(true)
|
||||
if err := decoder.Decode(&raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &raw, nil
|
||||
}
|
||||
|
||||
func promptDefinitionDataHasID(data []byte, id string) bool {
|
||||
var raw struct {
|
||||
ID string `yaml:"id"`
|
||||
}
|
||||
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&raw); err != nil {
|
||||
return false
|
||||
}
|
||||
return strings.TrimSpace(raw.ID) == id
|
||||
}
|
||||
|
||||
func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*domain.PromptDefinition, error) {
|
||||
promptDir := filepath.Dir(sourcePath)
|
||||
return normalizePromptDefinitionWithContent(raw, func(contentFile string) (string, string, error) {
|
||||
resolvedPath := strings.TrimSpace(contentFile)
|
||||
if !filepath.IsAbs(resolvedPath) {
|
||||
resolvedPath = filepath.Join(promptDir, resolvedPath)
|
||||
}
|
||||
resolvedPath = filepath.Clean(resolvedPath)
|
||||
|
||||
body, err := os.ReadFile(resolvedPath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return string(body), resolvedPath, nil
|
||||
})
|
||||
}
|
||||
|
||||
func normalizePromptDefinitionFromFS(raw *promptDefinitionFile, fsys fs.FS, sourcePath string) (*domain.PromptDefinition, error) {
|
||||
promptDir := path.Dir(sourcePath)
|
||||
return normalizePromptDefinitionWithContent(raw, func(contentFile string) (string, string, error) {
|
||||
resolvedPath := strings.TrimSpace(contentFile)
|
||||
if !path.IsAbs(resolvedPath) {
|
||||
resolvedPath = path.Join(promptDir, resolvedPath)
|
||||
}
|
||||
resolvedPath = strings.TrimPrefix(path.Clean(resolvedPath), "/")
|
||||
|
||||
body, err := fs.ReadFile(fsys, resolvedPath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return string(body), resolvedPath, nil
|
||||
})
|
||||
}
|
||||
|
||||
func normalizePromptDefinitionWithContent(raw *promptDefinitionFile, readContentFile func(string) (string, string, error)) (*domain.PromptDefinition, error) {
|
||||
if raw == nil {
|
||||
return nil, errors.New("prompt definition is nil")
|
||||
}
|
||||
@@ -199,7 +401,6 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
|
||||
}
|
||||
|
||||
templates := make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
|
||||
promptDir := filepath.Dir(sourcePath)
|
||||
for i, msg := range raw.Messages {
|
||||
role := strings.TrimSpace(msg.Role)
|
||||
if role == "" {
|
||||
@@ -212,27 +413,27 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
|
||||
return nil, fmt.Errorf("message %d (%s) must set exactly one of content or content_file", i, role)
|
||||
}
|
||||
|
||||
cacheControl, err := normalizeCacheControl(msg.CacheControl)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("message %d (%s) cache_control: %w", i, role, err)
|
||||
}
|
||||
|
||||
templateContent := msg.Content
|
||||
resolvedContentFile := ""
|
||||
if hasContentFile {
|
||||
resolvedPath := strings.TrimSpace(msg.ContentFile)
|
||||
if !filepath.IsAbs(resolvedPath) {
|
||||
resolvedPath = filepath.Join(promptDir, resolvedPath)
|
||||
}
|
||||
resolvedPath = filepath.Clean(resolvedPath)
|
||||
|
||||
body, err := os.ReadFile(resolvedPath)
|
||||
body, resolvedPath, err := readContentFile(msg.ContentFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prompt %q message %d (%s): failed to read content_file %q: %w", id, i, role, msg.ContentFile, err)
|
||||
}
|
||||
templateContent = string(body)
|
||||
templateContent = body
|
||||
resolvedContentFile = resolvedPath
|
||||
}
|
||||
|
||||
templates = append(templates, domain.PromptMessageTemplate{
|
||||
Role: role,
|
||||
Content: templateContent,
|
||||
ContentFile: resolvedContentFile,
|
||||
Role: role,
|
||||
Content: templateContent,
|
||||
ContentFile: resolvedContentFile,
|
||||
CacheControl: cacheControl,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -262,6 +463,7 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
|
||||
Version: version,
|
||||
DefaultProfile: defaultProfile,
|
||||
Description: strings.TrimSpace(raw.Description),
|
||||
SessionID: strings.TrimSpace(raw.SessionID),
|
||||
Inputs: inputs,
|
||||
Templates: templates,
|
||||
OutputFormat: raw.Output.Format,
|
||||
@@ -274,6 +476,30 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
|
||||
}, nil
|
||||
}
|
||||
|
||||
func normalizeCacheControl(raw *cacheControlFile) (*domain.CacheControl, error) {
|
||||
if raw == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
cacheType := strings.TrimSpace(raw.Type)
|
||||
if cacheType == "" {
|
||||
return nil, errors.New("type is required")
|
||||
}
|
||||
if domain.CacheControlType(cacheType) != domain.CacheControlEphemeral {
|
||||
return nil, fmt.Errorf("unsupported type %q", cacheType)
|
||||
}
|
||||
|
||||
ttl := strings.TrimSpace(raw.TTL)
|
||||
if ttl != "" && ttl != "1h" {
|
||||
return nil, fmt.Errorf("unsupported ttl %q", ttl)
|
||||
}
|
||||
|
||||
return &domain.CacheControl{
|
||||
Type: domain.CacheControlType(cacheType),
|
||||
TTL: ttl,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func isValidOutputFormat(f domain.OutputFormat) bool {
|
||||
switch f {
|
||||
case domain.FormatText, domain.FormatMarkdown, domain.FormatJSON:
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
)
|
||||
@@ -68,6 +69,44 @@ func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid cache control with ttl", func(t *testing.T) {
|
||||
p, err := repo.GetPromptDefinition(ctx, "valid-cache-control-ttl", "")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if len(p.Templates) != 2 {
|
||||
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
|
||||
}
|
||||
assertCacheControl(t, p.Templates[0].CacheControl, domain.CacheControlEphemeral, "1h")
|
||||
if p.Templates[1].CacheControl != nil {
|
||||
t.Fatalf("expected second message cache control to be nil, got %#v", p.Templates[1].CacheControl)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid cache control without ttl", func(t *testing.T) {
|
||||
p, err := repo.GetPromptDefinition(ctx, "valid-cache-control-without-ttl", "")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if len(p.Templates) != 2 {
|
||||
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
|
||||
}
|
||||
assertCacheControl(t, p.Templates[0].CacheControl, domain.CacheControlEphemeral, "")
|
||||
if p.Templates[1].CacheControl != nil {
|
||||
t.Fatalf("expected second message cache control to be nil, got %#v", p.Templates[1].CacheControl)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid session id template", func(t *testing.T) {
|
||||
p, err := repo.GetPromptDefinition(ctx, "valid-session-id", "")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if p.SessionID != "{{ .session_id }}" {
|
||||
t.Fatalf("expected trimmed session_id template, got %q", p.SessionID)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid nested file-backed prompt resolves content file relative to nested YAML", func(t *testing.T) {
|
||||
nestedDir := filepath.Join(tmpDir, "dnd", "recap")
|
||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
||||
@@ -258,6 +297,10 @@ output:
|
||||
{name: "invalid validation mode", id: "invalid_validation_mode", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"invalid validation mode"}},
|
||||
{name: "json_schema without schema_path", id: "json_schema_without_schema_path", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"schema_path"}},
|
||||
{name: "unknown input field", id: "unknown_input_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unknown_input_setting not found"}},
|
||||
{name: "empty cache control type", id: "empty_cache_control_type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "type is required"}},
|
||||
{name: "unsupported cache control type", id: "unsupported_cache_control_type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported type"}},
|
||||
{name: "unsupported cache control ttl", id: "unsupported_cache_control_ttl", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported ttl"}},
|
||||
{name: "unknown cache control field", id: "unknown_cache_control_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unexpected not found"}},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
@@ -282,6 +325,110 @@ output:
|
||||
})
|
||||
}
|
||||
|
||||
func TestFSRepositoryGetPromptDefinition(t *testing.T) {
|
||||
repo := NewFSRepository(fstest.MapFS{
|
||||
"prompts/nested/prompt.yaml": &fstest.MapFile{Data: []byte(`
|
||||
id: fs-prompt
|
||||
version: "1.0.0"
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./messages/user.tmpl
|
||||
output:
|
||||
format: markdown
|
||||
validation_mode: basic
|
||||
repair_attempts: 0
|
||||
`)},
|
||||
"prompts/nested/messages/user.tmpl": &fstest.MapFile{Data: []byte(`Summarize {{input "transcript"}}.`)},
|
||||
}, "prompts")
|
||||
|
||||
got, err := repo.GetPromptDefinition(context.Background(), "fs-prompt", "")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if got.ID != "fs-prompt" {
|
||||
t.Fatalf("unexpected prompt id: %q", got.ID)
|
||||
}
|
||||
if len(got.Templates) != 1 || !strings.Contains(got.Templates[0].Content, `{{input "transcript"}}`) {
|
||||
t.Fatalf("expected content_file body to be loaded, got %+v", got.Templates)
|
||||
}
|
||||
if got.Templates[0].ContentFile != "prompts/nested/messages/user.tmpl" {
|
||||
t.Fatalf("unexpected content file path: %q", got.Templates[0].ContentFile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSRepositoryRejectsDuplicatePromptIDs(t *testing.T) {
|
||||
repo := NewFSRepository(fstest.MapFS{
|
||||
"one.yaml": &fstest.MapFile{Data: []byte(`
|
||||
id: duplicate-fs-prompt
|
||||
version: "1.0.0"
|
||||
messages:
|
||||
- role: user
|
||||
content: First.
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
repair_attempts: 0
|
||||
`)},
|
||||
"nested/two.yaml": &fstest.MapFile{Data: []byte(`
|
||||
id: duplicate-fs-prompt
|
||||
version: "1.0.0"
|
||||
messages:
|
||||
- role: user
|
||||
content: Second.
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
repair_attempts: 0
|
||||
`)},
|
||||
}, ".")
|
||||
|
||||
_, err := repo.GetPromptDefinition(context.Background(), "duplicate-fs-prompt", "")
|
||||
if !errors.Is(err, ErrInvalidPromptDefinition) {
|
||||
t.Fatalf("expected ErrInvalidPromptDefinition, got %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "one.yaml") || !strings.Contains(err.Error(), "nested/two.yaml") {
|
||||
t.Fatalf("expected duplicate paths in error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSRepositoryRejectsUnknownYAMLFields(t *testing.T) {
|
||||
repo := NewFSRepository(fstest.MapFS{
|
||||
"not_named_like_id.yaml": &fstest.MapFile{Data: []byte(`
|
||||
id: strict-fs-prompt
|
||||
version: "1.0.0"
|
||||
unknown: true
|
||||
messages:
|
||||
- role: user
|
||||
content: Invalid.
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
repair_attempts: 0
|
||||
`)},
|
||||
}, ".")
|
||||
|
||||
_, err := repo.GetPromptDefinition(context.Background(), "strict-fs-prompt", "")
|
||||
if !errors.Is(err, ErrInvalidYAML) {
|
||||
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertCacheControl(t *testing.T, got *domain.CacheControl, wantType domain.CacheControlType, wantTTL string) {
|
||||
t.Helper()
|
||||
if got == nil {
|
||||
t.Fatal("expected cache control, got nil")
|
||||
}
|
||||
if got.Type != wantType {
|
||||
t.Fatalf("unexpected cache control type: got %q want %q", got.Type, wantType)
|
||||
}
|
||||
if got.TTL != wantTTL {
|
||||
t.Fatalf("unexpected cache control ttl: got %q want %q", got.TTL, wantTTL)
|
||||
}
|
||||
}
|
||||
|
||||
func writePromptTestFile(t *testing.T, path string, content string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil {
|
||||
|
||||
10
internal/promptdef/testdata/empty_cache_control_type.yaml
vendored
Normal file
10
internal/promptdef/testdata/empty_cache_control_type.yaml
vendored
Normal file
@@ -0,0 +1,10 @@
|
||||
id: empty-cache-control-type
|
||||
version: "1.0.0"
|
||||
messages:
|
||||
- role: system
|
||||
content: "Use cached instructions."
|
||||
cache_control: {}
|
||||
output:
|
||||
format: markdown
|
||||
validation_mode: basic
|
||||
repair_attempts: 0
|
||||
12
internal/promptdef/testdata/unknown_cache_control_field.yaml
vendored
Normal file
12
internal/promptdef/testdata/unknown_cache_control_field.yaml
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
id: unknown-cache-control-field
|
||||
version: "1.0.0"
|
||||
messages:
|
||||
- role: system
|
||||
content: "Use cached instructions."
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
unexpected: true
|
||||
output:
|
||||
format: markdown
|
||||
validation_mode: basic
|
||||
repair_attempts: 0
|
||||
12
internal/promptdef/testdata/unsupported_cache_control_ttl.yaml
vendored
Normal file
12
internal/promptdef/testdata/unsupported_cache_control_ttl.yaml
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
id: unsupported-cache-control-ttl
|
||||
version: "1.0.0"
|
||||
messages:
|
||||
- role: system
|
||||
content: "Use cached instructions."
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
ttl: 5m
|
||||
output:
|
||||
format: markdown
|
||||
validation_mode: basic
|
||||
repair_attempts: 0
|
||||
11
internal/promptdef/testdata/unsupported_cache_control_type.yaml
vendored
Normal file
11
internal/promptdef/testdata/unsupported_cache_control_type.yaml
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
id: unsupported-cache-control-type
|
||||
version: "1.0.0"
|
||||
messages:
|
||||
- role: system
|
||||
content: "Use cached instructions."
|
||||
cache_control:
|
||||
type: persistent
|
||||
output:
|
||||
format: markdown
|
||||
validation_mode: basic
|
||||
repair_attempts: 0
|
||||
14
internal/promptdef/testdata/valid_cache_control_ttl.yaml
vendored
Normal file
14
internal/promptdef/testdata/valid_cache_control_ttl.yaml
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
id: valid-cache-control-ttl
|
||||
version: "1.0.0"
|
||||
messages:
|
||||
- role: system
|
||||
content: "Use cached instructions."
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
ttl: 1h
|
||||
- role: user
|
||||
content: "Summarize the input."
|
||||
output:
|
||||
format: markdown
|
||||
validation_mode: basic
|
||||
repair_attempts: 0
|
||||
13
internal/promptdef/testdata/valid_cache_control_without_ttl.yaml
vendored
Normal file
13
internal/promptdef/testdata/valid_cache_control_without_ttl.yaml
vendored
Normal file
@@ -0,0 +1,13 @@
|
||||
id: valid-cache-control-without-ttl
|
||||
version: "1.0.0"
|
||||
messages:
|
||||
- role: system
|
||||
content: "Use cached instructions."
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content: "Summarize the input."
|
||||
output:
|
||||
format: markdown
|
||||
validation_mode: basic
|
||||
repair_attempts: 0
|
||||
10
internal/promptdef/testdata/valid_session_id.yaml
vendored
Normal file
10
internal/promptdef/testdata/valid_session_id.yaml
vendored
Normal file
@@ -0,0 +1,10 @@
|
||||
id: valid-session-id
|
||||
version: "1.0.0"
|
||||
session_id: " {{ .session_id }} "
|
||||
messages:
|
||||
- role: user
|
||||
content: Hello.
|
||||
output:
|
||||
format: markdown
|
||||
validation_mode: basic
|
||||
repair_attempts: 0
|
||||
@@ -27,6 +27,7 @@ var (
|
||||
ErrInvalidRequest = errors.New("invalid run request")
|
||||
ErrProfileRequired = errors.New("profile selection is required")
|
||||
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
||||
ErrAPIKeyRequired = errors.New("api key is required")
|
||||
ErrProfileLoad = errors.New("failed to load prompt definition")
|
||||
ErrArtifactLoad = errors.New("failed to load artifact")
|
||||
ErrPromptRender = errors.New("failed to render prompt")
|
||||
@@ -90,11 +91,15 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
||||
}
|
||||
|
||||
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: prepared.Messages},
|
||||
Prompt: domain.RenderedPrompt{SessionID: prepared.SessionID, Messages: prepared.Messages},
|
||||
Target: prepared.EffectiveModelParams,
|
||||
TargetPresence: prepared.TargetPresence,
|
||||
StructuredOutput: prepared.StructuredOutput,
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, llm.ErrInvalidRequest) {
|
||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err)
|
||||
}
|
||||
|
||||
@@ -187,14 +192,18 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||
}
|
||||
|
||||
effectiveModel := resolveExecutionTarget(execProfile, req.Execution)
|
||||
effectiveModel, targetPresence, err := resolveExecutionTarget(execProfile, req.Execution)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||
}
|
||||
effectiveModel.APIKey = req.APIKey
|
||||
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
|
||||
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
|
||||
}
|
||||
if strings.TrimSpace(effectiveModel.Model) == "" {
|
||||
return nil, fmt.Errorf("%w: execution model is required", ErrInvalidRequest)
|
||||
}
|
||||
if err := validateAPIKeyEnv(effectiveModel.APIKeyEnv); err != nil {
|
||||
if err := validateAPIKey(effectiveModel.APIKeyEnv, effectiveModel.APIKey, effectiveModel.APIKeyRequired); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
@@ -230,9 +239,11 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
||||
PromptHash: promptDefinitionHash,
|
||||
SelectedProfileID: selectedProfileID,
|
||||
EffectiveModelParams: effectiveModel,
|
||||
TargetPresence: targetPresence,
|
||||
OutputContract: effectiveContract,
|
||||
StructuredOutput: structuredOutput,
|
||||
InputHashes: inputHashes,
|
||||
SessionID: renderedPrompt.SessionID,
|
||||
RenderedPromptHash: hashRenderedPrompt(*renderedPrompt),
|
||||
Messages: renderedPrompt.Messages,
|
||||
StartTime: start,
|
||||
@@ -353,28 +364,90 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution
|
||||
if strings.TrimSpace(override.APIKeyEnv) != "" {
|
||||
out.APIKeyEnv = override.APIKeyEnv
|
||||
}
|
||||
if override.APIKeyRequired {
|
||||
out.APIKeyRequired = true
|
||||
}
|
||||
if len(override.ExtraParams) > 0 {
|
||||
cp := make(map[string]string, len(override.ExtraParams))
|
||||
for k, v := range override.ExtraParams {
|
||||
cp[k] = v
|
||||
}
|
||||
out.ExtraParams = cp
|
||||
out.ExtraParams = copyExtraParams(override.ExtraParams)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTarget) domain.ExecutionTarget {
|
||||
func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
|
||||
out := base
|
||||
var presence domain.ExecutionTargetPresence
|
||||
if override.Endpoint != "" {
|
||||
out.Endpoint = override.Endpoint
|
||||
}
|
||||
if override.Model != "" {
|
||||
out.Model = override.Model
|
||||
}
|
||||
if override.Temperature != nil {
|
||||
if *override.Temperature < 0 || *override.Temperature > 2 {
|
||||
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("temperature must be between 0 and 2")
|
||||
}
|
||||
out.Temperature = *override.Temperature
|
||||
presence.Temperature = true
|
||||
}
|
||||
if override.MaxTokens != nil {
|
||||
if *override.MaxTokens < 0 {
|
||||
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("max_tokens must be greater than or equal to 0")
|
||||
}
|
||||
out.MaxTokens = *override.MaxTokens
|
||||
presence.MaxTokens = true
|
||||
}
|
||||
if override.TopP != nil {
|
||||
if *override.TopP < 0 || *override.TopP > 1 {
|
||||
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("top_p must be between 0 and 1")
|
||||
}
|
||||
out.TopP = *override.TopP
|
||||
presence.TopP = true
|
||||
}
|
||||
if override.TimeoutSeconds != nil {
|
||||
if *override.TimeoutSeconds < 0 {
|
||||
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("timeout_seconds must be greater than or equal to 0")
|
||||
}
|
||||
out.TimeoutSeconds = *override.TimeoutSeconds
|
||||
presence.TimeoutSeconds = true
|
||||
}
|
||||
if strings.TrimSpace(override.ServiceTier) != "" {
|
||||
out.ServiceTier = override.ServiceTier
|
||||
}
|
||||
if strings.TrimSpace(override.ReasoningEffort) != "" {
|
||||
out.ReasoningEffort = override.ReasoningEffort
|
||||
}
|
||||
if strings.TrimSpace(override.APIKeyEnv) != "" {
|
||||
out.APIKeyEnv = override.APIKeyEnv
|
||||
}
|
||||
if len(override.ExtraParams) > 0 {
|
||||
out.ExtraParams = copyExtraParams(override.ExtraParams)
|
||||
}
|
||||
return out, presence, nil
|
||||
}
|
||||
|
||||
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
|
||||
out := defaults.ExecutionTargetDefault()
|
||||
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
|
||||
var presence domain.ExecutionTargetPresence
|
||||
if override != nil {
|
||||
out = mergeExecutionTarget(out, *override)
|
||||
var err error
|
||||
out, presence, err = mergeExecutionTargetOverride(out, *override)
|
||||
if err != nil {
|
||||
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, err
|
||||
}
|
||||
}
|
||||
return out
|
||||
return out, presence, nil
|
||||
}
|
||||
|
||||
func validateAPIKeyEnv(apiKeyEnv string) error {
|
||||
func validateAPIKey(apiKeyEnv string, apiKey string, apiKeyRequired bool) error {
|
||||
if strings.TrimSpace(apiKey) != "" {
|
||||
return nil
|
||||
}
|
||||
envName := strings.TrimSpace(apiKeyEnv)
|
||||
if envName == "" {
|
||||
if apiKeyRequired {
|
||||
return ErrAPIKeyRequired
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(os.Getenv(envName)) == "" {
|
||||
@@ -387,13 +460,6 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
|
||||
if p == nil {
|
||||
return domain.ExecutionTarget{}
|
||||
}
|
||||
cp := map[string]string(nil)
|
||||
if len(p.ExtraParams) > 0 {
|
||||
cp = make(map[string]string, len(p.ExtraParams))
|
||||
for k, v := range p.ExtraParams {
|
||||
cp[k] = v
|
||||
}
|
||||
}
|
||||
return domain.ExecutionTarget{
|
||||
Endpoint: p.Endpoint,
|
||||
Model: p.Model,
|
||||
@@ -404,10 +470,22 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
|
||||
ServiceTier: p.ServiceTier,
|
||||
ReasoningEffort: p.ReasoningEffort,
|
||||
APIKeyEnv: p.APIKeyEnv,
|
||||
ExtraParams: cp,
|
||||
APIKeyRequired: p.APIKeyRequired,
|
||||
ExtraParams: copyExtraParams(p.ExtraParams),
|
||||
}
|
||||
}
|
||||
|
||||
func copyExtraParams(src map[string]any) map[string]any {
|
||||
if len(src) == 0 {
|
||||
return nil
|
||||
}
|
||||
cp := make(map[string]any, len(src))
|
||||
for k, v := range src {
|
||||
cp[k] = v
|
||||
}
|
||||
return cp
|
||||
}
|
||||
|
||||
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract {
|
||||
contract := def.Validation
|
||||
if contract.Format == "" {
|
||||
@@ -424,10 +502,23 @@ func resolveOutputContract(def *domain.PromptDefinition, override *domain.Output
|
||||
|
||||
func hashRenderedPrompt(p domain.RenderedPrompt) string {
|
||||
var b strings.Builder
|
||||
if p.SessionID != "" {
|
||||
b.WriteString("session_id=")
|
||||
b.WriteString(p.SessionID)
|
||||
b.WriteString("\n---\n")
|
||||
}
|
||||
for _, msg := range p.Messages {
|
||||
b.WriteString(msg.Role)
|
||||
b.WriteByte('\n')
|
||||
b.WriteString(msg.Content)
|
||||
if msg.CacheControl != nil {
|
||||
b.WriteString("\ncache_control.type=")
|
||||
b.WriteString(string(msg.CacheControl.Type))
|
||||
if msg.CacheControl.TTL != "" {
|
||||
b.WriteString("\ncache_control.ttl=")
|
||||
b.WriteString(msg.CacheControl.TTL)
|
||||
}
|
||||
}
|
||||
b.WriteString("\n---\n")
|
||||
}
|
||||
h := sha256.Sum256([]byte(b.String()))
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||
@@ -160,7 +161,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
|
||||
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
|
||||
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
|
||||
}}
|
||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||
llmClient := &fakeLLM{forbid: true}
|
||||
|
||||
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
||||
@@ -172,7 +173,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
||||
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
|
||||
},
|
||||
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
|
||||
Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
@@ -198,6 +199,9 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
|
||||
if len(prepared.Messages) != 2 {
|
||||
t.Fatalf("expected two messages, got %d", len(prepared.Messages))
|
||||
}
|
||||
if prepared.SessionID != "session-123" {
|
||||
t.Fatalf("expected prepared session id, got %q", prepared.SessionID)
|
||||
}
|
||||
if llmClient.calls != 0 {
|
||||
t.Fatalf("prepare should not call llm, calls=%d", llmClient.calls)
|
||||
}
|
||||
@@ -269,11 +273,11 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
Execution: &domain.ExecutionTarget{
|
||||
Execution: &domain.ExecutionTargetOverride{
|
||||
Endpoint: "http://override/v1",
|
||||
Model: "override-model",
|
||||
Temperature: 0.7,
|
||||
TimeoutSeconds: 30,
|
||||
Temperature: float64Ptr(0.7),
|
||||
TimeoutSeconds: intPtr(30),
|
||||
ServiceTier: "flex",
|
||||
},
|
||||
})
|
||||
@@ -291,6 +295,143 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPrepareRequestNumericOverridePresence(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
override *domain.ExecutionTargetOverride
|
||||
wantTemperature float64
|
||||
wantMaxTokens int
|
||||
wantTopP float64
|
||||
wantTimeoutSecs int
|
||||
wantPresence domain.ExecutionTargetPresence
|
||||
}{
|
||||
{
|
||||
name: "omitted preserves profile values",
|
||||
override: &domain.ExecutionTargetOverride{},
|
||||
wantTemperature: 0.7,
|
||||
wantMaxTokens: 321,
|
||||
wantTopP: 0.8,
|
||||
wantTimeoutSecs: 45,
|
||||
},
|
||||
{
|
||||
name: "explicit zero temperature",
|
||||
override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(0)},
|
||||
wantTemperature: 0,
|
||||
wantMaxTokens: 321,
|
||||
wantTopP: 0.8,
|
||||
wantTimeoutSecs: 45,
|
||||
wantPresence: domain.ExecutionTargetPresence{Temperature: true},
|
||||
},
|
||||
{
|
||||
name: "explicit zero max tokens",
|
||||
override: &domain.ExecutionTargetOverride{MaxTokens: intPtr(0)},
|
||||
wantTemperature: 0.7,
|
||||
wantMaxTokens: 0,
|
||||
wantTopP: 0.8,
|
||||
wantTimeoutSecs: 45,
|
||||
wantPresence: domain.ExecutionTargetPresence{MaxTokens: true},
|
||||
},
|
||||
{
|
||||
name: "explicit zero top p",
|
||||
override: &domain.ExecutionTargetOverride{TopP: float64Ptr(0)},
|
||||
wantTemperature: 0.7,
|
||||
wantMaxTokens: 321,
|
||||
wantTopP: 0,
|
||||
wantTimeoutSecs: 45,
|
||||
wantPresence: domain.ExecutionTargetPresence{TopP: true},
|
||||
},
|
||||
{
|
||||
name: "explicit zero timeout",
|
||||
override: &domain.ExecutionTargetOverride{TimeoutSeconds: intPtr(0)},
|
||||
wantTemperature: 0.7,
|
||||
wantMaxTokens: 321,
|
||||
wantTopP: 0.8,
|
||||
wantTimeoutSecs: 0,
|
||||
wantPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {
|
||||
ID: "exec",
|
||||
Endpoint: "http://profile/v1",
|
||||
Model: "profile-model",
|
||||
Temperature: 0.7,
|
||||
MaxTokens: 321,
|
||||
TopP: 0.8,
|
||||
TimeoutSeconds: 45,
|
||||
},
|
||||
}},
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
nil,
|
||||
)
|
||||
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
Execution: tc.override,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
got := prepared.EffectiveModelParams
|
||||
if got.Temperature != tc.wantTemperature ||
|
||||
got.MaxTokens != tc.wantMaxTokens ||
|
||||
got.TopP != tc.wantTopP ||
|
||||
got.TimeoutSeconds != tc.wantTimeoutSecs {
|
||||
t.Fatalf("unexpected effective numeric settings: %+v", got)
|
||||
}
|
||||
if prepared.TargetPresence != tc.wantPresence {
|
||||
t.Fatalf("unexpected target presence: got %+v want %+v", prepared.TargetPresence, tc.wantPresence)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPrepareInvalidRequestNumericOverridesFail(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
override *domain.ExecutionTargetOverride
|
||||
}{
|
||||
{name: "temperature below range", override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(-0.1)}},
|
||||
{name: "temperature above range", override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(2.1)}},
|
||||
{name: "max tokens below range", override: &domain.ExecutionTargetOverride{MaxTokens: intPtr(-1)}},
|
||||
{name: "top p below range", override: &domain.ExecutionTargetOverride{TopP: float64Ptr(-0.1)}},
|
||||
{name: "top p above range", override: &domain.ExecutionTargetOverride{TopP: float64Ptr(1.1)}},
|
||||
{name: "timeout below range", override: &domain.ExecutionTargetOverride{TimeoutSeconds: intPtr(-1)}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
nil,
|
||||
)
|
||||
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
Execution: tc.override,
|
||||
})
|
||||
if !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
|
||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
@@ -577,6 +718,100 @@ func TestDeriveStructuredSchemaName(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashRenderedPromptIncludesCacheControlWhenPresent(t *testing.T) {
|
||||
uncached := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||
{Role: "system", Content: "sys"},
|
||||
{Role: "user", Content: "usr"},
|
||||
}}
|
||||
wantLegacyHash := hashString("system\nsys\n---\nuser\nusr\n---\n")
|
||||
if got := hashRenderedPrompt(uncached); got != wantLegacyHash {
|
||||
t.Fatalf("expected no-cache hash to preserve legacy input, got %q want %q", got, wantLegacyHash)
|
||||
}
|
||||
|
||||
withCache := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "sys",
|
||||
CacheControl: &domain.CacheControl{
|
||||
Type: domain.CacheControlEphemeral,
|
||||
TTL: "1h",
|
||||
},
|
||||
},
|
||||
{Role: "user", Content: "usr"},
|
||||
}}
|
||||
alsoWithCache := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "sys",
|
||||
CacheControl: &domain.CacheControl{
|
||||
Type: domain.CacheControlEphemeral,
|
||||
TTL: "1h",
|
||||
},
|
||||
},
|
||||
{Role: "user", Content: "usr"},
|
||||
}}
|
||||
withoutTTL := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "sys",
|
||||
CacheControl: &domain.CacheControl{
|
||||
Type: domain.CacheControlEphemeral,
|
||||
},
|
||||
},
|
||||
{Role: "user", Content: "usr"},
|
||||
}}
|
||||
|
||||
cachedHash := hashRenderedPrompt(withCache)
|
||||
if cachedHash == hashRenderedPrompt(uncached) {
|
||||
t.Fatal("expected cache control to change rendered prompt hash")
|
||||
}
|
||||
if cachedHash != hashRenderedPrompt(alsoWithCache) {
|
||||
t.Fatal("expected identical cache control metadata to produce stable hash")
|
||||
}
|
||||
if cachedHash == hashRenderedPrompt(withoutTTL) {
|
||||
t.Fatal("expected ttl changes to affect rendered prompt hash")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashRenderedPromptIncludesSessionIDWhenPresent(t *testing.T) {
|
||||
withoutSession := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||
{Role: "system", Content: "sys"},
|
||||
{Role: "user", Content: "usr"},
|
||||
}}
|
||||
withSession := domain.RenderedPrompt{
|
||||
SessionID: "session-123",
|
||||
Messages: []domain.RenderedMessage{
|
||||
{Role: "system", Content: "sys"},
|
||||
{Role: "user", Content: "usr"},
|
||||
},
|
||||
}
|
||||
alsoWithSession := domain.RenderedPrompt{
|
||||
SessionID: "session-123",
|
||||
Messages: []domain.RenderedMessage{
|
||||
{Role: "system", Content: "sys"},
|
||||
{Role: "user", Content: "usr"},
|
||||
},
|
||||
}
|
||||
otherSession := domain.RenderedPrompt{
|
||||
SessionID: "session-456",
|
||||
Messages: []domain.RenderedMessage{
|
||||
{Role: "system", Content: "sys"},
|
||||
{Role: "user", Content: "usr"},
|
||||
},
|
||||
}
|
||||
|
||||
sessionHash := hashRenderedPrompt(withSession)
|
||||
if sessionHash == hashRenderedPrompt(withoutSession) {
|
||||
t.Fatal("expected session_id to change rendered prompt hash")
|
||||
}
|
||||
if sessionHash != hashRenderedPrompt(alsoWithSession) {
|
||||
t.Fatal("expected identical session_id to produce stable hash")
|
||||
}
|
||||
if sessionHash == hashRenderedPrompt(otherSession) {
|
||||
t.Fatal("expected session_id value changes to affect rendered prompt hash")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunSuccessful(t *testing.T) {
|
||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}
|
||||
@@ -584,7 +819,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
||||
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
|
||||
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
|
||||
}}
|
||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
|
||||
|
||||
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
||||
@@ -596,7 +831,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
||||
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
|
||||
},
|
||||
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
|
||||
Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
@@ -631,6 +866,48 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
||||
if llmClient.lastReq.Target.TimeoutSeconds != 90 {
|
||||
t.Fatalf("expected timeout propagation, got %d", llmClient.lastReq.Target.TimeoutSeconds)
|
||||
}
|
||||
if !llmClient.lastReq.TargetPresence.Temperature || !llmClient.lastReq.TargetPresence.TimeoutSeconds {
|
||||
t.Fatalf("expected numeric override presence to be sent to llm, got %+v", llmClient.lastReq.TargetPresence)
|
||||
}
|
||||
if llmClient.lastReq.Prompt.SessionID != "session-123" {
|
||||
t.Fatalf("expected session id to be sent to llm, got %q", llmClient.lastReq.Prompt.SessionID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunPassesExtraParamsToGenerateRequestTarget(t *testing.T) {
|
||||
extraParams := map[string]any{
|
||||
"string_value": "enabled",
|
||||
"number_value": 42,
|
||||
"boolean_value": true,
|
||||
"object_value": map[string]any{"nested": "value"},
|
||||
"array_value": []any{"first", 3, false},
|
||||
}
|
||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {
|
||||
ID: "exec",
|
||||
Endpoint: "http://profile/v1",
|
||||
Model: "profile-model",
|
||||
ExtraParams: extraParams,
|
||||
},
|
||||
}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(res.EffectiveModelParams.ExtraParams, extraParams) {
|
||||
t.Fatalf("expected run result extra_params to match profile values, got %#v", res.EffectiveModelParams.ExtraParams)
|
||||
}
|
||||
if !reflect.DeepEqual(llmClient.lastReq.Target.ExtraParams, extraParams) {
|
||||
t.Fatalf("expected generate request extra_params to match profile values, got %#v", llmClient.lastReq.Target.ExtraParams)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T) {
|
||||
@@ -649,7 +926,7 @@ func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T)
|
||||
Inputs: map[string]domain.ArtifactRef{
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
||||
},
|
||||
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
|
||||
Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)},
|
||||
}
|
||||
|
||||
prepared, err := runner.Prepare(context.Background(), req)
|
||||
@@ -777,11 +1054,11 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T)
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
Execution: &domain.ExecutionTarget{
|
||||
Execution: &domain.ExecutionTargetOverride{
|
||||
Endpoint: "http://override/v1",
|
||||
Model: "override-model",
|
||||
Temperature: 0.7,
|
||||
TimeoutSeconds: 30,
|
||||
Temperature: float64Ptr(0.7),
|
||||
TimeoutSeconds: intPtr(30),
|
||||
ServiceTier: "flex",
|
||||
},
|
||||
})
|
||||
@@ -902,6 +1179,75 @@ func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
|
||||
const directKey = "direct-runner-key"
|
||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "SCRIPTORIUM_MISSING_KEY"},
|
||||
}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
APIKey: directKey,
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if llmClient.lastReq.Target.APIKey != directKey {
|
||||
t.Fatalf("expected direct API key to reach LLM request")
|
||||
}
|
||||
if llmClient.lastReq.Target.APIKeyEnv != "SCRIPTORIUM_MISSING_KEY" {
|
||||
t.Fatalf("expected api_key_env name to remain on target, got %q", llmClient.lastReq.Target.APIKeyEnv)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPrepareAPIKeyRequiredFailsWithoutDirectKey(t *testing.T) {
|
||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if !errors.Is(err, ErrAPIKeyRequired) {
|
||||
t.Fatalf("expected ErrAPIKeyRequired, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunAPIKeyRequiredSucceedsWithDirectKey(t *testing.T) {
|
||||
const directKey = "direct-required-key"
|
||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
|
||||
}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
APIKey: directKey,
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if llmClient.lastReq.Target.APIKey != directKey {
|
||||
t.Fatalf("expected direct API key to reach LLM request")
|
||||
}
|
||||
if !llmClient.lastReq.Target.APIKeyRequired {
|
||||
t.Fatalf("expected APIKeyRequired to be carried to target")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
|
||||
const envName = "SCRIPTORIUM_RUNTIME_API_KEY"
|
||||
t.Setenv(envName, "runtime-secret")
|
||||
@@ -916,7 +1262,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
Execution: &domain.ExecutionTarget{APIKeyEnv: envName},
|
||||
Execution: &domain.ExecutionTargetOverride{APIKeyEnv: envName},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
@@ -941,7 +1287,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) {
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
Execution: &domain.ExecutionTarget{APIKeyEnv: runtimeEnv},
|
||||
Execution: &domain.ExecutionTargetOverride{APIKeyEnv: runtimeEnv},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
@@ -1040,6 +1386,28 @@ func TestRunnerRunLLMFailure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{err: llm.ErrInvalidRequest},
|
||||
nil,
|
||||
)
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||
}
|
||||
if errors.Is(err, ErrLLMGenerate) {
|
||||
t.Fatalf("did not expect ErrLLMGenerate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunValidationStillWorks(t *testing.T) {
|
||||
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
|
||||
runner := NewRunner(
|
||||
@@ -1082,7 +1450,7 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "override-model", TimeoutSeconds: 22},
|
||||
Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "override-model", TimeoutSeconds: intPtr(22)},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
@@ -1169,7 +1537,8 @@ func TestExecutionProfileToTargetPopulatesAllFieldsAndCopiesExtraParams(t *testi
|
||||
ServiceTier: "priority",
|
||||
ReasoningEffort: "medium",
|
||||
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
||||
ExtraParams: map[string]string{
|
||||
APIKeyRequired: true,
|
||||
ExtraParams: map[string]any{
|
||||
"provider_option": "on",
|
||||
},
|
||||
}
|
||||
@@ -1183,7 +1552,8 @@ func TestExecutionProfileToTargetPopulatesAllFieldsAndCopiesExtraParams(t *testi
|
||||
target.TimeoutSeconds != src.TimeoutSeconds ||
|
||||
target.ServiceTier != src.ServiceTier ||
|
||||
target.ReasoningEffort != src.ReasoningEffort ||
|
||||
target.APIKeyEnv != src.APIKeyEnv {
|
||||
target.APIKeyEnv != src.APIKeyEnv ||
|
||||
target.APIKeyRequired != src.APIKeyRequired {
|
||||
t.Fatalf("expected all profile fields to populate target, got %+v", target)
|
||||
}
|
||||
if !reflect.DeepEqual(target.ExtraParams, src.ExtraParams) {
|
||||
@@ -1208,12 +1578,19 @@ func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testin
|
||||
ServiceTier: "priority",
|
||||
ReasoningEffort: "low",
|
||||
APIKeyEnv: "PROFILE_KEY",
|
||||
ExtraParams: map[string]string{
|
||||
APIKeyRequired: true,
|
||||
ExtraParams: map[string]any{
|
||||
"profile_option": "enabled",
|
||||
},
|
||||
}
|
||||
|
||||
target := resolveExecutionTarget(profileValue, nil)
|
||||
target, presence, err := resolveExecutionTarget(profileValue, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if presence != (domain.ExecutionTargetPresence{}) {
|
||||
t.Fatalf("expected no request override presence, got %+v", presence)
|
||||
}
|
||||
if target.Endpoint != profileValue.Endpoint ||
|
||||
target.Model != profileValue.Model ||
|
||||
target.Temperature != profileValue.Temperature ||
|
||||
@@ -1222,7 +1599,8 @@ func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testin
|
||||
target.TimeoutSeconds != profileValue.TimeoutSeconds ||
|
||||
target.ServiceTier != profileValue.ServiceTier ||
|
||||
target.ReasoningEffort != profileValue.ReasoningEffort ||
|
||||
target.APIKeyEnv != profileValue.APIKeyEnv {
|
||||
target.APIKeyEnv != profileValue.APIKeyEnv ||
|
||||
target.APIKeyRequired != profileValue.APIKeyRequired {
|
||||
t.Fatalf("expected profile values to populate target, got %+v", target)
|
||||
}
|
||||
if !reflect.DeepEqual(target.ExtraParams, profileValue.ExtraParams) {
|
||||
@@ -1242,32 +1620,38 @@ func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFiel
|
||||
ServiceTier: "priority",
|
||||
ReasoningEffort: "medium",
|
||||
APIKeyEnv: "PROFILE_KEY",
|
||||
ExtraParams: map[string]string{
|
||||
ExtraParams: map[string]any{
|
||||
"profile_only": "yes",
|
||||
},
|
||||
}
|
||||
override := &domain.ExecutionTarget{
|
||||
override := &domain.ExecutionTargetOverride{
|
||||
Endpoint: "http://override/v1",
|
||||
Model: "override-model",
|
||||
Temperature: 0.9,
|
||||
MaxTokens: 111,
|
||||
TopP: 0.5,
|
||||
TimeoutSeconds: 30,
|
||||
Temperature: float64Ptr(0.9),
|
||||
MaxTokens: intPtr(111),
|
||||
TopP: float64Ptr(0.5),
|
||||
TimeoutSeconds: intPtr(30),
|
||||
ServiceTier: "flex",
|
||||
ReasoningEffort: "high",
|
||||
APIKeyEnv: "RUNTIME_KEY",
|
||||
ExtraParams: map[string]string{
|
||||
ExtraParams: map[string]any{
|
||||
"runtime_only": "yes",
|
||||
},
|
||||
}
|
||||
|
||||
target := resolveExecutionTarget(profileValue, override)
|
||||
target, presence, err := resolveExecutionTarget(profileValue, override)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if presence != (domain.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}) {
|
||||
t.Fatalf("unexpected override presence: %+v", presence)
|
||||
}
|
||||
if target.Endpoint != override.Endpoint ||
|
||||
target.Model != override.Model ||
|
||||
target.Temperature != override.Temperature ||
|
||||
target.MaxTokens != override.MaxTokens ||
|
||||
target.TopP != override.TopP ||
|
||||
target.TimeoutSeconds != override.TimeoutSeconds ||
|
||||
target.Temperature != *override.Temperature ||
|
||||
target.MaxTokens != *override.MaxTokens ||
|
||||
target.TopP != *override.TopP ||
|
||||
target.TimeoutSeconds != *override.TimeoutSeconds ||
|
||||
target.ServiceTier != override.ServiceTier ||
|
||||
target.ReasoningEffort != override.ReasoningEffort ||
|
||||
target.APIKeyEnv != override.APIKeyEnv {
|
||||
@@ -1311,12 +1695,12 @@ func TestMergeExecutionTargetEmptyStringOverridesDoNotErase(t *testing.T) {
|
||||
|
||||
func TestMergeExecutionTargetEmptyExtraParamsDoesNotErase(t *testing.T) {
|
||||
base := domain.ExecutionTarget{
|
||||
ExtraParams: map[string]string{
|
||||
ExtraParams: map[string]any{
|
||||
"keep": "value",
|
||||
},
|
||||
}
|
||||
override := domain.ExecutionTarget{
|
||||
ExtraParams: map[string]string{},
|
||||
ExtraParams: map[string]any{},
|
||||
}
|
||||
|
||||
merged := mergeExecutionTarget(base, override)
|
||||
@@ -1392,6 +1776,14 @@ func singleInputRef() map[string]domain.ArtifactRef {
|
||||
return map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}}
|
||||
}
|
||||
|
||||
func float64Ptr(v float64) *float64 {
|
||||
return &v
|
||||
}
|
||||
|
||||
func intPtr(v int) *int {
|
||||
return &v
|
||||
}
|
||||
|
||||
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
|
||||
return NewRunner(
|
||||
promptRepo,
|
||||
|
||||
@@ -5,7 +5,9 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
@@ -18,10 +20,19 @@ type StandardValidator struct {
|
||||
schemaBaseDir string
|
||||
}
|
||||
|
||||
type FSValidator struct {
|
||||
fsys fs.FS
|
||||
root string
|
||||
}
|
||||
|
||||
func NewStandardValidator(schemaBaseDir string) Validator {
|
||||
return &StandardValidator{schemaBaseDir: schemaBaseDir}
|
||||
}
|
||||
|
||||
func NewFSValidator(fsys fs.FS, root string) Validator {
|
||||
return &FSValidator{fsys: fsys, root: root}
|
||||
}
|
||||
|
||||
func (v *StandardValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -100,6 +111,88 @@ func (v *StandardValidator) Validate(ctx context.Context, artifact *domain.Artif
|
||||
}
|
||||
}
|
||||
|
||||
func (v *FSValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return domain.ValidationResult{}, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
res := domain.ValidationResult{
|
||||
Mode: contract.ValidationMode,
|
||||
SchemaPath: contract.SchemaPath,
|
||||
RepairAttempts: contract.RepairAttempts,
|
||||
}
|
||||
|
||||
if artifact == nil {
|
||||
return domain.ValidationResult{}, errors.New("artifact is required for validation")
|
||||
}
|
||||
|
||||
switch contract.ValidationMode {
|
||||
case domain.ValidationNone:
|
||||
res.Status = domain.ValidationSkipped
|
||||
res.IsValid = true
|
||||
return res, nil
|
||||
case domain.ValidationBasic:
|
||||
if strings.TrimSpace(string(artifact.Body)) == "" {
|
||||
res.Status = domain.ValidationFailed
|
||||
res.IsValid = false
|
||||
res.Errors = []string{"output is empty"}
|
||||
return res, nil
|
||||
}
|
||||
res.Status = domain.ValidationPassed
|
||||
res.IsValid = true
|
||||
return res, nil
|
||||
case domain.ValidationJSON:
|
||||
_, jsonErr := parseJSON(artifact.Body)
|
||||
if jsonErr != nil {
|
||||
res.Status = domain.ValidationFailed
|
||||
res.IsValid = false
|
||||
res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)}
|
||||
return res, nil
|
||||
}
|
||||
res.Status = domain.ValidationPassed
|
||||
res.IsValid = true
|
||||
return res, nil
|
||||
case domain.ValidationJSONSchema:
|
||||
instance, jsonErr := parseJSON(artifact.Body)
|
||||
if jsonErr != nil {
|
||||
res.Status = domain.ValidationFailed
|
||||
res.IsValid = false
|
||||
res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
schemaName, schemaDoc, err := v.loadSchemaDocument(contract.SchemaPath)
|
||||
if err != nil {
|
||||
return domain.ValidationResult{}, err
|
||||
}
|
||||
|
||||
resourceURL := fsSchemaResourceURL(schemaName)
|
||||
compiler := jsonschema.NewCompiler()
|
||||
if err := compiler.AddResource(resourceURL, schemaDoc); err != nil {
|
||||
return domain.ValidationResult{}, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
|
||||
}
|
||||
schema, err := compiler.Compile(resourceURL)
|
||||
if err != nil {
|
||||
return domain.ValidationResult{}, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
|
||||
}
|
||||
|
||||
if err := schema.Validate(instance); err != nil {
|
||||
res.Status = domain.ValidationFailed
|
||||
res.IsValid = false
|
||||
res.Errors = []string{fmt.Sprintf("json schema validation failed: %v", err)}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
res.Status = domain.ValidationPassed
|
||||
res.IsValid = true
|
||||
return res, nil
|
||||
default:
|
||||
return domain.ValidationResult{}, fmt.Errorf("unsupported validation mode: %q", contract.ValidationMode)
|
||||
}
|
||||
}
|
||||
|
||||
func parseJSON(body []byte) (any, error) {
|
||||
var v any
|
||||
if err := json.Unmarshal(body, &v); err != nil {
|
||||
@@ -132,6 +225,20 @@ func (v *StandardValidator) LoadSchemaDocument(ctx context.Context, schemaPath s
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
func (v *FSValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
_, doc, err := v.loadSchemaDocument(schemaPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error) {
|
||||
if strings.TrimSpace(schemaPath) == "" {
|
||||
return "", errors.New("schema path is required for json_schema validation")
|
||||
@@ -149,3 +256,70 @@ func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error)
|
||||
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func (v *FSValidator) loadSchemaDocument(schemaPath string) (string, any, error) {
|
||||
resolved, err := v.resolveSchemaPath(schemaPath)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
raw, err := fs.ReadFile(v.fsys, resolved)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err)
|
||||
}
|
||||
|
||||
var doc any
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return "", nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
|
||||
}
|
||||
return resolved, doc, nil
|
||||
}
|
||||
|
||||
func (v *FSValidator) resolveSchemaPath(schemaPath string) (string, error) {
|
||||
if strings.TrimSpace(schemaPath) == "" {
|
||||
return "", errors.New("schema path is required for json_schema validation")
|
||||
}
|
||||
if v.fsys == nil {
|
||||
return "", errors.New("schema filesystem is nil")
|
||||
}
|
||||
|
||||
cleanRoot := cleanFSRoot(v.root)
|
||||
rootInfo, err := fs.Stat(v.fsys, cleanRoot)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to access schema source %q: %w", cleanRoot, err)
|
||||
}
|
||||
|
||||
cleanSchemaPath := cleanSchemaFSPath(schemaPath)
|
||||
var resolved string
|
||||
if rootInfo.IsDir() {
|
||||
resolved = path.Join(cleanRoot, cleanSchemaPath)
|
||||
} else {
|
||||
if cleanSchemaPath != path.Base(cleanRoot) {
|
||||
return "", fmt.Errorf("schema path %q does not match schema file %q", cleanSchemaPath, path.Base(cleanRoot))
|
||||
}
|
||||
resolved = cleanRoot
|
||||
}
|
||||
|
||||
if _, err := fs.Stat(v.fsys, resolved); err != nil {
|
||||
return "", fmt.Errorf("failed to access schema file %q: %w", resolved, err)
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func cleanSchemaFSPath(schemaPath string) string {
|
||||
cleaned := strings.TrimSpace(schemaPath)
|
||||
cleaned = strings.TrimPrefix(path.Clean(cleaned), "/")
|
||||
return cleaned
|
||||
}
|
||||
|
||||
func cleanFSRoot(root string) string {
|
||||
root = strings.TrimSpace(root)
|
||||
if root == "" || root == "." {
|
||||
return "."
|
||||
}
|
||||
return strings.TrimPrefix(path.Clean(root), "/")
|
||||
}
|
||||
|
||||
func fsSchemaResourceURL(schemaName string) string {
|
||||
return "scriptorium-schema:///" + strings.TrimPrefix(path.Clean(schemaName), "/")
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
)
|
||||
@@ -250,3 +251,77 @@ func TestStandardValidatorLoadSchemaDocumentInvalidJSON(t *testing.T) {
|
||||
t.Fatal("expected decode error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSValidatorJSONSchemaSuccess(t *testing.T) {
|
||||
v := NewFSValidator(fstest.MapFS{
|
||||
"schemas/events.schema.json": &fstest.MapFile{Data: []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"required": ["events"],
|
||||
"properties": {
|
||||
"events": {"type": "array"}
|
||||
}
|
||||
}`)},
|
||||
}, "schemas")
|
||||
|
||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
|
||||
ValidationMode: domain.ValidationJSONSchema,
|
||||
SchemaPath: "events.schema.json",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if res.Status != domain.ValidationPassed || !res.IsValid {
|
||||
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSValidatorSingleSchemaFileUsesBaseName(t *testing.T) {
|
||||
v := NewFSValidator(fstest.MapFS{
|
||||
"events.schema.json": &fstest.MapFile{Data: []byte(`{
|
||||
"type": "object",
|
||||
"required": ["events"],
|
||||
"properties": {
|
||||
"events": {"type": "array"}
|
||||
}
|
||||
}`)},
|
||||
}, "events.schema.json")
|
||||
|
||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
|
||||
ValidationMode: domain.ValidationJSONSchema,
|
||||
SchemaPath: "events.schema.json",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if res.Status != domain.ValidationPassed || !res.IsValid {
|
||||
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
|
||||
}
|
||||
|
||||
_, err = v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
|
||||
ValidationMode: domain.ValidationJSONSchema,
|
||||
SchemaPath: "other.schema.json",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected schema path mismatch error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSValidatorLoadSchemaDocument(t *testing.T) {
|
||||
v := NewFSValidator(fstest.MapFS{
|
||||
"schemas/schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
|
||||
}, "schemas")
|
||||
loader, ok := v.(SchemaDocumentLoader)
|
||||
if !ok {
|
||||
t.Fatal("fs validator must implement SchemaDocumentLoader")
|
||||
}
|
||||
|
||||
doc, err := loader.LoadSchemaDocument(context.Background(), "schema.json")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
obj, ok := doc.(map[string]any)
|
||||
if !ok || obj["type"] != "object" {
|
||||
t.Fatalf("unexpected schema document: %#v", doc)
|
||||
}
|
||||
}
|
||||
|
||||
23
llm_adapter.go
Normal file
23
llm_adapter.go
Normal file
@@ -0,0 +1,23 @@
|
||||
package scriptorium
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
)
|
||||
|
||||
type publicLLMClientAdapter struct {
|
||||
client LLMClient
|
||||
}
|
||||
|
||||
func (a publicLLMClientAdapter) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
resp, err := a.client.Generate(ctx, fromDomainGenerateRequest(req))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp == nil {
|
||||
return nil, fmt.Errorf("%w: llm client returned nil response", ErrLLMGenerate)
|
||||
}
|
||||
return toDomainGenerateResponse(resp), nil
|
||||
}
|
||||
105
profiles.go
Normal file
105
profiles.go
Normal file
@@ -0,0 +1,105 @@
|
||||
package scriptorium
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
)
|
||||
|
||||
// OpenAICompatibleProfile returns an in-memory profile for an OpenAI-compatible
|
||||
// chat-completions endpoint.
|
||||
func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile {
|
||||
return Profile{
|
||||
ID: cfg.ID,
|
||||
Endpoint: cfg.Endpoint,
|
||||
Model: cfg.Model,
|
||||
Temperature: cfg.Temperature,
|
||||
MaxTokens: cfg.MaxTokens,
|
||||
TopP: cfg.TopP,
|
||||
TimeoutSeconds: cfg.TimeoutSeconds,
|
||||
ServiceTier: cfg.ServiceTier,
|
||||
ReasoningEffort: cfg.ReasoningEffort,
|
||||
APIKeyRequired: cfg.APIKeyRequired,
|
||||
ExtraParams: copyAnyMap(cfg.ExtraParams),
|
||||
}
|
||||
}
|
||||
|
||||
type memoryProfileRepository struct {
|
||||
profiles map[string]domain.ExecutionProfile
|
||||
}
|
||||
|
||||
func newMemoryProfileRepository(profiles []Profile) (*memoryProfileRepository, error) {
|
||||
repo := &memoryProfileRepository{profiles: make(map[string]domain.ExecutionProfile, len(profiles))}
|
||||
for _, publicProfile := range profiles {
|
||||
prof, err := toDomainProfile(publicProfile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, exists := repo.profiles[prof.ID]; exists {
|
||||
return nil, fmt.Errorf("duplicate profile id %q", prof.ID)
|
||||
}
|
||||
repo.profiles[prof.ID] = prof
|
||||
}
|
||||
return repo, nil
|
||||
}
|
||||
|
||||
func (r *memoryProfileRepository) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
if r == nil {
|
||||
return nil, profile.ErrProfileNotFound
|
||||
}
|
||||
prof, ok := r.profiles[id]
|
||||
if !ok {
|
||||
return nil, profile.ErrProfileNotFound
|
||||
}
|
||||
prof.ExtraParams = copyAnyMap(prof.ExtraParams)
|
||||
return &prof, nil
|
||||
}
|
||||
|
||||
func toDomainProfile(publicProfile Profile) (domain.ExecutionProfile, error) {
|
||||
prof := domain.ExecutionProfile{
|
||||
ID: strings.TrimSpace(publicProfile.ID),
|
||||
Endpoint: publicProfile.Endpoint,
|
||||
Model: publicProfile.Model,
|
||||
Temperature: publicProfile.Temperature,
|
||||
MaxTokens: publicProfile.MaxTokens,
|
||||
TopP: publicProfile.TopP,
|
||||
TimeoutSeconds: publicProfile.TimeoutSeconds,
|
||||
ServiceTier: publicProfile.ServiceTier,
|
||||
ReasoningEffort: publicProfile.ReasoningEffort,
|
||||
APIKeyRequired: publicProfile.APIKeyRequired,
|
||||
ExtraParams: copyAnyMap(publicProfile.ExtraParams),
|
||||
}
|
||||
if err := validatePublicProfile(prof); err != nil {
|
||||
return domain.ExecutionProfile{}, err
|
||||
}
|
||||
return prof, nil
|
||||
}
|
||||
|
||||
func validatePublicProfile(prof domain.ExecutionProfile) error {
|
||||
if strings.TrimSpace(prof.ID) == "" {
|
||||
return errors.New("id is required")
|
||||
}
|
||||
if strings.TrimSpace(prof.Endpoint) == "" {
|
||||
return errors.New("endpoint is required")
|
||||
}
|
||||
if strings.TrimSpace(prof.Model) == "" {
|
||||
return errors.New("model is required")
|
||||
}
|
||||
if prof.Temperature < 0 || prof.Temperature > 2 {
|
||||
return errors.New("temperature must be between 0 and 2")
|
||||
}
|
||||
if prof.MaxTokens < 0 {
|
||||
return errors.New("max_tokens must be greater than or equal to 0")
|
||||
}
|
||||
if prof.TopP < 0 || prof.TopP > 1 {
|
||||
return errors.New("top_p must be between 0 and 1")
|
||||
}
|
||||
if prof.TimeoutSeconds < 0 {
|
||||
return errors.New("timeout_seconds must be greater than or equal to 0")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
288
types.go
Normal file
288
types.go
Normal file
@@ -0,0 +1,288 @@
|
||||
package scriptorium
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ArtifactRefType defines how an artifact is referenced.
|
||||
type ArtifactRefType string
|
||||
|
||||
const (
|
||||
ArtifactRefInline ArtifactRefType = "inline"
|
||||
ArtifactRefFile ArtifactRefType = "file"
|
||||
)
|
||||
|
||||
// OutputFormat defines the desired output format.
|
||||
type OutputFormat string
|
||||
|
||||
const (
|
||||
FormatText OutputFormat = "text"
|
||||
FormatMarkdown OutputFormat = "markdown"
|
||||
FormatJSON OutputFormat = "json"
|
||||
)
|
||||
|
||||
// ValidationMode defines the output validation strategy.
|
||||
type ValidationMode string
|
||||
|
||||
const (
|
||||
ValidationNone ValidationMode = "none"
|
||||
ValidationBasic ValidationMode = "basic"
|
||||
ValidationJSON ValidationMode = "json"
|
||||
ValidationJSONSchema ValidationMode = "json_schema"
|
||||
)
|
||||
|
||||
// ValidationStatus defines the result of a validation check.
|
||||
type ValidationStatus string
|
||||
|
||||
const (
|
||||
ValidationPassed ValidationStatus = "passed"
|
||||
ValidationFailed ValidationStatus = "failed"
|
||||
ValidationSkipped ValidationStatus = "skipped"
|
||||
)
|
||||
|
||||
// CacheControlType defines provider cache behavior for prompt content.
|
||||
type CacheControlType string
|
||||
|
||||
const (
|
||||
CacheControlEphemeral CacheControlType = "ephemeral"
|
||||
)
|
||||
|
||||
// StructuredOutputType identifies provider-level structured output modes.
|
||||
type StructuredOutputType string
|
||||
|
||||
const (
|
||||
StructuredOutputJSONSchema StructuredOutputType = "json_schema"
|
||||
)
|
||||
|
||||
// RunRequest represents a request to prepare or run a single prompt.
|
||||
type RunRequest struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
APIKey string `json:"-"`
|
||||
Inputs map[string]ArtifactRef
|
||||
Vars map[string]string
|
||||
Execution *ExecutionTargetOverride
|
||||
Validation *OutputContract
|
||||
Metadata map[string]string
|
||||
}
|
||||
|
||||
// PreparedRun contains prepared prompt execution state. It does not include
|
||||
// resolved API key values, model output, validation results, or internal target
|
||||
// presence metadata.
|
||||
type PreparedRun struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
EndTime time.Time `json:"end_time,omitempty"`
|
||||
DurationMS int64 `json:"duration_ms,omitempty"`
|
||||
}
|
||||
|
||||
// RunResult contains generated output, validation state, and run metadata.
|
||||
type RunResult struct {
|
||||
RunID string `json:"run_id"`
|
||||
Artifact Artifact `json:"artifact"`
|
||||
RawOutput string `json:"raw_output"`
|
||||
Validation ValidationResult `json:"validation"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
ModelName string `json:"model_name"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
Usage TokenUsage `json:"usage"`
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
EndTime time.Time `json:"end_time,omitempty"`
|
||||
Duration time.Duration `json:"duration,omitempty"`
|
||||
}
|
||||
|
||||
// ArtifactRef represents a reference to prompt input content.
|
||||
type ArtifactRef struct {
|
||||
Type ArtifactRefType
|
||||
URI string
|
||||
Body string
|
||||
}
|
||||
|
||||
// Artifact represents loaded artifact content.
|
||||
type Artifact struct {
|
||||
Name string
|
||||
ContentType string
|
||||
Body []byte
|
||||
URI string
|
||||
Size int64
|
||||
Hash string
|
||||
}
|
||||
|
||||
// ExecutionTarget represents effective model runtime settings.
|
||||
type ExecutionTarget struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Model string `json:"model"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
TopP float64 `json:"top_p"`
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
ServiceTier string `json:"service_tier"`
|
||||
ReasoningEffort string `json:"reasoning_effort"`
|
||||
APIKeyEnv string `json:"api_key_env"`
|
||||
ExtraParams map[string]any `json:"extra_params"`
|
||||
}
|
||||
|
||||
// ExecutionTargetOverride represents per-request runtime setting overrides.
|
||||
type ExecutionTargetOverride struct {
|
||||
Endpoint string
|
||||
Model string
|
||||
Temperature *float64
|
||||
MaxTokens *int
|
||||
TopP *float64
|
||||
TimeoutSeconds *int
|
||||
ServiceTier string
|
||||
ReasoningEffort string
|
||||
APIKeyEnv string
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
|
||||
// Profile is an in-memory execution profile for library consumers.
|
||||
type Profile struct {
|
||||
ID string
|
||||
Endpoint string
|
||||
Model string
|
||||
Temperature float64
|
||||
MaxTokens int
|
||||
TopP float64
|
||||
TimeoutSeconds int
|
||||
ServiceTier string
|
||||
ReasoningEffort string
|
||||
APIKeyRequired bool
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
|
||||
// OpenAICompatibleProfileConfig configures an OpenAI-compatible in-memory profile.
|
||||
type OpenAICompatibleProfileConfig struct {
|
||||
ID string
|
||||
Endpoint string
|
||||
Model string
|
||||
APIKeyRequired bool
|
||||
Temperature float64
|
||||
MaxTokens int
|
||||
TopP float64
|
||||
TimeoutSeconds int
|
||||
ServiceTier string
|
||||
ReasoningEffort string
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
|
||||
// ExecutionTargetPresence tracks which numeric runtime settings were explicit
|
||||
// request overrides.
|
||||
type ExecutionTargetPresence struct {
|
||||
Temperature bool
|
||||
MaxTokens bool
|
||||
TopP bool
|
||||
TimeoutSeconds bool
|
||||
}
|
||||
|
||||
// OutputContract defines output and validation requirements.
|
||||
type OutputContract struct {
|
||||
Format OutputFormat `json:"format"`
|
||||
ValidationMode ValidationMode `json:"validation_mode"`
|
||||
SchemaPath string `json:"schema_path"`
|
||||
RepairAttempts int `json:"repair_attempts"`
|
||||
}
|
||||
|
||||
// ValidationResult represents output validation state.
|
||||
type ValidationResult struct {
|
||||
Status ValidationStatus `json:"status"`
|
||||
Mode ValidationMode `json:"mode"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
SchemaPath string `json:"schema_path,omitempty"`
|
||||
RepairAttempts int `json:"repair_attempts"`
|
||||
IsValid bool `json:"is_valid"`
|
||||
}
|
||||
|
||||
// TokenUsage tracks token consumption.
|
||||
type TokenUsage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||
}
|
||||
|
||||
// RenderedPrompt is the fully rendered prompt passed to an LLM client.
|
||||
type RenderedPrompt struct {
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
}
|
||||
|
||||
// RenderedMessage is a rendered chat message.
|
||||
type RenderedMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
CacheControl *CacheControl `json:"cache_control,omitempty"`
|
||||
}
|
||||
|
||||
// CacheControl describes provider cache metadata attached to prompt content.
|
||||
type CacheControl struct {
|
||||
Type CacheControlType `json:"type"`
|
||||
TTL string `json:"ttl,omitempty"`
|
||||
}
|
||||
|
||||
// StructuredOutputSpec describes provider-level structured output.
|
||||
type StructuredOutputSpec struct {
|
||||
Type StructuredOutputType `json:"type"`
|
||||
JSONSchema *StructuredOutputJSONSpec `json:"json_schema,omitempty"`
|
||||
}
|
||||
|
||||
// StructuredOutputJSONSpec contains JSON Schema output constraints.
|
||||
type StructuredOutputJSONSpec struct {
|
||||
Name string `json:"name"`
|
||||
Strict bool `json:"strict"`
|
||||
Schema any `json:"schema"`
|
||||
}
|
||||
|
||||
// LLMClient executes rendered prompts for Engine.Run.
|
||||
type LLMClient interface {
|
||||
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
|
||||
}
|
||||
|
||||
// GenerateRequest is passed to an injected LLM client.
|
||||
type GenerateRequest struct {
|
||||
Prompt RenderedPrompt `json:"prompt"`
|
||||
Target ExecutionTarget `json:"target"`
|
||||
TargetPresence ExecutionTargetPresence `json:"target_presence"`
|
||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||
APIKey string `json:"-"`
|
||||
}
|
||||
|
||||
// GenerateResponse is returned by an injected LLM client.
|
||||
type GenerateResponse struct {
|
||||
Content string `json:"content"`
|
||||
Usage TokenUsage `json:"usage"`
|
||||
}
|
||||
|
||||
// File returns a file-backed artifact reference.
|
||||
func File(path string) ArtifactRef {
|
||||
return ArtifactRef{Type: ArtifactRefFile, URI: path}
|
||||
}
|
||||
|
||||
// Inline returns an inline artifact reference.
|
||||
func Inline(body string) ArtifactRef {
|
||||
return ArtifactRef{Type: ArtifactRefInline, Body: body}
|
||||
}
|
||||
|
||||
// InlineWithURI returns an inline artifact reference with URI metadata.
|
||||
func InlineWithURI(uri string, body string) ArtifactRef {
|
||||
return ArtifactRef{Type: ArtifactRefInline, URI: uri, Body: body}
|
||||
}
|
||||
Reference in New Issue
Block a user