Compare commits
70 Commits
v0.9.0
...
a0e905ce46
| Author | SHA1 | Date | |
|---|---|---|---|
| a0e905ce46 | |||
| 719243e90c | |||
| a9e1b7435c | |||
| eb6dfb19b0 | |||
| d86b65adad | |||
| 31faaf4259 | |||
| 9932153b97 | |||
| f0ca233c25 | |||
| ff31f8daf8 | |||
| c927b7819d | |||
| 6d1fb66dd7 | |||
| e0b1d6a0dc | |||
| 33698903be | |||
| 90b76ddad3 | |||
| d5b3d1e061 | |||
| 41083de46a | |||
| 07ac7e54c5 | |||
| 879cb021b2 | |||
| 574f88bd6a | |||
| d5d7a222a4 | |||
| aabd89aea7 | |||
| 9189cbfc22 | |||
| 872c166ed7 | |||
| 6742def4d3 | |||
| 1b39f82117 | |||
| f7d821067f | |||
| a16f66cbc7 | |||
| 39485d87f6 | |||
| 61e5b0fe58 | |||
| f3c21c7d9f | |||
| bc5f5d3731 | |||
| 93a76f1d36 | |||
| 5c882f26a9 | |||
| 0d45ac6e3c | |||
| f7ad756fc3 | |||
| 4fe11b1b2b | |||
| 296f9b1817 | |||
| 7a8516b0c6 | |||
| 2df2f530b3 | |||
| 8b25ca72e5 | |||
| fa02791fe9 | |||
| 32767b4eb4 | |||
| e1e5351c5d | |||
| d60ef66f53 | |||
| 4669b73d38 | |||
| 6f91603168 | |||
| 3ad247039b | |||
| 32e2433628 | |||
| 712c6b92b8 | |||
| 89cafcefec | |||
| 1d7fac0a47 | |||
| 03d4f27d2b | |||
| 4ac2038331 | |||
| 14a7e7e04c | |||
| 5e522bad8b | |||
| 23872dd742 | |||
| 7ffbf5f6ca | |||
| d0dc30fcc9 | |||
| b38f7b4dc3 | |||
| 0512995931 | |||
| 049a5feadb | |||
| 1798e9c575 | |||
| 5d4bc8c2b9 | |||
| 63fb8fc132 | |||
| 4d4bb7a121 | |||
| 5dcb3cd4fc | |||
| efe346893c | |||
| c95d6fcfec | |||
| 0badb4364d | |||
| 1f63f8afbb |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1,6 +1,5 @@
|
|||||||
# ---> Codex
|
# ---> Codex
|
||||||
.codex
|
.codex
|
||||||
AGENTS.md
|
|
||||||
|
|
||||||
# ---> Go
|
# ---> Go
|
||||||
# If you prefer the allow list template instead of the deny list, see community template:
|
# If you prefer the allow list template instead of the deny list, see community template:
|
||||||
|
|||||||
@@ -28,10 +28,6 @@ steps:
|
|||||||
|
|
||||||
build_binary linux amd64 ""
|
build_binary linux amd64 ""
|
||||||
build_binary linux arm64 ""
|
build_binary linux arm64 ""
|
||||||
build_binary darwin amd64 ""
|
|
||||||
build_binary darwin arm64 ""
|
|
||||||
build_binary windows amd64 ".exe"
|
|
||||||
build_binary windows arm64 ".exe"
|
|
||||||
|
|
||||||
- name: publish-release
|
- name: publish-release
|
||||||
image: woodpeckerci/plugin-release
|
image: woodpeckerci/plugin-release
|
||||||
|
|||||||
1
AGENTS.md
Normal file
1
AGENTS.md
Normal file
@@ -0,0 +1 @@
|
|||||||
|
Please review `docs/development.md` for initial orientation in this repository and follow its task-specific reading guide.
|
||||||
2
LICENSE
2
LICENSE
@@ -1,4 +1,4 @@
|
|||||||
Copyright (c) 2026 eric.
|
Copyright (c) 2026 Eric Rakestraw.
|
||||||
|
|
||||||
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
||||||
|
|
||||||
|
|||||||
23
README.md
23
README.md
@@ -1,8 +1,12 @@
|
|||||||
# scriptorium
|
# scriptorium
|
||||||
|
|
||||||
Scriptorium is a config-driven prompt execution engine.
|
Scriptorium is a narrow prompt-execution application for rendering prompt
|
||||||
|
requests, running them against OpenAI-compatible chat-completions endpoints, and
|
||||||
|
serving the same run workflow over HTTP.
|
||||||
|
|
||||||
It separates prompt definitions (what to generate) from execution profiles (how to call an OpenAI-compatible model endpoint), then runs or renders a prepared request from named input artifacts.
|
It keeps prompt definitions, execution profiles, schemas, and input artifacts as
|
||||||
|
separate files so prompts can be reviewed and reused without baking model
|
||||||
|
runtime settings into application code.
|
||||||
|
|
||||||
## Quickstart
|
## Quickstart
|
||||||
|
|
||||||
@@ -18,19 +22,24 @@ go run ./cmd/scriptorium render \
|
|||||||
```
|
```
|
||||||
|
|
||||||
This command renders the prepared prompt and effective runtime settings without calling an LLM.
|
This command renders the prepared prompt and effective runtime settings without calling an LLM.
|
||||||
|
For complete invocation and output behavior, see the [CLI reference](docs/cli.md).
|
||||||
|
|
||||||
## Documentation
|
## Documentation
|
||||||
|
|
||||||
- [CLI reference](docs/cli.md)
|
- [CLI reference](docs/cli.md)
|
||||||
- [Configuration reference](docs/config.md)
|
- [Configuration reference](docs/config.md)
|
||||||
|
- [HTTP API reference](docs/api.md)
|
||||||
- [Operations guide](docs/operations.md)
|
- [Operations guide](docs/operations.md)
|
||||||
- [Troubleshooting](docs/troubleshooting.md)
|
- [Consumer integration overview](docs/consumers/api.md)
|
||||||
- [HTTP API integration](docs/integrations/http-api.md)
|
- [Go library package](docs/consumers/pkg-scriptorium.md)
|
||||||
|
- [Subprocess integration](docs/integrations/subprocess.md)
|
||||||
- [OpenAI-compatible chat integration](docs/integrations/openai-compatible-chat.md)
|
- [OpenAI-compatible chat integration](docs/integrations/openai-compatible-chat.md)
|
||||||
- [Narratio subprocess integration](docs/integrations/narratio.md)
|
|
||||||
- [Architecture policy](docs/policy/architecture.md)
|
- [Architecture policy](docs/policy/architecture.md)
|
||||||
|
|
||||||
## Examples
|
## Examples
|
||||||
|
|
||||||
- `examples/render-markdown-summary.sh`
|
- [Minimal configuration](examples/config.yml) and [complete configuration](examples/config.full.yml)
|
||||||
- `examples/http-run.json`
|
- [Prompt definitions](examples/prompts/), [execution profiles](examples/profiles/), [schemas](examples/schemas/), and [synthetic input fixtures](examples/fixtures/)
|
||||||
|
- [Render script](examples/render-markdown-summary.sh)
|
||||||
|
- [HTTP request](examples/http-run.json)
|
||||||
|
- [Go library example](examples/go-library/prepare/main.go)
|
||||||
|
|||||||
406
convert.go
Normal file
406
convert.go
Normal file
@@ -0,0 +1,406 @@
|
|||||||
|
package scriptorium
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
||||||
|
execution, err := toDomainExecutionTargetOverride(req.Execution)
|
||||||
|
if err != nil {
|
||||||
|
return domain.RunRequest{}, err
|
||||||
|
}
|
||||||
|
return domain.RunRequest{
|
||||||
|
PromptID: req.PromptID,
|
||||||
|
PromptVersion: req.PromptVersion,
|
||||||
|
ProfileID: req.ProfileID,
|
||||||
|
APIKey: req.APIKey,
|
||||||
|
Inputs: toDomainArtifactRefMap(req.Inputs),
|
||||||
|
Vars: copyStringMap(req.Vars),
|
||||||
|
Execution: execution,
|
||||||
|
Validation: toDomainOutputContractPtr(req.Validation),
|
||||||
|
Metadata: copyStringMap(req.Metadata),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
||||||
|
if prepared == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &PreparedRun{
|
||||||
|
PromptID: prepared.PromptID,
|
||||||
|
PromptVersion: prepared.PromptVersion,
|
||||||
|
PromptHash: prepared.PromptHash,
|
||||||
|
SelectedProfileID: prepared.SelectedProfileID,
|
||||||
|
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
|
||||||
|
OutputContract: fromDomainOutputContract(prepared.OutputContract),
|
||||||
|
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
|
||||||
|
InputHashes: copyStringMap(prepared.InputHashes),
|
||||||
|
SessionID: prepared.SessionID,
|
||||||
|
RenderedPromptHash: prepared.RenderedPromptHash,
|
||||||
|
Messages: fromDomainRenderedMessages(prepared.Messages),
|
||||||
|
StartTime: prepared.StartTime,
|
||||||
|
EndTime: prepared.EndTime,
|
||||||
|
DurationMS: prepared.DurationMS,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
||||||
|
if result == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &RunResult{
|
||||||
|
RunID: result.RunID,
|
||||||
|
Artifact: fromDomainArtifact(result.Artifact),
|
||||||
|
RawOutput: result.RawOutput,
|
||||||
|
Validation: fromDomainValidationResult(result.Validation),
|
||||||
|
PromptID: result.PromptID,
|
||||||
|
PromptVersion: result.PromptVersion,
|
||||||
|
PromptHash: result.PromptHash,
|
||||||
|
RenderedPromptHash: result.RenderedPromptHash,
|
||||||
|
SelectedProfileID: result.SelectedProfileID,
|
||||||
|
ModelName: result.ModelName,
|
||||||
|
Endpoint: result.Endpoint,
|
||||||
|
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
|
||||||
|
InputHashes: copyStringMap(result.InputHashes),
|
||||||
|
Usage: fromDomainTokenUsage(result.Usage),
|
||||||
|
StartTime: result.StartTime,
|
||||||
|
EndTime: result.EndTime,
|
||||||
|
Duration: result.Duration,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainGenerateRequest(req domain.GenerateRequest) GenerateRequest {
|
||||||
|
return GenerateRequest{
|
||||||
|
Prompt: fromDomainRenderedPrompt(req.Prompt),
|
||||||
|
Target: fromDomainExecutionTarget(req.Target),
|
||||||
|
TargetPresence: fromDomainExecutionTargetPresence(req.TargetPresence),
|
||||||
|
StructuredOutput: fromDomainStructuredOutputSpec(req.StructuredOutput),
|
||||||
|
APIKey: req.Target.APIKey,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse {
|
||||||
|
if resp == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &domain.GenerateResponse{
|
||||||
|
Content: resp.Content,
|
||||||
|
Usage: toDomainTokenUsage(resp.Usage),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainRenderedPrompt(prompt domain.RenderedPrompt) RenderedPrompt {
|
||||||
|
return RenderedPrompt{
|
||||||
|
SessionID: prompt.SessionID,
|
||||||
|
Messages: fromDomainRenderedMessages(prompt.Messages),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func toDomainArtifactRefMap(src map[string]ArtifactRef) map[string]domain.ArtifactRef {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]domain.ArtifactRef, len(src))
|
||||||
|
for k, v := range src {
|
||||||
|
out[k] = toDomainArtifactRef(v)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func toDomainArtifactRef(ref ArtifactRef) domain.ArtifactRef {
|
||||||
|
return domain.ArtifactRef{
|
||||||
|
Type: domain.ArtifactRefType(ref.Type),
|
||||||
|
URI: ref.URI,
|
||||||
|
Body: ref.Body,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainArtifact(artifact domain.Artifact) Artifact {
|
||||||
|
return Artifact{
|
||||||
|
Name: artifact.Name,
|
||||||
|
ContentType: artifact.ContentType,
|
||||||
|
Body: copyBytes(artifact.Body),
|
||||||
|
URI: artifact.URI,
|
||||||
|
Size: artifact.Size,
|
||||||
|
Hash: artifact.Hash,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain.ExecutionTargetOverride, error) {
|
||||||
|
if override == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
extraParams, err := copyPublicJSONMap(override.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &domain.ExecutionTargetOverride{
|
||||||
|
Endpoint: override.Endpoint,
|
||||||
|
Model: override.Model,
|
||||||
|
Temperature: copyFloat64Ptr(override.Temperature),
|
||||||
|
MaxTokens: copyIntPtr(override.MaxTokens),
|
||||||
|
TopP: copyFloat64Ptr(override.TopP),
|
||||||
|
TimeoutSeconds: copyIntPtr(override.TimeoutSeconds),
|
||||||
|
ServiceTier: override.ServiceTier,
|
||||||
|
ReasoningEffort: override.ReasoningEffort,
|
||||||
|
APIKeyEnv: override.APIKeyEnv,
|
||||||
|
ExtraParams: extraParams,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
||||||
|
return ExecutionTarget{
|
||||||
|
Endpoint: target.Endpoint,
|
||||||
|
Model: target.Model,
|
||||||
|
Temperature: target.Temperature,
|
||||||
|
MaxTokens: target.MaxTokens,
|
||||||
|
TopP: target.TopP,
|
||||||
|
TimeoutSeconds: target.TimeoutSeconds,
|
||||||
|
ServiceTier: target.ServiceTier,
|
||||||
|
ReasoningEffort: target.ReasoningEffort,
|
||||||
|
APIKeyEnv: target.APIKeyEnv,
|
||||||
|
ExtraParams: copyAnyMap(target.ExtraParams),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
||||||
|
return ExecutionTargetPresence{
|
||||||
|
Temperature: presence.Temperature,
|
||||||
|
MaxTokens: presence.MaxTokens,
|
||||||
|
TopP: presence.TopP,
|
||||||
|
TimeoutSeconds: presence.TimeoutSeconds,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func toDomainOutputContractPtr(contract *OutputContract) *domain.OutputContract {
|
||||||
|
if contract == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := toDomainOutputContract(*contract)
|
||||||
|
return &out
|
||||||
|
}
|
||||||
|
|
||||||
|
func toDomainOutputContract(contract OutputContract) domain.OutputContract {
|
||||||
|
return domain.OutputContract{
|
||||||
|
Format: domain.OutputFormat(contract.Format),
|
||||||
|
ValidationMode: domain.ValidationMode(contract.ValidationMode),
|
||||||
|
SchemaPath: contract.SchemaPath,
|
||||||
|
RepairAttempts: contract.RepairAttempts,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainOutputContract(contract domain.OutputContract) OutputContract {
|
||||||
|
return OutputContract{
|
||||||
|
Format: OutputFormat(contract.Format),
|
||||||
|
ValidationMode: ValidationMode(contract.ValidationMode),
|
||||||
|
SchemaPath: contract.SchemaPath,
|
||||||
|
RepairAttempts: contract.RepairAttempts,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainValidationResult(result domain.ValidationResult) ValidationResult {
|
||||||
|
return ValidationResult{
|
||||||
|
Status: ValidationStatus(result.Status),
|
||||||
|
Mode: ValidationMode(result.Mode),
|
||||||
|
Errors: copyStringSlice(result.Errors),
|
||||||
|
SchemaPath: result.SchemaPath,
|
||||||
|
RepairAttempts: result.RepairAttempts,
|
||||||
|
IsValid: result.IsValid,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainTokenUsage(usage domain.TokenUsage) TokenUsage {
|
||||||
|
return TokenUsage{
|
||||||
|
PromptTokens: usage.PromptTokens,
|
||||||
|
CompletionTokens: usage.CompletionTokens,
|
||||||
|
TotalTokens: usage.TotalTokens,
|
||||||
|
CachedTokens: usage.CachedTokens,
|
||||||
|
CacheWriteTokens: usage.CacheWriteTokens,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func toDomainTokenUsage(usage TokenUsage) domain.TokenUsage {
|
||||||
|
return domain.TokenUsage{
|
||||||
|
PromptTokens: usage.PromptTokens,
|
||||||
|
CompletionTokens: usage.CompletionTokens,
|
||||||
|
TotalTokens: usage.TotalTokens,
|
||||||
|
CachedTokens: usage.CachedTokens,
|
||||||
|
CacheWriteTokens: usage.CacheWriteTokens,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainRenderedMessages(messages []domain.RenderedMessage) []RenderedMessage {
|
||||||
|
if messages == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]RenderedMessage, len(messages))
|
||||||
|
for i, msg := range messages {
|
||||||
|
out[i] = RenderedMessage{
|
||||||
|
Role: msg.Role,
|
||||||
|
Content: msg.Content,
|
||||||
|
CacheControl: fromDomainCacheControl(msg.CacheControl),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl {
|
||||||
|
if cacheControl == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &CacheControl{
|
||||||
|
Type: CacheControlType(cacheControl.Type),
|
||||||
|
TTL: cacheControl.TTL,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainStructuredOutputSpec(spec *domain.StructuredOutputSpec) *StructuredOutputSpec {
|
||||||
|
if spec == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := &StructuredOutputSpec{
|
||||||
|
Type: StructuredOutputType(spec.Type),
|
||||||
|
}
|
||||||
|
if spec.JSONSchema != nil {
|
||||||
|
out.JSONSchema = &StructuredOutputJSONSpec{
|
||||||
|
Name: spec.JSONSchema.Name,
|
||||||
|
Strict: spec.JSONSchema.Strict,
|
||||||
|
Schema: copyAny(spec.JSONSchema.Schema),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyStringMap(src map[string]string) map[string]string {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]string, len(src))
|
||||||
|
for k, v := range src {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyAnyMap(src map[string]any) map[string]any {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]any, len(src))
|
||||||
|
for k, v := range src {
|
||||||
|
out[k] = copyAny(v)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyAny(value any) any {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
switch v := value.(type) {
|
||||||
|
case map[string]any:
|
||||||
|
return copyAnyMap(v)
|
||||||
|
case []any:
|
||||||
|
out := make([]any, len(v))
|
||||||
|
for i, item := range v {
|
||||||
|
out[i] = copyAny(item)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case []string:
|
||||||
|
return copyStringSlice(v)
|
||||||
|
case []byte:
|
||||||
|
return copyBytes(v)
|
||||||
|
default:
|
||||||
|
return copyReflectValue(reflect.ValueOf(value)).Interface()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyReflectValue(value reflect.Value) reflect.Value {
|
||||||
|
if !value.IsValid() {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
switch value.Kind() {
|
||||||
|
case reflect.Interface:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
copied := copyReflectValue(value.Elem())
|
||||||
|
if copied.IsValid() && copied.Type().AssignableTo(value.Type()) {
|
||||||
|
return copied
|
||||||
|
}
|
||||||
|
out := reflect.New(value.Type()).Elem()
|
||||||
|
out.Set(copied)
|
||||||
|
return out
|
||||||
|
case reflect.Pointer:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
out := reflect.New(value.Type().Elem())
|
||||||
|
out.Elem().Set(copyReflectValue(value.Elem()))
|
||||||
|
return out
|
||||||
|
case reflect.Map:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
out := reflect.MakeMapWithSize(value.Type(), value.Len())
|
||||||
|
iter := value.MapRange()
|
||||||
|
for iter.Next() {
|
||||||
|
out.SetMapIndex(copyReflectValue(iter.Key()), copyReflectValue(iter.Value()))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case reflect.Slice:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap())
|
||||||
|
for i := 0; i < value.Len(); i++ {
|
||||||
|
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case reflect.Array:
|
||||||
|
out := reflect.New(value.Type()).Elem()
|
||||||
|
for i := 0; i < value.Len(); i++ {
|
||||||
|
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
default:
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyStringSlice(src []string) []string {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]string, len(src))
|
||||||
|
copy(out, src)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyBytes(src []byte) []byte {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]byte, len(src))
|
||||||
|
copy(out, src)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyFloat64Ptr(src *float64) *float64 {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
v := *src
|
||||||
|
return &v
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyIntPtr(src *int) *int {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
v := *src
|
||||||
|
return &v
|
||||||
|
}
|
||||||
49
docs/adr/0001-adopt-canonical-documentation-ownership.md
Normal file
49
docs/adr/0001-adopt-canonical-documentation-ownership.md
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
# ADR 0001: Adopt Canonical Documentation Ownership
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Accepted
|
||||||
|
|
||||||
|
## Date
|
||||||
|
|
||||||
|
2026-07-26
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Scriptorium's documentation grew alongside its CLI, HTTP, public Go, and
|
||||||
|
integration interfaces. As a result, several documents repeated mutable
|
||||||
|
contracts such as flags, configuration fields, and status behavior. Those
|
||||||
|
parallel definitions made it unclear which document to update when behavior
|
||||||
|
changed and increased the risk of documentation drift.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Assign each documentation topic one canonical owner, as defined in
|
||||||
|
[`docs/policy/documentation.md`](../policy/documentation.md). Non-owning
|
||||||
|
documents may provide short orientation and links, but do not redefine volatile
|
||||||
|
contracts. Current behavior is documented outside `docs/roadmap/`; roadmaps own
|
||||||
|
future work, sequencing, and implementation status.
|
||||||
|
|
||||||
|
## Alternatives Considered
|
||||||
|
|
||||||
|
- Keep broad reference material in several audience-specific documents. This
|
||||||
|
would preserve local convenience but leave conflicting contract definitions
|
||||||
|
likely.
|
||||||
|
- Consolidate all documentation into one reference. This would reduce duplicate
|
||||||
|
text but would not serve the distinct needs of users, operators, consumers,
|
||||||
|
and contributors.
|
||||||
|
|
||||||
|
## Rationale
|
||||||
|
|
||||||
|
Canonical ownership retains audience-specific guidance while making the source
|
||||||
|
of truth for each contract discoverable. It also makes documentation changes
|
||||||
|
reviewable alongside the implementation change that requires them.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
- Changes to behavior must update the canonical owner in the same change.
|
||||||
|
- Cross-cutting documentation links to the owner instead of copying its
|
||||||
|
details.
|
||||||
|
- Documentation restructuring follows the implementation sequence in
|
||||||
|
[`docs/roadmap/documentation.md`](../roadmap/documentation.md); the roadmap,
|
||||||
|
not this ADR, records completion status.
|
||||||
143
docs/api.md
Normal file
143
docs/api.md
Normal file
@@ -0,0 +1,143 @@
|
|||||||
|
# HTTP API Reference
|
||||||
|
|
||||||
|
This is the canonical public HTTP contract for Scriptorium.
|
||||||
|
|
||||||
|
## Service And Route
|
||||||
|
|
||||||
|
`POST /v1/runs` runs one prompt request and returns generated output,
|
||||||
|
validation, and metadata. The service has no built-in authentication or
|
||||||
|
authorization; deploy it behind appropriate network and authentication controls.
|
||||||
|
|
||||||
|
The service address and HTTP limits are configured as described in the
|
||||||
|
[configuration reference](config.md). `serve` invocation is defined in the
|
||||||
|
[CLI reference](cli.md).
|
||||||
|
|
||||||
|
Requests and responses are JSON objects. Requests are decoded as JSON regardless
|
||||||
|
of their `Content-Type`; successful JSON responses use
|
||||||
|
`Content-Type: application/json`. There are no query parameters.
|
||||||
|
|
||||||
|
## Request Limits
|
||||||
|
|
||||||
|
The configured request-body limit includes inline artifact bodies. The artifact
|
||||||
|
limit applies to HTTP `file` inputs. The response limit applies to the encoded
|
||||||
|
response, including the artifact body and optional raw output. A limit of zero
|
||||||
|
disables that limit.
|
||||||
|
|
||||||
|
A request body over its limit returns `413 request_too_large`; an oversized
|
||||||
|
file input returns `413 artifact_too_large`; an oversized encoded response
|
||||||
|
returns `413 response_too_large`.
|
||||||
|
|
||||||
|
## `POST /v1/runs`
|
||||||
|
|
||||||
|
### Request Body
|
||||||
|
|
||||||
|
The maintained [request example](../examples/http-run.json) is a complete
|
||||||
|
copyable shape. The smallest valid shape is:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"prompt_id": "generic.markdown_summary",
|
||||||
|
"inputs": {
|
||||||
|
"transcript": {"type": "inline", "body": "Source text"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
| Field | Required | Meaning |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `prompt_id` | yes | Non-blank prompt ID. |
|
||||||
|
| `prompt_version` | no | Prompt version filter. |
|
||||||
|
| `profile_id` | no | Execution-profile ID; otherwise the prompt must set `default_profile`. |
|
||||||
|
| `inputs` | yes | Non-empty object mapping input names to references. |
|
||||||
|
| `vars` | no | Object mapping template-variable names to strings. |
|
||||||
|
| `model` | no | Runtime model-override object. |
|
||||||
|
| `include_raw_output` | no | Include `raw_model_output` when true. |
|
||||||
|
|
||||||
|
An input reference has a required `type` of `file` or `inline`. A `file`
|
||||||
|
reference requires `uri`; an `inline` reference requires `body`.
|
||||||
|
|
||||||
|
HTTP file references require a configured artifact root. Relative paths resolve
|
||||||
|
within that root. Absolute paths must be lexically within it; traversal outside
|
||||||
|
it is rejected with `400 artifact_not_allowed`. This lexical check does not
|
||||||
|
resolve symlinks: the operating system follows symlinks inside the root,
|
||||||
|
including ones that target outside it. Keep the root narrow and inaccessible to
|
||||||
|
untrusted writers.
|
||||||
|
|
||||||
|
The optional `model` object accepts `endpoint`, `model`, `temperature`,
|
||||||
|
`max_tokens`, `top_p`, `timeout_seconds`, `service_tier`,
|
||||||
|
`reasoning_effort`, `api_key_env`, and `extra_params`. Numeric ranges and
|
||||||
|
credential supply are defined by the [configuration reference](config.md).
|
||||||
|
Explicit zero values for the numeric fields are overrides; zero
|
||||||
|
`timeout_seconds` disables the outbound client timeout.
|
||||||
|
|
||||||
|
Raw API-key values are not accepted. `api_key` and any other unknown model
|
||||||
|
field cause `400 invalid_json`.
|
||||||
|
|
||||||
|
### Strict JSON
|
||||||
|
|
||||||
|
Request decoding rejects malformed JSON, unknown fields at every request level,
|
||||||
|
and trailing JSON tokens with `400 invalid_json`. A blank `prompt_id` or
|
||||||
|
empty `inputs` object returns `400 invalid_request`.
|
||||||
|
|
||||||
|
### Success Response
|
||||||
|
|
||||||
|
A completed run returns `200 OK`, including when generated content fails its
|
||||||
|
validation contract. The response contains:
|
||||||
|
|
||||||
|
- `artifact`: `name`, `content_type`, `body`, `size`, `hash`, and
|
||||||
|
optional `uri`;
|
||||||
|
- `validation`: `status`, `mode`, `repair_attempts`, `is_valid`, plus
|
||||||
|
optional `errors` and `schema_path`;
|
||||||
|
- `metadata`: run, prompt, rendered-prompt, profile, model, input-hash, usage,
|
||||||
|
timing, validation, and repair-attempt metadata; and
|
||||||
|
- optional `raw_model_output` when requested.
|
||||||
|
|
||||||
|
`metadata.model_params` has `endpoint`, `model`, `temperature`,
|
||||||
|
`max_tokens`, `top_p`, and `timeout_seconds`, plus optional
|
||||||
|
`service_tier`, `reasoning_effort`, `api_key_env`, and `extra_params`.
|
||||||
|
`metadata.usage` always includes `prompt_tokens`, `completion_tokens`,
|
||||||
|
`total_tokens`, `cached_tokens`, and `cache_write_tokens`; unavailable
|
||||||
|
cache usage is reported as zero.
|
||||||
|
|
||||||
|
A validation failure has `validation.status: "failed"`, `is_valid: false`,
|
||||||
|
and any available diagnostic errors, while still returning the artifact and
|
||||||
|
metadata.
|
||||||
|
|
||||||
|
## Error Responses
|
||||||
|
|
||||||
|
Errors have this shape:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"error":{"code":"invalid_request","message":"prompt_id is required"}}
|
||||||
|
```
|
||||||
|
|
||||||
|
Messages are concise and do not expose wrapped internal causes.
|
||||||
|
|
||||||
|
| Status | Code | Meaning |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `400` | `invalid_json` | Malformed JSON, unknown field, or trailing JSON. |
|
||||||
|
| `400` | `invalid_request` | Missing or invalid request data or runtime override. |
|
||||||
|
| `400` | `profile_required` | No profile ID and no prompt default profile. |
|
||||||
|
| `400` | `prompt_load_failed` | Prompt definition failed to load. |
|
||||||
|
| `400` | `profile_load_failed` | Profile failed to load. |
|
||||||
|
| `400` | `artifact_not_allowed` | HTTP file input is disabled or outside the artifact root. |
|
||||||
|
| `400` | `artifact_read_failed` | Input artifact is invalid or cannot be read. |
|
||||||
|
| `400` | `prompt_render_failed` | Prompt template rendering failed. |
|
||||||
|
| `400` | `api_key_env_missing` | The selected credential environment variable is unset or empty. |
|
||||||
|
| `404` | `not_found` | Route does not exist. |
|
||||||
|
| `404` | `prompt_not_found` | Prompt ID or version does not exist. |
|
||||||
|
| `404` | `profile_not_found` | Profile ID does not exist. |
|
||||||
|
| `405` | `method_not_allowed` | The route does not accept the method. |
|
||||||
|
| `413` | `request_too_large` | Encoded request exceeds its limit. |
|
||||||
|
| `413` | `artifact_too_large` | File input exceeds its limit. |
|
||||||
|
| `413` | `response_too_large` | Encoded response exceeds its limit. |
|
||||||
|
| `500` | `validation_runtime_failed` | Schema or validator runtime failure. |
|
||||||
|
| `500` | `internal_error` | Unclassified server failure. |
|
||||||
|
| `502` | `llm_failed` | Outbound model request failed. |
|
||||||
|
|
||||||
|
## Retry And Idempotency
|
||||||
|
|
||||||
|
Scriptorium provides no idempotency keys, pagination, caching headers, or rate
|
||||||
|
limits. Clients may retry transport failures or `5xx` responses only when
|
||||||
|
their workflow tolerates another model call: a retry can produce different
|
||||||
|
output and incur another provider request.
|
||||||
218
docs/cli.md
218
docs/cli.md
@@ -1,5 +1,10 @@
|
|||||||
# CLI Reference
|
# CLI Reference
|
||||||
|
|
||||||
|
This is the canonical contract for invoking Scriptorium. Configuration discovery,
|
||||||
|
precedence, directories, profiles, and schemas are defined in the
|
||||||
|
[configuration reference](config.md). The [HTTP API reference](api.md) owns
|
||||||
|
service request and response behavior.
|
||||||
|
|
||||||
## Shortest Useful Command
|
## Shortest Useful Command
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -10,139 +15,132 @@ go run ./cmd/scriptorium render \
|
|||||||
--input glossary=./examples/fixtures/glossary.yml
|
--input glossary=./examples/fixtures/glossary.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
`render` prepares and formats the prompt without calling an LLM.
|
`render` prepares a request without calling an LLM.
|
||||||
|
|
||||||
## Command Overview
|
## Commands
|
||||||
|
|
||||||
- `scriptorium run`: prepare prompt, call the configured LLM, write generated output, print a run summary.
|
- `scriptorium run`: prepare a prompt, call the configured LLM, and write the
|
||||||
- `scriptorium render`: prepare prompt only; write prepared-run output as `text` or `json`.
|
generated artifact.
|
||||||
|
- `scriptorium render`: prepare a prompt and write prepared-run output.
|
||||||
- `scriptorium serve`: start the HTTP server.
|
- `scriptorium serve`: start the HTTP server.
|
||||||
|
|
||||||
Integration references:
|
All commands accept `--config <path>` and reject positional arguments. An
|
||||||
|
effective `prompt_dir` is required for every command. Supply it through the
|
||||||
|
configuration contract or the command's `--prompt-dir` flag.
|
||||||
|
|
||||||
- [HTTP contract](integrations/http-api.md)
|
## `scriptorium run`
|
||||||
- [Narratio subprocess contract](integrations/narratio.md)
|
|
||||||
|
|
||||||
## Common Argument Rules
|
```text
|
||||||
|
scriptorium run [flags]
|
||||||
|
```
|
||||||
|
|
||||||
- `--config` is supported by `run`, `render`, and `serve`.
|
Required flags:
|
||||||
- `run` and `render` require:
|
|
||||||
- `--prompt`
|
|
||||||
- at least one `--input`
|
|
||||||
- an effective `prompt_dir` and `profile_dir` (from flags or config)
|
|
||||||
- `serve` requires an effective `prompt_dir` and `profile_dir` (from flags or config).
|
|
||||||
- Positional arguments are rejected.
|
|
||||||
|
|
||||||
## Flag Reference
|
| Flag | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `--prompt <id>` | Prompt ID to execute. |
|
||||||
|
| `--input name=path` | Input file mapping; repeat or use comma-separated mappings. |
|
||||||
|
|
||||||
### `scriptorium run`
|
Optional flags:
|
||||||
|
|
||||||
- `--config <path>`: app config file path.
|
| Flag | Meaning |
|
||||||
- `--prompt-dir <dir>`: prompt definition directory.
|
| --- | --- |
|
||||||
- `--profile-dir <dir>`: profile definition directory.
|
| `--config <path>` | Application configuration file. |
|
||||||
- `--schema-dir <dir>`: schema base directory for `json_schema` validation.
|
| `--prompt-dir <dir>` | Prompt-definition directory override. |
|
||||||
- `--prompt <id>`: prompt ID to execute. Required.
|
| `--profile-dir <dir>` | Custom profile-directory override. |
|
||||||
- `--prompt-id <id>`: deprecated alias for `--prompt`.
|
| `--schema-dir <dir>` | Schema base-directory override. |
|
||||||
- `--profile <id>`: explicit profile override.
|
| `--profile <id>` | Execution-profile override. |
|
||||||
- `--profile-id <id>`: deprecated alias for `--profile`.
|
| `--var name=value` | Template-variable mapping; repeat or use comma-separated mappings. |
|
||||||
- `--input name=path`: input mapping (repeatable, comma-separated accepted).
|
| `--out <path>` | Write generated content to this file instead of stdout. |
|
||||||
- `--var name=value`: template variable mapping (repeatable, comma-separated accepted).
|
| `--llm-base-url <url>` | Runtime endpoint override. |
|
||||||
- `--out <path>`: write artifact body to file instead of stdout.
|
| `--model <name>` | Runtime model override. |
|
||||||
- `--llm-base-url <url>`: runtime endpoint override.
|
| `--api-key-env <name>` | Runtime API-key environment-variable name override. |
|
||||||
- `--model <name>`: runtime model override.
|
| `--temperature <float>` | Runtime temperature override. |
|
||||||
- `--api-key-env <name>`: runtime API key environment-variable name override.
|
| `--max-tokens <int>` | Runtime maximum-token override. |
|
||||||
- `--temperature <float>`: runtime temperature override.
|
| `--top-p <float>` | Runtime top-p override. |
|
||||||
- `--max-tokens <int>`: runtime max tokens override.
|
| `--timeout <duration>` | Runtime timeout override using Go duration syntax. |
|
||||||
- `--top-p <float>`: runtime top-p override.
|
|
||||||
- `--timeout <duration>`: runtime timeout override (Go duration syntax, for example `30s`, `2m`).
|
|
||||||
|
|
||||||
### `scriptorium render`
|
Deprecated aliases: `--prompt-id` for `--prompt`, and `--profile-id` for
|
||||||
|
`--profile`.
|
||||||
|
|
||||||
- Supports the same flags as `run`, except:
|
Omitted numeric runtime flags preserve the selected effective value; explicit
|
||||||
- no `--schema-dir` flag.
|
zero values override it. `--timeout 0s` disables the outbound HTTP-client
|
||||||
- Adds:
|
timeout. CLI durations are converted to whole seconds by truncation toward
|
||||||
- `--format text|json`: prepared-run output format.
|
zero, so any duration whose absolute value is below one second becomes an
|
||||||
|
explicit zero-second override.
|
||||||
|
|
||||||
Notes:
|
There is no raw API-key flag. Use `--api-key-env`.
|
||||||
- `render` still resolves profile and runtime settings.
|
|
||||||
- `render` still validates that `api_key_env` exists if the selected profile or overrides require it.
|
|
||||||
|
|
||||||
### `scriptorium serve`
|
## `scriptorium render`
|
||||||
|
|
||||||
- `--config <path>`: app config file path.
|
```text
|
||||||
- `--addr <listen-address>`: HTTP listen address.
|
scriptorium render [flags]
|
||||||
- `--prompt-dir <dir>`: prompt definition directory.
|
```
|
||||||
- `--profile-dir <dir>`: profile definition directory.
|
|
||||||
- `--schema-dir <dir>`: schema base directory for `json_schema` validation.
|
|
||||||
|
|
||||||
Notes:
|
`--prompt <id>` and at least one `--input name=path` are required. The
|
||||||
- `serve` does not accept runtime model override flags such as `--model` or `--llm-base-url`.
|
following optional flags are supported: `--config`, `--prompt-dir`,
|
||||||
|
`--profile-dir`, `--profile`, `--var`, `--out`, `--llm-base-url`,
|
||||||
|
`--model`, `--api-key-env`, `--temperature`, `--max-tokens`, `--top-p`,
|
||||||
|
`--timeout`, and `--format text|json`. Their meanings match the corresponding
|
||||||
|
`run` flags; `--format` selects prepared-run output and otherwise uses
|
||||||
|
`defaults.render_format`.
|
||||||
|
|
||||||
|
The same deprecated aliases and numeric/timeout behavior as `run` apply.
|
||||||
|
`render` does not accept `--schema-dir`; configure `schema_dir` through the
|
||||||
|
configuration file. It resolves profiles and schemas as part of preparation but
|
||||||
|
does not call an LLM.
|
||||||
|
|
||||||
|
## `scriptorium serve`
|
||||||
|
|
||||||
|
```text
|
||||||
|
scriptorium serve [flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
Optional flags:
|
||||||
|
|
||||||
|
| Flag | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `--config <path>` | Application configuration file. |
|
||||||
|
| `--addr <listen-address>` | HTTP listen-address override. |
|
||||||
|
| `--prompt-dir <dir>` | Prompt-definition directory override. |
|
||||||
|
| `--profile-dir <dir>` | Custom profile-directory override. |
|
||||||
|
| `--schema-dir <dir>` | Schema base-directory override. |
|
||||||
|
| `--artifact-root <dir>` | Root for HTTP `file` input references. |
|
||||||
|
| `--max-request-bytes <n>` | Maximum encoded HTTP request-body bytes; `0` disables the limit. |
|
||||||
|
| `--max-artifact-bytes <n>` | Maximum HTTP file-input artifact bytes; `0` disables the limit. |
|
||||||
|
| `--max-response-bytes <n>` | Maximum encoded HTTP response bytes; `0` disables the limit. |
|
||||||
|
|
||||||
|
`serve` accepts no runtime model override flags. HTTP request fields, response
|
||||||
|
schemas, and error codes are defined in the [HTTP API reference](api.md).
|
||||||
|
|
||||||
## Input And Variable Syntax
|
## Input And Variable Syntax
|
||||||
|
|
||||||
- `--input name=path` maps prompt input names to local file paths.
|
`--input name=path` maps an input name to a local file; `--var name=value`
|
||||||
- `--var name=value` maps template variable names to values.
|
maps a template variable to a string. Both flags can be repeated or contain
|
||||||
- Both flags can be repeated.
|
comma-separated mappings. Values may contain `=` after the first separator.
|
||||||
- Both flags also support comma-separated batches, for example:
|
Empty names and values are rejected.
|
||||||
- `--input transcript=./t.md,glossary=./g.yml`
|
|
||||||
- `--var session_id=42,session_date=2026-05-04`
|
|
||||||
|
|
||||||
## Output Behavior
|
CLI inputs are file references. HTTP inline inputs are defined by the
|
||||||
|
[HTTP API reference](api.md).
|
||||||
|
|
||||||
`run`:
|
## Output And Exit Behavior
|
||||||
- Writes generated artifact content to stdout by default.
|
|
||||||
- Writes generated artifact content to `--out` when provided.
|
|
||||||
- Prints run summary metadata to stderr on success.
|
|
||||||
- Prints errors to stderr on failure.
|
|
||||||
|
|
||||||
`render`:
|
- `run` writes generated content to stdout, or to `--out` when supplied, and
|
||||||
- Writes prepared-run output to stdout by default.
|
writes a concise summary to stderr.
|
||||||
- Writes prepared-run output to `--out` when provided.
|
- `render` writes prepared-run output to stdout, or to `--out` when supplied,
|
||||||
- Does not print a success summary line.
|
without a success summary.
|
||||||
|
- `serve` writes startup and server errors to stderr.
|
||||||
|
|
||||||
`serve`:
|
Exit statuses:
|
||||||
- Logs startup and server errors to stderr.
|
|
||||||
|
|
||||||
## Exit Codes
|
| Status | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `0` | Success. |
|
||||||
|
| `1` | Parse, configuration, loading, rendering, generation, output-write, or other runtime error. |
|
||||||
|
| `2` | `run` generated and wrote output, but validation failed. |
|
||||||
|
|
||||||
- `0`: success.
|
## Workflows And Examples
|
||||||
- `1`: runtime/parse/config/load/render/generation/output-write error.
|
|
||||||
- `2`: `run` completed, output was generated, but validation status is `failed`.
|
|
||||||
|
|
||||||
When `run` exits `2`, output may already be written to stdout or `--out`.
|
The [maintained render script](../examples/render-markdown-summary.sh) is a
|
||||||
|
copyable render workflow. The [HTTP request example](../examples/http-run.json)
|
||||||
## Common Workflows
|
is for a running `serve` process.
|
||||||
|
|
||||||
Render prompt inputs and template variables as JSON:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go run ./cmd/scriptorium render \
|
|
||||||
--config ./examples/config.yml \
|
|
||||||
--prompt generic.markdown_summary \
|
|
||||||
--input transcript=./examples/fixtures/transcript.md \
|
|
||||||
--input glossary=./examples/fixtures/glossary.yml \
|
|
||||||
--var session_date=2026-05-04 \
|
|
||||||
--format json
|
|
||||||
```
|
|
||||||
|
|
||||||
Run a prompt with profile override and file output:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go run ./cmd/scriptorium run \
|
|
||||||
--config ./examples/config.yml \
|
|
||||||
--prompt generic.markdown_summary \
|
|
||||||
--profile local-fast \
|
|
||||||
--input transcript=./examples/fixtures/transcript.md \
|
|
||||||
--input glossary=./examples/fixtures/glossary.yml \
|
|
||||||
--out ./summary.md
|
|
||||||
```
|
|
||||||
|
|
||||||
Start the HTTP server with explicit config:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go run ./cmd/scriptorium serve --config ./examples/config.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
Copyable example script:
|
|
||||||
|
|
||||||
- `examples/render-markdown-summary.sh`
|
|
||||||
|
|||||||
326
docs/config.md
326
docs/config.md
@@ -1,231 +1,169 @@
|
|||||||
# Configuration Reference
|
# Configuration Reference
|
||||||
|
|
||||||
## Config Discovery And Precedence
|
This is the canonical reference for Scriptorium application settings and the
|
||||||
|
prompt, profile, and schema files those settings select. For command syntax,
|
||||||
|
see the [CLI reference](cli.md); for HTTP request shapes, limits, and outcomes,
|
||||||
|
see the [HTTP API reference](api.md).
|
||||||
|
|
||||||
Application settings are loaded in this order:
|
## Discovery And Precedence
|
||||||
|
|
||||||
1. Built-in defaults
|
Application settings are resolved in this order:
|
||||||
2. `config.yml` values
|
|
||||||
3. CLI overrides
|
|
||||||
|
|
||||||
When `--config` is not provided, Scriptorium searches for config files in this order:
|
1. built-in defaults;
|
||||||
|
2. a configuration file; then
|
||||||
|
3. CLI overrides.
|
||||||
|
|
||||||
1. `/usr/local/etc/scriptorium/config.yml`
|
When `--config` is omitted, Scriptorium searches
|
||||||
2. `/etc/scriptorium/config.yml`
|
`/usr/local/etc/scriptorium/config.yml` and then `/etc/scriptorium/config.yml`.
|
||||||
|
If neither exists, it uses built-in defaults. An explicit `--config` path must
|
||||||
|
exist and decode successfully.
|
||||||
|
|
||||||
If neither file exists, Scriptorium continues with built-in defaults.
|
The maintained [minimal configuration](../examples/config.yml) and
|
||||||
|
[full configuration](../examples/config.full.yml) are copyable examples.
|
||||||
|
|
||||||
When `--config <path>` is provided, that file is required.
|
## Application Configuration File
|
||||||
|
|
||||||
## Minimal App Config
|
Configuration is strict YAML: unknown fields are rejected. Empty string values
|
||||||
|
do not override a prior value. Raw API-key fields are not accepted.
|
||||||
|
|
||||||
```yaml
|
| Field | Default | Meaning |
|
||||||
prompt_dir: ./examples/prompts
|
| --- | --- | --- |
|
||||||
profile_dir: ./examples/profiles
|
| `prompt_dir` | unset | Directory containing prompt-definition YAML. `run`, `render`, and `serve` require an effective value. |
|
||||||
```
|
| `profile_dir` | unset | Directory containing custom profile YAML. Built-in profiles remain available. |
|
||||||
|
| `schema_dir` | `.` | Base directory for relative JSON Schema paths. |
|
||||||
|
| `server.addr` | `:8080` | Address used by `serve`. |
|
||||||
|
| `server.artifact_root` | unset | Root that enables HTTP `file` input references. |
|
||||||
|
| `server.max_request_bytes` | `16777216` | Maximum encoded HTTP request body bytes; `0` disables the limit. |
|
||||||
|
| `server.max_artifact_bytes` | `16777216` | Maximum HTTP file-input artifact bytes; `0` disables the limit. |
|
||||||
|
| `server.max_response_bytes` | `16777216` | Maximum encoded HTTP response bytes; `0` disables the limit. |
|
||||||
|
| `defaults.render_format` | `text` | Default `render` output format: `text` or `json`. |
|
||||||
|
|
||||||
This is enough to use `run` and `render` when prompt/profile files are valid.
|
The three size fields must be zero or greater. The HTTP contract defines how
|
||||||
|
each limit is enforced and reported. `server.artifact_root` configures the
|
||||||
## Production-Oriented App Config
|
deployment boundary; see the [HTTP API reference](api.md) for request-path and
|
||||||
|
containment behavior, and [operations](operations.md) for deployment handling.
|
||||||
```yaml
|
|
||||||
prompt_dir: /opt/scriptorium/prompts
|
|
||||||
profile_dir: /opt/scriptorium/profiles
|
|
||||||
schema_dir: /opt/scriptorium/schemas
|
|
||||||
|
|
||||||
server:
|
|
||||||
addr: 127.0.0.1:8080
|
|
||||||
|
|
||||||
defaults:
|
|
||||||
render_format: text
|
|
||||||
```
|
|
||||||
|
|
||||||
## App Config File (`config.yml`)
|
|
||||||
|
|
||||||
Top-level fields:
|
|
||||||
|
|
||||||
- `prompt_dir` (optional): default prompt definition directory.
|
|
||||||
- `profile_dir` (optional): default profile definition directory.
|
|
||||||
- `schema_dir` (optional): base directory for schema files used by `json_schema` validation.
|
|
||||||
- `server.addr` (optional): default listen address for `serve`.
|
|
||||||
- `defaults.render_format` (optional): default `render` output format (`text` or `json`).
|
|
||||||
|
|
||||||
Built-in defaults:
|
|
||||||
|
|
||||||
- `schema_dir`: `.`
|
|
||||||
- `server.addr`: `:8080`
|
|
||||||
- `defaults.render_format`: `text`
|
|
||||||
|
|
||||||
Validation behavior:
|
|
||||||
|
|
||||||
- Config decoding is strict; unknown YAML fields are rejected.
|
|
||||||
- Raw API key fields are not supported in `config.yml`.
|
|
||||||
|
|
||||||
## Prompt Definition Files
|
## Prompt Definition Files
|
||||||
|
|
||||||
Prompt definitions are YAML files anywhere under `prompt_dir`, including nested subdirectories.
|
Prompt definitions are strict YAML files anywhere below `prompt_dir`. A prompt
|
||||||
|
is selected by its YAML `id`, not by file path; nested directories are only for
|
||||||
|
organization. See [maintained prompt examples](../examples/prompts/).
|
||||||
|
|
||||||
Subdirectories are organizational only. Callers still select prompts by the YAML `id`, not by file path. For example, `prompts/dnd/recap.yaml` may still declare `id: dnd.recap`, and callers use `--prompt dnd.recap`.
|
| Field | Required | Meaning |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `id` | yes | Prompt identifier. |
|
||||||
|
| `version` | yes | Prompt version. |
|
||||||
|
| `default_profile` | no | Profile used when a request omits a profile ID. |
|
||||||
|
| `description` | no | Human-readable description. |
|
||||||
|
| `session_id` | no | Go-template string rendered from request variables and sent to a compatible provider when non-empty. |
|
||||||
|
| `inputs` | no | Declared input metadata. |
|
||||||
|
| `messages` | yes | Chat-message templates. |
|
||||||
|
| `output` | yes | Output format and validation contract. |
|
||||||
|
|
||||||
Example:
|
### Inputs And Messages
|
||||||
|
|
||||||
```yaml
|
Each `inputs` item has a required `name` and optional `required`,
|
||||||
id: generic.structured_events
|
`content_type`, and `description` fields. Input names must be unique.
|
||||||
version: "1.0.0"
|
|
||||||
default_profile: local-quality
|
|
||||||
description: Produce structured event JSON from a transcript.
|
|
||||||
|
|
||||||
inputs:
|
Each message has a required `role`, exactly one of `content` or `content_file`,
|
||||||
- name: transcript
|
and optional `cache_control`. A `content_file` path is relative to the prompt
|
||||||
required: true
|
file. `cache_control.type` must be `ephemeral`; its optional `ttl` is `1h`.
|
||||||
content_type: text/markdown
|
|
||||||
description: Source transcript content
|
|
||||||
- name: glossary
|
|
||||||
required: false
|
|
||||||
content_type: text/yaml
|
|
||||||
description: Optional glossary context
|
|
||||||
|
|
||||||
messages:
|
`session_id` uses the same template variables as messages. Empty rendered
|
||||||
- role: system
|
values are omitted. A rendered value may contain at most 256 Unicode code
|
||||||
content_file: ./generic.structured_events.system.md
|
points.
|
||||||
- role: user
|
|
||||||
content_file: ./generic.structured_events.user.md
|
|
||||||
|
|
||||||
output:
|
### Output Contract
|
||||||
format: json
|
|
||||||
validation_mode: json_schema
|
|
||||||
schema_path: structured_events.schema.json
|
|
||||||
repair_attempts: 0
|
|
||||||
```
|
|
||||||
|
|
||||||
Field reference:
|
| Field | Required | Values or behavior |
|
||||||
|
| --- | --- | --- |
|
||||||
- `id` (required): prompt identifier.
|
| `format` | yes | `text`, `markdown`, or `json`. |
|
||||||
- `version` (required): prompt version.
|
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema`. |
|
||||||
- `default_profile` (optional): profile ID used when request does not provide `profile_id`.
|
| `schema_path` | for `json_schema` | Schema path, relative to `schema_dir` unless absolute. |
|
||||||
- `description` (optional): prompt description.
|
| `repair_attempts` | no | Integer greater than or equal to `0`; omitted means `0`. |
|
||||||
- `inputs` (optional list): expected named inputs.
|
|
||||||
- `messages` (required list): prompt message templates.
|
|
||||||
- `output` (required object): output contract.
|
|
||||||
|
|
||||||
`inputs[]` fields:
|
|
||||||
|
|
||||||
- `name` (required)
|
|
||||||
- `required` (optional, boolean)
|
|
||||||
- `content_type` (optional metadata)
|
|
||||||
- `description` (optional)
|
|
||||||
|
|
||||||
`messages[]` fields:
|
|
||||||
|
|
||||||
- `role` (required)
|
|
||||||
- `content` or `content_file` (exactly one is required)
|
|
||||||
|
|
||||||
Message rules:
|
|
||||||
|
|
||||||
- Repeated roles are allowed.
|
|
||||||
- `content_file` is resolved relative to the prompt YAML file location.
|
|
||||||
- Nested prompt files keep the same relative `content_file` behavior; `./recap.user.md` next to `dnd/recap.yaml` resolves from `dnd/`.
|
|
||||||
- Prompt decoding is strict; unknown YAML fields are rejected.
|
|
||||||
- Duplicate prompt IDs are invalid. If multiple files declare the requested prompt ID, Scriptorium fails instead of choosing one.
|
|
||||||
|
|
||||||
`output` fields:
|
|
||||||
|
|
||||||
- `format` (required): `text`, `markdown`, or `json`.
|
|
||||||
- `validation_mode` (required): `none`, `basic`, `json`, or `json_schema`.
|
|
||||||
- `schema_path` (required when `validation_mode: json_schema`).
|
|
||||||
- `repair_attempts` (required): integer `>= 0`.
|
|
||||||
|
|
||||||
Repair behavior boundary:
|
|
||||||
|
|
||||||
- `repair_attempts` is part of the prompt contract.
|
|
||||||
- CLI and HTTP currently construct the runner without a repairer, so normal `run`/`serve` execution does not perform output repair attempts.
|
|
||||||
|
|
||||||
## Profile Definition Files
|
## Profile Definition Files
|
||||||
|
|
||||||
Execution profiles are YAML files anywhere under `profile_dir`, including nested subdirectories.
|
Profiles are strict YAML files anywhere below `profile_dir`. A profile is
|
||||||
|
selected by YAML `id`; nested directories are organizational. See the
|
||||||
|
[maintained profile examples](../examples/profiles/).
|
||||||
|
|
||||||
Subdirectories are organizational only. Callers still select profiles by the YAML `id`, not by file path. For example, `profiles/local/local-quality.yaml` may still declare `id: local-quality`, and callers use `--profile local-quality`.
|
| Field | Required | Meaning |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `id` | yes | Profile identifier. |
|
||||||
|
| `endpoint` | yes | OpenAI-compatible base URL, including its API version path when needed. |
|
||||||
|
| `model` | yes | Provider model name. |
|
||||||
|
| `temperature` | no | Number from `0` through `2`. |
|
||||||
|
| `max_tokens` | no | Integer zero or greater. |
|
||||||
|
| `top_p` | no | Number from `0` through `1`. |
|
||||||
|
| `timeout_seconds` | no | Integer zero or greater. |
|
||||||
|
| `service_tier` | no | Non-empty provider-specific request tier. |
|
||||||
|
| `reasoning_effort` | no | Non-empty provider-specific reasoning setting. |
|
||||||
|
| `api_key_env` | no | Environment-variable name containing the API key. |
|
||||||
|
| `extra_params` | no | JSON-compatible provider-specific outbound request fields. |
|
||||||
|
|
||||||
Example:
|
Execution defaults before profile and request overrides are `temperature: 0`,
|
||||||
|
`max_tokens: 0`, `top_p: 1`, and `timeout_seconds: 600`. Profile numeric values
|
||||||
|
merge by non-zero value. Request overrides preserve presence, so an explicit
|
||||||
|
zero can override a profile value.
|
||||||
|
|
||||||
```yaml
|
Custom profiles take precedence over built-ins with the same ID. Invalid custom
|
||||||
id: local-fast
|
profiles are errors; they do not fall back to a built-in profile. Raw `api_key`
|
||||||
endpoint: http://localhost:8000/v1
|
is rejected. Use `api_key_env`, or the public Go package's request-scoped key
|
||||||
model: gpt-4o-mini
|
mechanism described in the [package contract](consumers/pkg-scriptorium.md).
|
||||||
temperature: 0.2
|
|
||||||
max_tokens: 500
|
|
||||||
top_p: 1.0
|
|
||||||
timeout_seconds: 90
|
|
||||||
api_key_env: SCRIPTORIUM_API_KEY
|
|
||||||
service_tier: priority
|
|
||||||
```
|
|
||||||
|
|
||||||
Field reference:
|
`extra_params` keys must be non-empty and cannot be `model`, `session_id`,
|
||||||
|
`messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`,
|
||||||
|
`reasoning_effort`, or `response_format`.
|
||||||
|
|
||||||
- `id` (required)
|
### Built-In Profile Catalog
|
||||||
- `endpoint` (required)
|
|
||||||
- `model` (required)
|
|
||||||
- `temperature` (optional): range `0..2`
|
|
||||||
- `max_tokens` (optional): `>= 0`
|
|
||||||
- `top_p` (optional): range `0..1`
|
|
||||||
- `timeout_seconds` (optional): `>= 0`
|
|
||||||
- `service_tier` (optional): provider-specific request tier such as OpenRouter `flex` or `priority`
|
|
||||||
- `reasoning_effort` (optional)
|
|
||||||
- `api_key_env` (optional)
|
|
||||||
- `extra_params` (optional map of strings)
|
|
||||||
|
|
||||||
Profile rules:
|
Each embedded profile uses `OPENROUTER_API_KEY`.
|
||||||
|
|
||||||
- Profile decoding is strict; unknown YAML fields are rejected.
|
| Provider | ID | Model |
|
||||||
- Raw `api_key` is rejected; use `api_key_env`.
|
| --- | --- | --- |
|
||||||
- If `api_key_env` is set, that environment variable must be set when preparing/running.
|
| aion-labs | `aion-2` | `aion-labs/aion-2.0` |
|
||||||
- Duplicate profile IDs are invalid. If multiple files declare the requested profile ID, Scriptorium fails instead of choosing one.
|
| anthropic | `claude-fable-latest` | `~anthropic/claude-fable-latest` |
|
||||||
|
| anthropic | `claude-haiku-latest` | `~anthropic/claude-haiku-latest` |
|
||||||
|
| anthropic | `claude-opus-latest` | `~anthropic/claude-opus-latest` |
|
||||||
|
| anthropic | `claude-sonnet-latest` | `~anthropic/claude-sonnet-latest` |
|
||||||
|
| deepseek | `deepseek-3-2` | `deepseek/deepseek-v3.2` |
|
||||||
|
| deepseek | `deepseek-4-flash` | `deepseek/deepseek-v4-flash` |
|
||||||
|
| deepseek | `deepseek-4-pro` | `deepseek/deepseek-v4-pro` |
|
||||||
|
| google | `gemini-2-flash` | `google/gemini-2.5-flash` |
|
||||||
|
| google | `gemini-2-flash-lite` | `google/gemini-2.5-flash-lite` |
|
||||||
|
| google | `gemini-2-pro` | `google/gemini-2.5-pro` |
|
||||||
|
| google | `gemini-3-flash-lite` | `google/gemini-3.1-flash-lite` |
|
||||||
|
| google | `gemini-flash-latest` | `~google/gemini-flash-latest` |
|
||||||
|
| google | `gemini-pro-latest` | `~google/gemini-pro-latest` |
|
||||||
|
| google | `gemma-4-31b` | `google/gemma-4-31b-it:exacto` |
|
||||||
|
| minimax | `minimax-m2` | `minimax/minimax-m2.5` |
|
||||||
|
| minimax | `minimax-m3` | `minimax/minimax-m3` |
|
||||||
|
| mistral | `mistral-large-2512` | `mistralai/mistral-large-2512` |
|
||||||
|
| mistral | `mistral-medium-3-5` | `mistralai/mistral-medium-3-5` |
|
||||||
|
| mistral | `mistral-small-3` | `mistralai/mistral-small-3.2-24b-instruct` |
|
||||||
|
| mistral | `mistral-small-4` | `mistralai/mistral-small-2603` |
|
||||||
|
| nvidia | `nemotron-3-ultra` | `nvidia/nemotron-3-ultra-550b-a55b` |
|
||||||
|
| openai | `gpt-5-mini` | `openai/gpt-5.4-mini` |
|
||||||
|
| openai | `gpt-5-nano` | `openai/gpt-5.4-nano` |
|
||||||
|
|
||||||
Current outbound request behavior:
|
## Schemas
|
||||||
|
|
||||||
- The OpenAI-compatible client currently serializes: `model`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, and optional `response_format` for `json_schema` prompts.
|
Schemas are JSON files, normally below `schema_dir`. `json_schema` output
|
||||||
- `reasoning_effort` and `extra_params` are parsed and carried in effective settings, but are not currently serialized into outbound chat-completions requests.
|
requires a `schema_path`. Relative paths resolve from `schema_dir`; absolute
|
||||||
|
paths are used directly. Referenced nested schemas use relative paths and are
|
||||||
|
not discovered by basename. An unreadable or invalid schema is a runtime
|
||||||
|
validation error; generated content that fails JSON or schema validation is a
|
||||||
|
validation result.
|
||||||
|
|
||||||
## Schema Behavior
|
## Credentials
|
||||||
|
|
||||||
Schemas are JSON files, typically in `schema_dir`.
|
Keep secrets in environment variables. Store only an environment-variable name
|
||||||
|
in `api_key_env`; do not place raw keys in configuration, prompt or profile
|
||||||
|
files, CLI arguments, examples, or HTTP payloads.
|
||||||
|
|
||||||
Rules:
|
## Related References
|
||||||
|
|
||||||
- `output.validation_mode: json_schema` requires `output.schema_path`.
|
- [CLI reference](cli.md)
|
||||||
- Relative `schema_path` values resolve from `schema_dir`, including explicit nested paths such as `dnd/structured_events.schema.json`.
|
- [HTTP API reference](api.md)
|
||||||
- Absolute `schema_path` values are used directly.
|
- [OpenAI-compatible outbound contract](integrations/openai-compatible-chat.md)
|
||||||
- Scriptorium does not recursively search schemas by basename; nested schemas must be referenced by their relative path.
|
|
||||||
- Missing or invalid schema documents cause runtime validation errors.
|
|
||||||
- Invalid generated JSON causes validation status `failed` (not a runtime error).
|
|
||||||
|
|
||||||
Supported artifact reference types for request inputs are `file` and `inline`.
|
|
||||||
|
|
||||||
## Secrets Handling
|
|
||||||
|
|
||||||
- Keep secret values in environment variables.
|
|
||||||
- Store only environment-variable names in profile `api_key_env`.
|
|
||||||
- Do not put raw API keys in config, prompts, profiles, CLI flags, or HTTP request bodies.
|
|
||||||
|
|
||||||
## Maintained Examples
|
|
||||||
|
|
||||||
- App config: `examples/config.yml`
|
|
||||||
- Prompt examples: `examples/prompts/`
|
|
||||||
- Profile examples: `examples/profiles/`
|
|
||||||
- Schema examples: `examples/schemas/`
|
|
||||||
- Input fixtures: `examples/fixtures/`
|
|
||||||
- Render example script: `examples/render-markdown-summary.sh`
|
|
||||||
- HTTP request example: `examples/http-run.json`
|
|
||||||
|
|
||||||
Example organizational layout:
|
|
||||||
|
|
||||||
```text
|
|
||||||
examples/prompts/dnd/recap.yaml
|
|
||||||
examples/profiles/local/local-quality.yaml
|
|
||||||
examples/schemas/dnd/structured_events.schema.json
|
|
||||||
```
|
|
||||||
|
|
||||||
## Integration References
|
|
||||||
|
|
||||||
- [Inbound HTTP contract](integrations/http-api.md)
|
|
||||||
- [Outbound OpenAI-compatible contract](integrations/openai-compatible-chat.md)
|
|
||||||
|
|||||||
59
docs/consumers/api.md
Normal file
59
docs/consumers/api.md
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
# Consumer Integration Overview
|
||||||
|
|
||||||
|
This guide helps applications choose a Scriptorium interface and understand
|
||||||
|
their responsibilities. The linked contracts own interface syntax and wire
|
||||||
|
semantics.
|
||||||
|
|
||||||
|
| Interface | Use when |
|
||||||
|
| --- | --- |
|
||||||
|
| Go package | The consumer is Go and needs typed requests, results, or an injected LLM client. |
|
||||||
|
| CLI subprocess | The consumer needs process isolation or is not written in Go. |
|
||||||
|
| HTTP API | The consumer needs a service boundary or remote access. |
|
||||||
|
|
||||||
|
- Go package: [package contract](pkg-scriptorium.md)
|
||||||
|
- CLI subprocess: [subprocess integration](../integrations/subprocess.md)
|
||||||
|
- HTTP service: [HTTP API reference](../api.md)
|
||||||
|
- Prompt, profile, schema, and credential configuration: [configuration reference](../config.md)
|
||||||
|
|
||||||
|
## Minimal Go Use
|
||||||
|
|
||||||
|
```go
|
||||||
|
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||||
|
PromptDir: "./examples/prompts",
|
||||||
|
ProfileDir: "./examples/profiles",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := engine.Prepare(ctx, scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_ = prepared
|
||||||
|
```
|
||||||
|
|
||||||
|
For a maintained program, see
|
||||||
|
[`examples/go-library/prepare`](../../examples/go-library/prepare).
|
||||||
|
|
||||||
|
## Consumer Responsibilities
|
||||||
|
|
||||||
|
Consumers are responsible for:
|
||||||
|
|
||||||
|
- selecting and deploying prompt, profile, and schema assets;
|
||||||
|
- supplying required inputs and template variables;
|
||||||
|
- supplying credentials through the applicable interface;
|
||||||
|
- protecting rendered prompts and generated artifacts as potentially sensitive;
|
||||||
|
- deciding whether validation-failed output is usable; and
|
||||||
|
- retrying only when another model call is acceptable.
|
||||||
|
|
||||||
|
Scriptorium does not persist run state. A retry can produce different output and
|
||||||
|
can incur another provider request. CLI exit behavior belongs to the
|
||||||
|
[CLI reference](../cli.md); HTTP status behavior belongs to the
|
||||||
|
[HTTP API reference](../api.md); package errors and results belong to the
|
||||||
|
[package contract](pkg-scriptorium.md).
|
||||||
182
docs/consumers/pkg-scriptorium.md
Normal file
182
docs/consumers/pkg-scriptorium.md
Normal file
@@ -0,0 +1,182 @@
|
|||||||
|
# Package `scriptorium`
|
||||||
|
|
||||||
|
Import path:
|
||||||
|
|
||||||
|
```go
|
||||||
|
import "gitea.maximumdirect.net/eric/scriptorium"
|
||||||
|
```
|
||||||
|
|
||||||
|
This is the canonical public Go contract for in-process prompt preparation and
|
||||||
|
execution. Prompt, profile, and schema file formats are defined in the
|
||||||
|
[configuration reference](../config.md).
|
||||||
|
|
||||||
|
## Engine Construction
|
||||||
|
|
||||||
|
`NewEngine(Config, ...Option)` constructs an engine. `Config` has these
|
||||||
|
fields:
|
||||||
|
|
||||||
|
| Field | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `PromptDir` | Prompt-definition directory, required unless a prompt source option is supplied. |
|
||||||
|
| `ProfileDir` | Optional custom profile directory over built-ins. |
|
||||||
|
| `SchemaDir` | Schema directory; empty uses `.`. |
|
||||||
|
| `Timeout` | Base timeout for the built-in OpenAI-compatible client when `HTTPClient` is absent or has a zero timeout. A non-positive value uses the internal default. |
|
||||||
|
| `HTTPClient` | Optional HTTP client for that built-in client. It is cloned; a non-zero `Timeout` on it takes precedence over `Config.Timeout` as the base timeout. |
|
||||||
|
|
||||||
|
Nil options are ignored. Invalid construction, including
|
||||||
|
`WithLLMClient(nil)`, returns an error matching `ErrInvalidConfig`.
|
||||||
|
|
||||||
|
An effective positive `timeout_seconds` replaces the base timeout. An explicit
|
||||||
|
request override of zero disables the HTTP-client timeout. The timeout is
|
||||||
|
otherwise inherited from the supplied client, `Config.Timeout`, or the internal
|
||||||
|
default in that order.
|
||||||
|
|
||||||
|
Source options replace their matching directory source:
|
||||||
|
|
||||||
|
- prompts: `WithPromptFS(fsys, root)`, `WithPromptFile(path)`;
|
||||||
|
- profiles: `WithProfileFS(fsys, root)`, `WithProfileFile(path)`, and
|
||||||
|
`WithProfiles(profiles...)`;
|
||||||
|
- schemas: `WithSchemaFS(fsys, root)`, `WithSchemaFile(path)`; and
|
||||||
|
- LLM client: `WithLLMClient(client)`.
|
||||||
|
|
||||||
|
`fs.FS` prompt-content and schema paths stay inside their configured roots.
|
||||||
|
Single-file prompt and profile sources are selected by their YAML `id`, not
|
||||||
|
their file names. `WithPromptFile` resolves relative `content_file` paths from
|
||||||
|
the prompt file's directory. `WithSchemaFile` exposes its schema by the schema
|
||||||
|
file's base name. In-memory profiles take precedence over an explicit or
|
||||||
|
directory-backed profile source, which in turn takes precedence over built-ins.
|
||||||
|
File and filesystem sources use the format and credential rules in the
|
||||||
|
[configuration reference](../config.md).
|
||||||
|
|
||||||
|
## Prepare And Run
|
||||||
|
|
||||||
|
`Prepare(ctx, request)` resolves the prompt, profile, input artifacts,
|
||||||
|
validation contract, and rendered messages without calling an LLM.
|
||||||
|
`Run(ctx, request)` performs that preparation, calls the configured client,
|
||||||
|
and validates generated content.
|
||||||
|
|
||||||
|
```go
|
||||||
|
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||||
|
PromptDir: "./examples/prompts",
|
||||||
|
ProfileDir: "./examples/profiles",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := engine.Prepare(ctx, scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_ = prepared.Messages
|
||||||
|
```
|
||||||
|
|
||||||
|
The maintained package example is
|
||||||
|
[`examples/go-library/prepare`](../../examples/go-library/prepare).
|
||||||
|
|
||||||
|
`PreparedRun` exposes prompt, selected-profile, effective-model, output
|
||||||
|
contract, structured-output, input-hash, rendered-message, and timing
|
||||||
|
information. It does not include a resolved API key, model output, validation
|
||||||
|
result, or target-presence metadata.
|
||||||
|
|
||||||
|
`RunResult` adds run ID, artifact, raw output, validation, model metadata,
|
||||||
|
usage, and duration. Generated-content validation failures return a result with
|
||||||
|
`Validation.Status == ValidationFailed`; schema or validator runtime failures
|
||||||
|
return an error matching `ErrValidation`.
|
||||||
|
|
||||||
|
## Public Values
|
||||||
|
|
||||||
|
`ArtifactRef` has `Type`, `URI`, and `Body`; `Artifact` has `Name`,
|
||||||
|
`ContentType`, `Body`, `URI`, `Size`, and `Hash`. `ExecutionTarget` exposes the
|
||||||
|
effective endpoint, model, numeric settings, credential-environment name,
|
||||||
|
service tier, reasoning effort, and extra parameters. `ValidationResult`
|
||||||
|
contains status, mode, errors, schema path, repair attempts, and validity.
|
||||||
|
|
||||||
|
The exported constants define these serialized values:
|
||||||
|
|
||||||
|
- artifact types: `inline` and `file`;
|
||||||
|
- output formats: `text`, `markdown`, and `json`;
|
||||||
|
- validation modes: `none`, `basic`, `json`, and `json_schema`; and
|
||||||
|
- validation statuses: `passed`, `failed`, and `skipped`.
|
||||||
|
|
||||||
|
`TokenUsage` reports prompt, completion, total, cached, and cache-write token
|
||||||
|
counts. `RenderedPrompt`, `RenderedMessage`, `CacheControl`, and
|
||||||
|
`StructuredOutputSpec` are the public shapes used by injected LLM clients.
|
||||||
|
|
||||||
|
## Requests, Inputs, And Overrides
|
||||||
|
|
||||||
|
`RunRequest` fields are `PromptID`, `PromptVersion`, `ProfileID`,
|
||||||
|
`APIKey`, `Inputs`, `Vars`, `Execution`, `Validation`, and
|
||||||
|
`Metadata`.
|
||||||
|
|
||||||
|
Input helpers are:
|
||||||
|
|
||||||
|
- `File(path)` for a file-backed artifact;
|
||||||
|
- `Inline(body)` for inline content; and
|
||||||
|
- `InlineWithURI(uri, body)` for inline content with URI metadata.
|
||||||
|
|
||||||
|
Required declared inputs must be supplied. Template rendering must also resolve
|
||||||
|
every input name the prompt actually references. Extra entries in `Inputs`
|
||||||
|
are not rejected solely because they are undeclared.
|
||||||
|
|
||||||
|
`ExecutionTargetOverride` supplies endpoint, model, credential-environment,
|
||||||
|
service-tier, reasoning-effort, and extra-parameter overrides. Its numeric
|
||||||
|
fields (`Temperature`, `MaxTokens`, `TopP`, and `TimeoutSeconds`) are
|
||||||
|
pointers so explicit zero values are preserved. `OutputContract` supplies
|
||||||
|
`Format`, `ValidationMode`, `SchemaPath`, and `RepairAttempts`.
|
||||||
|
|
||||||
|
`ExtraParams` accepts JSON-compatible values: strings, booleans, finite
|
||||||
|
numbers, objects with string keys, arrays or slices, and nil. Unsupported
|
||||||
|
values, non-string map keys, non-finite floats, and cycles return
|
||||||
|
`ErrInvalidConfig` for profiles or `ErrInvalidRequest` for request
|
||||||
|
overrides.
|
||||||
|
|
||||||
|
## Profiles And Credentials
|
||||||
|
|
||||||
|
`OpenAICompatibleProfile(OpenAICompatibleProfileConfig)` creates an
|
||||||
|
in-memory `Profile`. Its public fields are `ID`, `Endpoint`, `Model`,
|
||||||
|
`Temperature`, `MaxTokens`, `TopP`, `TimeoutSeconds`, `ServiceTier`,
|
||||||
|
`ReasoningEffort`, `APIKeyRequired`, and `ExtraParams`.
|
||||||
|
`WithProfiles` rejects duplicate IDs in one call.
|
||||||
|
|
||||||
|
A direct `RunRequest.APIKey` is request-scoped and takes precedence over
|
||||||
|
`api_key_env` for the built-in client. It is excluded from JSON output and
|
||||||
|
from `PreparedRun` and `RunResult`. The package's `String` and
|
||||||
|
`GoString` methods report only whether a direct key is set. Do not use
|
||||||
|
reflection-based dumps of request structs, which can bypass that redaction.
|
||||||
|
|
||||||
|
## Injected LLM Clients
|
||||||
|
|
||||||
|
`LLMClient` implements:
|
||||||
|
|
||||||
|
```go
|
||||||
|
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
|
||||||
|
```
|
||||||
|
|
||||||
|
Injected clients receive the rendered prompt, effective execution target, numeric
|
||||||
|
target-presence metadata, optional structured-output specification, and direct
|
||||||
|
request API key. `GenerateResponse` returns content and `TokenUsage`.
|
||||||
|
Custom clients should avoid logging raw prompts or credentials.
|
||||||
|
|
||||||
|
## Errors
|
||||||
|
|
||||||
|
Public methods preserve these sentinel checks through `errors.Is`:
|
||||||
|
|
||||||
|
- `ErrInvalidConfig`
|
||||||
|
- `ErrInvalidRequest`
|
||||||
|
- `ErrPromptNotFound`
|
||||||
|
- `ErrProfileNotFound`
|
||||||
|
- `ErrPromptLoad`
|
||||||
|
- `ErrProfileLoad`
|
||||||
|
- `ErrArtifactLoad`
|
||||||
|
- `ErrPromptRender`
|
||||||
|
- `ErrLLMGenerate`
|
||||||
|
- `ErrValidation`
|
||||||
|
|
||||||
|
For interface selection and operational responsibilities, see the
|
||||||
|
[consumer integration overview](api.md).
|
||||||
63
docs/development.md
Normal file
63
docs/development.md
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
# Development
|
||||||
|
|
||||||
|
This is the contributor entry point for Scriptorium. Use the task-specific
|
||||||
|
reading guide below before making changes. Canonical architecture, contracts,
|
||||||
|
component behavior, and policies remain in their owning documents.
|
||||||
|
|
||||||
|
## Initial Orientation
|
||||||
|
|
||||||
|
Before starting work:
|
||||||
|
|
||||||
|
1. inspect the working tree and preserve unrelated changes;
|
||||||
|
2. read the architecture policy for code or design work;
|
||||||
|
3. read the policy, contract, and internal documents listed for the task;
|
||||||
|
4. inspect the relevant implementation and tests before deciding how to change
|
||||||
|
them.
|
||||||
|
|
||||||
|
Start with:
|
||||||
|
|
||||||
|
- [Architecture policy](policy/architecture.md) for system boundaries,
|
||||||
|
invariants, and non-goals;
|
||||||
|
- [Internal component overview](internal/overview.md) for the current package
|
||||||
|
and component map;
|
||||||
|
- [Documentation policy](policy/documentation.md) before changing
|
||||||
|
documentation;
|
||||||
|
- [Testing policy](policy/testing.md) before adding, rewriting, or deleting
|
||||||
|
tests.
|
||||||
|
|
||||||
|
## Task-Specific Reading Guide
|
||||||
|
|
||||||
|
| Task | Read before changing |
|
||||||
|
| --- | --- |
|
||||||
|
| Repository orientation or component responsibility | [Internal component overview](internal/overview.md) and [architecture policy](policy/architecture.md) |
|
||||||
|
| Public Go package or engine behavior | [Go package consumer contract](consumers/pkg-scriptorium.md), [internal component overview](internal/overview.md), [runner internals](internal/runner.md), [adapter internals](internal/adapters.md), and [source internals](internal/sources.md) |
|
||||||
|
| CLI commands, flags, output, or exit behavior | [CLI contract](cli.md), [internal component overview](internal/overview.md), and [adapter internals](internal/adapters.md) |
|
||||||
|
| HTTP routes, DTOs, limits, or status mapping | [HTTP API contract](api.md), [internal component overview](internal/overview.md), [adapter internals](internal/adapters.md), and [source internals](internal/sources.md) |
|
||||||
|
| Application configuration | [Configuration contract](config.md), [internal component overview](internal/overview.md), [adapter internals](internal/adapters.md), and [source internals](internal/sources.md) |
|
||||||
|
| Prompt, profile, schema, or artifact loading | [Configuration contract](config.md), [internal component overview](internal/overview.md), and [source internals](internal/sources.md) |
|
||||||
|
| Runner orchestration, rendering, validation, or repair | [Runner internals](internal/runner.md) and [source internals](internal/sources.md) |
|
||||||
|
| OpenAI-compatible request or response behavior | [OpenAI-compatible integration](integrations/openai-compatible-chat.md), [LLM internals](internal/llm.md), [runner internals](internal/runner.md), and [adapter internals](internal/adapters.md) |
|
||||||
|
| Subprocess behavior | [Subprocess integration](integrations/subprocess.md) and [CLI contract](cli.md) |
|
||||||
|
| Runtime operation or recovery | [Operations](operations.md) |
|
||||||
|
| Examples or copyable assets | The owning contract for the demonstrated behavior and the related files under `examples/` |
|
||||||
|
| Architecture decisions or future work | The [documentation policy](policy/documentation.md), relevant accepted ADRs such as [ADR 0001](adr/0001-adopt-canonical-documentation-ownership.md), and relevant roadmap documents under `roadmap/` |
|
||||||
|
|
||||||
|
For cross-cutting changes, follow every applicable row. Internal component
|
||||||
|
documents own detailed subsystem change recipes.
|
||||||
|
|
||||||
|
## Baseline Validation
|
||||||
|
|
||||||
|
Use focused checks while iterating, then run validation proportionate to the
|
||||||
|
change and the risks described by the testing policy.
|
||||||
|
|
||||||
|
The repository-level baseline for code changes is:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go test ./...
|
||||||
|
go vet ./...
|
||||||
|
go build ./cmd/scriptorium
|
||||||
|
```
|
||||||
|
|
||||||
|
Documentation-only work does not require the full Go suite unless it changes
|
||||||
|
commands, examples, generated output, or another behavior that the suite
|
||||||
|
validates. Always check changed links, paths, examples, and canonical ownership.
|
||||||
@@ -1,192 +0,0 @@
|
|||||||
# HTTP API Integration
|
|
||||||
|
|
||||||
## Scope
|
|
||||||
|
|
||||||
This document defines the implemented inbound HTTP contract for Scriptorium.
|
|
||||||
|
|
||||||
Current scope is only:
|
|
||||||
|
|
||||||
- `POST /v1/runs`
|
|
||||||
|
|
||||||
For CLI behavior, see the [CLI reference](../cli.md).
|
|
||||||
|
|
||||||
## Endpoint
|
|
||||||
|
|
||||||
- Method: `POST`
|
|
||||||
- Path: `/v1/runs`
|
|
||||||
- Content type: JSON request/response
|
|
||||||
|
|
||||||
Route behavior:
|
|
||||||
|
|
||||||
- unknown path: `404 not_found`
|
|
||||||
- unsupported method on `/v1/runs`: `405 method_not_allowed`
|
|
||||||
|
|
||||||
Copyable request example file:
|
|
||||||
|
|
||||||
- `examples/http-run.json`
|
|
||||||
|
|
||||||
## Request Body
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"prompt_id": "generic.structured_events",
|
|
||||||
"profile_id": "local-quality",
|
|
||||||
"prompt_version": "1.0.0",
|
|
||||||
"inputs": {
|
|
||||||
"transcript": {"type": "file", "uri": "./examples/fixtures/transcript.md"},
|
|
||||||
"glossary": {"type": "inline", "body": "party:\n - Rin"}
|
|
||||||
},
|
|
||||||
"vars": {
|
|
||||||
"session_date": "2026-05-04"
|
|
||||||
},
|
|
||||||
"model": {
|
|
||||||
"endpoint": "http://localhost:8000/v1",
|
|
||||||
"model": "gpt-4o-mini",
|
|
||||||
"temperature": 0.0,
|
|
||||||
"max_tokens": 800,
|
|
||||||
"top_p": 1.0,
|
|
||||||
"timeout_seconds": 120,
|
|
||||||
"service_tier": "priority",
|
|
||||||
"reasoning_effort": "medium",
|
|
||||||
"api_key_env": "SCRIPTORIUM_API_KEY",
|
|
||||||
"extra_params": {
|
|
||||||
"route": "primary"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"include_raw_output": false
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Required fields:
|
|
||||||
|
|
||||||
- `prompt_id`
|
|
||||||
- `inputs` (must contain at least one named input)
|
|
||||||
|
|
||||||
Input reference types currently supported by runtime artifact loading:
|
|
||||||
|
|
||||||
- `file`
|
|
||||||
- `inline`
|
|
||||||
|
|
||||||
## Strict JSON Rules
|
|
||||||
|
|
||||||
Request decoding uses strict JSON field checks:
|
|
||||||
|
|
||||||
- unknown request fields are rejected with `400 invalid_json`
|
|
||||||
- unknown `model` fields are rejected with `400 invalid_json`
|
|
||||||
- raw API-key payload fields such as `api_key` are rejected as unknown fields
|
|
||||||
|
|
||||||
## Success Response
|
|
||||||
|
|
||||||
Status: `200 OK`
|
|
||||||
|
|
||||||
Response shape:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"artifact": {
|
|
||||||
"name": "output",
|
|
||||||
"content_type": "application/json",
|
|
||||||
"body": "{\"summary\":\"...\"}",
|
|
||||||
"uri": "",
|
|
||||||
"size": 123,
|
|
||||||
"hash": "..."
|
|
||||||
},
|
|
||||||
"validation": {
|
|
||||||
"status": "passed",
|
|
||||||
"mode": "json_schema",
|
|
||||||
"errors": [],
|
|
||||||
"schema_path": "structured_events.schema.json",
|
|
||||||
"repair_attempts": 0,
|
|
||||||
"is_valid": true
|
|
||||||
},
|
|
||||||
"metadata": {
|
|
||||||
"run_id": "...",
|
|
||||||
"prompt_id": "generic.structured_events",
|
|
||||||
"prompt_version": "1.0.0",
|
|
||||||
"prompt_hash": "...",
|
|
||||||
"rendered_prompt_hash": "...",
|
|
||||||
"selected_profile_id": "local-quality",
|
|
||||||
"model_name": "gpt-4o-mini",
|
|
||||||
"endpoint": "http://localhost:8000/v1",
|
|
||||||
"model_params": {
|
|
||||||
"endpoint": "http://localhost:8000/v1",
|
|
||||||
"model": "gpt-4o-mini",
|
|
||||||
"temperature": 0,
|
|
||||||
"max_tokens": 800,
|
|
||||||
"top_p": 1,
|
|
||||||
"timeout_seconds": 120,
|
|
||||||
"service_tier": "priority",
|
|
||||||
"reasoning_effort": "medium",
|
|
||||||
"api_key_env": "SCRIPTORIUM_API_KEY",
|
|
||||||
"extra_params": {
|
|
||||||
"route": "primary"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"input_hashes": {
|
|
||||||
"transcript": "..."
|
|
||||||
},
|
|
||||||
"usage": {
|
|
||||||
"prompt_tokens": 11,
|
|
||||||
"completion_tokens": 22,
|
|
||||||
"total_tokens": 33
|
|
||||||
},
|
|
||||||
"start_time": "2026-05-04T12:00:00Z",
|
|
||||||
"end_time": "2026-05-04T12:00:01Z",
|
|
||||||
"duration_ms": 1000,
|
|
||||||
"validation_mode": "json_schema",
|
|
||||||
"validation_status": "passed",
|
|
||||||
"repair_attempts_used": 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
`raw_model_output` is omitted by default.
|
|
||||||
|
|
||||||
To include it, send:
|
|
||||||
|
|
||||||
- `"include_raw_output": true`
|
|
||||||
|
|
||||||
## Validation Failure Behavior
|
|
||||||
|
|
||||||
Validation content failures do not map to HTTP error status.
|
|
||||||
|
|
||||||
Behavior:
|
|
||||||
|
|
||||||
- status remains `200 OK`
|
|
||||||
- `validation.status` is `failed`
|
|
||||||
- validation errors are returned in `validation.errors`
|
|
||||||
|
|
||||||
## Error Responses
|
|
||||||
|
|
||||||
Error body shape:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"error": {
|
|
||||||
"code": "invalid_request",
|
|
||||||
"message": "prompt_id is required"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Current error mapping (non-exhaustive):
|
|
||||||
|
|
||||||
- `400 invalid_json`: malformed JSON or unknown JSON fields
|
|
||||||
- `400 invalid_request`: missing/invalid request fields
|
|
||||||
- `400 profile_required`: no explicit `profile_id` and prompt has no `default_profile`
|
|
||||||
- `400 prompt_load_failed`: prompt definition invalid/unloadable
|
|
||||||
- `400 profile_load_failed`: profile invalid/unloadable
|
|
||||||
- `400 artifact_read_failed`: input artifact loading failed
|
|
||||||
- `400 prompt_render_failed`: template render failed
|
|
||||||
- `400 api_key_env_missing`: named API-key environment variable is missing
|
|
||||||
- `404 prompt_not_found`
|
|
||||||
- `404 profile_not_found`
|
|
||||||
- `502 llm_failed`: outbound model request failed
|
|
||||||
- `500 validation_runtime_failed`: validator runtime/schema-load failure
|
|
||||||
- `500 internal_error`
|
|
||||||
|
|
||||||
## Security And Deployment Note
|
|
||||||
|
|
||||||
The HTTP adapter has no built-in authentication or authorization.
|
|
||||||
|
|
||||||
Deploy behind trusted controls (for example authenticated gateway/reverse proxy and network boundaries).
|
|
||||||
@@ -1,114 +0,0 @@
|
|||||||
# Narratio Subprocess Integration
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
This document defines the supported subprocess contract for Narratio invoking Scriptorium through the public CLI.
|
|
||||||
|
|
||||||
This is a CLI contract, not an internal Go package integration.
|
|
||||||
|
|
||||||
## Supported Commands
|
|
||||||
|
|
||||||
Narratio should invoke:
|
|
||||||
|
|
||||||
- `scriptorium run`
|
|
||||||
- `scriptorium render`
|
|
||||||
|
|
||||||
Use `run` for generation.
|
|
||||||
|
|
||||||
Use `render` for preflight/debug output without LLM execution.
|
|
||||||
|
|
||||||
## Recommended Invocation Shapes
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
scriptorium run \
|
|
||||||
--prompt <prompt_id> \
|
|
||||||
--input transcript=<path> \
|
|
||||||
--out <artifact_path>
|
|
||||||
```
|
|
||||||
|
|
||||||
Render:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
scriptorium render \
|
|
||||||
--prompt <prompt_id> \
|
|
||||||
--input transcript=<path> \
|
|
||||||
--format json
|
|
||||||
```
|
|
||||||
|
|
||||||
Narratio may add:
|
|
||||||
|
|
||||||
- `--config <path>`
|
|
||||||
- `--profile <profile_id>`
|
|
||||||
- repeatable `--input name=path`
|
|
||||||
- repeatable `--var name=value`
|
|
||||||
- runtime overrides when explicitly needed (`--model`, `--llm-base-url`, `--timeout`, etc.)
|
|
||||||
|
|
||||||
## Config And Directory Behavior
|
|
||||||
|
|
||||||
Narratio can rely on resolved app config or pass explicit paths.
|
|
||||||
|
|
||||||
- default config search order:
|
|
||||||
1. `/usr/local/etc/scriptorium/config.yml`
|
|
||||||
2. `/etc/scriptorium/config.yml`
|
|
||||||
- explicit `--config` requires file existence and valid syntax
|
|
||||||
- CLI flags override config values
|
|
||||||
|
|
||||||
## Profile Selection
|
|
||||||
|
|
||||||
Profile selection follows runner behavior:
|
|
||||||
|
|
||||||
1. explicit `--profile`
|
|
||||||
2. prompt `default_profile`
|
|
||||||
3. error if neither is available
|
|
||||||
|
|
||||||
Narratio should treat prompt/profile IDs as deployment configuration, not hardcoded logic.
|
|
||||||
|
|
||||||
## Input And Variable Contract
|
|
||||||
|
|
||||||
- Inputs use repeated `--input name=path`.
|
|
||||||
- Input names must match prompt definition input names.
|
|
||||||
- Variables use repeated `--var name=value` for small metadata values.
|
|
||||||
- Prefer file inputs for large content.
|
|
||||||
|
|
||||||
## Environment Contract
|
|
||||||
|
|
||||||
- Pass through required API-key environment variables referenced by `api_key_env`.
|
|
||||||
- Never pass raw API keys via CLI arguments.
|
|
||||||
- Keep subprocess environment scoped to required variables.
|
|
||||||
|
|
||||||
## Output And Error Handling
|
|
||||||
|
|
||||||
`run`:
|
|
||||||
|
|
||||||
- stdout: artifact body unless `--out` is used
|
|
||||||
- `--out`: writes artifact to file
|
|
||||||
- stderr: success summary and errors
|
|
||||||
|
|
||||||
`render`:
|
|
||||||
|
|
||||||
- stdout: prepared-run output unless `--out` is used
|
|
||||||
- stderr: errors
|
|
||||||
|
|
||||||
Narratio should capture stdout and stderr separately.
|
|
||||||
|
|
||||||
## Exit Status Contract
|
|
||||||
|
|
||||||
- `0`: success
|
|
||||||
- `1`: parse/config/load/render/generation/IO/runtime error
|
|
||||||
- `2`: run completed but validation failed
|
|
||||||
|
|
||||||
A `run` exit code `2` can still produce output (stdout or `--out`).
|
|
||||||
|
|
||||||
## Security Notes
|
|
||||||
|
|
||||||
- Treat generated artifacts and stderr logs as potentially sensitive.
|
|
||||||
- Avoid logging full rendered prompts by default in production contexts.
|
|
||||||
- Use controlled output paths and access controls for persisted artifacts.
|
|
||||||
|
|
||||||
## Canonical References
|
|
||||||
|
|
||||||
- CLI behavior: [CLI reference](../cli.md)
|
|
||||||
- Config behavior: [Configuration reference](../config.md)
|
|
||||||
- Operations and failure handling: [Operations guide](../operations.md), [Troubleshooting](../troubleshooting.md)
|
|
||||||
@@ -1,48 +1,67 @@
|
|||||||
# OpenAI-Compatible Chat Integration
|
# OpenAI-Compatible Chat Integration
|
||||||
|
|
||||||
## Scope
|
This is the outbound wire contract for Scriptorium's OpenAI-compatible
|
||||||
|
chat-completions client.
|
||||||
|
|
||||||
This document defines the outbound LLM contract implemented by `internal/llm/openai_compatible_client.go`.
|
## Endpoint And Method
|
||||||
|
|
||||||
It documents only fields and behaviors currently serialized by code.
|
Scriptorium uses the request endpoint override when present; otherwise it uses
|
||||||
|
the configured client base URL. It removes a trailing slash and sends
|
||||||
|
`POST /chat/completions`.
|
||||||
|
|
||||||
## Endpoint Construction
|
For example, `http://localhost:8000/v1` becomes
|
||||||
|
`http://localhost:8000/v1/chat/completions`.
|
||||||
|
|
||||||
Request endpoint is built as:
|
## Request Payload
|
||||||
|
|
||||||
1. choose base URL:
|
The payload always contains `model` and rendered `messages`. It additionally
|
||||||
- `GenerateRequest.Target.Endpoint` if set
|
contains these fields when applicable:
|
||||||
- otherwise client config `BaseURL`
|
|
||||||
2. trim trailing slash
|
|
||||||
3. append `/chat/completions`
|
|
||||||
|
|
||||||
Example:
|
| Field | Inclusion |
|
||||||
|
| --- | --- |
|
||||||
|
| `session_id` | Non-empty rendered prompt session ID. |
|
||||||
|
| `temperature` | Non-zero effective value or an explicit zero override. |
|
||||||
|
| `max_tokens` | Non-zero effective value or an explicit zero override. |
|
||||||
|
| `top_p` | Non-zero effective value or an explicit zero override. |
|
||||||
|
| `service_tier` | Any non-empty configured value. |
|
||||||
|
| `reasoning_effort` | Any non-empty configured value. |
|
||||||
|
| `response_format` | Structured output is requested. |
|
||||||
|
| provider-specific fields | Flattened from `extra_params`. |
|
||||||
|
|
||||||
- base URL: `http://localhost:8000/v1`
|
`service_tier` and `reasoning_effort` are forwarded without a provider value
|
||||||
- final URL: `http://localhost:8000/v1/chat/completions`
|
catalog; the selected backend decides which values it supports.
|
||||||
|
|
||||||
## Request Fields Sent
|
`extra_params` are top-level JSON fields, not a nested object. Keys cannot be
|
||||||
|
empty or collide with `model`, `session_id`, `messages`, `temperature`,
|
||||||
|
`max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, or
|
||||||
|
`response_format`. Values must be JSON-serializable.
|
||||||
|
|
||||||
Serialized JSON fields:
|
A rendered `session_id` is sent as a top-level JSON field, not as a header.
|
||||||
|
Empty values are omitted. The maximum length is 256 Unicode code points.
|
||||||
|
|
||||||
- `model` (required after fallback resolution)
|
Messages without cache control use string `content`. A message with cache
|
||||||
- `messages` (role/content pairs from rendered prompt)
|
control uses one text block:
|
||||||
- `temperature` (only when non-zero)
|
|
||||||
- `max_tokens` (only when non-zero)
|
|
||||||
- `top_p` (only when non-zero)
|
|
||||||
- `service_tier` (only when non-empty)
|
|
||||||
- `response_format` (only when structured output is provided)
|
|
||||||
|
|
||||||
`service_tier` is provider-specific. OpenRouter currently documents request values such as `flex` and `priority`; Scriptorium forwards any non-empty configured value and lets the backend validate support.
|
```json
|
||||||
|
{
|
||||||
|
"role": "system",
|
||||||
|
"content": [{
|
||||||
|
"type": "text",
|
||||||
|
"text": "rendered text",
|
||||||
|
"cache_control": {"type": "ephemeral", "ttl": "1h"}
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
Structured output is currently `json_schema` only, serialized as:
|
When the prompt omits cache-control `ttl`, the payload omits `ttl`.
|
||||||
|
Structured JSON Schema output is sent as:
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"response_format": {
|
"response_format": {
|
||||||
"type": "json_schema",
|
"type": "json_schema",
|
||||||
"json_schema": {
|
"json_schema": {
|
||||||
"name": "...",
|
"name": "schema name",
|
||||||
"strict": true,
|
"strict": true,
|
||||||
"schema": {"type": "object"}
|
"schema": {"type": "object"}
|
||||||
}
|
}
|
||||||
@@ -50,64 +69,43 @@ Structured output is currently `json_schema` only, serialized as:
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
## Authentication Header
|
## Authentication And Timeout
|
||||||
|
|
||||||
If `Target.APIKeyEnv` is set:
|
When a direct request API key is present, Scriptorium sends
|
||||||
|
`Authorization: Bearer <key>` and does not read `api_key_env`. Otherwise, it
|
||||||
|
resolves the configured non-empty `api_key_env` at request time and sends the
|
||||||
|
same header. If neither mechanism supplies a key, it sends no
|
||||||
|
`Authorization` header.
|
||||||
|
|
||||||
- resolve environment variable value at request time
|
The client base timeout is chosen at construction: a non-zero timeout on a
|
||||||
- set `Authorization: Bearer <value>`
|
supplied `http.Client` takes precedence over `Config.Timeout`; otherwise a
|
||||||
|
positive `Config.Timeout` is used, then the internal default. A positive
|
||||||
|
effective `timeout_seconds` replaces that base. An explicit request override
|
||||||
|
of zero disables the HTTP-client timeout; negative values are rejected before a
|
||||||
|
request is sent.
|
||||||
|
|
||||||
If the environment variable is unset/empty:
|
## Response Subset And Failures
|
||||||
|
|
||||||
- request fails before HTTP call (`ErrInvalidRequest`)
|
A successful provider response must supply non-empty
|
||||||
|
`choices[0].message.content`. Scriptorium reads these optional or required
|
||||||
|
usage fields when present:
|
||||||
|
|
||||||
If `Target.APIKeyEnv` is empty:
|
|
||||||
|
|
||||||
- no `Authorization` header is sent
|
|
||||||
|
|
||||||
## Timeout Behavior
|
|
||||||
|
|
||||||
Base timeout comes from client configuration.
|
|
||||||
|
|
||||||
Per-request override:
|
|
||||||
|
|
||||||
- if `Target.TimeoutSeconds > 0`, use that value for request timeout
|
|
||||||
- if `Target.TimeoutSeconds < 0`, request is rejected (`ErrInvalidRequest`)
|
|
||||||
|
|
||||||
## Response Expectations
|
|
||||||
|
|
||||||
Expected successful response shape (subset used):
|
|
||||||
|
|
||||||
- `choices[0].message.content`
|
|
||||||
- `usage.prompt_tokens`
|
- `usage.prompt_tokens`
|
||||||
- `usage.completion_tokens`
|
- `usage.completion_tokens`
|
||||||
- `usage.total_tokens`
|
- `usage.total_tokens`
|
||||||
|
- `usage.prompt_tokens_details.cached_tokens`
|
||||||
|
- `usage.cache_write_tokens`
|
||||||
|
|
||||||
Malformed response conditions include:
|
Missing cache usage is reported as zero. Invalid JSON, an empty choices array,
|
||||||
|
or empty first-choice content is a malformed provider response. Network and
|
||||||
|
request-construction failures, non-2xx responses, and malformed responses fail
|
||||||
|
the outbound call. Provider response bodies are not exposed by this client.
|
||||||
|
|
||||||
- invalid JSON
|
The client does not implement built-in retries, tool calls, top-level
|
||||||
- empty `choices`
|
`cache_control`, or multi-request payload modes.
|
||||||
- empty `choices[0].message.content`
|
|
||||||
|
|
||||||
Malformed responses return `ErrMalformedResponse`.
|
## Related References
|
||||||
|
|
||||||
## Error Handling
|
Prompt schema preparation and runner orchestration are described in
|
||||||
|
[runner internals](../internal/runner.md). Prompt and profile configuration is
|
||||||
- network/request-construction failures: `ErrRequestFailed`
|
defined by the [configuration reference](../config.md).
|
||||||
- non-2xx HTTP status: `ErrUnexpectedStatus` (includes status code and trimmed response body snippet)
|
|
||||||
- malformed response shape/content: `ErrMalformedResponse`
|
|
||||||
|
|
||||||
## Unsupported Or Non-Serialized Fields
|
|
||||||
|
|
||||||
The following fields may exist in profile/effective settings but are not currently serialized into outbound chat-completions payloads:
|
|
||||||
|
|
||||||
- `reasoning_effort`
|
|
||||||
- `extra_params`
|
|
||||||
|
|
||||||
No built-in retries, tool-calls, or multi-request payload modes are implemented in this client.
|
|
||||||
|
|
||||||
## Relationship To Runner
|
|
||||||
|
|
||||||
When prompt validation mode is `json_schema`, runner prepares a structured-output schema spec and passes it to the client as `StructuredOutput`.
|
|
||||||
|
|
||||||
The client only serializes the provider request payload; it does not load schema files itself.
|
|
||||||
|
|||||||
41
docs/integrations/subprocess.md
Normal file
41
docs/integrations/subprocess.md
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
# Subprocess Integration
|
||||||
|
|
||||||
|
This document covers process-boundary behavior for callers that invoke
|
||||||
|
Scriptorium as a child process. Command syntax, flags, output, and exit codes
|
||||||
|
are defined by the [CLI reference](../cli.md). Interface selection belongs in
|
||||||
|
the [consumer integration overview](../consumers/api.md).
|
||||||
|
|
||||||
|
## Process Contract
|
||||||
|
|
||||||
|
Use `scriptorium render` when the caller needs prepared output without a model
|
||||||
|
call, and `scriptorium run` for generation. Pass an explicit `--config` or
|
||||||
|
make the configuration search paths available to the child process; configuration
|
||||||
|
discovery, fields, profile selection, and credential mechanisms are defined in
|
||||||
|
the [configuration reference](../config.md).
|
||||||
|
|
||||||
|
Pass required API-key environment variables through the child environment. Do
|
||||||
|
not place raw API keys in arguments. Keep the environment limited to the values
|
||||||
|
needed for the selected profile.
|
||||||
|
|
||||||
|
## Streams And Output Ownership
|
||||||
|
|
||||||
|
Capture stdout and stderr separately. Stdout contains the requested artifact or
|
||||||
|
prepared output unless the caller selects an output file; stderr contains
|
||||||
|
summaries, diagnostics, and server messages. The exact destinations and status
|
||||||
|
meanings are part of the [CLI reference](../cli.md), not a stable stderr data
|
||||||
|
protocol.
|
||||||
|
|
||||||
|
When using `--out`, the caller owns the output path, its permissions, and
|
||||||
|
cleanup. Treat rendered prompts, generated artifacts, stdout, and stderr as
|
||||||
|
potentially sensitive.
|
||||||
|
|
||||||
|
## Cancellation And Recovery
|
||||||
|
|
||||||
|
A CLI invocation performs one synchronous request and creates no durable run
|
||||||
|
state. A supervising process that needs cancellation must terminate the child
|
||||||
|
process according to its own process-management policy. A later invocation is a
|
||||||
|
new request and can make another model call; there is no resume or checkpoint
|
||||||
|
protocol.
|
||||||
|
|
||||||
|
For deployment, filesystem permissions, and sensitive-artifact handling, see
|
||||||
|
the [operations guide](../operations.md).
|
||||||
@@ -1,144 +1,113 @@
|
|||||||
# Adapter And Repository Internals
|
# Adapter Internals
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
This document describes implemented adapter/repository boundaries and their current behavior.
|
Adapters translate external inputs into domain requests, compose dependencies,
|
||||||
|
and translate domain results or errors back to their interface. They own IO and
|
||||||
|
presentation mechanics; use-case decisions remain in `internal/usecase`.
|
||||||
|
|
||||||
## Adapter Map
|
External contracts are canonical in the [CLI reference](../cli.md), [HTTP API
|
||||||
|
reference](../api.md), and [Go package contract](../consumers/pkg-scriptorium.md).
|
||||||
|
|
||||||
- `internal/adapter/cli`: CLI command parsing, app wiring, stdout/stderr handling, exit codes.
|
## Components And Collaborators
|
||||||
- `internal/adapter/http`: HTTP request/response mapping for `POST /v1/runs`.
|
|
||||||
- `internal/promptdef`: filesystem prompt-definition repository.
|
|
||||||
- `internal/profile`: filesystem execution-profile repository.
|
|
||||||
- `internal/artifact`: input artifact reader.
|
|
||||||
- `internal/prompt`: Go-template renderer.
|
|
||||||
- `internal/llm`: OpenAI-compatible LLM client implementation.
|
|
||||||
- `internal/validate`: output validator.
|
|
||||||
- `internal/format`: prepared-run formatters for `render` output.
|
|
||||||
|
|
||||||
## Inputs And Outputs
|
- `cmd/scriptorium` passes process arguments and streams to
|
||||||
|
`internal/adapter/cli`.
|
||||||
|
- `internal/adapter/cli` parses commands, resolves application settings through
|
||||||
|
`internal/config`, constructs a runner, and owns process output handling.
|
||||||
|
- `internal/adapter/http` decodes DTOs, maps them to `domain.RunRequest`, calls
|
||||||
|
a runner interface, and maps errors and results to HTTP DTOs.
|
||||||
|
- The root `scriptorium` package maps its public types and options to internal
|
||||||
|
collaborators and maps selected internal errors to public sentinels.
|
||||||
|
- `internal/format` formats prepared runs for the CLI; `internal/llm`,
|
||||||
|
`internal/prompt`, and source packages supply runner dependencies.
|
||||||
|
|
||||||
CLI adapter:
|
## Wiring Flows
|
||||||
|
|
||||||
- Input: process args, filesystem config/assets, environment.
|
### CLI
|
||||||
- Output: exit code, stdout artifact/prepared output, stderr summaries/errors.
|
|
||||||
|
|
||||||
HTTP adapter:
|
The CLI resolves configuration before constructing dependencies. `run` builds a
|
||||||
|
runner with the ordinary composite artifact reader and invokes `Runner.Run`;
|
||||||
|
`render` uses the same wiring and invokes `Runner.Prepare`; `serve` replaces the
|
||||||
|
file reader with the restricted artifact reader, builds an HTTP handler, and
|
||||||
|
starts the server.
|
||||||
|
|
||||||
- Input: JSON request body (`runRequestDTO`).
|
Parser state records whether numeric runtime values were explicitly supplied.
|
||||||
- Output: JSON success/error body with mapped status codes.
|
That presence is carried into `domain.ExecutionTargetOverride`, allowing the
|
||||||
|
runner to distinguish omitted values from explicit zero overrides.
|
||||||
|
|
||||||
Filesystem repositories:
|
### HTTP
|
||||||
|
|
||||||
- Input: prompt/profile YAML files under configured directories.
|
The handler first enforces transport limits, strict JSON decoding, and the
|
||||||
- Output: normalized domain definitions/profiles or typed errors.
|
minimal request shape. It maps DTO values to domain types without deciding
|
||||||
|
prompt selection, source behavior, or validation semantics. On success it maps
|
||||||
|
the domain result to the response DTO; on failure it uses `errors.Is` over
|
||||||
|
runner, source, artifact, and profile errors to choose the public error mapping.
|
||||||
|
|
||||||
Artifact reader:
|
The [HTTP API reference](../api.md) owns the route, DTO schema, status codes,
|
||||||
|
and externally observable limit behavior.
|
||||||
|
|
||||||
- Input: `domain.ArtifactRef`.
|
### Public Go Facade
|
||||||
- Output: loaded `domain.Artifact`.
|
|
||||||
|
|
||||||
LLM adapter:
|
`NewEngine` applies public options, selects filesystem, `fs.FS`, single-file,
|
||||||
|
or in-memory dependencies, and constructs a runner. The conversion functions
|
||||||
|
copy maps and slices across the boundary so callers do not receive internal
|
||||||
|
domain values. The facade maps selected internal errors to the public sentinel
|
||||||
|
set and keeps direct request API keys out of public results.
|
||||||
|
|
||||||
- Input: `domain.GenerateRequest`.
|
## Package-Local Guarantees
|
||||||
- Output: `domain.GenerateResponse`.
|
|
||||||
|
|
||||||
Validator:
|
- Adapters do not embed runner orchestration or source-loading decisions.
|
||||||
|
- Configuration is resolved before adapter dependency composition.
|
||||||
|
- CLI and HTTP create runners without a repairer; a repairer is available only
|
||||||
|
through explicit internal runner construction.
|
||||||
|
- DTO conversion preserves explicit numeric-override presence.
|
||||||
|
- Error mapping matches error identities, not error text.
|
||||||
|
- No adapter creates durable run state; caller-selected output files are not
|
||||||
|
application state.
|
||||||
|
|
||||||
- Input: artifact body + output contract.
|
## Failure And Verification Boundaries
|
||||||
- Output: validation result or runtime validation error.
|
|
||||||
|
|
||||||
## Boundaries
|
Keep external error payloads concise, preserve strict external decoding, and do
|
||||||
|
not serialize resolved secret values. Validation content failures remain result
|
||||||
|
state; runtime failures remain errors for the relevant adapter to map.
|
||||||
|
|
||||||
- Adapters convert external representations to domain requests and back.
|
Inspect focused tests when changing this area:
|
||||||
- Use-case decisions remain in `internal/usecase`.
|
|
||||||
- External dependency details stay scoped to adapter packages.
|
|
||||||
|
|
||||||
## Config Fields Used
|
|
||||||
|
|
||||||
Primary app settings consumed by adapters:
|
|
||||||
|
|
||||||
- `prompt_dir`
|
|
||||||
- `profile_dir`
|
|
||||||
- `schema_dir`
|
|
||||||
- `server.addr`
|
|
||||||
- `defaults.render_format`
|
|
||||||
|
|
||||||
Execution profile/request settings used through runner:
|
|
||||||
|
|
||||||
- `endpoint`, `model`, `temperature`, `max_tokens`, `top_p`, `timeout_seconds`, `service_tier`, `api_key_env`, `reasoning_effort`, `extra_params`
|
|
||||||
|
|
||||||
## External Dependencies
|
|
||||||
|
|
||||||
- YAML decoding: `gopkg.in/yaml.v3` (strict known-fields mode in config/prompt/profile loaders).
|
|
||||||
- JSON Schema validation: `github.com/santhosh-tekuri/jsonschema/v6`.
|
|
||||||
- HTTP client/server: Go standard library.
|
|
||||||
|
|
||||||
## Failure Behavior
|
|
||||||
|
|
||||||
Strict decoding and input checks:
|
|
||||||
|
|
||||||
- config/prompt/profile loaders reject unknown YAML fields.
|
|
||||||
- prompt/profile repositories scan nested subdirectories recursively.
|
|
||||||
- prompt/profile lookup uses YAML `id` values; subdirectory paths are organizational only.
|
|
||||||
- duplicate prompt/profile IDs are invalid and fail instead of using first-match behavior.
|
|
||||||
- HTTP DTO decoder rejects unknown JSON fields.
|
|
||||||
- raw API key payload fields are rejected by strict decoding in profile/http paths.
|
|
||||||
|
|
||||||
Artifact refs:
|
|
||||||
|
|
||||||
- Supported reference types: `inline`, `file`.
|
|
||||||
- Unsupported types return `ErrUnsupportedRefType`.
|
|
||||||
|
|
||||||
LLM adapter:
|
|
||||||
|
|
||||||
- endpoint appends `/chat/completions`.
|
|
||||||
- non-2xx responses map to request failure errors.
|
|
||||||
- malformed responses (including missing/empty first choice content) are errors.
|
|
||||||
|
|
||||||
Validator:
|
|
||||||
|
|
||||||
- `basic`, `json`, `json_schema` content failures return `ValidationFailed` results.
|
|
||||||
- schema load/compile/path failures are runtime errors.
|
|
||||||
- schema lookup uses explicit `schema_path` values relative to `schema_dir`; it does not recursively search by basename.
|
|
||||||
|
|
||||||
HTTP error mapping:
|
|
||||||
|
|
||||||
- maps domain/use-case errors to stable HTTP code + error code/message.
|
|
||||||
- distinguishes missing profile selection and missing `api_key_env` variable using stable use-case sentinel errors.
|
|
||||||
- avoids returning internal wrapped-cause details in response payload.
|
|
||||||
|
|
||||||
## CLI Adapter Semantics
|
|
||||||
|
|
||||||
Implemented commands:
|
|
||||||
|
|
||||||
- `run`
|
|
||||||
- `render`
|
|
||||||
- `serve`
|
|
||||||
|
|
||||||
Behavior highlights:
|
|
||||||
|
|
||||||
- `run` exit `2` indicates validation failed after generation.
|
|
||||||
- `render` does not call the LLM.
|
|
||||||
- `serve` exposes HTTP handler only; no built-in auth.
|
|
||||||
- `render` supports `--format text|json`; `render` does not expose `--schema-dir`.
|
|
||||||
- deprecated aliases `--prompt-id` and `--profile-id` are still accepted.
|
|
||||||
|
|
||||||
## Tests To Inspect Before Changing
|
|
||||||
|
|
||||||
- `internal/adapter/cli/run_test.go`
|
- `internal/adapter/cli/run_test.go`
|
||||||
- `internal/adapter/http/handler_test.go`
|
- `internal/adapter/http/handler_test.go`
|
||||||
- `internal/promptdef/repository_test.go`
|
- `engine_test.go`
|
||||||
- `internal/profile/repository_test.go`
|
|
||||||
- `internal/artifact/reader_test.go`
|
|
||||||
- `internal/prompt/renderer_test.go`
|
|
||||||
- `internal/llm/openai_compatible_client_test.go`
|
|
||||||
- `internal/validate/standard_validator_test.go`
|
|
||||||
- `internal/format/prepared_run_test.go`
|
- `internal/format/prepared_run_test.go`
|
||||||
|
|
||||||
## Architectural Invariants
|
Run the affected adapter package tests and recheck the relevant canonical
|
||||||
|
contract. The [testing policy](../policy/testing.md) owns global test
|
||||||
|
sufficiency guidance.
|
||||||
|
|
||||||
- Adapter packages do not own runner decision logic.
|
## Change Recipes
|
||||||
- External request/response strictness is part of contract stability.
|
|
||||||
- Prepared-render output never includes resolved API key values.
|
### Application Configuration Fields
|
||||||
- Outbound OpenAI-compatible request includes only currently serialized fields (`model`, `messages`, optional `temperature`, `max_tokens`, `top_p`, optional `service_tier`, optional `response_format`).
|
|
||||||
|
1. Add the field to the relevant `internal/config` shape and default handling.
|
||||||
|
2. Parse and validate it, then preserve configuration and CLI-override
|
||||||
|
precedence while wiring it through its consuming adapter.
|
||||||
|
3. Add focused configuration and adapter tests for parsing, mapping, and
|
||||||
|
effective behavior.
|
||||||
|
4. Update the [configuration contract](../config.md) and any affected external
|
||||||
|
contract.
|
||||||
|
|
||||||
|
### CLI Flags
|
||||||
|
|
||||||
|
1. Add the flag to the relevant parser in `internal/adapter/cli/run.go`.
|
||||||
|
2. Keep command scope and application-configuration precedence intentional.
|
||||||
|
3. Add or update parser and command tests in
|
||||||
|
`internal/adapter/cli/run_test.go`.
|
||||||
|
4. Update the [CLI contract](../cli.md) and affected maintained examples.
|
||||||
|
|
||||||
|
### Adapter Capabilities
|
||||||
|
|
||||||
|
1. Define or reuse the appropriate domain or use-case interface boundary.
|
||||||
|
2. Implement translation and IO behavior without moving use-case decisions out
|
||||||
|
of `internal/usecase`.
|
||||||
|
3. Add focused mapping, parsing, and error-behavior tests.
|
||||||
|
4. Update this document and the affected public or integration contract. Update
|
||||||
|
[source internals](sources.md) when source-loading behavior changes.
|
||||||
|
|||||||
86
docs/internal/llm.md
Normal file
86
docs/internal/llm.md
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
# LLM Internals
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
`internal/llm` defines the provider-neutral `Client` interface and the
|
||||||
|
OpenAI-compatible client implementation. The [OpenAI-compatible integration
|
||||||
|
contract](../integrations/openai-compatible-chat.md) owns the outbound HTTP wire
|
||||||
|
format and protocol behavior.
|
||||||
|
|
||||||
|
## Construction
|
||||||
|
|
||||||
|
`NewOpenAICompatibleClient` validates a non-empty configured base URL, records
|
||||||
|
an optional default model, and resolves one base timeout. A supplied client with
|
||||||
|
a non-zero timeout supplies that base; otherwise a positive configured timeout
|
||||||
|
is used, then the internal default.
|
||||||
|
|
||||||
|
When callers supply an `http.Client`, construction clones it rather than
|
||||||
|
mutating the caller's instance. A supplied client with no timeout receives the
|
||||||
|
resolved base timeout in the clone. The client stores the trimmed base URL,
|
||||||
|
default model, and cloned client.
|
||||||
|
|
||||||
|
## Generate Flow
|
||||||
|
|
||||||
|
`Generate` receives a `domain.GenerateRequest` from the runner:
|
||||||
|
|
||||||
|
1. validate the effective timeout and choose the request endpoint;
|
||||||
|
2. map the domain request to the internal wire-request representation;
|
||||||
|
3. validate and flatten extra parameters, encode JSON, and create the HTTP
|
||||||
|
request;
|
||||||
|
4. prefer a direct API key, otherwise resolve the configured key environment
|
||||||
|
variable;
|
||||||
|
5. derive a request HTTP client only when an explicit timeout changes the base
|
||||||
|
client;
|
||||||
|
6. execute the request, reject non-success status responses without returning
|
||||||
|
provider response bodies; and
|
||||||
|
7. decode the response subset into `domain.GenerateResponse`.
|
||||||
|
|
||||||
|
`openAIChatRequestFromGenerateRequest` is the conversion boundary for effective
|
||||||
|
model defaults, explicit numeric-presence state, rendered messages, structured
|
||||||
|
output, and session-ID validation. `openAIChatRequestPayload` protects reserved
|
||||||
|
fields and JSON encoding before an HTTP call. The external payload shape is
|
||||||
|
defined only in the [integration contract](../integrations/openai-compatible-chat.md).
|
||||||
|
|
||||||
|
## Error Categories
|
||||||
|
|
||||||
|
The package uses these internal sentinels:
|
||||||
|
|
||||||
|
- `ErrInvalidConfig` for invalid client construction;
|
||||||
|
- `ErrInvalidRequest` for invalid effective generation input;
|
||||||
|
- `ErrRequestFailed` for request construction or transport failures;
|
||||||
|
- `ErrUnexpectedStatus` for non-success HTTP responses; and
|
||||||
|
- `ErrMalformedResponse` for invalid or incomplete successful-response data.
|
||||||
|
|
||||||
|
The runner maps an invalid LLM request to its invalid-request category and
|
||||||
|
other LLM failures to its generation category. Adapters then apply their public
|
||||||
|
error contracts.
|
||||||
|
|
||||||
|
## Package-Local Guarantees
|
||||||
|
|
||||||
|
- The default-model fallback happens before wire encoding.
|
||||||
|
- Per-request timeout handling clones a configured HTTP client when needed; it
|
||||||
|
does not mutate shared client state.
|
||||||
|
- Direct API keys take precedence over environment lookup within this client.
|
||||||
|
- Provider response bodies are discarded for non-success status responses.
|
||||||
|
- The client does not implement retries, tool calls, or a stateful session
|
||||||
|
store.
|
||||||
|
|
||||||
|
## Verification And Change Recipe
|
||||||
|
|
||||||
|
Inspect:
|
||||||
|
|
||||||
|
- `internal/llm/openai_compatible_client_test.go`
|
||||||
|
- `internal/usecase/runner_test.go`
|
||||||
|
- `internal/adapter/http/handler_test.go`
|
||||||
|
|
||||||
|
When changing the client:
|
||||||
|
|
||||||
|
1. keep domain-to-wire mapping inside `internal/llm` and preserve the `Client`
|
||||||
|
interface;
|
||||||
|
2. test construction, timeout selection, mapping, and error categorization;
|
||||||
|
3. update the [OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
|
||||||
|
for any observable wire or protocol change; and
|
||||||
|
4. update [runner internals](runner.md) if the client boundary or structured
|
||||||
|
output handoff changes.
|
||||||
|
|
||||||
|
The [testing policy](../policy/testing.md) owns global test sufficiency.
|
||||||
48
docs/internal/overview.md
Normal file
48
docs/internal/overview.md
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
# Internal Component Overview
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This is the inventory of Scriptorium's implemented components for contributors.
|
||||||
|
The [architecture policy](../policy/architecture.md) owns normative boundaries
|
||||||
|
and invariants; public behavior belongs in the linked contracts.
|
||||||
|
|
||||||
|
## Public And Command Entrypoints
|
||||||
|
|
||||||
|
| Component | Implemented responsibility | References |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Root package `scriptorium` | Public Go facade that constructs the engine, exposes request/result types and options, and maps internal errors. | [Go package contract](../consumers/pkg-scriptorium.md), [adapter internals](adapters.md) |
|
||||||
|
| `cmd/scriptorium` | Process entrypoint that delegates command execution to the CLI adapter. | [CLI contract](../cli.md), [adapter internals](adapters.md) |
|
||||||
|
|
||||||
|
## Adapters, Domain, And Use Case
|
||||||
|
|
||||||
|
| Component | Implemented responsibility | References |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `internal/adapter/cli` | Parses CLI commands, applies application wiring, and handles process input and output. | [CLI contract](../cli.md), [adapter internals](adapters.md) |
|
||||||
|
| `internal/adapter/http` | Maps HTTP requests and responses to domain operations and maps public errors. | [HTTP API contract](../api.md), [adapter internals](adapters.md) |
|
||||||
|
| `internal/domain` | Defines core request, result, output-contract, and LLM-boundary types. | [runner internals](runner.md) |
|
||||||
|
| `internal/usecase` | Implements `Runner` preparation, execution, validation coordination, and the repairer boundary. | [runner internals](runner.md) |
|
||||||
|
|
||||||
|
## Configuration And Sources
|
||||||
|
|
||||||
|
| Component | Implemented responsibility | References |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `internal/config` | Loads application settings, applies defaults, and applies CLI overrides. | [configuration contract](../config.md), [adapter internals](adapters.md) |
|
||||||
|
| `internal/defaults` | Holds compile-time default values used when application settings are resolved. | [configuration contract](../config.md) |
|
||||||
|
| `internal/promptdef` | Loads prompt definitions from filesystem and `fs.FS` sources. | [configuration contract](../config.md), [source internals](sources.md) |
|
||||||
|
| `internal/profile` | Loads filesystem and `fs.FS` execution profiles and combines profile repositories. | [configuration contract](../config.md), [source internals](sources.md) |
|
||||||
|
| `internal/profile/builtin` | Provides embedded built-in execution profiles as a repository. | [configuration contract](../config.md), [source internals](sources.md) |
|
||||||
|
| `internal/filecatalog` | Provides shared YAML discovery and source-root helpers. | [source internals](sources.md) |
|
||||||
|
| `internal/artifact` | Reads inline and file-backed input artifacts. | [configuration contract](../config.md), [HTTP API contract](../api.md), [source internals](sources.md) |
|
||||||
|
| `internal/prompt` | Renders prompt templates into messages. | [runner internals](runner.md) |
|
||||||
|
|
||||||
|
## Formatting, Validation, And Model Access
|
||||||
|
|
||||||
|
| Component | Implemented responsibility | References |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `internal/format` | Formats prepared-run information for CLI output. | [CLI contract](../cli.md), [adapter internals](adapters.md) |
|
||||||
|
| `internal/validate` | Defines validation interfaces and provides standard filesystem and `fs.FS` schema validation. | [configuration contract](../config.md), [source internals](sources.md), [runner internals](runner.md) |
|
||||||
|
| `internal/llm` | Defines the provider-neutral LLM client boundary and its OpenAI-compatible implementation. | [OpenAI-compatible integration](../integrations/openai-compatible-chat.md), [LLM internals](llm.md), [runner internals](runner.md) |
|
||||||
|
|
||||||
|
Focused internal documents describe the components that have detailed
|
||||||
|
orchestration, adapter, or source behavior. Package tests live alongside the
|
||||||
|
implementation and are identified in those focused documents where relevant.
|
||||||
@@ -2,154 +2,118 @@
|
|||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
`internal/usecase.Runner` is the core use case orchestrator for prompt preparation and execution.
|
`internal/usecase.Runner` is the prompt-execution orchestrator. It prepares
|
||||||
|
domain requests, invokes an injected LLM client, validates output, and returns
|
||||||
|
domain results. Transport parsing, response mapping, and public type conversion
|
||||||
|
remain outside this package.
|
||||||
|
|
||||||
It owns request validation, prompt/profile resolution, runtime-parameter merge, artifact loading, prompt rendering, structured-output setup, LLM invocation, output validation, and result metadata.
|
The [configuration reference](../config.md) owns prompt, profile, schema, and
|
||||||
|
runtime-setting definitions. Public error behavior is defined by the
|
||||||
|
[HTTP API](../api.md) and [Go package](../consumers/pkg-scriptorium.md)
|
||||||
|
contracts.
|
||||||
|
|
||||||
## Inputs And Outputs
|
## Dependencies And Construction
|
||||||
|
|
||||||
Primary input type:
|
`Runner` receives these collaborators:
|
||||||
|
|
||||||
- `domain.RunRequest`
|
- `promptdef.Repository`;
|
||||||
|
- `profile.Repository`;
|
||||||
|
- `artifact.Reader`;
|
||||||
|
- `prompt.Renderer`;
|
||||||
|
- `llm.Client`;
|
||||||
|
- `validate.Validator`; and
|
||||||
|
- an optional `OutputRepairer`.
|
||||||
|
|
||||||
Primary output types:
|
`NewRunner` constructs a runner without a repairer. `NewRunnerWithRepairer`
|
||||||
|
accepts one explicitly. Adapters and the public engine choose concrete
|
||||||
- `domain.PreparedRun` from `Prepare`
|
repositories and readers; the runner does not load application configuration.
|
||||||
- `domain.RunResult` from `Run`
|
|
||||||
|
|
||||||
LLM boundary types:
|
|
||||||
|
|
||||||
- `domain.GenerateRequest`
|
|
||||||
- `domain.GenerateResponse`
|
|
||||||
|
|
||||||
## Boundaries
|
|
||||||
|
|
||||||
`Runner` coordinates the following interfaces:
|
|
||||||
|
|
||||||
- `promptdef.Repository`
|
|
||||||
- `profile.Repository`
|
|
||||||
- `artifact.Reader`
|
|
||||||
- `prompt.Renderer`
|
|
||||||
- `llm.Client`
|
|
||||||
- `validate.Validator`
|
|
||||||
- optional `usecase.OutputRepairer`
|
|
||||||
|
|
||||||
Transport concerns (CLI flags, HTTP DTO parsing, status-code mapping) stay outside runner.
|
|
||||||
|
|
||||||
## Config Fields Used
|
|
||||||
|
|
||||||
`Runner` does not read app config files directly.
|
|
||||||
|
|
||||||
It receives fully constructed repositories/readers/validators from adapters. Effective behavior depends on adapter wiring, including:
|
|
||||||
|
|
||||||
- prompt/profile directories
|
|
||||||
- schema base directory
|
|
||||||
- selected profile/runtime overrides in request
|
|
||||||
|
|
||||||
## External Adapters Used
|
|
||||||
|
|
||||||
`Runner` works with adapter implementations via interfaces. Current wiring from CLI/HTTP uses:
|
|
||||||
|
|
||||||
- filesystem prompt/profile repositories
|
|
||||||
- composite artifact reader
|
|
||||||
- Go-template prompt renderer
|
|
||||||
- OpenAI-compatible LLM client
|
|
||||||
- standard validator
|
|
||||||
|
|
||||||
## State And Resume Behavior
|
|
||||||
|
|
||||||
`Runner` is stateless across requests.
|
|
||||||
|
|
||||||
- No durable run-state storage.
|
|
||||||
- No built-in resume/skip checkpoints.
|
|
||||||
- Each `Run`/`Prepare` executes from request inputs and current repositories.
|
|
||||||
|
|
||||||
## Failure Behavior
|
|
||||||
|
|
||||||
Primary runner error classes:
|
|
||||||
|
|
||||||
- `ErrInvalidRequest`: invalid run request envelope.
|
|
||||||
- `ErrProfileRequired`: specific invalid-request reason when neither request `profile_id` nor prompt `default_profile` is available.
|
|
||||||
- `ErrAPIKeyEnvMissing`: specific invalid-request reason when `api_key_env` is set but the named environment variable is unset/empty.
|
|
||||||
- `ErrProfileLoad`: prompt/profile repository load failures.
|
|
||||||
- `ErrArtifactLoad`: artifact read failures.
|
|
||||||
- `ErrPromptRender`: template render failures.
|
|
||||||
- `ErrLLMGenerate`: outbound model request failures.
|
|
||||||
- `ErrValidation`: validation runtime failures (including structured-output schema load/compile failures).
|
|
||||||
|
|
||||||
Reason sentinel behavior:
|
|
||||||
|
|
||||||
- `ErrProfileRequired` and `ErrAPIKeyEnvMissing` are wrapped with `ErrInvalidRequest`.
|
|
||||||
- Adapters can use `errors.Is` for stable reason mapping without matching runner prose.
|
|
||||||
|
|
||||||
Validation content failures are not run errors:
|
|
||||||
|
|
||||||
- `Run` can succeed with `Validation.Status == failed`.
|
|
||||||
- CLI maps this to exit code `2`.
|
|
||||||
- HTTP returns `200` with failed validation details.
|
|
||||||
|
|
||||||
## Prepare Flow
|
## Prepare Flow
|
||||||
|
|
||||||
`Prepare` performs:
|
`Prepare` performs one deterministic preparation pass for a request:
|
||||||
|
|
||||||
1. validate request basics (prompt ID present).
|
1. validate the prompt ID and load the prompt definition;
|
||||||
2. load prompt definition by ID/version.
|
2. hash the definition and select the explicit or default profile;
|
||||||
3. select profile ID:
|
3. load the profile and resolve effective execution settings;
|
||||||
- explicit request profile ID
|
4. validate endpoint, model, and credential availability;
|
||||||
- prompt `default_profile`
|
5. resolve the output contract and, for JSON Schema output, load a structured
|
||||||
- otherwise return an invalid request with `ErrProfileRequired`
|
schema document before model execution;
|
||||||
4. load execution profile.
|
6. read and hash input artifacts;
|
||||||
5. merge effective runtime target:
|
7. render messages and the session ID; and
|
||||||
- built-in execution defaults
|
8. return a `PreparedRun` containing the effective state and rendered-prompt
|
||||||
- selected profile values
|
hash.
|
||||||
- request overrides
|
|
||||||
6. verify required `api_key_env` environment variable:
|
|
||||||
- missing/empty env value returns an invalid request with `ErrAPIKeyEnvMissing`
|
|
||||||
- only the environment-variable name is retained; secret value is never returned
|
|
||||||
7. resolve output contract and structured-output schema payload when `json_schema` mode is active.
|
|
||||||
8. read input artifacts.
|
|
||||||
9. render prompt messages.
|
|
||||||
10. compute prompt/input/render hashes and return `PreparedRun`.
|
|
||||||
|
|
||||||
`Prepare` does not call the LLM.
|
Execution settings merge defaults, profile values, and a request override.
|
||||||
|
Numeric override presence is retained so explicit zero values are not confused
|
||||||
|
with omissions.
|
||||||
|
|
||||||
## Run Flow
|
## Run And Validation Flow
|
||||||
|
|
||||||
`Run` performs:
|
`Run` creates a run ID and timestamps, then calls `Prepare` rather than
|
||||||
|
duplicating preparation. It sends the prepared prompt, effective target,
|
||||||
|
target-presence state, and optional structured-output specification to the LLM
|
||||||
|
client. It converts the returned content to an output artifact, validates it,
|
||||||
|
and returns the artifact, validation, hashes, usage, and timing metadata.
|
||||||
|
|
||||||
1. generate run ID.
|
A validator can return a content result or an operational error. Content
|
||||||
2. call `Prepare`.
|
failures stay in the result; schema loading, compilation, and validator
|
||||||
3. call LLM with prepared messages/effective target/structured-output spec.
|
operational failures are returned as `ErrValidation`. The canonical distinction
|
||||||
4. build output artifact content type from output format.
|
for callers is documented by the public contracts.
|
||||||
5. validate output.
|
|
||||||
6. optionally attempt bounded repair when repairer is injected and contract allows it.
|
|
||||||
7. return `RunResult` with artifact, raw output, validation, hashes, profile/model metadata, usage, and timestamps.
|
|
||||||
|
|
||||||
## Repair Hook Boundary
|
## Repair Boundary
|
||||||
|
|
||||||
Repair attempts occur only when all are true:
|
Repair is an internal optional loop. It starts only when a repairer is present,
|
||||||
|
the output contract permits one or more attempts, validation failed, and the
|
||||||
|
validation mode is JSON or JSON Schema. Each repair receives the previous
|
||||||
|
output, validation errors, effective target, structured-output specification,
|
||||||
|
and attempt metadata; every repaired result is validated again.
|
||||||
|
|
||||||
- repairer is injected
|
`NewDefaultOutputRepairer` delegates to the injected LLM client. CLI, HTTP, and
|
||||||
- `repair_attempts > 0`
|
the public engine use `NewRunner` and therefore do not inject this repairer.
|
||||||
- validation status is `failed`
|
|
||||||
- validation mode is `json` or `json_schema`
|
|
||||||
|
|
||||||
Current production wiring boundary:
|
## Error Translation
|
||||||
|
|
||||||
- CLI and HTTP adapters call `usecase.NewRunner(...)` (no repairer argument).
|
Runner sentinels identify failure categories for adapters:
|
||||||
- Therefore normal CLI/HTTP execution does not perform repair attempts today.
|
|
||||||
|
|
||||||
## Tests To Inspect Before Changing
|
- `ErrInvalidRequest`
|
||||||
|
- `ErrProfileRequired`
|
||||||
|
- `ErrAPIKeyEnvMissing` and `ErrAPIKeyRequired`
|
||||||
|
- `ErrPromptLoad`, `ErrProfileLoad`, and `ErrArtifactLoad`
|
||||||
|
- `ErrPromptRender`
|
||||||
|
- `ErrLLMGenerate`
|
||||||
|
- `ErrValidation`
|
||||||
|
|
||||||
|
Wrap errors with those sentinels and preserve their identities through
|
||||||
|
`errors.Is`; adapters must not classify errors by message text. The runner
|
||||||
|
passes direct keys only to the LLM boundary and never includes resolved key
|
||||||
|
values in prepared or run results.
|
||||||
|
|
||||||
|
## Package-Local Guarantees
|
||||||
|
|
||||||
|
- `Run` always reuses `Prepare`.
|
||||||
|
- Schema documents are loaded before the initial LLM call when structured output
|
||||||
|
is required.
|
||||||
|
- Output validation records attempts used, including repair attempts.
|
||||||
|
- Runner state is per request; the package does not create a durable run store
|
||||||
|
or manifest.
|
||||||
|
- Source, renderer, validator, and LLM implementations remain injected
|
||||||
|
boundaries.
|
||||||
|
|
||||||
|
## Verification And Change Recipe
|
||||||
|
|
||||||
|
Inspect:
|
||||||
|
|
||||||
- `internal/usecase/runner_test.go`
|
- `internal/usecase/runner_test.go`
|
||||||
- `internal/usecase/integration_test.go`
|
- `internal/usecase/integration_test.go`
|
||||||
- `internal/adapter/cli/run_test.go`
|
- `engine_test.go`
|
||||||
- `internal/adapter/http/handler_test.go`
|
|
||||||
|
|
||||||
## Architectural Invariants
|
When changing orchestration:
|
||||||
|
|
||||||
- `Run` reuses `Prepare`; prepare logic is not duplicated.
|
1. identify the collaborator boundary and the affected `Prepare` or `Run` state;
|
||||||
- Effective API-key environment-variable name may appear; resolved secret value must not.
|
2. preserve the `Run`-through-`Prepare` path and error identity;
|
||||||
- Structured-output schema document must load before LLM call for `json_schema` mode.
|
3. add focused runner or integration tests for changed state transitions,
|
||||||
- Repair loops are bounded by `repair_attempts` and repairer presence.
|
validation, or repair behavior; and
|
||||||
- Runner stays transport-agnostic.
|
4. update the owning external contract and any affected source or LLM internal
|
||||||
|
document.
|
||||||
|
|
||||||
|
The [testing policy](../policy/testing.md) owns global test sufficiency.
|
||||||
|
|||||||
97
docs/internal/sources.md
Normal file
97
docs/internal/sources.md
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
# Source Internals
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document describes how source packages load prompt definitions, profiles,
|
||||||
|
schemas, and artifacts. The [configuration reference](../config.md) owns their
|
||||||
|
user-facing formats and settings. The [HTTP API reference](../api.md) owns
|
||||||
|
HTTP-visible artifact outcomes; [operations](../operations.md) owns deployment
|
||||||
|
handling.
|
||||||
|
|
||||||
|
## Prompt Definitions
|
||||||
|
|
||||||
|
`internal/promptdef` provides filesystem and `fs.FS` repositories. Both use
|
||||||
|
`internal/filecatalog` for recursive YAML discovery, deterministic ordering,
|
||||||
|
display paths, and root cleaning.
|
||||||
|
|
||||||
|
Repositories select a prompt by YAML ID and optional version rather than by
|
||||||
|
path. They decode through strict YAML handling, reject duplicate matching
|
||||||
|
definitions, and resolve `content_file` relative to the definition. The `fs.FS`
|
||||||
|
implementation resolves content paths inside its source root; absolute paths and
|
||||||
|
traversal outside that root are rejected before file access.
|
||||||
|
|
||||||
|
## Profiles And Built-Ins
|
||||||
|
|
||||||
|
`internal/profile` provides filesystem, `fs.FS`, and overlay repositories.
|
||||||
|
`internal/profile/builtin` exposes embedded assets through the same repository
|
||||||
|
interface.
|
||||||
|
|
||||||
|
An overlay asks its primary source first. It falls back only when the primary
|
||||||
|
reports `ErrProfileNotFound`; invalid YAML, duplicate IDs, validation failures,
|
||||||
|
and raw-key failures are returned rather than hidden by fallback. This makes a
|
||||||
|
custom ID override a built-in ID while retaining errors in the custom source.
|
||||||
|
|
||||||
|
The public engine can overlay in-memory profiles ahead of both file-backed and
|
||||||
|
built-in repositories. Profile field definitions, validation ranges, and the
|
||||||
|
built-in catalog remain in the [configuration reference](../config.md).
|
||||||
|
|
||||||
|
## Schemas
|
||||||
|
|
||||||
|
`internal/validate` supplies `StandardValidator` for filesystem sources and
|
||||||
|
`FSValidator` for `fs.FS` sources. Directory-backed validation loads the named
|
||||||
|
schema path; it does not search directories by basename. `fs.FS` schema paths
|
||||||
|
are cleaned and checked against their configured root, while a single-file
|
||||||
|
source matches its file base name.
|
||||||
|
|
||||||
|
The runner requests a schema document before generation when it needs
|
||||||
|
structured output. JSON and schema mismatches in generated content are
|
||||||
|
validation results; source access, decoding, registration, and compilation
|
||||||
|
failures are operational errors.
|
||||||
|
|
||||||
|
## Artifacts
|
||||||
|
|
||||||
|
`internal/artifact` composes inline and file readers. The ordinary composite
|
||||||
|
reader used by CLI and the public engine reads file references from the process
|
||||||
|
filesystem. The restricted composite reader used by the HTTP adapter combines
|
||||||
|
inline reading with a rooted file reader and optional byte limit.
|
||||||
|
|
||||||
|
The rooted reader cleans paths and applies lexical containment without resolving
|
||||||
|
symlinks. It checks relative references against the configured root and accepts
|
||||||
|
absolute references only when they remain inside that lexical root. The OS still
|
||||||
|
follows symlinks after that check. The public containment outcome is documented
|
||||||
|
by the [HTTP API reference](../api.md); deployment permissions belong in
|
||||||
|
[operations](../operations.md).
|
||||||
|
|
||||||
|
## Failure Boundaries
|
||||||
|
|
||||||
|
Source packages report repository, decoding, duplicate, validation, and read
|
||||||
|
failures to their callers. They do not select public status codes or response
|
||||||
|
schemas. The runner wraps source failures with use-case categories; adapters map
|
||||||
|
them to their own external contract.
|
||||||
|
|
||||||
|
Source reads use current filesystem or `fs.FS` content for each request. These
|
||||||
|
packages create no manifests, checkpoints, or durable run state.
|
||||||
|
|
||||||
|
## Verification And Change Recipe
|
||||||
|
|
||||||
|
Inspect:
|
||||||
|
|
||||||
|
- `internal/promptdef/repository_test.go`
|
||||||
|
- `internal/profile/repository_test.go`
|
||||||
|
- `internal/profile/builtin/repository_test.go`
|
||||||
|
- `internal/artifact/reader_test.go`
|
||||||
|
- `internal/validate/standard_validator_test.go`
|
||||||
|
- `internal/usecase/integration_test.go`
|
||||||
|
- `engine_test.go`
|
||||||
|
|
||||||
|
When updating prompt, profile, schema, or built-in assets:
|
||||||
|
|
||||||
|
1. keep assets valid for the strict loader and the relevant source boundary;
|
||||||
|
2. update the [configuration reference](../config.md) when a file-format,
|
||||||
|
catalog, or default changes;
|
||||||
|
3. run focused source and integration tests, including the built-in repository
|
||||||
|
test when embedded assets change; and
|
||||||
|
4. update this document when discovery, precedence, containment, or failure
|
||||||
|
mechanics change.
|
||||||
|
|
||||||
|
The [testing policy](../policy/testing.md) owns global test sufficiency.
|
||||||
@@ -1,125 +1,157 @@
|
|||||||
# Operations Guide
|
# Operations Guide
|
||||||
|
|
||||||
## Scope
|
## Scope And References
|
||||||
|
|
||||||
This document covers day-to-day operation of the CLI and HTTP service for currently implemented behavior.
|
This runbook covers deployment, normal operation, capacity planning, and safe
|
||||||
|
recovery for Scriptorium. It does not redefine invocation syntax, configuration
|
||||||
|
fields, or HTTP wire behavior.
|
||||||
|
|
||||||
For command syntax, see [CLI reference](cli.md). For file formats and defaults, see [Configuration reference](config.md).
|
- [CLI reference](cli.md): commands, output destinations, and exit codes.
|
||||||
|
- [Configuration reference](config.md): configuration, prompt/profile/schema
|
||||||
|
formats, defaults, and credentials.
|
||||||
|
- [HTTP API reference](api.md): route, request/response schema, status codes,
|
||||||
|
limits, and HTTP artifact access.
|
||||||
|
- [Consumer integration overview](consumers/api.md): caller responsibilities.
|
||||||
|
|
||||||
## Operational Model
|
## Operational Model And State
|
||||||
|
|
||||||
Scriptorium executes one request at a time per CLI invocation or HTTP request.
|
Scriptorium handles one prompt request for each CLI invocation or HTTP request.
|
||||||
|
It has no durable run store, archive, checkpoint, cache, or resume mechanism.
|
||||||
|
A failed or interrupted request is recovered by correcting its inputs,
|
||||||
|
configuration, or environment and submitting a new request.
|
||||||
|
|
||||||
Important boundaries:
|
Generated artifacts, rendered prompts, model output, and run metadata are
|
||||||
|
caller-owned data. Retention, encryption, backup, and deletion are deployment
|
||||||
|
responsibilities.
|
||||||
|
|
||||||
- No durable run state is stored.
|
## Deploy The Filesystem And Process
|
||||||
- No built-in resume, checkpoint, archive, or backup workflow exists.
|
|
||||||
- Recovery is rerun-based: fix inputs/config, then rerun.
|
|
||||||
|
|
||||||
## Filesystem Layout And Config
|
Provide the process with readable prompt, profile, and schema sources. Keep
|
||||||
|
prompt templates adjacent to the prompt definitions that reference them. For an
|
||||||
|
HTTP deployment that accepts file artifacts, use a dedicated, narrow artifact
|
||||||
|
directory rather than a general-purpose or sensitive filesystem tree.
|
||||||
|
|
||||||
Scriptorium depends on:
|
Run Scriptorium under an identity that can:
|
||||||
|
|
||||||
- prompt definition files (`prompt_dir`)
|
- read only the prompt, profile, schema, and allowed input-artifact paths it
|
||||||
- execution profile files (`profile_dir`)
|
needs;
|
||||||
- optional JSON schemas (`schema_dir`)
|
- read the required credential environment variables without writing them to
|
||||||
|
files or logs; and
|
||||||
|
- write only caller-selected output locations when CLI output files are used.
|
||||||
|
|
||||||
Config discovery order when `--config` is omitted:
|
Do not make the HTTP artifact directory writable by untrusted users. The HTTP
|
||||||
|
artifact containment behavior is lexical and the operating system follows
|
||||||
|
symlinks; account for that when choosing ownership and mount boundaries. See
|
||||||
|
the [HTTP API reference](api.md) for the externally observable behavior.
|
||||||
|
|
||||||
1. `/usr/local/etc/scriptorium/config.yml`
|
## Supply Credentials And Protect Runtime Data
|
||||||
2. `/etc/scriptorium/config.yml`
|
|
||||||
|
|
||||||
If neither exists, built-in defaults are used. If `--config <path>` is provided, that file must exist and parse successfully.
|
Set secret values in the process environment and configure only their
|
||||||
|
environment-variable names. Do not put raw keys in configuration, prompt or
|
||||||
|
profile files, process arguments, HTTP payloads, captured command lines, or
|
||||||
|
debug dumps.
|
||||||
|
|
||||||
Built-in defaults relevant to operations:
|
Treat stdout, stderr, prepared-run output, generated artifacts, and HTTP
|
||||||
|
responses as potentially sensitive. Send service logs to a controlled collector
|
||||||
|
and apply the same retention and access rules as for model input and output.
|
||||||
|
|
||||||
- `schema_dir: .`
|
## Run A Normal Workflow
|
||||||
- `server.addr: :8080`
|
|
||||||
- `defaults.render_format: text`
|
|
||||||
|
|
||||||
## Normal CLI Workflow
|
Before changing production inputs, profiles, or schemas:
|
||||||
|
|
||||||
Use `render` first when you need to verify prompt resolution and runtime settings without calling a model.
|
1. confirm the deployed configuration selects the intended sources and model
|
||||||
|
credentials;
|
||||||
|
2. use [`render`](cli.md) with the same request inputs and variables to confirm
|
||||||
|
preparation without a model call;
|
||||||
|
3. use [`run`](cli.md) for generation; and
|
||||||
|
4. retain or discard validation-failed output according to the caller's
|
||||||
|
policy.
|
||||||
|
|
||||||
Use `run` for generation.
|
The [maintained render script](../examples/render-markdown-summary.sh) is a
|
||||||
|
copyable preflight example. The CLI reference owns its complete invocation and
|
||||||
|
exit semantics.
|
||||||
|
|
||||||
Typical sequence:
|
## Expose The HTTP Service
|
||||||
|
|
||||||
1. Confirm prompt/profile directories resolve through config or flags.
|
The HTTP service has no built-in authentication or authorization. Place it on a
|
||||||
2. Confirm required input files exist and map to prompt input names.
|
trusted network or behind an authenticated reverse proxy, API gateway, or
|
||||||
3. Confirm required API-key environment variables are set.
|
equivalent access control. Restrict who can reach it and who can read the
|
||||||
4. Confirm the selected profile's model endpoint is reachable from the process environment.
|
artifact root.
|
||||||
5. Run `render` for preflight when changing prompt/profile/input wiring.
|
|
||||||
6. Run `run` for actual generation.
|
|
||||||
|
|
||||||
## Secrets Handling
|
Use a service manager or supervisor appropriate to the deployment to manage
|
||||||
|
process lifetime, restart policy, log capture, and environment injection. The
|
||||||
|
[HTTP API reference](api.md) owns client request shapes, status behavior, and
|
||||||
|
artifact-access outcomes.
|
||||||
|
|
||||||
Raw API keys are not accepted in config files, profile files as `api_key`, CLI flags, or HTTP request bodies.
|
## Plan Capacity And Limits
|
||||||
|
|
||||||
Operational pattern:
|
Capacity is primarily determined by concurrent model calls, input and output
|
||||||
|
sizes, schema complexity, provider latency, and network behavior. Size limits
|
||||||
|
protect request bodies, HTTP file artifacts, and encoded responses; configure
|
||||||
|
them through the [configuration reference](config.md) and rely on the
|
||||||
|
[HTTP API reference](api.md) for their response effects.
|
||||||
|
|
||||||
- Set environment variables that hold secret values.
|
Before increasing a limit:
|
||||||
- Set profile `api_key_env` (or runtime override `api_key_env`) to the environment variable name.
|
|
||||||
- Keep process environments scoped to only required variables.
|
|
||||||
|
|
||||||
## HTTP Service Operation
|
1. measure representative input, generated-output, and optional raw-output
|
||||||
|
sizes;
|
||||||
|
2. confirm memory, network, and upstream-provider capacity;
|
||||||
|
3. retain an upstream request-size and authentication boundary; and
|
||||||
|
4. test the intended workload in a non-production environment.
|
||||||
|
|
||||||
Start service with:
|
For large local inputs, prefer a controlled file-artifact directory over
|
||||||
|
placing arbitrary paths on the service host. Avoid disabling a limit unless an
|
||||||
|
equivalent trusted control exists elsewhere.
|
||||||
|
|
||||||
```bash
|
## Diagnose And Recover
|
||||||
go run ./cmd/scriptorium serve --config ./examples/config.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
Current inbound API behavior:
|
### Preparation Or Configuration Failure
|
||||||
|
|
||||||
- Route: `POST /v1/runs`
|
Capture the CLI diagnostic or HTTP error response, then verify the selected
|
||||||
- JSON request parsing rejects unknown fields.
|
configuration, prompt ID, profile selection, source readability, and input
|
||||||
- Validation content failures still return `200 OK` with `validation.status: "failed"`.
|
mapping. Use `render` with the same request when it is unclear whether failure
|
||||||
|
occurs before model execution. Consult the [CLI reference](cli.md), the
|
||||||
|
[configuration reference](config.md), and the [HTTP API reference](api.md) for
|
||||||
|
the exact interface contract.
|
||||||
|
|
||||||
Security caveat:
|
### Credential Or Provider Failure
|
||||||
|
|
||||||
- `serve` has no built-in authentication or authorization.
|
Confirm that the process environment contains the configured credential name
|
||||||
- Deploy only behind trusted controls (private network boundary, authenticated reverse proxy, API gateway, or equivalent).
|
without printing the secret. Check endpoint reachability and provider health
|
||||||
|
from the process network. If preparation succeeds but generation fails, inspect
|
||||||
|
the selected model settings in prepared output and the service's controlled
|
||||||
|
logs. Correct the deployment or provider issue, then submit a new request.
|
||||||
|
|
||||||
## Output, Logs, And Exit Codes
|
### Artifact Or Permission Failure
|
||||||
|
|
||||||
`run` command:
|
Verify that the process can read the intended local input. For HTTP file
|
||||||
|
artifacts, verify the deployment's artifact root, ownership, path layout, and
|
||||||
|
file size. Do not widen filesystem permissions or the allowed root merely to
|
||||||
|
make an arbitrary path work; move or copy the required artifact into the
|
||||||
|
controlled location instead.
|
||||||
|
|
||||||
- Generated artifact body goes to stdout by default.
|
### Validation Failure
|
||||||
- `--out` writes generated artifact to a file.
|
|
||||||
- Summary metadata line is written to stderr on success.
|
|
||||||
- Exit code `2` means generation completed but validation failed.
|
|
||||||
|
|
||||||
`render` command:
|
A generated-content validation failure is distinct from a runtime failure.
|
||||||
|
CLI `run` reports the validation result and error count in its success summary;
|
||||||
|
it does not print the individual validation messages. For HTTP, inspect the
|
||||||
|
validation object in the response according to the [HTTP API reference](api.md).
|
||||||
|
|
||||||
- Prepared-run output goes to stdout by default.
|
Use rendered input and generated output to determine whether prompt instructions,
|
||||||
- `--out` writes prepared-run output to a file.
|
the selected model, or the schema needs correction. If schema loading or
|
||||||
- Exit code is `0` on success and `1` on failure.
|
compilation itself fails, correct the source deployment or schema document
|
||||||
|
before rerunning.
|
||||||
|
|
||||||
`serve` command:
|
### HTTP Limit Or Request Failure
|
||||||
|
|
||||||
- Startup and server errors are written to stderr.
|
Compare the request, artifact, or expected response size with the deployed
|
||||||
|
configuration, and validate the request against the [HTTP API reference](api.md).
|
||||||
|
Reduce the payload, use an appropriate controlled artifact source, omit
|
||||||
|
unneeded raw output, or adjust the deployment limit after capacity review.
|
||||||
|
|
||||||
## Validation Behavior In Operations
|
## Cleanup And Reruns
|
||||||
|
|
||||||
Validation modes (`none`, `basic`, `json`, `json_schema`) are defined by prompt output contract.
|
Because no run state is retained, cleanup concerns caller-owned output files,
|
||||||
|
logs, and artifacts only. Remove or rotate them using the deployment's normal
|
||||||
Operational interpretation:
|
retention policy. After a correction, rerun the request from the beginning;
|
||||||
|
there is no safe resume point.
|
||||||
- Validation runtime errors are hard failures (`run` exit `1`; HTTP error response).
|
|
||||||
- Validation content failures are soft failures (`run` exit `2`; HTTP `200` with failed status).
|
|
||||||
|
|
||||||
A failed validation run can still produce output. Decide whether to keep or discard that output in your surrounding workflow.
|
|
||||||
|
|
||||||
## Safe Recovery Steps
|
|
||||||
|
|
||||||
For failed runs or requests:
|
|
||||||
|
|
||||||
1. Capture stderr output or HTTP error code/message.
|
|
||||||
2. Confirm config path and directory settings.
|
|
||||||
3. Verify prompt/profile IDs and input mappings.
|
|
||||||
4. Verify API-key environment-variable presence when required.
|
|
||||||
5. Reproduce with `render --format json` when prompt/profile/input resolution is uncertain.
|
|
||||||
6. Rerun after correction.
|
|
||||||
|
|
||||||
Because Scriptorium does not persist run state, rerun is the canonical recovery path.
|
|
||||||
|
|||||||
@@ -4,13 +4,12 @@ This document is the development architecture policy for Scriptorium.
|
|||||||
|
|
||||||
It is for developers and LLM coding agents. User-facing behavior belongs in `README.md` and the docs under `docs/` that target operators/users.
|
It is for developers and LLM coding agents. User-facing behavior belongs in `README.md` and the docs under `docs/` that target operators/users.
|
||||||
|
|
||||||
## Project Shape
|
## System Shape
|
||||||
|
|
||||||
Scriptorium is a narrow prompt-execution application with three entry paths:
|
Scriptorium is a narrow prompt-execution application with three executable
|
||||||
|
entry paths: CLI `run`, CLI `render`, and the HTTP service started by `serve`.
|
||||||
- CLI `run`
|
It also provides a public Go package for in-process use. Its current component
|
||||||
- CLI `render`
|
inventory is maintained in the [internal overview](../internal/overview.md).
|
||||||
- HTTP `POST /v1/runs` through `serve`
|
|
||||||
|
|
||||||
Domain behavior is centralized in `internal/usecase` and `internal/domain`.
|
Domain behavior is centralized in `internal/usecase` and `internal/domain`.
|
||||||
|
|
||||||
@@ -19,44 +18,21 @@ Domain behavior is centralized in `internal/usecase` and `internal/domain`.
|
|||||||
- Keep orchestration narrow: Scriptorium executes one prompt request; it is not a multi-step workflow engine.
|
- Keep orchestration narrow: Scriptorium executes one prompt request; it is not a multi-step workflow engine.
|
||||||
- Keep adapter logic thin: adapters map external shapes to domain requests/results and should not hold domain decisions.
|
- Keep adapter logic thin: adapters map external shapes to domain requests/results and should not hold domain decisions.
|
||||||
- Keep boundaries explicit: repositories/loaders/renderers/validators/LLM client stay behind package interfaces.
|
- Keep boundaries explicit: repositories/loaders/renderers/validators/LLM client stay behind package interfaces.
|
||||||
- Keep config strict: YAML/JSON decoding for external inputs should reject unknown fields.
|
- Keep external decoding strict: configuration, prompt, and profile YAML and
|
||||||
|
HTTP JSON should reject unknown fields.
|
||||||
- Keep secrets out of payloads: raw API key values must not be accepted or emitted.
|
- Keep secrets out of payloads: raw API key values must not be accepted or emitted.
|
||||||
|
|
||||||
## Package Boundaries
|
## Dependency Direction
|
||||||
|
|
||||||
Current package map:
|
- Adapters translate external shapes and IO concerns; they do not make
|
||||||
|
use-case decisions.
|
||||||
- `cmd/scriptorium`: process entrypoint.
|
- Use-case and domain code depend on explicit repository, renderer, validator,
|
||||||
- `internal/adapter/cli`: command parsing, app wiring for CLI commands, output behavior.
|
and LLM interfaces rather than adapter implementations.
|
||||||
- `internal/adapter/http`: HTTP DTO mapping and error/status mapping.
|
- Source, rendering, validation, and LLM implementations remain behind their
|
||||||
- `internal/config`: application settings loading and CLI override precedence.
|
package boundaries.
|
||||||
- `internal/defaults`: compile-time default constants.
|
- Dependency-specific types must not leak across unrelated package boundaries.
|
||||||
- `internal/domain`: core request/result and contract types.
|
- Prefer the standard library; add an external dependency only when it
|
||||||
- `internal/usecase`: `Runner` prepare/run orchestration and repair-hook boundary.
|
materially reduces risk or complexity.
|
||||||
- `internal/promptdef`: filesystem prompt-definition repository.
|
|
||||||
- `internal/profile`: filesystem execution-profile repository.
|
|
||||||
- `internal/artifact`: artifact reference readers.
|
|
||||||
- `internal/prompt`: template renderer.
|
|
||||||
- `internal/llm`: provider-neutral LLM client interface and OpenAI-compatible implementation.
|
|
||||||
- `internal/validate`: validator interfaces and standard implementation.
|
|
||||||
- `internal/format`: prepared-run output formatting.
|
|
||||||
|
|
||||||
Detailed component behavior is documented in:
|
|
||||||
|
|
||||||
- `docs/internal/runner.md`
|
|
||||||
- `docs/internal/adapters.md`
|
|
||||||
|
|
||||||
## Configuration And Precedence
|
|
||||||
|
|
||||||
Application settings are resolved as:
|
|
||||||
|
|
||||||
1. built-in defaults
|
|
||||||
2. config file values
|
|
||||||
3. CLI overrides
|
|
||||||
|
|
||||||
`config.yml` is for application wiring (directories, server address, render default format), not prompt/profile runtime execution settings.
|
|
||||||
|
|
||||||
Profile selection and runtime model resolution remain use-case concerns.
|
|
||||||
|
|
||||||
## State And Persistence Policy
|
## State And Persistence Policy
|
||||||
|
|
||||||
@@ -65,43 +41,31 @@ Scriptorium has no durable run-state store.
|
|||||||
- No built-in resume/checkpoint/archive behavior.
|
- No built-in resume/checkpoint/archive behavior.
|
||||||
- Recovery model is rerun after correcting inputs/config/environment.
|
- Recovery model is rerun after correcting inputs/config/environment.
|
||||||
|
|
||||||
## External Integration Policy
|
## Contract Ownership
|
||||||
|
|
||||||
Current external contracts:
|
The [CLI](../cli.md), [configuration](../config.md), [HTTP API](../api.md),
|
||||||
|
[public Go package](../consumers/pkg-scriptorium.md), and
|
||||||
- inbound HTTP contract: `POST /v1/runs`
|
[integration](../integrations/) documents own their respective external
|
||||||
- outbound model contract: OpenAI-compatible chat completions subset
|
contracts. This policy keeps only the architectural boundaries that govern
|
||||||
- subprocess contract for integrators: CLI `run`/`render`
|
their implementation.
|
||||||
|
|
||||||
Integration docs belong under `docs/integrations/`.
|
|
||||||
|
|
||||||
## Error Handling And Logging
|
## Error Handling And Logging
|
||||||
|
|
||||||
- Wrap errors with domain/operation context.
|
- Wrap errors with domain/operation context.
|
||||||
- Map domain errors to adapter-appropriate statuses/codes without leaking sensitive internals.
|
- Map domain errors to adapter-appropriate statuses/codes without leaking sensitive internals.
|
||||||
- Keep stderr summaries concise for CLI success/error paths.
|
|
||||||
- Never emit raw secret values.
|
- Never emit raw secret values.
|
||||||
|
|
||||||
## Testing Expectations
|
## Testing And Documentation
|
||||||
|
|
||||||
- Core runner behavior should be covered with isolated unit tests and fixture-based integration tests.
|
Testing philosophy and change-validation expectations are defined by the
|
||||||
- Adapter behavior should be tested for parse/mapping/error semantics.
|
[testing policy](testing.md). Documentation ownership and maintenance rules are
|
||||||
- Config parsing, prompt/profile loading, validator behavior, and LLM client error handling should remain covered by package tests.
|
defined by the [documentation policy](documentation.md).
|
||||||
- Repository-level docs/examples that claim runnable behavior should be validated by tests or smoke commands.
|
|
||||||
|
|
||||||
## Documentation Expectations
|
|
||||||
|
|
||||||
- Document implemented behavior only outside `docs/roadmap/`.
|
|
||||||
- Keep canonical reference locations stable (`docs/cli.md`, `docs/config.md`, `docs/operations.md`, `docs/troubleshooting.md`, `docs/internal/`).
|
|
||||||
- Update docs in the same change when architecture-relevant behavior changes.
|
|
||||||
|
|
||||||
## Architectural Invariants
|
## Architectural Invariants
|
||||||
|
|
||||||
- `Runner.Run` reuses `Runner.Prepare` flow.
|
- `Runner.Run` reuses `Runner.Prepare` flow.
|
||||||
- CLI and HTTP currently instantiate `Runner` without a repairer.
|
- Raw API key values must not be accepted through external configuration or
|
||||||
- Artifact reading supports `inline` and `file` references.
|
request payloads, and resolved secret values must not be emitted.
|
||||||
- Unknown input fields in config/prompt/profile/http JSON should be rejected by strict decoding.
|
|
||||||
- Raw API key values must not be accepted through config/HTTP payloads.
|
|
||||||
|
|
||||||
## Non-Goals
|
## Non-Goals
|
||||||
|
|
||||||
|
|||||||
@@ -1,103 +0,0 @@
|
|||||||
# Development Guide
|
|
||||||
|
|
||||||
This document defines contributor workflow for Scriptorium.
|
|
||||||
|
|
||||||
## Repository Layout
|
|
||||||
|
|
||||||
- `cmd/scriptorium`: application entrypoint.
|
|
||||||
- `internal/domain`: core contracts.
|
|
||||||
- `internal/usecase`: runner orchestration.
|
|
||||||
- `internal/adapter/cli`: CLI adapter.
|
|
||||||
- `internal/adapter/http`: HTTP adapter.
|
|
||||||
- `internal/config`: application settings loading and precedence.
|
|
||||||
- `internal/defaults`: default constants.
|
|
||||||
- `internal/promptdef`: prompt-definition repository.
|
|
||||||
- `internal/profile`: execution-profile repository.
|
|
||||||
- `internal/artifact`: artifact readers.
|
|
||||||
- `internal/prompt`: prompt rendering.
|
|
||||||
- `internal/llm`: LLM client interface and OpenAI-compatible implementation.
|
|
||||||
- `internal/validate`: validation interfaces and implementation.
|
|
||||||
- `internal/format`: prepared-run formatting.
|
|
||||||
- `docs/`: canonical documentation.
|
|
||||||
- `examples/`: copyable maintained examples and fixtures.
|
|
||||||
|
|
||||||
## Common Commands
|
|
||||||
|
|
||||||
Build:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go build ./cmd/scriptorium
|
|
||||||
```
|
|
||||||
|
|
||||||
Test:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go test ./...
|
|
||||||
```
|
|
||||||
|
|
||||||
Targeted test runs commonly used during changes:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go test ./internal/adapter/cli ./internal/adapter/http ./internal/usecase
|
|
||||||
```
|
|
||||||
|
|
||||||
## Coding Conventions
|
|
||||||
|
|
||||||
- Prefer small interfaces at package boundaries.
|
|
||||||
- Keep adapter packages focused on translation and IO concerns.
|
|
||||||
- Keep domain/use-case logic outside adapters.
|
|
||||||
- Wrap errors with operation context.
|
|
||||||
- Use strict decoding for user-provided YAML/JSON where applicable.
|
|
||||||
- Avoid introducing dependencies unless they materially reduce risk/complexity.
|
|
||||||
|
|
||||||
## Dependency Policy
|
|
||||||
|
|
||||||
- Prefer standard library unless an external library is clearly justified.
|
|
||||||
- Current non-stdlib dependencies are intentionally small:
|
|
||||||
- `gopkg.in/yaml.v3` for YAML decoding.
|
|
||||||
- `github.com/santhosh-tekuri/jsonschema/v6` for JSON Schema validation.
|
|
||||||
- Do not leak dependency-specific types across unrelated package boundaries.
|
|
||||||
|
|
||||||
## How To Add App Config Fields
|
|
||||||
|
|
||||||
1. Add fields in `internal/config/config.go` (`Config`, `AppSettings`, and/or `CLIOverrides` as needed).
|
|
||||||
2. Apply defaults in `BuiltInDefaults()` when required.
|
|
||||||
3. Parse and validate in `applyConfig` / `ApplyCLIOverrides`.
|
|
||||||
4. Wire the field through the consuming adapter(s).
|
|
||||||
5. Add/update config tests in `internal/config/config_test.go`.
|
|
||||||
6. Update canonical docs (`docs/config.md`, and other affected docs).
|
|
||||||
|
|
||||||
## How To Add CLI Flags
|
|
||||||
|
|
||||||
1. Add flags in `internal/adapter/cli/run.go` for the relevant command.
|
|
||||||
2. Ensure precedence behavior remains consistent with app config rules.
|
|
||||||
3. Keep `run`, `render`, and `serve` flag surfaces intentionally scoped.
|
|
||||||
4. Add/update parser and command tests in `internal/adapter/cli/run_test.go`.
|
|
||||||
5. Update `docs/cli.md` and any related docs/examples.
|
|
||||||
|
|
||||||
## How To Add Adapters Or Adapter Capabilities
|
|
||||||
|
|
||||||
1. Define or reuse the appropriate interface boundary in domain/use-case packages.
|
|
||||||
2. Implement adapter code under `internal/adapter/<name>` (or relevant boundary package).
|
|
||||||
3. Keep business decisions in `internal/usecase`.
|
|
||||||
4. Add focused adapter tests for mapping, parse, and error behavior.
|
|
||||||
5. Document the new/changed boundary in `docs/internal/adapters.md`.
|
|
||||||
6. If external contract changes, update `docs/integrations/` in the same change.
|
|
||||||
|
|
||||||
## How To Update Prompt/Profile/Schema Assets
|
|
||||||
|
|
||||||
1. Keep prompt/profile/schema files valid under strict loaders.
|
|
||||||
2. Keep examples secret-free.
|
|
||||||
3. Re-run tests that cover prompt/profile/validation behavior.
|
|
||||||
4. Update `docs/config.md` and any docs that reference changed contracts.
|
|
||||||
|
|
||||||
## Documentation Update Expectations
|
|
||||||
|
|
||||||
When behavior changes:
|
|
||||||
|
|
||||||
1. Update canonical doc locations, not duplicate files.
|
|
||||||
2. Keep non-roadmap docs limited to implemented behavior.
|
|
||||||
3. Update links after file moves/renames.
|
|
||||||
4. Re-run relevant tests and smoke commands.
|
|
||||||
|
|
||||||
Docs work is complete only when code/tests/examples/docs agree.
|
|
||||||
@@ -1,356 +1,163 @@
|
|||||||
# Go Project Documentation Policy
|
# Documentation Policy
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
Project documentation must help four audiences:
|
This policy assigns each documentation topic to one canonical owner. Its goal is
|
||||||
|
to keep this repository's documentation accurate, concise, discoverable, and
|
||||||
1. users who need to run the application;
|
resistant to drift for users, operators, developers, integrators, and LLM
|
||||||
2. administrators/operators who need to configure and operate it;
|
coding agents.
|
||||||
3. developers who need to understand and change it safely;
|
|
||||||
4. LLM coding agents that need clear scope, boundaries, and invariants.
|
|
||||||
|
|
||||||
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
|
||||||
|
|
||||||
## Core Rules
|
## Core Rules
|
||||||
|
|
||||||
### 1. Keep docs concise
|
### One Canonical Owner
|
||||||
|
|
||||||
Each document should cover a defined scope and only the essentials for that scope.
|
Each authoritative fact belongs in one document. A non-owning document may give
|
||||||
|
a short, stable summary for orientation, but it must link to the canonical owner
|
||||||
|
instead of repeating volatile details.
|
||||||
|
|
||||||
Avoid:
|
Volatile details include commands, flags, configuration fields and defaults,
|
||||||
- long background explanations;
|
module keys, schemas, file names, paths, status codes, retry behavior, and
|
||||||
- repeated reference material;
|
runtime guarantees. If readers could reasonably treat a statement as a
|
||||||
- implementation detail in user-facing docs;
|
contract, maintain it only in the owning document.
|
||||||
- aspirational language outside roadmap docs;
|
|
||||||
- verbose examples where one minimal example is clearer.
|
|
||||||
|
|
||||||
### 2. Document only implemented behavior outside roadmap files
|
Minimal tested usage examples are allowed outside the owning contract when this
|
||||||
|
policy assigns them an orientation or instructional purpose. They must link to
|
||||||
|
the canonical contract and must not redefine complete syntax, defaults, or
|
||||||
|
semantics.
|
||||||
|
|
||||||
|
### Current And Future Behavior
|
||||||
|
|
||||||
|
Outside `docs/roadmap/`, documentation describes implemented behavior only.
|
||||||
|
Partial features may be described only to their implemented boundary.
|
||||||
|
|
||||||
Unimplemented, planned, aspirational, experimental, or future work may be described only under:
|
ADRs are the narrow exception: an ADR may record an accepted architectural
|
||||||
|
decision before implementation, but acceptance must not be presented as proof
|
||||||
|
that the behavior exists. The roadmap owns implementation status and sequencing
|
||||||
|
until the decision is implemented. Current architecture, user, operator,
|
||||||
|
integration, and internal documentation are updated when the behavior lands.
|
||||||
|
|
||||||
- `docs/roadmap/`
|
### Audience And Detail
|
||||||
|
|
||||||
|
Write for the document's stated audience and include only the detail needed for
|
||||||
|
its owned topic. User and operator docs should not expose implementation detail.
|
||||||
|
Developer docs should link to user-facing and external contracts rather than
|
||||||
|
restate them.
|
||||||
|
|
||||||
|
### Examples
|
||||||
|
|
||||||
|
Complete copyable files belong in `examples/`. Documentation may use the
|
||||||
|
smallest illustrative snippet needed to explain its owned topic, but should link
|
||||||
|
to maintained examples instead of embedding a second complete copy.
|
||||||
|
|
||||||
|
Examples must be valid, secret-free, and tested where practical. Commands and
|
||||||
|
configuration used in documentation should match the application.
|
||||||
|
|
||||||
No other documentation file, including `README.md`, should describe code, features, modules, stages, commands, config fields, or behaviors that do not currently exist.
|
### Security And Privacy
|
||||||
|
|
||||||
If a feature is partial, non-roadmap docs may describe only the implemented portion and its current boundary.
|
Documentation and examples must not contain real credentials, private keys,
|
||||||
|
private environment dumps, sensitive source material, or private infrastructure
|
||||||
### 3. Use canonical homes
|
details unless intentionally public. Document secret-handling mechanisms, not
|
||||||
|
secret values.
|
||||||
Each type of information should have one canonical location.
|
|
||||||
|
## Canonical Ownership
|
||||||
Canonical homes:
|
|
||||||
|
| Topic | Canonical owner | Owned content | Content owned elsewhere |
|
||||||
- project purpose and quickstart: `README.md`
|
| --- | --- | --- | --- |
|
||||||
- development principles: `docs/policy/architecture.md`
|
| Product orientation and minimal end-to-end quickstart | `README.md` | What this project is, why it is useful, one shortest successful invocation, and links onward. | Complete command reference, configuration reference, operational procedures, implementation detail. |
|
||||||
- configuration reference: `docs/config.md`
|
| Contributor entry point | `docs/development.md` | Task-oriented reading guide, minimal contributor orientation, baseline validation commands, and links to canonical docs. | Package inventory, architecture rules, subsystem behavior, and detailed change recipes, which belong in the relevant internal component document. |
|
||||||
- CLI reference: `docs/cli.md`
|
| Current application architecture | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, architectural boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
|
||||||
- operations and recovery: `docs/operations.md`
|
| Documentation organization | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
|
||||||
- troubleshooting: `docs/troubleshooting.md`
|
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression-test policy, and criteria for adding, rewriting, or deleting tests. | Subsystem behavior, application contracts, subsystem-specific test inventories, and implementation plans. |
|
||||||
- implemented internals: `docs/internal/`
|
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
|
||||||
- future work: `docs/roadmap/`
|
| Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. |
|
||||||
- contributor workflow: `docs/policy/development.md`
|
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
|
||||||
- copyable examples: `examples/`
|
| Public HTTP contract | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
|
||||||
|
| Consumer guidance | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
|
||||||
Other files should summarize briefly and link to the canonical source.
|
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
||||||
|
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
||||||
### 4. Keep examples real
|
| Internal component behavior | Other files under `docs/internal/` | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
||||||
|
| Architectural decision history | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
|
||||||
Examples should be valid, maintained, and free of secrets.
|
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status; sequencing; and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
||||||
|
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other files intended to be copied or run. | Field-by-field reference, command reference, prose explanation. |
|
||||||
Where practical:
|
|
||||||
- example configs should load successfully;
|
Documents that do not exist are required only when the corresponding interface
|
||||||
- example commands should match real CLI syntax;
|
or responsibility exists. Do not create placeholder API, consumer, integration,
|
||||||
- important examples should be covered by tests.
|
or operations documents for behavior the application does not have.
|
||||||
|
|
||||||
## Documentation Profiles
|
## Boundary Rules
|
||||||
|
|
||||||
All projects require:
|
### Orientation
|
||||||
|
|
||||||
- `README.md`
|
The README owns product orientation. The developer guide routes contributors.
|
||||||
- `docs/policy/architecture.md`
|
Architecture owns normative structure. Internal overview owns the current
|
||||||
|
concrete component map. These documents may link to one another but should not
|
||||||
Additional docs depend on the project.
|
maintain parallel package or behavior descriptions.
|
||||||
|
|
||||||
### Small library
|
### Commands, Configuration, And Operations
|
||||||
|
|
||||||
Recommended:
|
CLI documentation answers how to invoke the application. Configuration
|
||||||
- `docs/policy/development.md`, if contributor conventions are non-obvious
|
documentation answers what settings mean. Operations answers what happens to
|
||||||
|
runtime state and how to operate or recover the application. When a workflow
|
||||||
### Simple CLI
|
crosses these topics, choose the document that owns the task and link to the
|
||||||
|
other contracts.
|
||||||
Required:
|
|
||||||
- `docs/cli.md`
|
### Contracts And Implementation
|
||||||
|
|
||||||
Recommended:
|
Integration and API documents define externally observable shapes and
|
||||||
- `docs/policy/development.md`
|
semantics. Internal documents explain how this project implements or consumes
|
||||||
|
those contracts. Internal docs may name a field, file, or protocol to identify
|
||||||
### Config-driven CLI
|
a dependency, but must link to its canonical contract for the definition.
|
||||||
|
|
||||||
Required:
|
### Security Topics
|
||||||
- `docs/cli.md`
|
|
||||||
- `docs/config.md`
|
This policy owns what documentation and examples may contain. Architecture owns
|
||||||
|
application security invariants. Configuration owns credential-supply
|
||||||
Recommended:
|
mechanisms. Operations owns permissions and handling of sensitive runtime
|
||||||
- `examples/`
|
artifacts. Internal docs own implementation mechanisms only.
|
||||||
- `docs/policy/development.md`
|
|
||||||
|
## Architecture Decision Records
|
||||||
### Stateful or operator-facing application
|
|
||||||
|
Use sequentially numbered ADR filenames such as
|
||||||
Required:
|
`0001-record-architecture-decisions.md`. Follow the lightweight Nygard format:
|
||||||
- `docs/cli.md`, if CLI-based
|
|
||||||
- `docs/config.md`, if config-driven
|
1. title;
|
||||||
- `docs/operations.md`
|
2. status;
|
||||||
|
3. date;
|
||||||
Recommended:
|
4. context;
|
||||||
- `docs/troubleshooting.md`
|
5. decision;
|
||||||
- `examples/`
|
6. alternatives considered;
|
||||||
- `docs/policy/development.md`
|
7. consequences.
|
||||||
|
|
||||||
### Modular, staged, service-oriented, or orchestration application
|
Use one of these statuses:
|
||||||
|
|
||||||
Required:
|
- **Proposed:** the decision is under consideration and may change;
|
||||||
- `docs/cli.md`, if CLI-based
|
- **Accepted:** the decision is approved, whether or not implementation is
|
||||||
- `docs/config.md`, if config-driven
|
complete;
|
||||||
- `docs/operations.md`
|
- **Rejected:** the proposed decision was considered and not adopted;
|
||||||
- `docs/internal/`
|
- **Superseded:** a later ADR replaces the accepted decision.
|
||||||
- `docs/policy/development.md`
|
|
||||||
|
A proposed ADR transitions to accepted or rejected. An accepted ADR transitions
|
||||||
Recommended:
|
to superseded only when a later accepted ADR replaces it. An ADR may be created
|
||||||
- `docs/troubleshooting.md`
|
as accepted when the decision has already been made.
|
||||||
- validated examples under `examples/`
|
|
||||||
|
Treat the decision content of an accepted ADR as immutable. Its status and
|
||||||
## Required Documents
|
supersession metadata may be updated, but a changed decision requires a new ADR.
|
||||||
|
A superseded ADR must link to its replacement, and the replacement must link
|
||||||
### README.md
|
back to the superseded ADR. Rejected architectural alternatives belong in the
|
||||||
|
ADR; rejected product ideas belong in the roadmap.
|
||||||
**Audience:** users, administrators, operators
|
|
||||||
|
## Maintenance
|
||||||
The README is the outward-facing project orientation page.
|
|
||||||
|
When behavior changes, update its canonical owner in the same change. If
|
||||||
It should include, in order:
|
ownership moves, remove the old definition and replace it with a link where
|
||||||
|
navigation remains useful.
|
||||||
1. concise description;
|
|
||||||
2. elevator pitch;
|
Before completing documentation work:
|
||||||
3. shortest useful command or usage example;
|
|
||||||
4. links to targeted docs.
|
- verify affected behavior and examples;
|
||||||
|
- check commands, flags, fields, defaults, schemas, and paths against their
|
||||||
The README should be short. It is not a manual.
|
implementation;
|
||||||
|
- keep unimplemented behavior in the roadmap, subject to the ADR exception;
|
||||||
The “shortest useful command” means the simplest command that performs the project’s core use case. (It does not mean `app --help`.)
|
- remove stale references and validate links;
|
||||||
|
- confirm that non-owning documents summarize and link rather than redefine;
|
||||||
### docs/policy/architecture.md
|
- confirm that no secrets or sensitive private data were added.
|
||||||
|
|
||||||
**Audience:** developers, LLM coding agents
|
|
||||||
|
|
||||||
`docs/policy/architecture.md` is required for every project.
|
|
||||||
|
|
||||||
It is an inward-facing development policy document. It should describe how the project is intended to be built and changed.
|
|
||||||
|
|
||||||
It should include:
|
|
||||||
|
|
||||||
- project shape;
|
|
||||||
- core design principles;
|
|
||||||
- package and boundary philosophy;
|
|
||||||
- state/persistence philosophy, if applicable;
|
|
||||||
- external integration philosophy, if applicable;
|
|
||||||
- error-handling and logging principles;
|
|
||||||
- testing expectations;
|
|
||||||
- documentation expectations;
|
|
||||||
- architectural invariants;
|
|
||||||
- explicit non-goals, if useful.
|
|
||||||
|
|
||||||
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
|
|
||||||
|
|
||||||
### docs/policy/development.md
|
|
||||||
|
|
||||||
**Audience:** developers, LLM coding agents
|
|
||||||
|
|
||||||
Required for projects maintained by humans and LLM coding agents.
|
|
||||||
|
|
||||||
It should include:
|
|
||||||
|
|
||||||
- repository layout;
|
|
||||||
- build/test commands;
|
|
||||||
- coding conventions;
|
|
||||||
- dependency policy;
|
|
||||||
- how to add config fields;
|
|
||||||
- how to add CLI flags;
|
|
||||||
- how to add stages/modules/adapters, if applicable;
|
|
||||||
- how to update examples;
|
|
||||||
- documentation update expectations.
|
|
||||||
|
|
||||||
### docs/config.md
|
|
||||||
|
|
||||||
**Audience:** administrators, operators, advanced users
|
|
||||||
|
|
||||||
Required for applications with configuration files.
|
|
||||||
|
|
||||||
It should include, in order:
|
|
||||||
|
|
||||||
1. config file locations and discovery precedence;
|
|
||||||
2. minimal working config;
|
|
||||||
3. production-oriented config;
|
|
||||||
4. full configuration reference;
|
|
||||||
5. secrets handling, if applicable;
|
|
||||||
6. links to maintained examples.
|
|
||||||
|
|
||||||
The full configuration reference should be canonical.
|
|
||||||
|
|
||||||
### docs/cli.md
|
|
||||||
|
|
||||||
**Audience:** users, administrators, operators
|
|
||||||
|
|
||||||
Required for CLI applications.
|
|
||||||
|
|
||||||
It should include, in order:
|
|
||||||
|
|
||||||
1. shortest useful command;
|
|
||||||
2. command overview;
|
|
||||||
3. complete flag reference;
|
|
||||||
4. common workflows;
|
|
||||||
5. diagnostic or recovery commands, if applicable.
|
|
||||||
|
|
||||||
Explain when commands are useful, not just their syntax.
|
|
||||||
|
|
||||||
### docs/operations.md
|
|
||||||
|
|
||||||
**Audience:** administrators, operators
|
|
||||||
|
|
||||||
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
|
|
||||||
|
|
||||||
It should cover:
|
|
||||||
|
|
||||||
- normal workflow;
|
|
||||||
- filesystem layout;
|
|
||||||
- remote storage layout, if applicable;
|
|
||||||
- logs and manifests;
|
|
||||||
- resume/retry behavior;
|
|
||||||
- cleanup behavior;
|
|
||||||
- archive/backup behavior;
|
|
||||||
- safe recovery procedures;
|
|
||||||
- operational caveats.
|
|
||||||
|
|
||||||
### docs/troubleshooting.md
|
|
||||||
|
|
||||||
**Audience:** administrators, operators
|
|
||||||
|
|
||||||
Recommended once recurring failure modes exist.
|
|
||||||
|
|
||||||
Each entry should include:
|
|
||||||
|
|
||||||
- symptom;
|
|
||||||
- likely cause;
|
|
||||||
- diagnostic command or inspection step;
|
|
||||||
- safe fix;
|
|
||||||
- relevant links.
|
|
||||||
|
|
||||||
### docs/internal/
|
|
||||||
|
|
||||||
**Audience:** developers, LLM coding agents
|
|
||||||
|
|
||||||
Required for modular, staged, service-oriented, or orchestration projects.
|
|
||||||
|
|
||||||
This directory describes implemented internal components. It is not the roadmap.
|
|
||||||
|
|
||||||
Use one file per major component where useful.
|
|
||||||
|
|
||||||
Each component doc should include:
|
|
||||||
|
|
||||||
1. purpose;
|
|
||||||
2. inputs and outputs;
|
|
||||||
3. boundaries;
|
|
||||||
4. config fields used;
|
|
||||||
5. external adapters used;
|
|
||||||
6. state or manifest behavior, if applicable;
|
|
||||||
7. skip/resume behavior, if applicable;
|
|
||||||
8. failure behavior;
|
|
||||||
9. tests to inspect before changing;
|
|
||||||
10. architectural invariants.
|
|
||||||
|
|
||||||
### docs/roadmap/
|
|
||||||
|
|
||||||
**Audience:** maintainers, developers, LLM coding agents
|
|
||||||
|
|
||||||
This is the only place for planned, future, aspirational, experimental, or unimplemented work.
|
|
||||||
|
|
||||||
Roadmap docs should clearly distinguish:
|
|
||||||
|
|
||||||
- proposed work;
|
|
||||||
- accepted plans;
|
|
||||||
- deferred ideas;
|
|
||||||
- rejected ideas;
|
|
||||||
- implementation prompts or task breakdowns, if useful.
|
|
||||||
|
|
||||||
Roadmap docs should not be confused with current behavior.
|
|
||||||
|
|
||||||
### docs/integrations/
|
|
||||||
|
|
||||||
**Audience:** developers, LLM coding agents
|
|
||||||
|
|
||||||
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
|
||||||
|
|
||||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
|
|
||||||
|
|
||||||
Use one file per integration where useful.
|
|
||||||
|
|
||||||
## Examples Directory
|
|
||||||
|
|
||||||
Projects with non-trivial configuration or workflows should include `examples/`.
|
|
||||||
|
|
||||||
Useful examples include:
|
|
||||||
|
|
||||||
- minimal working config;
|
|
||||||
- production-oriented config;
|
|
||||||
- full annotated config;
|
|
||||||
- local development config;
|
|
||||||
- remote/object-storage config;
|
|
||||||
- minimal session/input file.
|
|
||||||
|
|
||||||
Examples should be valid, maintained, tested when practical, and linked from relevant docs.
|
|
||||||
|
|
||||||
## Security and Privacy
|
|
||||||
|
|
||||||
Docs and examples must not include:
|
|
||||||
|
|
||||||
- real API keys;
|
|
||||||
- tokens;
|
|
||||||
- passwords;
|
|
||||||
- private keys;
|
|
||||||
- private environment dumps;
|
|
||||||
- sensitive user data;
|
|
||||||
- raw private transcripts;
|
|
||||||
- private infrastructure details unless intentionally public.
|
|
||||||
|
|
||||||
Document secret-handling mechanisms, not actual secret values.
|
|
||||||
|
|
||||||
## Maintenance Rules
|
|
||||||
|
|
||||||
When docs change, verify the affected behavior.
|
|
||||||
|
|
||||||
Where practical:
|
|
||||||
|
|
||||||
- load example config files in tests;
|
|
||||||
- test CLI examples or command parser behavior;
|
|
||||||
- validate documented flags against real flags;
|
|
||||||
- remove stale references;
|
|
||||||
- update links after renames;
|
|
||||||
- keep roadmap content out of non-roadmap docs.
|
|
||||||
|
|
||||||
If documentation and code disagree, fix the documentation and/or open a roadmap item; do not leave aspirational behavior in current-behavior docs.
|
|
||||||
|
|
||||||
Documentation is complete only when it matches the current code.
|
|
||||||
|
|
||||||
## Documentation Change Checklist
|
|
||||||
|
|
||||||
Before merging documentation changes, verify:
|
|
||||||
|
|
||||||
- README is concise and orientation-focused.
|
|
||||||
- `docs/policy/architecture.md` describes development principles.
|
|
||||||
- Future work appears only under `docs/roadmap/`.
|
|
||||||
- User-facing docs avoid unnecessary internals.
|
|
||||||
- Developer-facing docs preserve boundaries and invariants.
|
|
||||||
- Config examples match the schema.
|
|
||||||
- CLI examples match real commands and flags.
|
|
||||||
- Defaults appear in the canonical config reference.
|
|
||||||
- No secrets or private data are included.
|
|
||||||
- Links are accurate.
|
|
||||||
|
|||||||
296
docs/policy/testing.md
Normal file
296
docs/policy/testing.md
Normal file
@@ -0,0 +1,296 @@
|
|||||||
|
# Testing Policy
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Our tests exist to make **incorrect changes expensive and correct changes cheap**.
|
||||||
|
|
||||||
|
We do not optimize for test count, line coverage, exhaustive isolation, or the fewest possible tests. We optimize for sufficient confidence in important behavior while imposing as little unnecessary friction as possible on future development.
|
||||||
|
|
||||||
|
## Every test has a cost
|
||||||
|
|
||||||
|
Testing is not an unqualified good. Every test imposes both an immediate cost and a continuing lifetime cost.
|
||||||
|
|
||||||
|
A test must be:
|
||||||
|
|
||||||
|
- written and reviewed;
|
||||||
|
- understood by future maintainers and coding agents;
|
||||||
|
- executed in local and CI workflows;
|
||||||
|
- diagnosed when it fails;
|
||||||
|
- updated when legitimate behavior changes;
|
||||||
|
- maintained as fixtures, APIs, and dependencies evolve; and
|
||||||
|
- removed or rewritten when it becomes redundant, brittle, misleading, or obsolete.
|
||||||
|
|
||||||
|
Tests also create cognitive and architectural friction. They can constrain refactoring, duplicate policy, slow feedback loops, add noise to failures, and cause harmless implementation changes to require unrelated edits across the suite.
|
||||||
|
|
||||||
|
A test is warranted only when the confidence it provides justifies these costs.
|
||||||
|
|
||||||
|
Apply this cost-benefit analysis at two levels:
|
||||||
|
|
||||||
|
1. **Per test:** What realistic defect does this test detect, how consequential would that defect be, and is that protection worth the test's lifetime cost?
|
||||||
|
2. **Across the suite:** Does this collection provide materially more confidence than a smaller, simpler suite would?
|
||||||
|
|
||||||
|
The preferred test suite is a **lean suite that provides sufficient confidence in the risks that matter, without redundant or low-value tests**. We seek sufficient confidence with the least unnecessary testing friction, not the fewest possible tests.
|
||||||
|
|
||||||
|
Some friction is intentional. Tests should make dangerous changes—such as breaking compatibility, corrupting data, violating security boundaries, or reintroducing subtle bugs—require deliberate review. They should not make ordinary internal changes needlessly expensive.
|
||||||
|
|
||||||
|
The cost of a test is not a reason to omit testing by default. Do not cite maintenance cost abstractly. When omitting a plausible test, be able to state why the protected failure is low-risk, already covered, obvious, reversible, or cheaper to detect elsewhere. For consequential, subtle, or difficult-to-observe behavior, the presumption should favor testing.
|
||||||
|
|
||||||
|
## Default testing style
|
||||||
|
|
||||||
|
Use a **classical/Detroit-style** approach:
|
||||||
|
|
||||||
|
- Test observable behavior, resulting state, contracts, and invariants.
|
||||||
|
- Use real internal collaborators when they are fast and deterministic.
|
||||||
|
- Use fakes, stubs, or mocks primarily at expensive, nondeterministic, destructive, or external boundaries.
|
||||||
|
- Prefer package-level behavioral tests over tests coupled to private helpers or internal call sequences.
|
||||||
|
- Treat exact collaborator interactions as testable behavior only when the interaction itself is a requirement.
|
||||||
|
|
||||||
|
Examples of appropriate seams include clocks, randomness, subprocesses, remote APIs, object storage, email, and paid LLM calls.
|
||||||
|
|
||||||
|
## Test execution requirements
|
||||||
|
|
||||||
|
Tests in the default suite must be deterministic, offline, and independent of real credentials. They must not invoke paid APIs or depend on mutable external services. Tests that require live infrastructure must be explicitly opt-in and clearly separated from the default suite.
|
||||||
|
|
||||||
|
Control clocks, randomness, environment variables, and other process-global or machine-specific state when they affect behavior. Tests should be safe to run repeatedly and alongside other tests without depending on execution order or state left by an earlier test.
|
||||||
|
|
||||||
|
## What deserves tests
|
||||||
|
|
||||||
|
Prioritize tests for:
|
||||||
|
|
||||||
|
1. Public and package-level contracts.
|
||||||
|
2. Domain rules and important invariants.
|
||||||
|
3. Boundary conditions and malformed input.
|
||||||
|
4. Failure handling, cancellation, retries, recovery, and partial success.
|
||||||
|
5. Serialization, schemas, compatibility, and round trips.
|
||||||
|
6. Previously observed or plausible regressions.
|
||||||
|
7. Representative integration and end-to-end workflows.
|
||||||
|
|
||||||
|
A package-level contract is behavior relied upon by another package or major collaborator, not every observable detail of a package implementation.
|
||||||
|
|
||||||
|
For behavior involving **data integrity, destructive operations, compatibility, security, concurrency, idempotency, or recovery**, presume that durable tests are required unless the behavior is already credibly protected at another layer.
|
||||||
|
|
||||||
|
Do not add tests merely because a function, branch, or line exists. Do not add a test when the same meaningful risk is already adequately protected elsewhere.
|
||||||
|
|
||||||
|
## Choose the right test boundary
|
||||||
|
|
||||||
|
Test through the narrowest stable boundary that expresses the behavior clearly.
|
||||||
|
|
||||||
|
This is often the package API, but it may instead be:
|
||||||
|
|
||||||
|
- a smaller pure function when dense domain logic is most clearly isolated there;
|
||||||
|
- a package-level operation when several internal collaborators jointly produce the behavior; or
|
||||||
|
- a larger integration boundary when correctness emerges from interaction with a real dependency.
|
||||||
|
|
||||||
|
Do not force all behavior through oversized end-to-end tests. Do not test every private helper merely because it exists. Choose the boundary that gives durable confidence with the least incidental coupling.
|
||||||
|
|
||||||
|
## Test behavior, not implementation
|
||||||
|
|
||||||
|
A test should protect a decision, contract, or invariant—not memorialize the current implementation.
|
||||||
|
|
||||||
|
Before adding or retaining a test, ask:
|
||||||
|
|
||||||
|
> What realistic defect would this test catch?
|
||||||
|
|
||||||
|
A test is suspect when its main purpose is to detect that someone:
|
||||||
|
|
||||||
|
- changed an internal constant;
|
||||||
|
- renamed or split a private helper;
|
||||||
|
- reordered equivalent internal operations;
|
||||||
|
- changed incidental formatting;
|
||||||
|
- replaced one correct algorithm with another; or
|
||||||
|
- refactored internal object structure without changing behavior.
|
||||||
|
|
||||||
|
Refactoring should normally require no test edits unless the refactored structure is itself part of the contract.
|
||||||
|
|
||||||
|
A test can be factually correct and still have negative value. Accurately describing current behavior is not enough; the protected behavior must be important enough to justify the future friction.
|
||||||
|
|
||||||
|
## Expected effects of different changes
|
||||||
|
|
||||||
|
Use the following expectations when evaluating test failures and test maintenance:
|
||||||
|
|
||||||
|
| Change | Expected effect on tests |
|
||||||
|
|---|---|
|
||||||
|
| Internal refactor that preserves behavior | Existing tests should normally remain unchanged and continue to pass. |
|
||||||
|
| Change to an internal default with no contractual significance | Behavioral tests should normally remain unchanged; tests should derive expectations from configuration or relationships rather than duplicate the old value. |
|
||||||
|
| Intentional change to public behavior, policy, schema, or compatibility guarantees | The relevant tests should be reviewed and changed deliberately. |
|
||||||
|
| Accidental violation of a contract or invariant | Tests should fail; fix the production code rather than rewriting the tests to accept the defect. |
|
||||||
|
|
||||||
|
A test failing is not the same as a test needing to be edited. Many tests may correctly fail because of one production defect. The maintenance smell is a correct internal change that requires unrelated expectation updates throughout the suite.
|
||||||
|
|
||||||
|
## Separate mechanism from policy
|
||||||
|
|
||||||
|
Configurable thresholds and defaults must not be duplicated throughout the test suite.
|
||||||
|
|
||||||
|
For example, do not encode an internal concurrency limit indirectly:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// Production policy:
|
||||||
|
const maxConcurrency = 4
|
||||||
|
|
||||||
|
// Brittle test:
|
||||||
|
err := startProcesses(5)
|
||||||
|
require.Error(t, err)
|
||||||
|
```
|
||||||
|
|
||||||
|
Instead, test the mechanism relationally:
|
||||||
|
|
||||||
|
```go
|
||||||
|
const limit = 2
|
||||||
|
runner := NewRunner(limit)
|
||||||
|
|
||||||
|
require.NoError(t, runner.Start(limit))
|
||||||
|
require.ErrorIs(t, runner.Start(limit+1), ErrTooMuchConcurrency)
|
||||||
|
```
|
||||||
|
|
||||||
|
The test should prove:
|
||||||
|
|
||||||
|
- the configured limit is accepted; and
|
||||||
|
- one beyond the configured limit is rejected.
|
||||||
|
|
||||||
|
The production default should be tested exactly only when its literal value is itself a public, operational, safety, protocol, or compatibility requirement.
|
||||||
|
|
||||||
|
Apply the same rule to limits, timeouts, capacities, retry counts, and ranges: test relationships and behavior, not duplicated literals.
|
||||||
|
|
||||||
|
For concurrency limits, test both kinds of behavior when relevant:
|
||||||
|
|
||||||
|
1. **Configuration enforcement:** invalid or excessive requested values are handled correctly.
|
||||||
|
2. **Runtime enforcement:** observed peak concurrency never exceeds the configured limit.
|
||||||
|
|
||||||
|
Use a test-controlled limit and measure the behavior relative to that limit. Do not merely assert today's default value.
|
||||||
|
|
||||||
|
## Avoid semantic duplication across layers
|
||||||
|
|
||||||
|
Each behavior should have a clear test owner.
|
||||||
|
|
||||||
|
- Parser tests own parsing cases.
|
||||||
|
- Validator tests own validation rules.
|
||||||
|
- Domain tests own transformations and invariants.
|
||||||
|
- Adapter tests own external integration behavior.
|
||||||
|
- Orchestrator tests own coordination and failure propagation.
|
||||||
|
- CLI tests own argument and configuration mapping.
|
||||||
|
- End-to-end tests prove that representative assembled workflows work.
|
||||||
|
|
||||||
|
Higher-level tests should not repeat every lower-level case. A single intentional policy change should not require unrelated edits across many test files.
|
||||||
|
|
||||||
|
Tests that are individually reasonable may still be collectively redundant. Evaluate the marginal value of each additional test in light of the protection already provided by the rest of the suite.
|
||||||
|
|
||||||
|
## Use test doubles deliberately
|
||||||
|
|
||||||
|
Choose the least elaborate test double that provides the required control or observation.
|
||||||
|
|
||||||
|
As a default:
|
||||||
|
|
||||||
|
1. Prefer real collaborators when they are fast and deterministic.
|
||||||
|
2. Use small in-memory fakes when realistic stateful behavior is helpful.
|
||||||
|
3. Use stubs when a dependency only needs to provide controlled responses.
|
||||||
|
4. Use mocks when the interaction itself is contractual.
|
||||||
|
|
||||||
|
Mocks are appropriate when the contract includes facts such as:
|
||||||
|
|
||||||
|
- a notification is sent exactly once;
|
||||||
|
- a transaction is committed only after successful writes;
|
||||||
|
- cancellation reaches a subprocess;
|
||||||
|
- an expensive API is called no more than once; or
|
||||||
|
- a security audit event is emitted.
|
||||||
|
|
||||||
|
Do not use mocks merely to isolate every object or reproduce the implementation's call graph.
|
||||||
|
|
||||||
|
## Go-specific guidance
|
||||||
|
|
||||||
|
Use:
|
||||||
|
|
||||||
|
- table-driven tests for meaningful behavioral categories and boundaries;
|
||||||
|
- `t.TempDir()` for real filesystem behavior;
|
||||||
|
- `httptest.Server` for realistic HTTP interactions;
|
||||||
|
- fuzz tests for parsers, normalization, path handling, and broad input spaces;
|
||||||
|
- golden files only when the complete output is intentionally stable;
|
||||||
|
- integration tests where correctness depends on component interaction; and
|
||||||
|
- a small number of representative end-to-end tests.
|
||||||
|
|
||||||
|
Avoid exact error-string assertions unless the wording is itself contractual. Prefer `errors.Is`, `errors.As`, typed errors, or structured error fields.
|
||||||
|
|
||||||
|
At CLI boundaries, prefer exit classifications, structured output, and the smallest stable semantic fragment needed to identify the error. Do not snapshot complete diagnostic wording unless it is contractual.
|
||||||
|
|
||||||
|
Golden-file updates must require an explicit local flag. CI must not update golden files automatically, and reviewers must inspect the semantic diff before accepting an update.
|
||||||
|
|
||||||
|
Keep tests readable and direct. Test helpers and fixture frameworks must earn their own maintenance cost; do not build elaborate test infrastructure for small or isolated needs.
|
||||||
|
|
||||||
|
## Coverage
|
||||||
|
|
||||||
|
Coverage is a diagnostic, not a target.
|
||||||
|
|
||||||
|
Use it to find untested critical branches and unexpectedly weak packages. Do not write low-value tests solely to increase a percentage, and do not infer test quality from coverage alone.
|
||||||
|
|
||||||
|
Pure domain logic will often warrant higher coverage than CLI wiring or external adapters. Uneven coverage is acceptable when it reflects risk.
|
||||||
|
|
||||||
|
Increasing coverage is valuable only when the newly covered behavior protects a meaningful risk at an acceptable cost.
|
||||||
|
|
||||||
|
## Regression tests
|
||||||
|
|
||||||
|
A bug fix should normally include a regression test that fails before the fix and passes afterward.
|
||||||
|
|
||||||
|
Retain the test when the defect could realistically recur and its consequences justify the ongoing cost. Prefer the narrowest durable test of the violated contract or invariant; do not preserve accidental implementation details from the original bug.
|
||||||
|
|
||||||
|
Not every historical bug requires a permanent test. If the underlying design has made recurrence impossible, the test has become redundant, or a stronger invariant test now subsumes it, remove or consolidate it.
|
||||||
|
|
||||||
|
## Deleting or rewriting tests
|
||||||
|
|
||||||
|
Tests are maintained code, not permanent historical artifacts.
|
||||||
|
|
||||||
|
Delete or rewrite a test when its maintenance cost exceeds the confidence it provides.
|
||||||
|
|
||||||
|
Strong candidates include tests that:
|
||||||
|
|
||||||
|
- require updates after harmless internal changes;
|
||||||
|
- directly assert private constants without protecting a real contract;
|
||||||
|
- duplicate the same policy across several layers;
|
||||||
|
- verify mock choreography rather than outcomes;
|
||||||
|
- snapshot large amounts of incidental output;
|
||||||
|
- test trivial private helpers already exercised through stable package behavior;
|
||||||
|
- protect risks already covered more effectively elsewhere;
|
||||||
|
- are flaky, misleading, obsolete, or disproportionately expensive to diagnose; or
|
||||||
|
- no longer correspond to a plausible failure mode.
|
||||||
|
|
||||||
|
Several brittle tests may encode one genuine requirement. Replace them with one durable behavior-level or invariant test rather than preserving all of them.
|
||||||
|
|
||||||
|
Deleting a low-value test can improve the quality of the suite by reducing noise, maintenance burden, and friction around legitimate change.
|
||||||
|
|
||||||
|
## Reviewing a proposed test
|
||||||
|
|
||||||
|
Use the following questions when the value, boundary, or durability of a proposed test is not self-evident. Significant test additions should be reviewable against them, but written answers are not required for every routine test.
|
||||||
|
|
||||||
|
1. What realistic defect would it catch?
|
||||||
|
2. How likely is that defect?
|
||||||
|
3. How consequential would it be?
|
||||||
|
4. Is the behavior already protected elsewhere?
|
||||||
|
5. At which layer should this behavior be owned?
|
||||||
|
6. Does the test assert a durable contract or an incidental implementation detail?
|
||||||
|
7. Could the implementation be refactored without changing the behavior and without editing this test?
|
||||||
|
8. What should cause this test to fail?
|
||||||
|
9. What legitimate changes should not cause this test to fail?
|
||||||
|
10. What ongoing maintenance, execution, and diagnostic cost will the test impose?
|
||||||
|
11. Is there a smaller or more direct test that protects the same risk?
|
||||||
|
|
||||||
|
Do not add the test when its expected lifetime cost exceeds its expected protective value.
|
||||||
|
|
||||||
|
When deciding not to test plausible behavior, record or be able to explain why the risk is low, already protected, obvious, reversible, or cheaper to detect elsewhere.
|
||||||
|
|
||||||
|
## Definition of sufficient
|
||||||
|
|
||||||
|
A test suite is sufficient when:
|
||||||
|
|
||||||
|
- important contracts and invariants are protected;
|
||||||
|
- meaningful boundaries and failure modes are exercised;
|
||||||
|
- realistic and consequential regressions are credibly protected against silent recurrence;
|
||||||
|
- behavior involving data integrity, destructive operations, compatibility, security, concurrency, idempotency, and recovery is credibly protected;
|
||||||
|
- important external boundaries have realistic integration coverage;
|
||||||
|
- representative complete workflows are tested;
|
||||||
|
- failures provide useful signal rather than redundant noise;
|
||||||
|
- legitimate internal changes usually do not require test edits; and
|
||||||
|
- additional tests would mostly repeat existing protection or preserve inconsequential implementation details.
|
||||||
|
|
||||||
|
Sufficiency is a risk judgment, not a coverage percentage or test count. Reassess it as the application, its users, and the consequences of failure evolve.
|
||||||
|
|
||||||
|
The governing rule is:
|
||||||
|
|
||||||
|
> Test heavily where failure is consequential, subtle, or difficult to detect after the fact. Test lightly where failure is obvious, reversible, and inexpensive—and retain no test whose lifetime cost exceeds the confidence it provides.
|
||||||
141
docs/roadmap/documentation.md
Normal file
141
docs/roadmap/documentation.md
Normal file
@@ -0,0 +1,141 @@
|
|||||||
|
# Documentation Compliance Roadmap
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
The initial documentation refresh and its follow-up compliance work were
|
||||||
|
completed on 2026-07-26. The documentation prerequisite for Step 2 of the
|
||||||
|
[Promptkit migration roadmap](migration.md) is satisfied.
|
||||||
|
|
||||||
|
## Objective
|
||||||
|
|
||||||
|
Maintain an accurate, concise documentation baseline under the canonical
|
||||||
|
ownership policy, and close the remaining review findings before architectural
|
||||||
|
migration begins.
|
||||||
|
|
||||||
|
## Completed Refresh
|
||||||
|
|
||||||
|
The initial refresh:
|
||||||
|
|
||||||
|
- established the documentation and ADR policies and recorded
|
||||||
|
[ADR 0001](../adr/0001-adopt-canonical-documentation-ownership.md);
|
||||||
|
- made `docs/development.md` the contributor entry point and added a canonical
|
||||||
|
internal component overview;
|
||||||
|
- refocused architecture on normative boundaries and moved implementation
|
||||||
|
detail into focused internal documents;
|
||||||
|
- consolidated CLI, configuration, HTTP, public Go, and integration contracts
|
||||||
|
under distinct canonical owners;
|
||||||
|
- rewrote operations as a runbook, incorporated useful recovery guidance, and
|
||||||
|
removed the standalone troubleshooting document;
|
||||||
|
- normalized maintained examples and added an offline HTTP request-example
|
||||||
|
check;
|
||||||
|
- corrected the factual discrepancies found by the original audit, including
|
||||||
|
the entry-path count, built-in profile catalog, prompt output optionality,
|
||||||
|
input-map behavior, validation diagnostics, session-ID units, and CLI flag
|
||||||
|
descriptions; and
|
||||||
|
- validated local links, configuration examples, smoke examples, tests, vet,
|
||||||
|
and the documented build.
|
||||||
|
|
||||||
|
## Follow-Up Findings
|
||||||
|
|
||||||
|
The post-completion review found:
|
||||||
|
|
||||||
|
1. The effective precedence between `Config.Timeout` and a supplied
|
||||||
|
`HTTPClient.Timeout` is not consistently expressed by the implementation,
|
||||||
|
tests, and documentation.
|
||||||
|
2. The public Go contract overgeneralizes basename selection for single-file
|
||||||
|
sources; prompt and profile files remain selected by YAML ID, while
|
||||||
|
`WithSchemaFile` uses the schema file's base name.
|
||||||
|
3. The maintained HTTP request-example test pins incidental fixture values
|
||||||
|
beyond the structural contract it needs to protect.
|
||||||
|
4. The Promptkit migration roadmap's overall status says no work is implemented
|
||||||
|
even though its documentation-refresh step is complete.
|
||||||
|
5. One architecture invariant incorrectly describes configuration, prompt, and
|
||||||
|
profile inputs as JSON rather than YAML.
|
||||||
|
|
||||||
|
## Follow-Up Implementation Sequence
|
||||||
|
|
||||||
|
### Stage 1: Resolve The Timeout Contract
|
||||||
|
|
||||||
|
1. Decide and document the precedence among:
|
||||||
|
- `Config.Timeout`;
|
||||||
|
- a non-zero timeout on a supplied `http.Client`; and
|
||||||
|
- an explicit per-request `timeout_seconds` override.
|
||||||
|
2. Align `internal/llm` with that decision, removing any constructor state that
|
||||||
|
is immediately discarded during generation.
|
||||||
|
3. Add or revise a behavior-level test that performs `Generate` with a supplied
|
||||||
|
HTTP client and proves the selected timeout without relying only on private
|
||||||
|
constructor state.
|
||||||
|
4. Update the canonical public Go contract, outbound OpenAI-compatible
|
||||||
|
integration contract, and internal LLM document together.
|
||||||
|
|
||||||
|
**Gate:** There is one documented timeout-precedence rule, the implementation
|
||||||
|
follows it, and a request-level test protects it.
|
||||||
|
|
||||||
|
### Stage 2: Correct The Remaining Contract And Architecture Text
|
||||||
|
|
||||||
|
1. Clarify in the public Go contract that:
|
||||||
|
- prompt and profile single-file sources are selected by their YAML IDs;
|
||||||
|
- `WithPromptFile` resolves relative `content_file` paths from the prompt
|
||||||
|
file's directory; and
|
||||||
|
- `WithSchemaFile` exposes the schema through its base name.
|
||||||
|
2. Correct or remove the duplicate architecture invariant so it accurately
|
||||||
|
distinguishes strict YAML configuration, prompt, and profile inputs from
|
||||||
|
strict JSON HTTP input.
|
||||||
|
3. Recheck the affected statements against the public facade and source
|
||||||
|
implementations without duplicating file-format definitions outside the
|
||||||
|
configuration contract.
|
||||||
|
|
||||||
|
**Gate:** Public source-option behavior and strict-decoding invariants agree
|
||||||
|
with the implementation and retain one canonical owner.
|
||||||
|
|
||||||
|
### Stage 3: Refine The Maintained HTTP Example Check
|
||||||
|
|
||||||
|
1. Preserve an offline check that `examples/http-run.json` is accepted by the
|
||||||
|
HTTP request contract.
|
||||||
|
2. Remove assertions that freeze prompt IDs, profile IDs, fixture paths, or
|
||||||
|
example variable values unless a specific value protects an independently
|
||||||
|
identified contract.
|
||||||
|
3. Rely on focused handler tests, rather than the maintained-example check, for
|
||||||
|
exhaustive DTO-to-domain mapping behavior.
|
||||||
|
4. Run the affected HTTP adapter tests and confirm that the revised test still
|
||||||
|
fails for structurally invalid example content.
|
||||||
|
|
||||||
|
**Gate:** The maintained example has durable structural protection without
|
||||||
|
duplicating mapping coverage or making legitimate fixture edits unnecessarily
|
||||||
|
expensive.
|
||||||
|
|
||||||
|
### Stage 4: Reconcile Status And Revalidate The Baseline
|
||||||
|
|
||||||
|
1. Update the Promptkit migration roadmap's overall status to distinguish the
|
||||||
|
completed documentation step from the unimplemented migration steps.
|
||||||
|
2. Recheck local Markdown links and paths.
|
||||||
|
3. Verify both maintained configuration files through the real configuration
|
||||||
|
loader.
|
||||||
|
4. Run the maintained render script and Go package example.
|
||||||
|
5. Run `go test ./...`, `go vet ./...`, and `go build ./cmd/scriptorium`.
|
||||||
|
6. Record completion of these follow-up stages here and confirm the
|
||||||
|
documentation gate in the Promptkit migration roadmap.
|
||||||
|
|
||||||
|
**Gate:** Code, tests, examples, canonical contracts, architecture, and both
|
||||||
|
roadmap status documents agree. Promptkit migration Step 2 may then begin.
|
||||||
|
|
||||||
|
## Completion Criteria
|
||||||
|
|
||||||
|
This follow-up is complete when:
|
||||||
|
|
||||||
|
- timeout precedence is unambiguous, implemented, and protected by an
|
||||||
|
execution-level test;
|
||||||
|
- public single-file source behavior is accurately documented;
|
||||||
|
- strict YAML and JSON boundaries are accurately stated;
|
||||||
|
- the HTTP example check protects structure without pinning incidental content;
|
||||||
|
- roadmap status accurately distinguishes completed and pending migration work;
|
||||||
|
and
|
||||||
|
- links, examples, tests, vet, and build validation pass.
|
||||||
|
|
||||||
|
## Completion Record
|
||||||
|
|
||||||
|
Completed on 2026-07-26. The timeout contract, single-file source contract,
|
||||||
|
strict-decoding boundary, and maintained HTTP example check were reconciled.
|
||||||
|
Final validation confirmed local links and paths, both maintained configuration
|
||||||
|
files through the real loader, the maintained render and Go package examples,
|
||||||
|
`go test ./...`, `go vet ./...`, and `go build ./cmd/scriptorium`.
|
||||||
296
docs/roadmap/migration.md
Normal file
296
docs/roadmap/migration.md
Normal file
@@ -0,0 +1,296 @@
|
|||||||
|
# Promptkit Migration Roadmap
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Accepted plan. Step 1, the documentation refresh, is complete. Steps 2 through
|
||||||
|
9 remain proposed and are not yet implemented.
|
||||||
|
|
||||||
|
## Objective
|
||||||
|
|
||||||
|
Split the current repository into two projects:
|
||||||
|
|
||||||
|
- **Promptkit**: the reusable Go framework, public Go facade, execution engine,
|
||||||
|
source and validation support, OpenAI-compatible client, extension
|
||||||
|
interfaces, and built-in execution-profile registry.
|
||||||
|
- **Scriptorium**: a slim runnable application that imports Promptkit and
|
||||||
|
provides the CLI and HTTP interfaces.
|
||||||
|
|
||||||
|
Scriptorium will become another downstream Promptkit consumer rather than the
|
||||||
|
owner of the framework.
|
||||||
|
|
||||||
|
## Compatibility And Migration Policy
|
||||||
|
|
||||||
|
This is an intentionally breaking change.
|
||||||
|
|
||||||
|
- New and migrated Go consumers must import Promptkit instead of Scriptorium.
|
||||||
|
- Scriptorium will not retain type aliases, forwarding packages, deprecated
|
||||||
|
facade APIs, or other source-compatibility shims.
|
||||||
|
- Existing consumers may continue using a previously tagged Scriptorium module
|
||||||
|
version until they are migrated.
|
||||||
|
- The migration does not need to preserve compatibility between intermediate
|
||||||
|
development states. Each completed phase must instead leave the affected
|
||||||
|
repository internally consistent and tested.
|
||||||
|
- Promptkit should initially preserve the useful shape and behavior of the
|
||||||
|
current public Go facade where doing so reduces extraction risk. Broader API
|
||||||
|
redesign should follow the split unless required to establish the new
|
||||||
|
boundary.
|
||||||
|
|
||||||
|
## Target Ownership
|
||||||
|
|
||||||
|
Promptkit should own application-neutral framework behavior:
|
||||||
|
|
||||||
|
- public engine, request, result, option, extension, and error APIs;
|
||||||
|
- prompt-definition loading and rendering;
|
||||||
|
- execution profiles, overlays, and the built-in profile registry;
|
||||||
|
- artifact-loading interfaces and general-purpose `file` and `inline` support;
|
||||||
|
- schema loading and output validation;
|
||||||
|
- LLM client boundaries and the OpenAI-compatible implementation;
|
||||||
|
- preparation and execution orchestration;
|
||||||
|
- framework and execution defaults.
|
||||||
|
|
||||||
|
Scriptorium should own executable and transport concerns:
|
||||||
|
|
||||||
|
- the `scriptorium` command and its `run`, `render`, and `serve` interfaces;
|
||||||
|
- CLI parsing, output formatting, exit codes, and process behavior;
|
||||||
|
- application-config discovery and CLI precedence;
|
||||||
|
- HTTP routing, request and response DTOs, limits, and error/status mapping;
|
||||||
|
- HTTP artifact-root and deployment security policy;
|
||||||
|
- server and adapter defaults;
|
||||||
|
- executable examples, operations guidance, and transport documentation.
|
||||||
|
|
||||||
|
The intended dependency direction is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Scriptorium CLI and HTTP adapters
|
||||||
|
|
|
||||||
|
v
|
||||||
|
Promptkit
|
||||||
|
|
|
||||||
|
v
|
||||||
|
consumer-supplied sources and clients
|
||||||
|
```
|
||||||
|
|
||||||
|
Scriptorium must use Promptkit's public API. It must not depend on Promptkit
|
||||||
|
implementation packages or reproduce framework orchestration.
|
||||||
|
|
||||||
|
## Migration Steps
|
||||||
|
|
||||||
|
### Step 1: Refresh And Synchronize Documentation
|
||||||
|
|
||||||
|
Perform a repository-wide documentation refresh before migration development.
|
||||||
|
|
||||||
|
At minimum:
|
||||||
|
|
||||||
|
- reconcile all current-behavior documentation with the code, tests, examples,
|
||||||
|
defaults, and current public contracts;
|
||||||
|
- introduce the planned documentation-policy updates;
|
||||||
|
- establish an architecture decision record policy and canonical ADR location;
|
||||||
|
- resolve stale, duplicated, or misplaced material;
|
||||||
|
- validate documentation links and maintained examples;
|
||||||
|
- leave future migration behavior in `docs/roadmap/` until implemented.
|
||||||
|
|
||||||
|
**Gate:** Do not begin architectural migration work until the documentation
|
||||||
|
refresh and policy updates are merged and the repository has an agreed,
|
||||||
|
accurate baseline.
|
||||||
|
|
||||||
|
**Gate status:** Complete as of 2026-07-26. The completed documentation
|
||||||
|
refresh and follow-up verification record are in the [documentation compliance
|
||||||
|
roadmap](documentation.md).
|
||||||
|
|
||||||
|
### Step 2: Record The Architectural Decision And Detailed Boundary
|
||||||
|
|
||||||
|
Create an ADR, under the policy established in Step 1, that records:
|
||||||
|
|
||||||
|
- the decision to split Promptkit from Scriptorium;
|
||||||
|
- the target ownership and dependency direction;
|
||||||
|
- the selected Promptkit repository and Go module paths;
|
||||||
|
- the breaking-change and versioning policy;
|
||||||
|
- ownership of configuration fields and defaults;
|
||||||
|
- artifact-reader and HTTP containment responsibilities;
|
||||||
|
- local multi-repository development and release coordination;
|
||||||
|
- documentation ownership after the split.
|
||||||
|
|
||||||
|
Use the ADR to resolve any remaining public-boundary decisions before code is
|
||||||
|
moved.
|
||||||
|
|
||||||
|
**Gate:** The ADR is accepted, and every existing package, public contract,
|
||||||
|
configuration category, and maintained asset has a target owner.
|
||||||
|
|
||||||
|
### Step 3: Characterize Existing Framework Behavior
|
||||||
|
|
||||||
|
Strengthen or add contract-focused tests where needed so extraction can be
|
||||||
|
verified without relying on package placement.
|
||||||
|
|
||||||
|
Preserve coverage of:
|
||||||
|
|
||||||
|
- `Prepare` and `Run` behavior;
|
||||||
|
- prompt, profile, execution-default, and request-override precedence;
|
||||||
|
- presence-aware numeric overrides;
|
||||||
|
- built-in profile fallback and custom-profile overlays;
|
||||||
|
- strict YAML and JSON decoding;
|
||||||
|
- prompt, profile, schema, and artifact source behavior;
|
||||||
|
- structured-output requests and output validation;
|
||||||
|
- validation failures versus validation runtime errors;
|
||||||
|
- secret handling and redaction;
|
||||||
|
- public error classification;
|
||||||
|
- HTTP artifact restrictions and transport mappings.
|
||||||
|
|
||||||
|
**Gate:** Current framework and adapter contracts are represented by passing
|
||||||
|
tests sufficient to detect behavioral regressions during the split.
|
||||||
|
|
||||||
|
### Step 4: Make Scriptorium Adapters Consume The Public Facade
|
||||||
|
|
||||||
|
Within the current repository, refactor the CLI and HTTP adapters to use the
|
||||||
|
public framework facade rather than constructing or importing internal runner
|
||||||
|
components directly.
|
||||||
|
|
||||||
|
Add only the minimum public capabilities needed to support this boundary. These
|
||||||
|
may include:
|
||||||
|
|
||||||
|
- a small `Run`/`Prepare` consumer interface;
|
||||||
|
- injectable artifact-reading behavior for Scriptorium's restricted HTTP
|
||||||
|
policy;
|
||||||
|
- source options currently available only through internal constructors;
|
||||||
|
- prepared-run formatting based on public types;
|
||||||
|
- stable public error classification required by CLI and HTTP mappings.
|
||||||
|
|
||||||
|
Do not broadly export internal repositories, domain types, or use-case
|
||||||
|
implementations.
|
||||||
|
|
||||||
|
**Gate:** The CLI and HTTP adapters use only the public framework API for
|
||||||
|
framework behavior, and all tests and documented smoke commands pass.
|
||||||
|
|
||||||
|
### Step 5: Create The Promptkit Repository
|
||||||
|
|
||||||
|
Create the Promptkit repository and Go module as an explicit out-of-band
|
||||||
|
operation.
|
||||||
|
|
||||||
|
Establish:
|
||||||
|
|
||||||
|
- repository access, ownership, and branch protections;
|
||||||
|
- the module path selected by the ADR;
|
||||||
|
- baseline development, architecture, documentation, and release policies;
|
||||||
|
- CI for build, test, vet, and other agreed checks;
|
||||||
|
- an initial package layout centered on a small public facade with internal
|
||||||
|
implementation packages;
|
||||||
|
- a local development workflow for coordinated Promptkit and Scriptorium
|
||||||
|
changes, using a workspace or temporary uncommitted module replacement where
|
||||||
|
appropriate.
|
||||||
|
|
||||||
|
Do not commit local filesystem `replace` directives to release branches.
|
||||||
|
|
||||||
|
**Gate:** The Promptkit repository exists, is accessible to maintainers, has
|
||||||
|
working CI and policy scaffolding, and can receive the extracted framework.
|
||||||
|
Do not begin cross-repository extraction until this out-of-band work is
|
||||||
|
confirmed complete.
|
||||||
|
|
||||||
|
### Step 6: Extract And Stabilize Promptkit
|
||||||
|
|
||||||
|
Move the application-neutral framework and built-in profile assets into
|
||||||
|
Promptkit. Preserve implementation packages as internal where practical.
|
||||||
|
|
||||||
|
The initial public API should remain focused on the established engine workflow
|
||||||
|
and the source and client extension points required by real consumers. Avoid
|
||||||
|
combining the extraction with unrelated API redesign.
|
||||||
|
|
||||||
|
Move or recreate the relevant:
|
||||||
|
|
||||||
|
- framework implementation;
|
||||||
|
- public package tests and framework contract tests;
|
||||||
|
- built-in profile assets and registry tests;
|
||||||
|
- Go consumer examples;
|
||||||
|
- framework, consumer, configuration-format, and integration documentation.
|
||||||
|
|
||||||
|
Verify that Promptkit can be built, tested, and consumed independently of the
|
||||||
|
Scriptorium repository.
|
||||||
|
|
||||||
|
**Gate:** Promptkit independently provides the agreed framework contract,
|
||||||
|
passes its CI checks, and has a tagged version that Scriptorium and other
|
||||||
|
consumers can import.
|
||||||
|
|
||||||
|
### Step 7: Slim Scriptorium And Adopt Promptkit
|
||||||
|
|
||||||
|
Update Scriptorium to import the tagged Promptkit module and remove the
|
||||||
|
framework implementation and public Go facade that Promptkit replaces.
|
||||||
|
|
||||||
|
Retain only Scriptorium-owned executable and transport behavior. In particular:
|
||||||
|
|
||||||
|
- wire CLI and HTTP requests through Promptkit's public API;
|
||||||
|
- keep application config and transport defaults in Scriptorium;
|
||||||
|
- keep restricted HTTP artifact policy in Scriptorium while injecting it
|
||||||
|
through Promptkit's supported boundary;
|
||||||
|
- remove obsolete framework packages, tests, and documentation;
|
||||||
|
- update Scriptorium examples and docs to describe the CLI and HTTP application;
|
||||||
|
- direct Go framework consumers to Promptkit without providing compatibility
|
||||||
|
aliases or forwarding APIs.
|
||||||
|
|
||||||
|
**Gate:** Scriptorium builds and passes all tests using a tagged Promptkit
|
||||||
|
dependency, contains no duplicate framework implementation, and its current
|
||||||
|
documentation describes only the slimmed application.
|
||||||
|
|
||||||
|
### Step 8: Migrate Downstream Consumers To Promptkit
|
||||||
|
|
||||||
|
Inventory downstream Go consumers and migrate each from the Scriptorium package
|
||||||
|
to Promptkit. This work may occur in external repositories and must be tracked
|
||||||
|
explicitly.
|
||||||
|
|
||||||
|
For each consumer:
|
||||||
|
|
||||||
|
- update module imports and dependencies;
|
||||||
|
- adapt to any intentionally changed public API;
|
||||||
|
- run its tests and relevant integration or smoke checks;
|
||||||
|
- confirm configuration, source, validation, and error behavior;
|
||||||
|
- release or deploy the migrated consumer through its normal process.
|
||||||
|
|
||||||
|
Consumers that cannot migrate immediately may remain pinned to the last
|
||||||
|
framework-bearing Scriptorium tag. No compatibility work is required in the new
|
||||||
|
Scriptorium project for those consumers.
|
||||||
|
|
||||||
|
**Gate:** All in-scope downstream consumers are either migrated and verified or
|
||||||
|
explicitly recorded as remaining on the previous Scriptorium version with an
|
||||||
|
owner and follow-up plan. Do not declare the ecosystem migration complete until
|
||||||
|
the required out-of-band consumer changes are confirmed.
|
||||||
|
|
||||||
|
### Step 9: Complete Release And Documentation Cutover
|
||||||
|
|
||||||
|
Complete the coordinated project transition:
|
||||||
|
|
||||||
|
- publish Promptkit before dependent Scriptorium releases;
|
||||||
|
- release the breaking Scriptorium version against the tagged Promptkit
|
||||||
|
dependency;
|
||||||
|
- publish migration guidance that maps the former Scriptorium Go API to
|
||||||
|
Promptkit;
|
||||||
|
- update cross-project links, examples, package documentation, and release
|
||||||
|
notes;
|
||||||
|
- verify that no release artifact depends on local workspaces or replacements;
|
||||||
|
- archive completed roadmap material according to the documentation policy in
|
||||||
|
effect at that time.
|
||||||
|
|
||||||
|
**Gate:** Promptkit and Scriptorium are independently releasable, their
|
||||||
|
documentation has distinct and accurate ownership, and the migration status of
|
||||||
|
all identified downstream consumers is recorded.
|
||||||
|
|
||||||
|
## Cross-Cutting Constraints
|
||||||
|
|
||||||
|
- Preserve the invariant that execution orchestration remains narrow and
|
||||||
|
application-neutral.
|
||||||
|
- Keep adapter-specific decisions out of Promptkit.
|
||||||
|
- Keep Scriptorium dependent only on Promptkit's supported public API.
|
||||||
|
- Preserve strict external decoding, error classification, validation
|
||||||
|
semantics, and secret redaction throughout the migration.
|
||||||
|
- Keep each repository buildable and testable at merged phase boundaries.
|
||||||
|
- Coordinate cross-repository changes through tagged dependencies and explicit
|
||||||
|
gates rather than assuming atomic commits across repositories.
|
||||||
|
- Document only implemented behavior outside roadmap files.
|
||||||
|
|
||||||
|
## Completion Criteria
|
||||||
|
|
||||||
|
The migration is complete when:
|
||||||
|
|
||||||
|
- Promptkit is the independent owner of the reusable framework and built-in
|
||||||
|
profiles;
|
||||||
|
- Scriptorium is a slim CLI and HTTP consumer of Promptkit;
|
||||||
|
- Scriptorium no longer exposes or maintains the former public Go framework;
|
||||||
|
- all required downstream migrations and external repository work have been
|
||||||
|
completed or explicitly dispositioned;
|
||||||
|
- both repositories build, test, document, version, and release independently.
|
||||||
@@ -1,334 +0,0 @@
|
|||||||
# Troubleshooting
|
|
||||||
|
|
||||||
This guide lists recurring implemented failure modes and safe fixes.
|
|
||||||
|
|
||||||
For command syntax, see [CLI reference](cli.md). For configuration and file formats, see [Configuration reference](config.md). For operational behavior, see [Operations guide](operations.md).
|
|
||||||
|
|
||||||
## Missing Or Invalid Config File
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI errors such as `application config error: config file not found` or `invalid config YAML`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- `--config` points to a missing file.
|
|
||||||
- Config YAML has syntax errors or unknown fields.
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go run ./cmd/scriptorium render --config /path/to/config.yml --prompt generic.markdown_summary --input transcript=./examples/fixtures/transcript.md --input glossary=./examples/fixtures/glossary.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Correct file path.
|
|
||||||
- Remove unknown fields.
|
|
||||||
- Fix YAML syntax.
|
|
||||||
- Keep secrets out of config.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
- [CLI reference](cli.md)
|
|
||||||
|
|
||||||
## Missing Prompt/Profile Directory Settings
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI parse errors saying prompt directory or profile directory is required.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Neither CLI flags nor config provide effective `prompt_dir` / `profile_dir`.
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
- Run the failing command with explicit `--prompt-dir` and `--profile-dir` once to verify.
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Set `prompt_dir` and `profile_dir` in config, or always pass both flags.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
- [CLI reference](cli.md)
|
|
||||||
|
|
||||||
## Unknown Or Unsupported Flags
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI parse error for an unknown flag.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Typo or command mismatch (for example, `serve` with runtime model override flags).
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
- Compare command against the command-specific flag list.
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Remove unsupported flags.
|
|
||||||
- Use `run`/`render` for runtime model overrides.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [CLI reference](cli.md)
|
|
||||||
|
|
||||||
## Prompt Definition Load Failures
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI run/render error from prompt loading.
|
|
||||||
- HTTP `404 prompt_not_found` or `400 prompt_load_failed`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Prompt ID not found.
|
|
||||||
- Invalid prompt YAML.
|
|
||||||
- Invalid prompt contract (for example bad validation mode, message content/content_file rule violation, missing schema path for `json_schema`).
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go run ./cmd/scriptorium render --config ./examples/config.yml --prompt <prompt-id> --input transcript=./examples/fixtures/transcript.md --input glossary=./examples/fixtures/glossary.yml --format json
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Correct prompt ID.
|
|
||||||
- Fix prompt YAML and contract fields.
|
|
||||||
- Ensure referenced `content_file` paths exist.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
- [CLI reference](cli.md)
|
|
||||||
|
|
||||||
## Profile Definition Load Failures
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI run/render error from profile loading.
|
|
||||||
- HTTP `404 profile_not_found` or `400 profile_load_failed`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Profile ID missing/not found.
|
|
||||||
- Invalid profile YAML.
|
|
||||||
- Invalid profile values.
|
|
||||||
- Raw `api_key` field present (rejected).
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go run ./cmd/scriptorium render --config ./examples/config.yml --prompt generic.markdown_summary --profile <profile-id> --input transcript=./examples/fixtures/transcript.md --input glossary=./examples/fixtures/glossary.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Correct profile ID.
|
|
||||||
- Fix profile YAML and value ranges.
|
|
||||||
- Replace `api_key` with `api_key_env`.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
- [CLI reference](cli.md)
|
|
||||||
|
|
||||||
## Input Artifact Read Failures
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI run/render error reading input artifacts.
|
|
||||||
- HTTP `400 artifact_read_failed`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- File path in input mapping does not exist or is unreadable.
|
|
||||||
- Unsupported artifact reference type in HTTP request.
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
- Verify every mapped file path exists and is readable by the process.
|
|
||||||
- For HTTP, verify each input uses supported `type` values.
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Correct file paths and permissions.
|
|
||||||
- Use supported input types (`file`, `inline`).
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [CLI reference](cli.md)
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
|
|
||||||
## Prompt Template Render Failures
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI run/render error from prompt rendering.
|
|
||||||
- HTTP `400 prompt_render_failed`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Template references missing input names.
|
|
||||||
- Template syntax or data reference issues.
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
- Run `render --format json` with the same prompt, inputs, vars, and profile selection.
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Align template `{{input "name"}}` references with actual input mappings.
|
|
||||||
- Fix template syntax and variable names.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [CLI reference](cli.md)
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
|
|
||||||
## Missing API-Key Environment Variable
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI run/render invalid request error about missing API-key environment variable.
|
|
||||||
- HTTP `400 api_key_env_missing`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Selected profile or override sets `api_key_env`, but that environment variable is unset/empty.
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
printenv SCRIPTORIUM_API_KEY
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Set the required environment variable before invoking CLI/service.
|
|
||||||
- Or use a profile that does not require API key auth for the target endpoint.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
- [Operations guide](operations.md)
|
|
||||||
|
|
||||||
## LLM Request Failures
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI `run` fails with LLM generation errors.
|
|
||||||
- HTTP returns `502 llm_failed`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Endpoint unreachable.
|
|
||||||
- Non-2xx response from provider.
|
|
||||||
- Timeout.
|
|
||||||
- Malformed provider response.
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
- Confirm endpoint URL and model in selected profile/overrides.
|
|
||||||
- Retry with `render` first to confirm pre-LLM preparation works.
|
|
||||||
- Check provider/network logs for non-2xx responses and timeouts.
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Correct endpoint/model settings.
|
|
||||||
- Adjust timeout if needed.
|
|
||||||
- Resolve provider-side or network issues.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [CLI reference](cli.md)
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
- [Operations guide](operations.md)
|
|
||||||
|
|
||||||
## Validation Status Failed (`run` Exit 2 Or HTTP 200 With Failed Status)
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI exits with code `2`.
|
|
||||||
- HTTP returns `200`, but `validation.status` is `failed`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Generated output failed `basic`, `json`, or `json_schema` content validation.
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
- Inspect validation mode and validation errors in CLI summary/HTTP response.
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Refine prompt constraints.
|
|
||||||
- Tighten schema or adjust model/profile settings.
|
|
||||||
- Rerun after correction.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
- [Operations guide](operations.md)
|
|
||||||
|
|
||||||
## Validation Runtime Failure
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI `run` fails with validation runtime error.
|
|
||||||
- HTTP returns `500 validation_runtime_failed`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- `json_schema` schema file missing/inaccessible.
|
|
||||||
- Invalid schema JSON document.
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
- Verify `schema_dir` and `output.schema_path` resolution.
|
|
||||||
- Check schema file readability and valid JSON syntax.
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Correct schema path.
|
|
||||||
- Fix schema JSON content.
|
|
||||||
- Rerun.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
- [Operations guide](operations.md)
|
|
||||||
|
|
||||||
## HTTP Request Parsing/Contract Errors
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- HTTP `400 invalid_json` or `400 invalid_request`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Malformed JSON body.
|
|
||||||
- Unknown JSON fields.
|
|
||||||
- Missing required `prompt_id` or `inputs`.
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
- Revalidate request JSON.
|
|
||||||
- Confirm required request fields are present.
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Send valid JSON with only supported fields.
|
|
||||||
- Ensure `prompt_id` and at least one input mapping are included.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [Operations guide](operations.md)
|
|
||||||
- [CLI reference](cli.md)
|
|
||||||
318
engine.go
Normal file
318
engine.go
Normal file
@@ -0,0 +1,318 @@
|
|||||||
|
package scriptorium
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
artifactadapter "gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile/builtin"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrInvalidConfig indicates invalid public engine configuration.
|
||||||
|
var ErrInvalidConfig = errors.New("invalid engine configuration")
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrInvalidRequest = errors.New("invalid run request")
|
||||||
|
ErrPromptNotFound = errors.New("prompt not found")
|
||||||
|
ErrProfileNotFound = errors.New("profile not found")
|
||||||
|
ErrPromptLoad = errors.New("failed to load prompt definition")
|
||||||
|
ErrProfileLoad = errors.New("failed to load execution profile")
|
||||||
|
ErrArtifactLoad = errors.New("failed to load artifact")
|
||||||
|
ErrPromptRender = errors.New("failed to render prompt")
|
||||||
|
ErrLLMGenerate = errors.New("failed to generate output")
|
||||||
|
ErrValidation = errors.New("failed to validate output")
|
||||||
|
)
|
||||||
|
|
||||||
|
// Engine prepares and runs Scriptorium prompt requests.
|
||||||
|
type Engine struct {
|
||||||
|
runner *usecase.Runner
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config configures a public Scriptorium engine.
|
||||||
|
type Config struct {
|
||||||
|
PromptDir string
|
||||||
|
ProfileDir string
|
||||||
|
SchemaDir string
|
||||||
|
Timeout time.Duration
|
||||||
|
HTTPClient *http.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
// Option customizes engine construction.
|
||||||
|
type Option interface {
|
||||||
|
apply(*engineOptions) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type optionFunc func(*engineOptions) error
|
||||||
|
|
||||||
|
func (f optionFunc) apply(options *engineOptions) error {
|
||||||
|
return f(options)
|
||||||
|
}
|
||||||
|
|
||||||
|
type engineOptions struct {
|
||||||
|
llmClient llm.Client
|
||||||
|
promptDefs promptdef.Repository
|
||||||
|
profiles profile.Repository
|
||||||
|
memoryProfiles profile.Repository
|
||||||
|
validator validate.Validator
|
||||||
|
promptSource bool
|
||||||
|
profileSource bool
|
||||||
|
memorySource bool
|
||||||
|
validatorSource bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithLLMClient injects a custom LLM client for execution.
|
||||||
|
func WithLLMClient(client LLMClient) Option {
|
||||||
|
return optionFunc(func(options *engineOptions) error {
|
||||||
|
if client == nil {
|
||||||
|
return ErrInvalidConfig
|
||||||
|
}
|
||||||
|
options.llmClient = publicLLMClientAdapter{client: client}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithPromptFS loads prompt definitions from fsys under root.
|
||||||
|
//
|
||||||
|
// The source uses the same strict prompt YAML rules as configured prompt
|
||||||
|
// directories, and prompt content_file paths resolve within this source.
|
||||||
|
func WithPromptFS(fsys fs.FS, root string) Option {
|
||||||
|
return optionFunc(func(options *engineOptions) error {
|
||||||
|
if fsys == nil {
|
||||||
|
return ErrInvalidConfig
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(root) == "" {
|
||||||
|
return ErrInvalidConfig
|
||||||
|
}
|
||||||
|
options.promptDefs = promptdef.NewFSRepository(fsys, root)
|
||||||
|
options.promptSource = true
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithPromptFile loads prompt definitions from the single prompt file at path.
|
||||||
|
//
|
||||||
|
// Relative prompt content_file paths resolve from the file's directory.
|
||||||
|
func WithPromptFile(path string) Option {
|
||||||
|
return optionFunc(func(options *engineOptions) error {
|
||||||
|
fsys, root, err := fileSource(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
options.promptDefs = promptdef.NewFSRepository(fsys, root)
|
||||||
|
options.promptSource = true
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithProfileFS loads execution profiles from fsys under root.
|
||||||
|
//
|
||||||
|
// Profiles from this source overlay built-in profiles. Profile YAML must use
|
||||||
|
// api_key_env for environment-based credentials; raw API keys are rejected.
|
||||||
|
func WithProfileFS(fsys fs.FS, root string) Option {
|
||||||
|
return optionFunc(func(options *engineOptions) error {
|
||||||
|
if fsys == nil {
|
||||||
|
return ErrInvalidConfig
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(root) == "" {
|
||||||
|
return ErrInvalidConfig
|
||||||
|
}
|
||||||
|
options.profiles = profile.NewFSRepository(fsys, root)
|
||||||
|
options.profileSource = true
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithProfileFile loads execution profiles from the single profile file at path.
|
||||||
|
//
|
||||||
|
// The profile overlays built-in profiles. Profile YAML must use api_key_env for
|
||||||
|
// environment-based credentials; raw API keys are rejected.
|
||||||
|
func WithProfileFile(path string) Option {
|
||||||
|
return optionFunc(func(options *engineOptions) error {
|
||||||
|
fsys, root, err := fileSource(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
options.profiles = profile.NewFSRepository(fsys, root)
|
||||||
|
options.profileSource = true
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithProfiles configures in-memory profiles that take precedence over
|
||||||
|
// configured profile files and built-in profiles.
|
||||||
|
func WithProfiles(profiles ...Profile) Option {
|
||||||
|
return optionFunc(func(options *engineOptions) error {
|
||||||
|
repo, err := newMemoryProfileRepository(profiles)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
options.memoryProfiles = repo
|
||||||
|
options.memorySource = true
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithSchemaFS loads JSON Schema documents from fsys under root.
|
||||||
|
//
|
||||||
|
// Prompt schema_path values resolve within this source when schema validation
|
||||||
|
// or structured output is requested.
|
||||||
|
func WithSchemaFS(fsys fs.FS, root string) Option {
|
||||||
|
return optionFunc(func(options *engineOptions) error {
|
||||||
|
if fsys == nil {
|
||||||
|
return ErrInvalidConfig
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(root) == "" {
|
||||||
|
return ErrInvalidConfig
|
||||||
|
}
|
||||||
|
options.validator = validate.NewFSValidator(fsys, root)
|
||||||
|
options.validatorSource = true
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithSchemaFile loads JSON Schema documents from the single schema file at path.
|
||||||
|
//
|
||||||
|
// Prompt schema_path values refer to the file's base name.
|
||||||
|
func WithSchemaFile(path string) Option {
|
||||||
|
return optionFunc(func(options *engineOptions) error {
|
||||||
|
fsys, root, err := fileSource(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
options.validator = validate.NewFSValidator(fsys, root)
|
||||||
|
options.validatorSource = true
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewEngine constructs an Engine using the same default internal components as
|
||||||
|
// the CLI and HTTP adapters.
|
||||||
|
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||||
|
var options engineOptions
|
||||||
|
for _, opt := range opts {
|
||||||
|
if opt == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := opt.apply(&options); err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
promptDefs := options.promptDefs
|
||||||
|
if !options.promptSource {
|
||||||
|
if strings.TrimSpace(cfg.PromptDir) == "" {
|
||||||
|
return nil, fmt.Errorf("%w: prompt directory is required", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
profiles := builtin.NewRepositoryWithDirectory(cfg.ProfileDir)
|
||||||
|
if options.profileSource {
|
||||||
|
profiles = builtin.NewRepositoryWithPrimary(options.profiles)
|
||||||
|
}
|
||||||
|
if options.memorySource {
|
||||||
|
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
|
||||||
|
}
|
||||||
|
|
||||||
|
validator := options.validator
|
||||||
|
if !options.validatorSource {
|
||||||
|
schemaDir := cfg.SchemaDir
|
||||||
|
if strings.TrimSpace(schemaDir) == "" {
|
||||||
|
schemaDir = defaults.SchemaDirDefault
|
||||||
|
}
|
||||||
|
validator = validate.NewStandardValidator(schemaDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
llmClient := options.llmClient
|
||||||
|
if llmClient == nil {
|
||||||
|
var err error
|
||||||
|
llmClient, err = llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
|
||||||
|
Timeout: cfg.Timeout,
|
||||||
|
HTTPClient: cfg.HTTPClient,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Engine{
|
||||||
|
runner: usecase.NewRunner(
|
||||||
|
promptDefs,
|
||||||
|
profiles,
|
||||||
|
artifactadapter.NewCompositeReader(),
|
||||||
|
prompt.NewGoRenderer(),
|
||||||
|
llmClient,
|
||||||
|
validator,
|
||||||
|
),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileSource(name string) (fs.FS, string, error) {
|
||||||
|
cleanName := strings.TrimSpace(name)
|
||||||
|
if cleanName == "" {
|
||||||
|
return nil, "", ErrInvalidConfig
|
||||||
|
}
|
||||||
|
dir := filepath.Dir(cleanName)
|
||||||
|
base := filepath.Base(cleanName)
|
||||||
|
if base == "." || base == string(filepath.Separator) || strings.TrimSpace(base) == "" {
|
||||||
|
return nil, "", ErrInvalidConfig
|
||||||
|
}
|
||||||
|
info, err := os.Stat(cleanName)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, cleanName, err)
|
||||||
|
}
|
||||||
|
if info.IsDir() {
|
||||||
|
return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, cleanName)
|
||||||
|
}
|
||||||
|
return os.DirFS(dir), filepath.ToSlash(base), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prepare resolves a prompt request without calling an LLM.
|
||||||
|
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
||||||
|
if e == nil || e.runner == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
domainReq, err := toDomainRunRequest(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := e.runner.Prepare(ctx, domainReq)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return fromDomainPreparedRun(prepared), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run executes a prompt request and returns the generated artifact and metadata.
|
||||||
|
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||||
|
if e == nil || e.runner == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
domainReq, err := toDomainRunRequest(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := e.runner.Run(ctx, domainReq)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return fromDomainRunResult(result), nil
|
||||||
|
}
|
||||||
1799
engine_test.go
Normal file
1799
engine_test.go
Normal file
File diff suppressed because it is too large
Load Diff
79
errors.go
Normal file
79
errors.go
Normal file
@@ -0,0 +1,79 @@
|
|||||||
|
package scriptorium
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||||
|
)
|
||||||
|
|
||||||
|
func mapPublicError(err error) error {
|
||||||
|
if err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if hasPublicError(err) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
publicErr := publicErrorFor(err)
|
||||||
|
if publicErr == nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%w: %w", publicErr, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasPublicError(err error) bool {
|
||||||
|
for _, publicErr := range []error{
|
||||||
|
ErrInvalidConfig,
|
||||||
|
ErrInvalidRequest,
|
||||||
|
ErrPromptNotFound,
|
||||||
|
ErrProfileNotFound,
|
||||||
|
ErrPromptLoad,
|
||||||
|
ErrProfileLoad,
|
||||||
|
ErrArtifactLoad,
|
||||||
|
ErrPromptRender,
|
||||||
|
ErrLLMGenerate,
|
||||||
|
ErrValidation,
|
||||||
|
} {
|
||||||
|
if errors.Is(err, publicErr) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func publicErrorFor(err error) error {
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, promptdef.ErrPromptDefinitionNotFound):
|
||||||
|
return ErrPromptNotFound
|
||||||
|
case errors.Is(err, profile.ErrProfileNotFound):
|
||||||
|
return ErrProfileNotFound
|
||||||
|
case errors.Is(err, usecase.ErrPromptLoad):
|
||||||
|
return ErrPromptLoad
|
||||||
|
case errors.Is(err, usecase.ErrProfileLoad):
|
||||||
|
return ErrProfileLoad
|
||||||
|
case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition):
|
||||||
|
return ErrPromptLoad
|
||||||
|
case isProfileLoadCause(err):
|
||||||
|
return ErrProfileLoad
|
||||||
|
case errors.Is(err, usecase.ErrArtifactLoad):
|
||||||
|
return ErrArtifactLoad
|
||||||
|
case errors.Is(err, usecase.ErrPromptRender):
|
||||||
|
return ErrPromptRender
|
||||||
|
case errors.Is(err, usecase.ErrLLMGenerate):
|
||||||
|
return ErrLLMGenerate
|
||||||
|
case errors.Is(err, usecase.ErrValidation):
|
||||||
|
return ErrValidation
|
||||||
|
case errors.Is(err, usecase.ErrInvalidRequest):
|
||||||
|
return ErrInvalidRequest
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func isProfileLoadCause(err error) bool {
|
||||||
|
return errors.Is(err, profile.ErrInvalidYAML) ||
|
||||||
|
errors.Is(err, profile.ErrInvalidProfile) ||
|
||||||
|
errors.Is(err, profile.ErrRawAPIKeyNotAllowed)
|
||||||
|
}
|
||||||
13
examples/config.full.yml
Normal file
13
examples/config.full.yml
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
prompt_dir: ./examples/prompts
|
||||||
|
profile_dir: ./examples/profiles
|
||||||
|
schema_dir: ./examples/schemas
|
||||||
|
|
||||||
|
server:
|
||||||
|
addr: 127.0.0.1:8080
|
||||||
|
artifact_root: .
|
||||||
|
max_request_bytes: 16777216
|
||||||
|
max_artifact_bytes: 16777216
|
||||||
|
max_response_bytes: 16777216
|
||||||
|
|
||||||
|
defaults:
|
||||||
|
render_format: text
|
||||||
@@ -4,6 +4,7 @@ schema_dir: ./examples/schemas
|
|||||||
|
|
||||||
server:
|
server:
|
||||||
addr: :8080
|
addr: :8080
|
||||||
|
artifact_root: .
|
||||||
|
|
||||||
defaults:
|
defaults:
|
||||||
render_format: text
|
render_format: text
|
||||||
|
|||||||
50
examples/go-library/prepare/main.go
Normal file
50
examples/go-library/prepare/main.go
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||||
|
PromptDir: "./examples/prompts",
|
||||||
|
ProfileDir: "./examples/profiles",
|
||||||
|
SchemaDir: "./examples/schemas",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||||
|
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
summary := struct {
|
||||||
|
PromptID string `json:"prompt_id"`
|
||||||
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
|
Model string `json:"model"`
|
||||||
|
MessageCount int `json:"message_count"`
|
||||||
|
InputHashes map[string]string `json:"input_hashes"`
|
||||||
|
}{
|
||||||
|
PromptID: prepared.PromptID,
|
||||||
|
SelectedProfileID: prepared.SelectedProfileID,
|
||||||
|
Model: prepared.EffectiveModelParams.Model,
|
||||||
|
MessageCount: len(prepared.Messages),
|
||||||
|
InputHashes: prepared.InputHashes,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := json.NewEncoder(os.Stdout).Encode(summary); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
51
formatting.go
Normal file
51
formatting.go
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
package scriptorium
|
||||||
|
|
||||||
|
import "fmt"
|
||||||
|
|
||||||
|
// String returns a concise request summary without exposing direct API keys.
|
||||||
|
func (r RunRequest) String() string {
|
||||||
|
return r.redactedString()
|
||||||
|
}
|
||||||
|
|
||||||
|
// GoString returns a concise request summary without exposing direct API keys.
|
||||||
|
func (r RunRequest) GoString() string {
|
||||||
|
return r.redactedString()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r RunRequest) redactedString() string {
|
||||||
|
return fmt.Sprintf(
|
||||||
|
"scriptorium.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t Metadata:%d}",
|
||||||
|
r.PromptID,
|
||||||
|
r.PromptVersion,
|
||||||
|
r.ProfileID,
|
||||||
|
r.APIKey != "",
|
||||||
|
len(r.Inputs),
|
||||||
|
len(r.Vars),
|
||||||
|
r.Execution != nil,
|
||||||
|
r.Validation != nil,
|
||||||
|
len(r.Metadata),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// String returns a concise request summary without exposing direct API keys or
|
||||||
|
// rendered prompt content.
|
||||||
|
func (r GenerateRequest) String() string {
|
||||||
|
return r.redactedString()
|
||||||
|
}
|
||||||
|
|
||||||
|
// GoString returns a concise request summary without exposing direct API keys or
|
||||||
|
// rendered prompt content.
|
||||||
|
func (r GenerateRequest) GoString() string {
|
||||||
|
return r.redactedString()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r GenerateRequest) redactedString() string {
|
||||||
|
return fmt.Sprintf(
|
||||||
|
"scriptorium.GenerateRequest{Messages:%d Model:%q APIKeySet:%t StructuredOutputSet:%t ExtraParams:%d}",
|
||||||
|
len(r.Prompt.Messages),
|
||||||
|
r.Target.Model,
|
||||||
|
r.APIKey != "",
|
||||||
|
r.StructuredOutput != nil,
|
||||||
|
len(r.Target.ExtraParams),
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -19,7 +19,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile/builtin"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||||
@@ -33,8 +33,7 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
errPromptDirRequired = "prompt directory is required; provide --prompt-dir or config.yml prompt_dir"
|
errPromptDirRequired = "prompt directory is required; provide --prompt-dir or config.yml prompt_dir"
|
||||||
errProfileDirRequired = "profile directory is required; provide --profile-dir or config.yml profile_dir"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type runConfig struct {
|
type runConfig struct {
|
||||||
@@ -75,10 +74,14 @@ type renderConfig struct {
|
|||||||
type serveConfig struct {
|
type serveConfig struct {
|
||||||
configPath string
|
configPath string
|
||||||
|
|
||||||
addr string
|
addr string
|
||||||
promptDir string
|
promptDir string
|
||||||
profileDir string
|
profileDir string
|
||||||
schemaDir string
|
schemaDir string
|
||||||
|
artifactRoot string
|
||||||
|
maxRequestBytes int64
|
||||||
|
maxArtifactBytes int64
|
||||||
|
maxResponseBytes int64
|
||||||
}
|
}
|
||||||
|
|
||||||
type commonCommandSettings struct {
|
type commonCommandSettings struct {
|
||||||
@@ -86,6 +89,10 @@ type commonCommandSettings struct {
|
|||||||
profileDir string
|
profileDir string
|
||||||
schemaDir string
|
schemaDir string
|
||||||
serverAddr string
|
serverAddr string
|
||||||
|
artifactRoot string
|
||||||
|
maxRequestBytes int64
|
||||||
|
maxArtifactBytes int64
|
||||||
|
maxResponseBytes int64
|
||||||
defaultRenderFormat renderformat.PreparedRunOutputFormat
|
defaultRenderFormat renderformat.PreparedRunOutputFormat
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -203,9 +210,18 @@ func serveCommand(args []string, stderr io.Writer) int {
|
|||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
|
|
||||||
runner := newRunner(cfg.promptDir, cfg.profileDir, cfg.schemaDir, llmClient)
|
artifactReader, err := artifactadapter.NewRestrictedCompositeReaderWithLimit(cfg.artifactRoot, cfg.maxArtifactBytes)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "artifact root error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
|
||||||
h := httpadapter.NewHandler(runner)
|
runner := newRunnerWithArtifactReader(cfg.promptDir, cfg.profileDir, cfg.schemaDir, llmClient, artifactReader)
|
||||||
|
|
||||||
|
h := httpadapter.NewHandlerWithOptions(runner, httpadapter.HandlerOptions{
|
||||||
|
MaxRequestBytes: cfg.maxRequestBytes,
|
||||||
|
MaxResponseBytes: cfg.maxResponseBytes,
|
||||||
|
})
|
||||||
srv := &http.Server{
|
srv := &http.Server{
|
||||||
Addr: cfg.addr,
|
Addr: cfg.addr,
|
||||||
Handler: h,
|
Handler: h,
|
||||||
@@ -282,6 +298,10 @@ func parseServeArgs(args []string) (*serveConfig, error) {
|
|||||||
fs.StringVar(&cfg.promptDir, "prompt-dir", "", "directory containing prompt definition YAML files")
|
fs.StringVar(&cfg.promptDir, "prompt-dir", "", "directory containing prompt definition YAML files")
|
||||||
fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing execution profile YAML files")
|
fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing execution profile YAML files")
|
||||||
fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas")
|
fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas")
|
||||||
|
fs.StringVar(&cfg.artifactRoot, "artifact-root", "", "base directory for HTTP file input artifacts")
|
||||||
|
fs.Int64Var(&cfg.maxRequestBytes, "max-request-bytes", 0, "maximum HTTP request body bytes; 0 disables the limit")
|
||||||
|
fs.Int64Var(&cfg.maxArtifactBytes, "max-artifact-bytes", 0, "maximum HTTP file artifact bytes; 0 disables the limit")
|
||||||
|
fs.Int64Var(&cfg.maxResponseBytes, "max-response-bytes", 0, "maximum HTTP response body bytes; 0 disables the limit")
|
||||||
|
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -291,10 +311,14 @@ func parseServeArgs(args []string) (*serveConfig, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
settings, err := resolveCommonSettings(fs, cfg.configPath, appconfig.CLIOverrides{
|
settings, err := resolveCommonSettings(fs, cfg.configPath, appconfig.CLIOverrides{
|
||||||
PromptDir: cfg.promptDirIfSet(fs),
|
PromptDir: cfg.promptDirIfSet(fs),
|
||||||
ProfileDir: cfg.profileDirIfSet(fs),
|
ProfileDir: cfg.profileDirIfSet(fs),
|
||||||
SchemaDir: cfg.schemaDirIfSet(fs),
|
SchemaDir: cfg.schemaDirIfSet(fs),
|
||||||
ServerAddr: cfg.addrIfSet(fs),
|
ServerAddr: cfg.addrIfSet(fs),
|
||||||
|
ArtifactRoot: cfg.artifactRootIfSet(fs),
|
||||||
|
MaxRequestBytes: cfg.maxRequestBytesIfSet(fs),
|
||||||
|
MaxArtifactBytes: cfg.maxArtifactBytesIfSet(fs),
|
||||||
|
MaxResponseBytes: cfg.maxResponseBytesIfSet(fs),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -304,14 +328,23 @@ func parseServeArgs(args []string) (*serveConfig, error) {
|
|||||||
cfg.profileDir = settings.profileDir
|
cfg.profileDir = settings.profileDir
|
||||||
cfg.schemaDir = settings.schemaDir
|
cfg.schemaDir = settings.schemaDir
|
||||||
cfg.addr = settings.serverAddr
|
cfg.addr = settings.serverAddr
|
||||||
|
cfg.artifactRoot = settings.artifactRoot
|
||||||
|
cfg.maxRequestBytes = settings.maxRequestBytes
|
||||||
|
cfg.maxArtifactBytes = settings.maxArtifactBytes
|
||||||
|
cfg.maxResponseBytes = settings.maxResponseBytes
|
||||||
|
|
||||||
if err := validateRequiredLibraryDirs(cfg.promptDir, cfg.profileDir); err != nil {
|
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg.promptDir = filepath.Clean(cfg.promptDir)
|
cfg.promptDir = filepath.Clean(cfg.promptDir)
|
||||||
cfg.profileDir = filepath.Clean(cfg.profileDir)
|
if strings.TrimSpace(cfg.profileDir) != "" {
|
||||||
|
cfg.profileDir = filepath.Clean(cfg.profileDir)
|
||||||
|
}
|
||||||
cfg.schemaDir = filepath.Clean(cfg.schemaDir)
|
cfg.schemaDir = filepath.Clean(cfg.schemaDir)
|
||||||
|
if strings.TrimSpace(cfg.artifactRoot) != "" {
|
||||||
|
cfg.artifactRoot = filepath.Clean(cfg.artifactRoot)
|
||||||
|
}
|
||||||
return cfg, nil
|
return cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -353,7 +386,7 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
|
|||||||
cfg.schemaDir = settings.schemaDir
|
cfg.schemaDir = settings.schemaDir
|
||||||
cfg.defaultRenderFormat = settings.defaultRenderFormat
|
cfg.defaultRenderFormat = settings.defaultRenderFormat
|
||||||
|
|
||||||
if err := validateRequiredLibraryDirs(cfg.promptDir, cfg.profileDir); err != nil {
|
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(cfg.promptID) == "" {
|
if strings.TrimSpace(cfg.promptID) == "" {
|
||||||
@@ -363,7 +396,9 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
|
|||||||
return errors.New("at least one --input is required")
|
return errors.New("at least one --input is required")
|
||||||
}
|
}
|
||||||
cfg.promptDir = filepath.Clean(cfg.promptDir)
|
cfg.promptDir = filepath.Clean(cfg.promptDir)
|
||||||
cfg.profileDir = filepath.Clean(cfg.profileDir)
|
if strings.TrimSpace(cfg.profileDir) != "" {
|
||||||
|
cfg.profileDir = filepath.Clean(cfg.profileDir)
|
||||||
|
}
|
||||||
if cfg.outputPath != "" {
|
if cfg.outputPath != "" {
|
||||||
cfg.outputPath = filepath.Clean(cfg.outputPath)
|
cfg.outputPath = filepath.Clean(cfg.outputPath)
|
||||||
}
|
}
|
||||||
@@ -426,6 +461,34 @@ func (c *serveConfig) addrIfSet(fs *flag.FlagSet) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *serveConfig) artifactRootIfSet(fs *flag.FlagSet) string {
|
||||||
|
if flagWasSet(fs, "artifact-root") {
|
||||||
|
return c.artifactRoot
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *serveConfig) maxRequestBytesIfSet(fs *flag.FlagSet) *int64 {
|
||||||
|
if flagWasSet(fs, "max-request-bytes") {
|
||||||
|
return &c.maxRequestBytes
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *serveConfig) maxArtifactBytesIfSet(fs *flag.FlagSet) *int64 {
|
||||||
|
if flagWasSet(fs, "max-artifact-bytes") {
|
||||||
|
return &c.maxArtifactBytes
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *serveConfig) maxResponseBytesIfSet(fs *flag.FlagSet) *int64 {
|
||||||
|
if flagWasSet(fs, "max-response-bytes") {
|
||||||
|
return &c.maxResponseBytes
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func registerConfigPathFlag(fs *flag.FlagSet, target *string) {
|
func registerConfigPathFlag(fs *flag.FlagSet, target *string) {
|
||||||
fs.StringVar(
|
fs.StringVar(
|
||||||
target,
|
target,
|
||||||
@@ -463,25 +526,33 @@ func resolveCommonSettings(fs *flag.FlagSet, configPath string, overrides appcon
|
|||||||
profileDir: settings.ProfileDir,
|
profileDir: settings.ProfileDir,
|
||||||
schemaDir: settings.SchemaDir,
|
schemaDir: settings.SchemaDir,
|
||||||
serverAddr: settings.ServerAddr,
|
serverAddr: settings.ServerAddr,
|
||||||
|
artifactRoot: settings.ArtifactRoot,
|
||||||
|
maxRequestBytes: settings.MaxRequestBytes,
|
||||||
|
maxArtifactBytes: settings.MaxArtifactBytes,
|
||||||
|
maxResponseBytes: settings.MaxResponseBytes,
|
||||||
defaultRenderFormat: settings.DefaultRenderFormat,
|
defaultRenderFormat: settings.DefaultRenderFormat,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateRequiredLibraryDirs(promptDir, profileDir string) error {
|
func validateRequiredLibraryDirs(promptDir string) error {
|
||||||
if strings.TrimSpace(promptDir) == "" {
|
if strings.TrimSpace(promptDir) == "" {
|
||||||
return errors.New(errPromptDirRequired)
|
return errors.New(errPromptDirRequired)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(profileDir) == "" {
|
|
||||||
return errors.New(errProfileDirRequired)
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func newRunner(promptDir, profileDir, schemaDir string, llmClient llm.Client) *usecase.Runner {
|
func newRunner(promptDir, profileDir, schemaDir string, llmClient llm.Client) *usecase.Runner {
|
||||||
|
return newRunnerWithArtifactReader(promptDir, profileDir, schemaDir, llmClient, artifactadapter.NewCompositeReader())
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRunnerWithArtifactReader(promptDir, profileDir, schemaDir string, llmClient llm.Client, artifactReader artifactadapter.Reader) *usecase.Runner {
|
||||||
|
if artifactReader == nil {
|
||||||
|
artifactReader = artifactadapter.NewCompositeReader()
|
||||||
|
}
|
||||||
return usecase.NewRunner(
|
return usecase.NewRunner(
|
||||||
promptdef.NewFilesystemRepository(promptDir),
|
promptdef.NewFilesystemRepository(promptDir),
|
||||||
profile.NewFilesystemRepository(profileDir),
|
builtin.NewRepositoryWithDirectory(profileDir),
|
||||||
artifactadapter.NewCompositeReader(),
|
artifactReader,
|
||||||
prompt.NewGoRenderer(),
|
prompt.NewGoRenderer(),
|
||||||
llmClient,
|
llmClient,
|
||||||
validate.NewStandardValidator(schemaDir),
|
validate.NewStandardValidator(schemaDir),
|
||||||
@@ -513,18 +584,25 @@ func buildRunRequestFromConfig(cfg *runConfig) (domain.RunRequest, error) {
|
|||||||
inputs[name] = domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path}
|
inputs[name] = domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path}
|
||||||
}
|
}
|
||||||
|
|
||||||
var modelOverride *domain.ExecutionTarget
|
var modelOverride *domain.ExecutionTargetOverride
|
||||||
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.apiKeyEnvSet || cfg.timeoutSet {
|
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.apiKeyEnvSet || cfg.timeoutSet {
|
||||||
modelOverride = &domain.ExecutionTarget{
|
modelOverride = &domain.ExecutionTargetOverride{
|
||||||
Endpoint: cfg.llmBaseURL,
|
Endpoint: cfg.llmBaseURL,
|
||||||
Model: cfg.model,
|
Model: cfg.model,
|
||||||
Temperature: cfg.temperature,
|
APIKeyEnv: cfg.apiKeyEnv,
|
||||||
MaxTokens: cfg.maxTokens,
|
}
|
||||||
TopP: cfg.topP,
|
if cfg.temperatureSet {
|
||||||
APIKeyEnv: cfg.apiKeyEnv,
|
modelOverride.Temperature = &cfg.temperature
|
||||||
|
}
|
||||||
|
if cfg.maxTokensSet {
|
||||||
|
modelOverride.MaxTokens = &cfg.maxTokens
|
||||||
|
}
|
||||||
|
if cfg.topPSet {
|
||||||
|
modelOverride.TopP = &cfg.topP
|
||||||
}
|
}
|
||||||
if cfg.timeoutSet {
|
if cfg.timeoutSet {
|
||||||
modelOverride.TimeoutSeconds = int(cfg.timeout.Seconds())
|
timeoutSeconds := int(cfg.timeout.Seconds())
|
||||||
|
modelOverride.TimeoutSeconds = &timeoutSeconds
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -606,7 +684,7 @@ func printSummary(stderr io.Writer, res *domain.RunResult) {
|
|||||||
if res == nil {
|
if res == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d\n",
|
fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d",
|
||||||
res.PromptID,
|
res.PromptID,
|
||||||
res.PromptVersion,
|
res.PromptVersion,
|
||||||
res.SelectedProfileID,
|
res.SelectedProfileID,
|
||||||
@@ -620,11 +698,15 @@ func printSummary(stderr io.Writer, res *domain.RunResult) {
|
|||||||
res.Usage.CompletionTokens,
|
res.Usage.CompletionTokens,
|
||||||
res.Usage.TotalTokens,
|
res.Usage.TotalTokens,
|
||||||
)
|
)
|
||||||
|
if res.Usage.CachedTokens != 0 || res.Usage.CacheWriteTokens != 0 {
|
||||||
|
fmt.Fprintf(stderr, " cached_tokens=%d cache_write_tokens=%d", res.Usage.CachedTokens, res.Usage.CacheWriteTokens)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
func printUsage(w io.Writer) {
|
func printUsage(w io.Writer) {
|
||||||
fmt.Fprintln(w, "usage: scriptorium <run|render|serve> ...")
|
fmt.Fprintln(w, "usage: scriptorium <run|render|serve> ...")
|
||||||
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID --input name=path [--input ...] [--profile ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--var k=v] [--out path] [--timeout 10m]")
|
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID --input name=path [--input ...] [--profile ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--var k=v] [--out path] [--timeout 10m]")
|
||||||
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID --input name=path [--input ...] [--profile ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--var k=v] [--format text|json] [--out path] [--timeout 10m]")
|
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID --input name=path [--input ...] [--profile ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--var k=v] [--format text|json] [--out path] [--timeout 10m]")
|
||||||
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR]\n", defaults.HTTPAddrDefault)
|
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -74,12 +74,12 @@ func TestParseRunArgsRequiredFlags(t *testing.T) {
|
|||||||
t.Fatalf("expected clear prompt-dir guidance, got %v", err)
|
t.Fatalf("expected clear prompt-dir guidance, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p", "--input", "a=b"})
|
cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p", "--input", "a=b"})
|
||||||
if err == nil {
|
if err != nil {
|
||||||
t.Fatal("expected missing --profile-dir error")
|
t.Fatalf("expected missing --profile-dir to be accepted, got %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "profile directory is required") {
|
if cfg.profileDir != "" {
|
||||||
t.Fatalf("expected clear profile-dir guidance, got %v", err)
|
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--input", "a=b"})
|
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--input", "a=b"})
|
||||||
@@ -161,17 +161,12 @@ func TestParseServeArgsRequiredFlags(t *testing.T) {
|
|||||||
t.Fatalf("expected clear prompt-dir guidance, got %v", err)
|
t.Fatalf("expected clear prompt-dir guidance, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts"})
|
cfg, err := parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts"})
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected missing --profile-dir error")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "profile directory is required") {
|
|
||||||
t.Fatalf("expected clear profile-dir guidance, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles"})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected valid serve args, got %v", err)
|
t.Fatalf("expected missing --profile-dir to be accepted, got %v", err)
|
||||||
|
}
|
||||||
|
if cfg.profileDir != "" {
|
||||||
|
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
||||||
}
|
}
|
||||||
if cfg.addr != defaults.HTTPAddrDefault {
|
if cfg.addr != defaults.HTTPAddrDefault {
|
||||||
t.Fatalf("expected default addr %s, got %q", defaults.HTTPAddrDefault, cfg.addr)
|
t.Fatalf("expected default addr %s, got %q", defaults.HTTPAddrDefault, cfg.addr)
|
||||||
@@ -197,6 +192,26 @@ func TestParseServeArgsRejectsRuntimeOverrideFlags(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestUsageIncludesServeFileAndSizeLimitFlags(t *testing.T) {
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Run(nil, io.Discard, &stderr)
|
||||||
|
if code != ExitRuntimeError {
|
||||||
|
t.Fatalf("expected usage path to return runtime error, got %d", code)
|
||||||
|
}
|
||||||
|
|
||||||
|
usage := stderr.String()
|
||||||
|
for _, want := range []string{
|
||||||
|
"--artifact-root",
|
||||||
|
"--max-request-bytes",
|
||||||
|
"--max-artifact-bytes",
|
||||||
|
"--max-response-bytes",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(usage, want) {
|
||||||
|
t.Fatalf("expected usage to include %q, got %q", want, usage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseRunArgsTimeout(t *testing.T) {
|
func TestParseRunArgsTimeout(t *testing.T) {
|
||||||
cfg, err := parseRunArgs([]string{
|
cfg, err := parseRunArgs([]string{
|
||||||
"--prompt-dir", "./prompts",
|
"--prompt-dir", "./prompts",
|
||||||
@@ -440,11 +455,19 @@ profile_dir: ./from-config/profiles
|
|||||||
schema_dir: ./from-config/schemas
|
schema_dir: ./from-config/schemas
|
||||||
server:
|
server:
|
||||||
addr: 127.0.0.1:9000
|
addr: 127.0.0.1:9000
|
||||||
|
artifact_root: ./from-config/artifacts
|
||||||
|
max_request_bytes: 1024
|
||||||
|
max_artifact_bytes: 2048
|
||||||
|
max_response_bytes: 4096
|
||||||
`)
|
`)
|
||||||
|
|
||||||
cfg, err := parseServeArgs([]string{
|
cfg, err := parseServeArgs([]string{
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
"--addr", ":7777",
|
"--addr", ":7777",
|
||||||
|
"--artifact-root", "./from-cli/artifacts",
|
||||||
|
"--max-request-bytes", "0",
|
||||||
|
"--max-artifact-bytes", "8192",
|
||||||
|
"--max-response-bytes", "16384",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected valid args, got %v", err)
|
t.Fatalf("expected valid args, got %v", err)
|
||||||
@@ -462,6 +485,18 @@ server:
|
|||||||
if cfg.addr != ":7777" {
|
if cfg.addr != ":7777" {
|
||||||
t.Fatalf("expected CLI addr override, got %q", cfg.addr)
|
t.Fatalf("expected CLI addr override, got %q", cfg.addr)
|
||||||
}
|
}
|
||||||
|
if cfg.artifactRoot != filepath.Clean("./from-cli/artifacts") {
|
||||||
|
t.Fatalf("expected CLI artifact root override, got %q", cfg.artifactRoot)
|
||||||
|
}
|
||||||
|
if cfg.maxRequestBytes != 0 {
|
||||||
|
t.Fatalf("expected CLI max request bytes override, got %d", cfg.maxRequestBytes)
|
||||||
|
}
|
||||||
|
if cfg.maxArtifactBytes != 8192 {
|
||||||
|
t.Fatalf("expected CLI max artifact bytes override, got %d", cfg.maxArtifactBytes)
|
||||||
|
}
|
||||||
|
if cfg.maxResponseBytes != 16384 {
|
||||||
|
t.Fatalf("expected CLI max response bytes override, got %d", cfg.maxResponseBytes)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseServeArgsWithConfigProvidesRequiredDirectoriesAndAddr(t *testing.T) {
|
func TestParseServeArgsWithConfigProvidesRequiredDirectoriesAndAddr(t *testing.T) {
|
||||||
@@ -471,6 +506,10 @@ profile_dir: ./from-config/profiles
|
|||||||
schema_dir: ./from-config/schemas
|
schema_dir: ./from-config/schemas
|
||||||
server:
|
server:
|
||||||
addr: 127.0.0.1:9000
|
addr: 127.0.0.1:9000
|
||||||
|
artifact_root: ./from-config/artifacts
|
||||||
|
max_request_bytes: 1024
|
||||||
|
max_artifact_bytes: 2048
|
||||||
|
max_response_bytes: 4096
|
||||||
`)
|
`)
|
||||||
|
|
||||||
cfg, err := parseServeArgs([]string{
|
cfg, err := parseServeArgs([]string{
|
||||||
@@ -492,6 +531,75 @@ server:
|
|||||||
if cfg.addr != "127.0.0.1:9000" {
|
if cfg.addr != "127.0.0.1:9000" {
|
||||||
t.Fatalf("expected addr from config, got %q", cfg.addr)
|
t.Fatalf("expected addr from config, got %q", cfg.addr)
|
||||||
}
|
}
|
||||||
|
if cfg.artifactRoot != filepath.Clean("./from-config/artifacts") {
|
||||||
|
t.Fatalf("expected artifact root from config, got %q", cfg.artifactRoot)
|
||||||
|
}
|
||||||
|
if cfg.maxRequestBytes != 1024 {
|
||||||
|
t.Fatalf("expected max request bytes from config, got %d", cfg.maxRequestBytes)
|
||||||
|
}
|
||||||
|
if cfg.maxArtifactBytes != 2048 {
|
||||||
|
t.Fatalf("expected max artifact bytes from config, got %d", cfg.maxArtifactBytes)
|
||||||
|
}
|
||||||
|
if cfg.maxResponseBytes != 4096 {
|
||||||
|
t.Fatalf("expected max response bytes from config, got %d", cfg.maxResponseBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseServeArgsRejectsNegativeSizeLimits(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
flag string
|
||||||
|
}{
|
||||||
|
{name: "request", flag: "--max-request-bytes"},
|
||||||
|
{name: "artifact", flag: "--max-artifact-bytes"},
|
||||||
|
{name: "response", flag: "--max-response-bytes"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := parseServeArgs([]string{
|
||||||
|
"--prompt-dir", "./prompts",
|
||||||
|
tc.flag, "-1",
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected negative size limit error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunAndRenderRejectServeSizeLimitFlags(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
parse func([]string) error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "run",
|
||||||
|
parse: func(args []string) error {
|
||||||
|
_, err := parseRunArgs(args)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "render",
|
||||||
|
parse: func(args []string) error {
|
||||||
|
_, err := parseRenderArgs(args)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
err := tc.parse([]string{
|
||||||
|
"--prompt-dir", "./prompts",
|
||||||
|
"--prompt", "p",
|
||||||
|
"--input", "a=b",
|
||||||
|
"--max-request-bytes", "1024",
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected unsupported flag error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *testing.T) {
|
func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *testing.T) {
|
||||||
@@ -565,21 +673,21 @@ profile_dir: ./profiles
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseRunArgsFailsClearlyWhenNoEffectiveProfileDir(t *testing.T) {
|
func TestParseRunArgsAcceptsMissingEffectiveProfileDir(t *testing.T) {
|
||||||
configPath := writeAppConfigFile(t, `
|
configPath := writeAppConfigFile(t, `
|
||||||
prompt_dir: ./prompts
|
prompt_dir: ./prompts
|
||||||
`)
|
`)
|
||||||
|
|
||||||
_, err := parseRunArgs([]string{
|
cfg, err := parseRunArgs([]string{
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
"--prompt", "p",
|
"--prompt", "p",
|
||||||
"--input", "a=b",
|
"--input", "a=b",
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err != nil {
|
||||||
t.Fatal("expected missing profile_dir error")
|
t.Fatalf("expected missing profile_dir to be accepted, got %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "profile directory is required") || !strings.Contains(err.Error(), "config.yml profile_dir") {
|
if cfg.profileDir != "" {
|
||||||
t.Fatalf("expected clear profile_dir guidance, got %v", err)
|
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -601,21 +709,21 @@ profile_dir: ./profiles
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseRenderArgsFailsClearlyWhenNoEffectiveProfileDir(t *testing.T) {
|
func TestParseRenderArgsAcceptsMissingEffectiveProfileDir(t *testing.T) {
|
||||||
configPath := writeAppConfigFile(t, `
|
configPath := writeAppConfigFile(t, `
|
||||||
prompt_dir: ./prompts
|
prompt_dir: ./prompts
|
||||||
`)
|
`)
|
||||||
|
|
||||||
_, err := parseRenderArgs([]string{
|
cfg, err := parseRenderArgs([]string{
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
"--prompt", "p",
|
"--prompt", "p",
|
||||||
"--input", "a=b",
|
"--input", "a=b",
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err != nil {
|
||||||
t.Fatal("expected missing profile_dir error")
|
t.Fatalf("expected missing profile_dir to be accepted, got %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "profile directory is required") || !strings.Contains(err.Error(), "config.yml profile_dir") {
|
if cfg.profileDir != "" {
|
||||||
t.Fatalf("expected clear profile_dir guidance, got %v", err)
|
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -747,6 +855,35 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRenderCommandExplicitZeroTemperatureReachesEffectiveSettings(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
||||||
|
|
||||||
|
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
||||||
|
profile := `id: local-default
|
||||||
|
endpoint: http://127.0.0.1:1/v1
|
||||||
|
model: profile-model
|
||||||
|
temperature: 0.7
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-default.yaml"), []byte(profile), 0o644); err != nil {
|
||||||
|
t.Fatalf("failed to write profile fixture: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--prompt-dir", lib.promptDir,
|
||||||
|
"--profile-dir", lib.profileDir,
|
||||||
|
"--prompt", "prompt.render",
|
||||||
|
"--input", "transcript=" + inputPath,
|
||||||
|
"--temperature", "0",
|
||||||
|
})
|
||||||
|
if code != ExitOK {
|
||||||
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout, "\n temperature: 0\n") {
|
||||||
|
t.Fatalf("expected explicit zero temperature in effective settings, got:\n%s", stdout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRenderCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
|
func TestRenderCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
||||||
@@ -900,6 +1037,29 @@ func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRenderCommandUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
|
||||||
|
t.Setenv("OPENROUTER_API_KEY", "test-key")
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||||
|
|
||||||
|
writePromptFile(t, lib.promptDir, "prompt.builtin", "mistral-small-3")
|
||||||
|
|
||||||
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--prompt-dir", lib.promptDir,
|
||||||
|
"--prompt", "prompt.builtin",
|
||||||
|
"--input", "transcript=" + inputPath,
|
||||||
|
})
|
||||||
|
if code != ExitOK {
|
||||||
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout, "selected_profile_id: mistral-small-3") {
|
||||||
|
t.Fatalf("expected built-in selected profile, got %q", stdout)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout, "model: mistralai/mistral-small-3.2-24b-instruct") {
|
||||||
|
t.Fatalf("expected built-in model, got %q", stdout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||||
@@ -1064,6 +1224,38 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
|
|||||||
if !strings.Contains(stderr.String(), "prompt=p@1") {
|
if !strings.Contains(stderr.String(), "prompt=p@1") {
|
||||||
t.Fatalf("expected summary on stderr, got %q", stderr.String())
|
t.Fatalf("expected summary on stderr, got %q", stderr.String())
|
||||||
}
|
}
|
||||||
|
if strings.Contains(stderr.String(), "cached_tokens=") || strings.Contains(stderr.String(), "cache_write_tokens=") {
|
||||||
|
t.Fatalf("expected zero cache usage to be omitted from summary, got %q", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) {
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
printSummary(&stderr, &domain.RunResult{
|
||||||
|
PromptID: "p",
|
||||||
|
PromptVersion: "1",
|
||||||
|
SelectedProfileID: "exec",
|
||||||
|
ModelName: "m",
|
||||||
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic},
|
||||||
|
RenderedPromptHash: "h",
|
||||||
|
InputHashes: map[string]string{"in": "x"},
|
||||||
|
Usage: domain.TokenUsage{
|
||||||
|
PromptTokens: 10,
|
||||||
|
CompletionTokens: 5,
|
||||||
|
TotalTokens: 15,
|
||||||
|
CachedTokens: 0,
|
||||||
|
CacheWriteTokens: 3,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
summary := stderr.String()
|
||||||
|
if !strings.Contains(summary, "usage=10/5/15") {
|
||||||
|
t.Fatalf("expected base usage summary, got %q", summary)
|
||||||
|
}
|
||||||
|
if !strings.Contains(summary, "cached_tokens=0 cache_write_tokens=3") {
|
||||||
|
t.Fatalf("expected cache usage in summary, got %q", summary)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type cliTestLibrary struct {
|
type cliTestLibrary struct {
|
||||||
|
|||||||
@@ -21,16 +21,16 @@ type inputRefDTO struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type modelOverrideRequestDTO struct {
|
type modelOverrideRequestDTO struct {
|
||||||
Endpoint string `json:"endpoint,omitempty"`
|
Endpoint string `json:"endpoint,omitempty"`
|
||||||
Model string `json:"model,omitempty"`
|
Model string `json:"model,omitempty"`
|
||||||
Temperature float64 `json:"temperature,omitempty"`
|
Temperature *float64 `json:"temperature,omitempty"`
|
||||||
MaxTokens int `json:"max_tokens,omitempty"`
|
MaxTokens *int `json:"max_tokens,omitempty"`
|
||||||
TopP float64 `json:"top_p,omitempty"`
|
TopP *float64 `json:"top_p,omitempty"`
|
||||||
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
|
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||||
ServiceTier string `json:"service_tier,omitempty"`
|
ServiceTier string `json:"service_tier,omitempty"`
|
||||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||||
ExtraParams map[string]string `json:"extra_params,omitempty"`
|
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type runResponseDTO struct {
|
type runResponseDTO struct {
|
||||||
@@ -70,22 +70,24 @@ type metadataDTO struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type modelParamsDTO struct {
|
type modelParamsDTO struct {
|
||||||
Endpoint string `json:"endpoint"`
|
Endpoint string `json:"endpoint"`
|
||||||
Model string `json:"model"`
|
Model string `json:"model"`
|
||||||
Temperature float64 `json:"temperature"`
|
Temperature float64 `json:"temperature"`
|
||||||
MaxTokens int `json:"max_tokens"`
|
MaxTokens int `json:"max_tokens"`
|
||||||
TopP float64 `json:"top_p"`
|
TopP float64 `json:"top_p"`
|
||||||
TimeoutSeconds int `json:"timeout_seconds"`
|
TimeoutSeconds int `json:"timeout_seconds"`
|
||||||
ServiceTier string `json:"service_tier,omitempty"`
|
ServiceTier string `json:"service_tier,omitempty"`
|
||||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||||
ExtraParams map[string]string `json:"extra_params,omitempty"`
|
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type tokenUsageDTO struct {
|
type tokenUsageDTO struct {
|
||||||
PromptTokens int `json:"prompt_tokens"`
|
PromptTokens int `json:"prompt_tokens"`
|
||||||
CompletionTokens int `json:"completion_tokens"`
|
CompletionTokens int `json:"completion_tokens"`
|
||||||
TotalTokens int `json:"total_tokens"`
|
TotalTokens int `json:"total_tokens"`
|
||||||
|
CachedTokens int `json:"cached_tokens"`
|
||||||
|
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type validationDTO struct {
|
type validationDTO struct {
|
||||||
|
|||||||
@@ -4,9 +4,12 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||||
@@ -18,11 +21,24 @@ type Runner interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Handler struct {
|
type Handler struct {
|
||||||
runner Runner
|
runner Runner
|
||||||
|
options HandlerOptions
|
||||||
|
}
|
||||||
|
|
||||||
|
type HandlerOptions struct {
|
||||||
|
MaxRequestBytes int64
|
||||||
|
MaxResponseBytes int64
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewHandler(runner Runner) *Handler {
|
func NewHandler(runner Runner) *Handler {
|
||||||
return &Handler{runner: runner}
|
return NewHandlerWithOptions(runner, HandlerOptions{
|
||||||
|
MaxRequestBytes: defaults.HTTPMaxRequestBytesDefault,
|
||||||
|
MaxResponseBytes: defaults.HTTPMaxResponseBytesDefault,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewHandlerWithOptions(runner Runner, options HandlerOptions) *Handler {
|
||||||
|
return &Handler{runner: runner, options: options}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -36,9 +52,26 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var req runRequestDTO
|
var req runRequestDTO
|
||||||
dec := json.NewDecoder(r.Body)
|
body := r.Body
|
||||||
|
if h.options.MaxRequestBytes > 0 {
|
||||||
|
body = http.MaxBytesReader(w, r.Body, h.options.MaxRequestBytes)
|
||||||
|
}
|
||||||
|
dec := json.NewDecoder(body)
|
||||||
dec.DisallowUnknownFields()
|
dec.DisallowUnknownFields()
|
||||||
if err := dec.Decode(&req); err != nil {
|
if err := dec.Decode(&req); err != nil {
|
||||||
|
if isRequestTooLarge(err) {
|
||||||
|
writeError(w, http.StatusRequestEntityTooLarge, "request_too_large", "request body is too large")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_json", "invalid JSON request body")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var trailing any
|
||||||
|
if err := dec.Decode(&trailing); err != io.EOF {
|
||||||
|
if isRequestTooLarge(err) {
|
||||||
|
writeError(w, http.StatusRequestEntityTooLarge, "request_too_large", "request body is too large")
|
||||||
|
return
|
||||||
|
}
|
||||||
writeError(w, http.StatusBadRequest, "invalid_json", "invalid JSON request body")
|
writeError(w, http.StatusBadRequest, "invalid_json", "invalid JSON request body")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -61,9 +94,9 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var model *domain.ExecutionTarget
|
var model *domain.ExecutionTargetOverride
|
||||||
if req.Model != nil {
|
if req.Model != nil {
|
||||||
model = executionTargetFromModelOverrideDTO(req.Model)
|
model = executionTargetOverrideFromModelOverrideDTO(req.Model)
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := h.runner.Run(r.Context(), domain.RunRequest{
|
res, err := h.runner.Run(r.Context(), domain.RunRequest{
|
||||||
@@ -105,6 +138,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
PromptTokens: res.Usage.PromptTokens,
|
PromptTokens: res.Usage.PromptTokens,
|
||||||
CompletionTokens: res.Usage.CompletionTokens,
|
CompletionTokens: res.Usage.CompletionTokens,
|
||||||
TotalTokens: res.Usage.TotalTokens,
|
TotalTokens: res.Usage.TotalTokens,
|
||||||
|
CachedTokens: res.Usage.CachedTokens,
|
||||||
|
CacheWriteTokens: res.Usage.CacheWriteTokens,
|
||||||
},
|
},
|
||||||
StartTime: res.StartTime,
|
StartTime: res.StartTime,
|
||||||
EndTime: res.EndTime,
|
EndTime: res.EndTime,
|
||||||
@@ -118,14 +153,14 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
raw := res.RawOutput
|
raw := res.RawOutput
|
||||||
resp.RawModelOutput = &raw
|
resp.RawModelOutput = &raw
|
||||||
}
|
}
|
||||||
writeJSON(w, http.StatusOK, resp)
|
writeLimitedJSON(w, http.StatusOK, resp, h.options.MaxResponseBytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
func executionTargetFromModelOverrideDTO(dto *modelOverrideRequestDTO) *domain.ExecutionTarget {
|
func executionTargetOverrideFromModelOverrideDTO(dto *modelOverrideRequestDTO) *domain.ExecutionTargetOverride {
|
||||||
if dto == nil {
|
if dto == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return &domain.ExecutionTarget{
|
return &domain.ExecutionTargetOverride{
|
||||||
Endpoint: dto.Endpoint,
|
Endpoint: dto.Endpoint,
|
||||||
Model: dto.Model,
|
Model: dto.Model,
|
||||||
Temperature: dto.Temperature,
|
Temperature: dto.Temperature,
|
||||||
@@ -173,7 +208,7 @@ func mapRunError(err error) (int, string, string) {
|
|||||||
return http.StatusNotFound, "profile_not_found", "execution profile not found"
|
return http.StatusNotFound, "profile_not_found", "execution profile not found"
|
||||||
case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition):
|
case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition):
|
||||||
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
|
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
|
||||||
case errors.Is(err, profile.ErrInvalidYAML), errors.Is(err, profile.ErrInvalidProfile):
|
case errors.Is(err, profile.ErrInvalidYAML), errors.Is(err, profile.ErrInvalidProfile), errors.Is(err, profile.ErrRawAPIKeyNotAllowed):
|
||||||
return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile"
|
return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile"
|
||||||
case errors.Is(err, usecase.ErrProfileRequired):
|
case errors.Is(err, usecase.ErrProfileRequired):
|
||||||
return http.StatusBadRequest, "profile_required", "profile_id is required when prompt default_profile is not set"
|
return http.StatusBadRequest, "profile_required", "profile_id is required when prompt default_profile is not set"
|
||||||
@@ -181,8 +216,14 @@ func mapRunError(err error) (int, string, string) {
|
|||||||
return http.StatusBadRequest, "api_key_env_missing", "api_key_env is set but the environment variable is missing"
|
return http.StatusBadRequest, "api_key_env_missing", "api_key_env is set but the environment variable is missing"
|
||||||
case errors.Is(err, usecase.ErrInvalidRequest):
|
case errors.Is(err, usecase.ErrInvalidRequest):
|
||||||
return http.StatusBadRequest, "invalid_request", "invalid run request"
|
return http.StatusBadRequest, "invalid_request", "invalid run request"
|
||||||
case errors.Is(err, usecase.ErrProfileLoad):
|
case errors.Is(err, usecase.ErrPromptLoad):
|
||||||
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
|
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
|
||||||
|
case errors.Is(err, usecase.ErrProfileLoad):
|
||||||
|
return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile"
|
||||||
|
case errors.Is(err, artifact.ErrFileNotAllowed), errors.Is(err, artifact.ErrFileOutsideRoot):
|
||||||
|
return http.StatusBadRequest, "artifact_not_allowed", "file input artifact is not allowed"
|
||||||
|
case errors.Is(err, artifact.ErrFileTooLarge):
|
||||||
|
return http.StatusRequestEntityTooLarge, "artifact_too_large", "file input artifact is too large"
|
||||||
case errors.Is(err, usecase.ErrArtifactLoad):
|
case errors.Is(err, usecase.ErrArtifactLoad):
|
||||||
return http.StatusBadRequest, "artifact_read_failed", "failed to read input artifact"
|
return http.StatusBadRequest, "artifact_read_failed", "failed to read input artifact"
|
||||||
case errors.Is(err, usecase.ErrPromptRender):
|
case errors.Is(err, usecase.ErrPromptRender):
|
||||||
@@ -197,9 +238,23 @@ func mapRunError(err error) (int, string, string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func writeJSON(w http.ResponseWriter, status int, v any) {
|
func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||||
|
writeLimitedJSON(w, status, v, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeLimitedJSON(w http.ResponseWriter, status int, v any, maxBytes int64) {
|
||||||
|
data, err := json.Marshal(v)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, "internal_error", "internal server error")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
data = append(data, '\n')
|
||||||
|
if maxBytes > 0 && int64(len(data)) > maxBytes {
|
||||||
|
writeError(w, http.StatusRequestEntityTooLarge, "response_too_large", "response body is too large")
|
||||||
|
return
|
||||||
|
}
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
w.WriteHeader(status)
|
w.WriteHeader(status)
|
||||||
_ = json.NewEncoder(w).Encode(v)
|
_, _ = w.Write(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeError(w http.ResponseWriter, status int, code, message string) {
|
func writeError(w http.ResponseWriter, status int, code, message string) {
|
||||||
@@ -210,3 +265,8 @@ func writeError(w http.ResponseWriter, status int, code, message string) {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isRequestTooLarge(err error) bool {
|
||||||
|
var maxBytesErr *http.MaxBytesError
|
||||||
|
return errors.As(err, &maxBytesErr)
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,12 +7,16 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||||
@@ -32,6 +36,72 @@ func (f *fakeRunner) Run(ctx context.Context, req domain.RunRequest) (*domain.Ru
|
|||||||
return f.result, nil
|
return f.result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMaintainedHTTPRunExampleMatchesRequestContract(t *testing.T) {
|
||||||
|
body, err := os.ReadFile(filepath.Join("..", "..", "..", "examples", "http-run.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read maintained HTTP request example: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
runner := &fakeRunner{result: &domain.RunResult{}}
|
||||||
|
h := NewHandler(runner)
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewReader(body))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected maintained HTTP request example to be accepted, got %d: %s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
var invalidExample map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(body, &invalidExample); err != nil {
|
||||||
|
t.Fatalf("decode maintained HTTP request example: %v", err)
|
||||||
|
}
|
||||||
|
invalidExample["unexpected"] = json.RawMessage(`true`)
|
||||||
|
invalidBody, err := json.Marshal(invalidExample)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("encode structurally invalid request example: %v", err)
|
||||||
|
}
|
||||||
|
invalidReq := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewReader(invalidBody))
|
||||||
|
invalidW := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(invalidW, invalidReq)
|
||||||
|
assertHTTPErrorCode(t, invalidW, http.StatusBadRequest, "invalid_json")
|
||||||
|
}
|
||||||
|
|
||||||
|
type handlerPromptRepo struct {
|
||||||
|
def *domain.PromptDefinition
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r handlerPromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
||||||
|
return r.def, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type handlerProfileRepo struct {
|
||||||
|
profile *domain.ExecutionProfile
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r handlerProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
return r.profile, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type handlerArtifactReader struct{}
|
||||||
|
|
||||||
|
func (handlerArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
||||||
|
return &domain.Artifact{Name: "input", Body: []byte("input"), Hash: "hash"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type handlerRenderer struct{}
|
||||||
|
|
||||||
|
func (handlerRenderer) Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
|
||||||
|
return &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type handlerLLMClient struct{}
|
||||||
|
|
||||||
|
func (handlerLLMClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
|
return &domain.GenerateResponse{Content: "ok"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
||||||
start := time.Now().UTC()
|
start := time.Now().UTC()
|
||||||
end := start.Add(2 * time.Second)
|
end := start.Add(2 * time.Second)
|
||||||
@@ -66,11 +136,17 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
APIKeyEnv: envName,
|
APIKeyEnv: envName,
|
||||||
},
|
},
|
||||||
InputHashes: map[string]string{"transcript": "h1"},
|
InputHashes: map[string]string{"transcript": "h1"},
|
||||||
Usage: domain.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3},
|
Usage: domain.TokenUsage{
|
||||||
StartTime: start,
|
PromptTokens: 1,
|
||||||
EndTime: end,
|
CompletionTokens: 2,
|
||||||
Duration: 2 * time.Second,
|
TotalTokens: 3,
|
||||||
RawOutput: "hello",
|
CachedTokens: 4,
|
||||||
|
CacheWriteTokens: 5,
|
||||||
|
},
|
||||||
|
StartTime: start,
|
||||||
|
EndTime: end,
|
||||||
|
Duration: 2 * time.Second,
|
||||||
|
RawOutput: "hello",
|
||||||
}}
|
}}
|
||||||
|
|
||||||
h := NewHandler(r)
|
h := NewHandler(r)
|
||||||
@@ -111,6 +187,13 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" {
|
if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" {
|
||||||
t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"])
|
t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"])
|
||||||
}
|
}
|
||||||
|
usage := metadata["usage"].(map[string]any)
|
||||||
|
if usage["prompt_tokens"] != float64(1) || usage["completion_tokens"] != float64(2) || usage["total_tokens"] != float64(3) {
|
||||||
|
t.Fatalf("unexpected base usage metadata: %#v", usage)
|
||||||
|
}
|
||||||
|
if usage["cached_tokens"] != float64(4) || usage["cache_write_tokens"] != float64(5) {
|
||||||
|
t.Fatalf("unexpected cache usage metadata: %#v", usage)
|
||||||
|
}
|
||||||
modelParams := metadata["model_params"].(map[string]any)
|
modelParams := metadata["model_params"].(map[string]any)
|
||||||
if modelParams["api_key_env"] != envName {
|
if modelParams["api_key_env"] != envName {
|
||||||
t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"])
|
t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"])
|
||||||
@@ -134,7 +217,7 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
if r.last.Execution == nil || r.last.Execution.Model != "gpt-x" {
|
if r.last.Execution == nil || r.last.Execution.Model != "gpt-x" {
|
||||||
t.Fatalf("expected model override, got %#v", r.last.Execution)
|
t.Fatalf("expected model override, got %#v", r.last.Execution)
|
||||||
}
|
}
|
||||||
if r.last.Execution.TimeoutSeconds != 120 {
|
if r.last.Execution.TimeoutSeconds == nil || *r.last.Execution.TimeoutSeconds != 120 {
|
||||||
t.Fatalf("expected timeout_seconds override 120, got %#v", r.last.Execution)
|
t.Fatalf("expected timeout_seconds override 120, got %#v", r.last.Execution)
|
||||||
}
|
}
|
||||||
if r.last.Execution.ServiceTier != "flex" {
|
if r.last.Execution.ServiceTier != "flex" {
|
||||||
@@ -142,6 +225,105 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandlerInlineRefsWorkWithoutArtifactRoot(t *testing.T) {
|
||||||
|
h := newArtifactRootHandler(t, "")
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id":"p",
|
||||||
|
"inputs":{"x":{"type":"inline","body":"inline body"}}
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerFileRefsWithoutArtifactRootAreRejected(t *testing.T) {
|
||||||
|
h := newArtifactRootHandler(t, "")
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id":"p",
|
||||||
|
"inputs":{"x":{"type":"file","uri":"input.txt"}}
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusBadRequest, "artifact_not_allowed")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerFileRefsUnderArtifactRootWork(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "input.txt"), []byte("allowed"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
h := newArtifactRootHandler(t, root)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id":"p",
|
||||||
|
"inputs":{"x":{"type":"file","uri":"input.txt"}}
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerFileRefsAboveArtifactLimitAreRejected(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "large.txt"), []byte("123456"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
h := newArtifactRootHandlerWithLimit(t, root, 5)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id":"p",
|
||||||
|
"inputs":{"x":{"type":"file","uri":"large.txt"}}
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusRequestEntityTooLarge, "artifact_too_large")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerFileRefsOutsideArtifactRootAreRejected(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
outside := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(outside, "secret.txt"), []byte("denied"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
h := newArtifactRootHandler(t, root)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
uri string
|
||||||
|
}{
|
||||||
|
{name: "relative traversal", uri: filepath.Join("..", filepath.Base(outside), "secret.txt")},
|
||||||
|
{name: "absolute outside root", uri: filepath.Join(outside, "secret.txt")},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
body := fmt.Sprintf(`{
|
||||||
|
"prompt_id":"p",
|
||||||
|
"inputs":{"x":{"type":"file","uri":%q}}
|
||||||
|
}`, tc.uri)
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(body))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusBadRequest, "artifact_not_allowed")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
|
func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
|
||||||
r := &fakeRunner{result: &domain.RunResult{
|
r := &fakeRunner{result: &domain.RunResult{
|
||||||
Artifact: domain.Artifact{Body: []byte("ok")},
|
Artifact: domain.Artifact{Body: []byte("ok")},
|
||||||
@@ -171,6 +353,10 @@ func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
|
|||||||
if metadata["selected_profile_id"] != "prompt-default" {
|
if metadata["selected_profile_id"] != "prompt-default" {
|
||||||
t.Fatalf("expected selected_profile_id from result, got %#v", metadata["selected_profile_id"])
|
t.Fatalf("expected selected_profile_id from result, got %#v", metadata["selected_profile_id"])
|
||||||
}
|
}
|
||||||
|
usage := metadata["usage"].(map[string]any)
|
||||||
|
if usage["cached_tokens"] != float64(0) || usage["cache_write_tokens"] != float64(0) {
|
||||||
|
t.Fatalf("expected zero cache usage fields to be included, got %#v", usage)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
|
func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
|
||||||
@@ -211,20 +397,136 @@ func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
|
|||||||
got := r.last.Execution
|
got := r.last.Execution
|
||||||
if got.Endpoint != "http://override/v1" ||
|
if got.Endpoint != "http://override/v1" ||
|
||||||
got.Model != "override-model" ||
|
got.Model != "override-model" ||
|
||||||
got.Temperature != 0.6 ||
|
|
||||||
got.MaxTokens != 250 ||
|
|
||||||
got.TopP != 0.85 ||
|
|
||||||
got.TimeoutSeconds != 33 ||
|
|
||||||
got.ServiceTier != "flex" ||
|
got.ServiceTier != "flex" ||
|
||||||
got.ReasoningEffort != "medium" ||
|
got.ReasoningEffort != "medium" ||
|
||||||
got.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
|
got.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
|
||||||
t.Fatalf("unexpected mapped execution target: %+v", got)
|
t.Fatalf("unexpected mapped execution target: %+v", got)
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(got.ExtraParams, map[string]string{"provider_option": "on"}) {
|
if got.Temperature == nil || *got.Temperature != 0.6 {
|
||||||
|
t.Fatalf("unexpected mapped temperature: %#v", got.Temperature)
|
||||||
|
}
|
||||||
|
if got.MaxTokens == nil || *got.MaxTokens != 250 {
|
||||||
|
t.Fatalf("unexpected mapped max_tokens: %#v", got.MaxTokens)
|
||||||
|
}
|
||||||
|
if got.TopP == nil || *got.TopP != 0.85 {
|
||||||
|
t.Fatalf("unexpected mapped top_p: %#v", got.TopP)
|
||||||
|
}
|
||||||
|
if got.TimeoutSeconds == nil || *got.TimeoutSeconds != 33 {
|
||||||
|
t.Fatalf("unexpected mapped timeout_seconds: %#v", got.TimeoutSeconds)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got.ExtraParams, map[string]any{"provider_option": "on"}) {
|
||||||
t.Fatalf("unexpected mapped extra_params: %#v", got.ExtraParams)
|
t.Fatalf("unexpected mapped extra_params: %#v", got.ExtraParams)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandlerModelOverrideAcceptsJSONCompatibleExtraParams(t *testing.T) {
|
||||||
|
r := &fakeRunner{result: &domain.RunResult{
|
||||||
|
Artifact: domain.Artifact{Body: []byte("ok")},
|
||||||
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
|
}}
|
||||||
|
h := NewHandler(r)
|
||||||
|
|
||||||
|
reqBody := `{
|
||||||
|
"prompt_id": "prompt-1",
|
||||||
|
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
|
||||||
|
"model": {
|
||||||
|
"extra_params": {
|
||||||
|
"string_value": "enabled",
|
||||||
|
"number_value": 42,
|
||||||
|
"boolean_value": true,
|
||||||
|
"object_value": {"nested": "value", "count": 2},
|
||||||
|
"array_value": ["first", 3, false]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}`
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(reqBody))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if r.last.Execution == nil {
|
||||||
|
t.Fatal("expected execution override in run request")
|
||||||
|
}
|
||||||
|
want := map[string]any{
|
||||||
|
"string_value": "enabled",
|
||||||
|
"number_value": float64(42),
|
||||||
|
"boolean_value": true,
|
||||||
|
"object_value": map[string]any{"nested": "value", "count": float64(2)},
|
||||||
|
"array_value": []any{"first", float64(3), false},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(r.last.Execution.ExtraParams, want) {
|
||||||
|
t.Fatalf("unexpected mapped extra_params:\ngot=%#v\nwant=%#v", r.last.Execution.ExtraParams, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerModelOverrideExplicitZeroTemperatureMapsAsPresent(t *testing.T) {
|
||||||
|
r := &fakeRunner{result: &domain.RunResult{
|
||||||
|
Artifact: domain.Artifact{Body: []byte("ok")},
|
||||||
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0},
|
||||||
|
}}
|
||||||
|
h := NewHandler(r)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id": "prompt-1",
|
||||||
|
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
|
||||||
|
"model": {"temperature": 0}
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if r.last.Execution == nil || r.last.Execution.Temperature == nil {
|
||||||
|
t.Fatalf("expected temperature override to be present, got %#v", r.last.Execution)
|
||||||
|
}
|
||||||
|
if *r.last.Execution.Temperature != 0 {
|
||||||
|
t.Fatalf("expected zero temperature override, got %v", *r.last.Execution.Temperature)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerModelOverrideOmittedTemperatureMapsAsAbsent(t *testing.T) {
|
||||||
|
r := &fakeRunner{result: &domain.RunResult{
|
||||||
|
Artifact: domain.Artifact{Body: []byte("ok")},
|
||||||
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0.7},
|
||||||
|
}}
|
||||||
|
h := NewHandler(r)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id": "prompt-1",
|
||||||
|
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
|
||||||
|
"model": {"model": "override-model"}
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if r.last.Execution == nil {
|
||||||
|
t.Fatal("expected model override")
|
||||||
|
}
|
||||||
|
if r.last.Execution.Temperature != nil {
|
||||||
|
t.Fatalf("expected omitted temperature to remain absent, got %#v", r.last.Execution.Temperature)
|
||||||
|
}
|
||||||
|
|
||||||
|
var resp map[string]any
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||||
|
t.Fatalf("invalid JSON response: %v", err)
|
||||||
|
}
|
||||||
|
metadata := resp["metadata"].(map[string]any)
|
||||||
|
params := metadata["model_params"].(map[string]any)
|
||||||
|
if params["temperature"] != 0.7 {
|
||||||
|
t.Fatalf("expected effective profile/default temperature in response, got %#v", params["temperature"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing.T) {
|
func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing.T) {
|
||||||
r := &fakeRunner{result: &domain.RunResult{
|
r := &fakeRunner{result: &domain.RunResult{
|
||||||
Artifact: domain.Artifact{
|
Artifact: domain.Artifact{
|
||||||
@@ -245,8 +547,10 @@ func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing
|
|||||||
ServiceTier: "priority",
|
ServiceTier: "priority",
|
||||||
ReasoningEffort: "high",
|
ReasoningEffort: "high",
|
||||||
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
||||||
ExtraParams: map[string]string{
|
ExtraParams: map[string]any{
|
||||||
"provider_option": "on",
|
"provider_option": "on",
|
||||||
|
"number_value": 42,
|
||||||
|
"object_value": map[string]any{"nested": "value"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}}
|
}}
|
||||||
@@ -301,6 +605,13 @@ func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing
|
|||||||
if extraParams["provider_option"] != "on" {
|
if extraParams["provider_option"] != "on" {
|
||||||
t.Fatalf("unexpected extra_params.provider_option: %#v", extraParams["provider_option"])
|
t.Fatalf("unexpected extra_params.provider_option: %#v", extraParams["provider_option"])
|
||||||
}
|
}
|
||||||
|
if extraParams["number_value"] != float64(42) {
|
||||||
|
t.Fatalf("unexpected extra_params.number_value: %#v", extraParams["number_value"])
|
||||||
|
}
|
||||||
|
objectValue, ok := extraParams["object_value"].(map[string]any)
|
||||||
|
if !ok || objectValue["nested"] != "value" {
|
||||||
|
t.Fatalf("unexpected extra_params.object_value: %#v", extraParams["object_value"])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerInvalidJSON(t *testing.T) {
|
func TestHandlerInvalidJSON(t *testing.T) {
|
||||||
@@ -315,6 +626,69 @@ func TestHandlerInvalidJSON(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandlerRejectsTrailingJSON(t *testing.T) {
|
||||||
|
h := NewHandler(&fakeRunner{})
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}} {}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusBadRequest, "invalid_json")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerRequestTooLarge(t *testing.T) {
|
||||||
|
h := NewHandlerWithOptions(&fakeRunner{}, HandlerOptions{MaxRequestBytes: 12})
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusRequestEntityTooLarge, "request_too_large")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerMalformedJSONBelowLimitStillBadRequest(t *testing.T) {
|
||||||
|
h := NewHandlerWithOptions(&fakeRunner{}, HandlerOptions{MaxRequestBytes: 1024})
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString("{"))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusBadRequest, "invalid_json")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerResponseTooLarge(t *testing.T) {
|
||||||
|
h := NewHandlerWithOptions(&fakeRunner{result: &domain.RunResult{
|
||||||
|
Artifact: domain.Artifact{Body: []byte(strings.Repeat("x", 128))},
|
||||||
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
|
}}, HandlerOptions{MaxRequestBytes: 1024, MaxResponseBytes: 64})
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusRequestEntityTooLarge, "response_too_large")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerRawOutputDoesNotBypassResponseLimit(t *testing.T) {
|
||||||
|
h := NewHandlerWithOptions(&fakeRunner{result: &domain.RunResult{
|
||||||
|
Artifact: domain.Artifact{Body: []byte("ok")},
|
||||||
|
RawOutput: strings.Repeat("raw", 80),
|
||||||
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
|
}}, HandlerOptions{MaxRequestBytes: 1024, MaxResponseBytes: 128})
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id":"p",
|
||||||
|
"inputs":{"x":{"type":"file","uri":"a"}},
|
||||||
|
"include_raw_output":true
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusRequestEntityTooLarge, "response_too_large")
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandlerMissingPromptID(t *testing.T) {
|
func TestHandlerMissingPromptID(t *testing.T) {
|
||||||
h := NewHandler(&fakeRunner{})
|
h := NewHandler(&fakeRunner{})
|
||||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"inputs":{"x":{"type":"file","uri":"a"}}}`))
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"inputs":{"x":{"type":"file","uri":"a"}}}`))
|
||||||
@@ -335,6 +709,54 @@ func TestHandlerMissingPromptID(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandlerReservedExtraParamsThroughRunnerMapsToInvalidRequest(t *testing.T) {
|
||||||
|
llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
runner := usecase.NewRunner(
|
||||||
|
handlerPromptRepo{def: &domain.PromptDefinition{
|
||||||
|
ID: "p",
|
||||||
|
Version: "1",
|
||||||
|
DefaultProfile: "exec",
|
||||||
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "hi"}},
|
||||||
|
OutputFormat: domain.FormatText,
|
||||||
|
Validation: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone},
|
||||||
|
}},
|
||||||
|
handlerProfileRepo{profile: &domain.ExecutionProfile{
|
||||||
|
ID: "exec",
|
||||||
|
Endpoint: "http://example.invalid/v1",
|
||||||
|
Model: "model",
|
||||||
|
}},
|
||||||
|
handlerArtifactReader{},
|
||||||
|
handlerRenderer{},
|
||||||
|
llmClient,
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
h := NewHandler(runner)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id":"p",
|
||||||
|
"inputs":{"x":{"type":"file","uri":"a"}},
|
||||||
|
"model":{"extra_params":{"model":"collision"}}
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("expected 400, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
var resp map[string]any
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||||
|
t.Fatalf("invalid JSON response: %v", err)
|
||||||
|
}
|
||||||
|
errBody := resp["error"].(map[string]any)
|
||||||
|
if errBody["code"] != "invalid_request" {
|
||||||
|
t.Fatalf("expected invalid_request code, got %#v", errBody["code"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandlerUsecaseErrorMapping(t *testing.T) {
|
func TestHandlerUsecaseErrorMapping(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -344,11 +766,13 @@ func TestHandlerUsecaseErrorMapping(t *testing.T) {
|
|||||||
message string
|
message string
|
||||||
avoidCause string
|
avoidCause string
|
||||||
}{
|
}{
|
||||||
{name: "prompt not found", err: wrap(usecase.ErrProfileLoad, promptdef.ErrPromptDefinitionNotFound), status: http.StatusNotFound, code: "prompt_not_found", message: "prompt definition not found"},
|
{name: "prompt not found", err: wrap(usecase.ErrPromptLoad, promptdef.ErrPromptDefinitionNotFound), status: http.StatusNotFound, code: "prompt_not_found", message: "prompt definition not found"},
|
||||||
{name: "prompt load invalid", err: wrap(usecase.ErrProfileLoad, promptdef.ErrInvalidPromptDefinition), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition"},
|
{name: "prompt load invalid", err: wrap(usecase.ErrPromptLoad, promptdef.ErrInvalidPromptDefinition), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition"},
|
||||||
|
{name: "prompt load generic", err: wrap(usecase.ErrPromptLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition", avoidCause: "read failed"},
|
||||||
{name: "missing profile/default", err: wrap(usecase.ErrInvalidRequest, usecase.ErrProfileRequired), status: http.StatusBadRequest, code: "profile_required", message: "profile_id is required when prompt default_profile is not set"},
|
{name: "missing profile/default", err: wrap(usecase.ErrInvalidRequest, usecase.ErrProfileRequired), status: http.StatusBadRequest, code: "profile_required", message: "profile_id is required when prompt default_profile is not set"},
|
||||||
{name: "profile not found", err: wrap(usecase.ErrProfileLoad, profile.ErrProfileNotFound), status: http.StatusNotFound, code: "profile_not_found", message: "execution profile not found"},
|
{name: "profile not found", err: wrap(usecase.ErrProfileLoad, profile.ErrProfileNotFound), status: http.StatusNotFound, code: "profile_not_found", message: "execution profile not found"},
|
||||||
{name: "profile invalid", err: wrap(usecase.ErrProfileLoad, profile.ErrInvalidProfile), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile"},
|
{name: "profile invalid", err: wrap(usecase.ErrProfileLoad, profile.ErrInvalidProfile), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile"},
|
||||||
|
{name: "profile load generic", err: wrap(usecase.ErrProfileLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile", avoidCause: "read failed"},
|
||||||
{name: "api key env missing", err: wrap(usecase.ErrInvalidRequest, usecase.ErrAPIKeyEnvMissing), status: http.StatusBadRequest, code: "api_key_env_missing", message: "api_key_env is set but the environment variable is missing"},
|
{name: "api key env missing", err: wrap(usecase.ErrInvalidRequest, usecase.ErrAPIKeyEnvMissing), status: http.StatusBadRequest, code: "api_key_env_missing", message: "api_key_env is set but the environment variable is missing"},
|
||||||
{name: "artifact", err: wrap(usecase.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"},
|
{name: "artifact", err: wrap(usecase.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"},
|
||||||
{name: "prompt render", err: wrap(usecase.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"},
|
{name: "prompt render", err: wrap(usecase.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"},
|
||||||
@@ -455,3 +879,54 @@ func TestHandlerValidationFailureStillSuccessAndRawOutputOptIn(t *testing.T) {
|
|||||||
func wrap(stage error, cause error) error {
|
func wrap(stage error, cause error) error {
|
||||||
return fmt.Errorf("%w: %w", stage, cause)
|
return fmt.Errorf("%w: %w", stage, cause)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newArtifactRootHandler(t *testing.T, root string) *Handler {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
return newArtifactRootHandlerWithLimit(t, root, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newArtifactRootHandlerWithLimit(t *testing.T, root string, maxArtifactBytes int64) *Handler {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
reader, err := artifact.NewRestrictedCompositeReaderWithLimit(root, maxArtifactBytes)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected restricted artifact reader: %v", err)
|
||||||
|
}
|
||||||
|
runner := usecase.NewRunner(
|
||||||
|
handlerPromptRepo{def: &domain.PromptDefinition{
|
||||||
|
ID: "p",
|
||||||
|
Version: "1",
|
||||||
|
DefaultProfile: "exec",
|
||||||
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "hi"}},
|
||||||
|
OutputFormat: domain.FormatText,
|
||||||
|
Validation: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone},
|
||||||
|
}},
|
||||||
|
handlerProfileRepo{profile: &domain.ExecutionProfile{
|
||||||
|
ID: "exec",
|
||||||
|
Endpoint: "http://example.invalid/v1",
|
||||||
|
Model: "model",
|
||||||
|
}},
|
||||||
|
reader,
|
||||||
|
handlerRenderer{},
|
||||||
|
handlerLLMClient{},
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
return NewHandler(runner)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertHTTPErrorCode(t *testing.T, w *httptest.ResponseRecorder, status int, code string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
if w.Code != status {
|
||||||
|
t.Fatalf("expected %d, got %d body=%s", status, w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
var resp map[string]any
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||||
|
t.Fatalf("invalid JSON response: %v", err)
|
||||||
|
}
|
||||||
|
errBody := resp["error"].(map[string]any)
|
||||||
|
if errBody["code"] != code {
|
||||||
|
t.Fatalf("expected code %q, got %#v", code, errBody["code"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,15 +7,20 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
"io"
|
||||||
"mime"
|
"mime"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
ErrUnsupportedRefType = errors.New("unsupported artifact reference type")
|
ErrUnsupportedRefType = errors.New("unsupported artifact reference type")
|
||||||
ErrMissingInlineBody = errors.New("missing body for inline artifact")
|
ErrMissingInlineBody = errors.New("missing body for inline artifact")
|
||||||
ErrMissingFilePath = errors.New("missing file path for file artifact")
|
ErrMissingFilePath = errors.New("missing file path for file artifact")
|
||||||
|
ErrFileNotAllowed = errors.New("file artifact references are not allowed")
|
||||||
|
ErrFileOutsideRoot = errors.New("file artifact path is outside artifact root")
|
||||||
|
ErrFileTooLarge = errors.New("file artifact exceeds size limit")
|
||||||
)
|
)
|
||||||
|
|
||||||
// Reader resolves artifact references into actual artifacts.
|
// Reader resolves artifact references into actual artifacts.
|
||||||
@@ -26,7 +31,7 @@ type Reader interface {
|
|||||||
// CompositeReader routes artifact resolution based on the reference type.
|
// CompositeReader routes artifact resolution based on the reference type.
|
||||||
type CompositeReader struct {
|
type CompositeReader struct {
|
||||||
inlineReader *inlineReader
|
inlineReader *inlineReader
|
||||||
fileReader *fileReader
|
fileReader Reader
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewCompositeReader() Reader {
|
func NewCompositeReader() Reader {
|
||||||
@@ -36,6 +41,21 @@ func NewCompositeReader() Reader {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func NewRestrictedCompositeReader(root string) (Reader, error) {
|
||||||
|
return NewRestrictedCompositeReaderWithLimit(root, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewRestrictedCompositeReaderWithLimit(root string, maxBytes int64) (Reader, error) {
|
||||||
|
fileReader, err := newRestrictedFileReader(root, maxBytes)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &CompositeReader{
|
||||||
|
inlineReader: &inlineReader{},
|
||||||
|
fileReader: fileReader,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (c *CompositeReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
func (c *CompositeReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
@@ -89,21 +109,133 @@ func (r *fileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.
|
|||||||
return nil, ErrMissingFilePath
|
return nil, ErrMissingFilePath
|
||||||
}
|
}
|
||||||
|
|
||||||
data, err := os.ReadFile(ref.URI)
|
return readFileArtifact(ref.URI)
|
||||||
if err != nil {
|
}
|
||||||
return nil, fmt.Errorf("failed to read file %s: %w", ref.URI, err)
|
|
||||||
|
type deniedFileReader struct{}
|
||||||
|
|
||||||
|
func (r deniedFileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
contentType := mime.TypeByExtension(filepath.Ext(ref.URI))
|
if ref.URI == "" {
|
||||||
|
return nil, ErrMissingFilePath
|
||||||
|
}
|
||||||
|
return nil, ErrFileNotAllowed
|
||||||
|
}
|
||||||
|
|
||||||
|
type restrictedFileReader struct {
|
||||||
|
root string
|
||||||
|
maxBytes int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRestrictedFileReader(root string, maxBytes int64) (Reader, error) {
|
||||||
|
if maxBytes < 0 {
|
||||||
|
return nil, fmt.Errorf("artifact size limit must be greater than or equal to 0")
|
||||||
|
}
|
||||||
|
cleanRoot := strings.TrimSpace(root)
|
||||||
|
if cleanRoot == "" {
|
||||||
|
return deniedFileReader{}, nil
|
||||||
|
}
|
||||||
|
absRoot, err := filepath.Abs(filepath.Clean(cleanRoot))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("resolve artifact root: %w", err)
|
||||||
|
}
|
||||||
|
return &restrictedFileReader{root: absRoot, maxBytes: maxBytes}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *restrictedFileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
if ref.URI == "" {
|
||||||
|
return nil, ErrMissingFilePath
|
||||||
|
}
|
||||||
|
|
||||||
|
path, err := r.resolveLexicalPath(ref.URI)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return readFileArtifactWithLimit(path, r.maxBytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveLexicalPath checks cleaned path containment without resolving symlinks.
|
||||||
|
func (r *restrictedFileReader) resolveLexicalPath(rawPath string) (string, error) {
|
||||||
|
cleanPath := filepath.Clean(strings.TrimSpace(rawPath))
|
||||||
|
var candidate string
|
||||||
|
if filepath.IsAbs(cleanPath) {
|
||||||
|
candidate = cleanPath
|
||||||
|
} else {
|
||||||
|
candidate = filepath.Join(r.root, cleanPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
absCandidate, err := filepath.Abs(candidate)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("resolve artifact path: %w", err)
|
||||||
|
}
|
||||||
|
absCandidate = filepath.Clean(absCandidate)
|
||||||
|
|
||||||
|
rel, err := filepath.Rel(r.root, absCandidate)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("compare artifact path to root: %w", err)
|
||||||
|
}
|
||||||
|
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) {
|
||||||
|
return "", ErrFileOutsideRoot
|
||||||
|
}
|
||||||
|
return absCandidate, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readFileArtifact(path string) (*domain.Artifact, error) {
|
||||||
|
return readFileArtifactWithLimit(path, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func readFileArtifactWithLimit(path string, maxBytes int64) (*domain.Artifact, error) {
|
||||||
|
if maxBytes < 0 {
|
||||||
|
return nil, fmt.Errorf("file size limit must be greater than or equal to 0")
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
info, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to stat file %s: %w", path, err)
|
||||||
|
}
|
||||||
|
if maxBytes > 0 && info.Size() > maxBytes {
|
||||||
|
return nil, ErrFileTooLarge
|
||||||
|
}
|
||||||
|
|
||||||
|
var reader io.Reader = file
|
||||||
|
if maxBytes > 0 {
|
||||||
|
reader = io.LimitReader(file, maxBytes+1)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(reader)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
||||||
|
}
|
||||||
|
if maxBytes > 0 && int64(len(data)) > maxBytes {
|
||||||
|
return nil, ErrFileTooLarge
|
||||||
|
}
|
||||||
|
|
||||||
|
contentType := mime.TypeByExtension(filepath.Ext(path))
|
||||||
if contentType == "" {
|
if contentType == "" {
|
||||||
contentType = defaults.ContentTypeTextPlain
|
contentType = defaults.ContentTypeTextPlain
|
||||||
}
|
}
|
||||||
|
|
||||||
return &domain.Artifact{
|
return &domain.Artifact{
|
||||||
Name: filepath.Base(ref.URI),
|
Name: filepath.Base(path),
|
||||||
ContentType: contentType,
|
ContentType: contentType,
|
||||||
Body: data,
|
Body: data,
|
||||||
URI: ref.URI,
|
URI: path,
|
||||||
Size: int64(len(data)),
|
Size: int64(len(data)),
|
||||||
Hash: fmt.Sprintf("%x", sha256.Sum256(data)),
|
Hash: fmt.Sprintf("%x", sha256.Sum256(data)),
|
||||||
}, nil
|
}, nil
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
@@ -56,6 +57,157 @@ func TestCompositeReader_Read(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRestrictedCompositeReader(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
root := t.TempDir()
|
||||||
|
outside := t.TempDir()
|
||||||
|
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "input.txt"), []byte("allowed"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Mkdir(filepath.Join(root, "nested"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(outside, "secret.txt"), []byte("denied"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
reader, err := NewRestrictedCompositeReader(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected restricted reader construction, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("accepts relative contained path", func(t *testing.T) {
|
||||||
|
art, err := reader.Read(ctx, domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: "nested/../input.txt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected contained relative path to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
if string(art.Body) != "allowed" {
|
||||||
|
t.Fatalf("unexpected artifact body: %q", string(art.Body))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("accepts absolute contained path", func(t *testing.T) {
|
||||||
|
art, err := reader.Read(ctx, domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: filepath.Join(root, "input.txt")})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected contained absolute path to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
if art.Name != "input.txt" {
|
||||||
|
t.Fatalf("unexpected artifact name: %q", art.Name)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("rejects relative traversal outside root", func(t *testing.T) {
|
||||||
|
_, err := reader.Read(ctx, domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: filepath.Join("..", filepath.Base(outside), "secret.txt")})
|
||||||
|
if !errors.Is(err, ErrFileOutsideRoot) {
|
||||||
|
t.Fatalf("expected ErrFileOutsideRoot, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("rejects absolute path outside root", func(t *testing.T) {
|
||||||
|
_, err := reader.Read(ctx, domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: filepath.Join(outside, "secret.txt")})
|
||||||
|
if !errors.Is(err, ErrFileOutsideRoot) {
|
||||||
|
t.Fatalf("expected ErrFileOutsideRoot, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRestrictedCompositeReaderFollowsSymlinkInsideRoot(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
root := t.TempDir()
|
||||||
|
outside := t.TempDir()
|
||||||
|
|
||||||
|
target := filepath.Join(outside, "linked.txt")
|
||||||
|
if err := os.WriteFile(target, []byte("linked outside root"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
link := filepath.Join(root, "linked.txt")
|
||||||
|
if err := os.Symlink(target, link); err != nil {
|
||||||
|
t.Skipf("symlink creation unavailable: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
reader, err := NewRestrictedCompositeReader(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected restricted reader construction, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
art, err := reader.Read(ctx, domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: "linked.txt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected symlink inside root to be followed, got %v", err)
|
||||||
|
}
|
||||||
|
if string(art.Body) != "linked outside root" {
|
||||||
|
t.Fatalf("unexpected artifact body: %q", string(art.Body))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRestrictedCompositeReaderWithoutRootDeniesFileRefs(t *testing.T) {
|
||||||
|
reader, err := NewRestrictedCompositeReader("")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected restricted reader construction, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
art, err := reader.Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: "inline"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected inline ref to work without artifact root, got %v", err)
|
||||||
|
}
|
||||||
|
if string(art.Body) != "inline" {
|
||||||
|
t.Fatalf("unexpected inline body: %q", string(art.Body))
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = reader.Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: "input.txt"})
|
||||||
|
if !errors.Is(err, ErrFileNotAllowed) {
|
||||||
|
t.Fatalf("expected ErrFileNotAllowed, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRestrictedCompositeReaderFileSizeLimit(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
root := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "exact.txt"), []byte("12345"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "large.txt"), []byte("123456"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
reader, err := NewRestrictedCompositeReaderWithLimit(root, 5)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected restricted reader construction, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
art, err := reader.Read(ctx, domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: "exact.txt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected file at limit to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
if string(art.Body) != "12345" {
|
||||||
|
t.Fatalf("unexpected artifact body: %q", string(art.Body))
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = reader.Read(ctx, domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: "large.txt"})
|
||||||
|
if !errors.Is(err, ErrFileTooLarge) {
|
||||||
|
t.Fatalf("expected ErrFileTooLarge, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRestrictedCompositeReaderFileSizeLimitZeroDisablesLimit(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "large.txt"), []byte("123456"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
reader, err := NewRestrictedCompositeReaderWithLimit(root, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected restricted reader construction, got %v", err)
|
||||||
|
}
|
||||||
|
art, err := reader.Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: "large.txt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected unlimited reader to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
if string(art.Body) != "123456" {
|
||||||
|
t.Fatalf("unexpected artifact body: %q", string(art.Body))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFileReader_Read(t *testing.T) {
|
func TestFileReader_Read(t *testing.T) {
|
||||||
content := []byte("test file content")
|
content := []byte("test file content")
|
||||||
tmpFile, err := os.CreateTemp("", "artifact_test_*.txt")
|
tmpFile, err := os.CreateTemp("", "artifact_test_*.txt")
|
||||||
|
|||||||
@@ -40,7 +40,11 @@ type Config struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ServerConfig struct {
|
type ServerConfig struct {
|
||||||
Addr string `yaml:"addr"`
|
Addr string `yaml:"addr"`
|
||||||
|
ArtifactRoot string `yaml:"artifact_root"`
|
||||||
|
MaxRequestBytes *int64 `yaml:"max_request_bytes"`
|
||||||
|
MaxArtifactBytes *int64 `yaml:"max_artifact_bytes"`
|
||||||
|
MaxResponseBytes *int64 `yaml:"max_response_bytes"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type DefaultsConfig struct {
|
type DefaultsConfig struct {
|
||||||
@@ -53,16 +57,24 @@ type AppSettings struct {
|
|||||||
ProfileDir string
|
ProfileDir string
|
||||||
SchemaDir string
|
SchemaDir string
|
||||||
ServerAddr string
|
ServerAddr string
|
||||||
|
ArtifactRoot string
|
||||||
|
MaxRequestBytes int64
|
||||||
|
MaxArtifactBytes int64
|
||||||
|
MaxResponseBytes int64
|
||||||
DefaultRenderFormat renderformat.PreparedRunOutputFormat
|
DefaultRenderFormat renderformat.PreparedRunOutputFormat
|
||||||
}
|
}
|
||||||
|
|
||||||
// CLIOverrides can be applied after config load to enforce precedence.
|
// CLIOverrides can be applied after config load to enforce precedence.
|
||||||
type CLIOverrides struct {
|
type CLIOverrides struct {
|
||||||
PromptDir string
|
PromptDir string
|
||||||
ProfileDir string
|
ProfileDir string
|
||||||
SchemaDir string
|
SchemaDir string
|
||||||
ServerAddr string
|
ServerAddr string
|
||||||
RenderFormat string
|
ArtifactRoot string
|
||||||
|
MaxRequestBytes *int64
|
||||||
|
MaxArtifactBytes *int64
|
||||||
|
MaxResponseBytes *int64
|
||||||
|
RenderFormat string
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuiltInDefaults returns compile-time application defaults.
|
// BuiltInDefaults returns compile-time application defaults.
|
||||||
@@ -70,6 +82,9 @@ func BuiltInDefaults() AppSettings {
|
|||||||
return AppSettings{
|
return AppSettings{
|
||||||
SchemaDir: defaults.SchemaDirDefault,
|
SchemaDir: defaults.SchemaDirDefault,
|
||||||
ServerAddr: defaults.HTTPAddrDefault,
|
ServerAddr: defaults.HTTPAddrDefault,
|
||||||
|
MaxRequestBytes: defaults.HTTPMaxRequestBytesDefault,
|
||||||
|
MaxArtifactBytes: defaults.HTTPMaxArtifactBytesDefault,
|
||||||
|
MaxResponseBytes: defaults.HTTPMaxResponseBytesDefault,
|
||||||
DefaultRenderFormat: renderformat.DefaultPreparedRunOutputFormat,
|
DefaultRenderFormat: renderformat.DefaultPreparedRunOutputFormat,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -142,6 +157,27 @@ func ApplyCLIOverrides(base AppSettings, overrides CLIOverrides) (AppSettings, e
|
|||||||
if v := strings.TrimSpace(overrides.ServerAddr); v != "" {
|
if v := strings.TrimSpace(overrides.ServerAddr); v != "" {
|
||||||
out.ServerAddr = v
|
out.ServerAddr = v
|
||||||
}
|
}
|
||||||
|
if v := strings.TrimSpace(overrides.ArtifactRoot); v != "" {
|
||||||
|
out.ArtifactRoot = filepath.Clean(v)
|
||||||
|
}
|
||||||
|
if overrides.MaxRequestBytes != nil {
|
||||||
|
if *overrides.MaxRequestBytes < 0 {
|
||||||
|
return AppSettings{}, fmt.Errorf("%w: server.max_request_bytes must be greater than or equal to 0", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
out.MaxRequestBytes = *overrides.MaxRequestBytes
|
||||||
|
}
|
||||||
|
if overrides.MaxArtifactBytes != nil {
|
||||||
|
if *overrides.MaxArtifactBytes < 0 {
|
||||||
|
return AppSettings{}, fmt.Errorf("%w: server.max_artifact_bytes must be greater than or equal to 0", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
out.MaxArtifactBytes = *overrides.MaxArtifactBytes
|
||||||
|
}
|
||||||
|
if overrides.MaxResponseBytes != nil {
|
||||||
|
if *overrides.MaxResponseBytes < 0 {
|
||||||
|
return AppSettings{}, fmt.Errorf("%w: server.max_response_bytes must be greater than or equal to 0", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
out.MaxResponseBytes = *overrides.MaxResponseBytes
|
||||||
|
}
|
||||||
if rawFormat := strings.TrimSpace(overrides.RenderFormat); rawFormat != "" {
|
if rawFormat := strings.TrimSpace(overrides.RenderFormat); rawFormat != "" {
|
||||||
parsed, err := renderformat.ParsePreparedRunOutputFormat(rawFormat)
|
parsed, err := renderformat.ParsePreparedRunOutputFormat(rawFormat)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -181,6 +217,27 @@ func applyConfig(base AppSettings, cfg Config) (AppSettings, error) {
|
|||||||
if v := strings.TrimSpace(cfg.Server.Addr); v != "" {
|
if v := strings.TrimSpace(cfg.Server.Addr); v != "" {
|
||||||
out.ServerAddr = v
|
out.ServerAddr = v
|
||||||
}
|
}
|
||||||
|
if v := strings.TrimSpace(cfg.Server.ArtifactRoot); v != "" {
|
||||||
|
out.ArtifactRoot = filepath.Clean(v)
|
||||||
|
}
|
||||||
|
if cfg.Server.MaxRequestBytes != nil {
|
||||||
|
if *cfg.Server.MaxRequestBytes < 0 {
|
||||||
|
return AppSettings{}, fmt.Errorf("%w: server.max_request_bytes must be greater than or equal to 0", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
out.MaxRequestBytes = *cfg.Server.MaxRequestBytes
|
||||||
|
}
|
||||||
|
if cfg.Server.MaxArtifactBytes != nil {
|
||||||
|
if *cfg.Server.MaxArtifactBytes < 0 {
|
||||||
|
return AppSettings{}, fmt.Errorf("%w: server.max_artifact_bytes must be greater than or equal to 0", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
out.MaxArtifactBytes = *cfg.Server.MaxArtifactBytes
|
||||||
|
}
|
||||||
|
if cfg.Server.MaxResponseBytes != nil {
|
||||||
|
if *cfg.Server.MaxResponseBytes < 0 {
|
||||||
|
return AppSettings{}, fmt.Errorf("%w: server.max_response_bytes must be greater than or equal to 0", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
out.MaxResponseBytes = *cfg.Server.MaxResponseBytes
|
||||||
|
}
|
||||||
if rawFormat := strings.TrimSpace(cfg.Defaults.RenderFormat); rawFormat != "" {
|
if rawFormat := strings.TrimSpace(cfg.Defaults.RenderFormat); rawFormat != "" {
|
||||||
parsed, err := renderformat.ParsePreparedRunOutputFormat(rawFormat)
|
parsed, err := renderformat.ParsePreparedRunOutputFormat(rawFormat)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -24,6 +25,20 @@ func TestLoadConfigMissingImplicitPathUsesBuiltInDefaults(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBuiltInDefaultsIncludeHTTPSizeLimits(t *testing.T) {
|
||||||
|
got := BuiltInDefaults()
|
||||||
|
|
||||||
|
if got.MaxRequestBytes != defaults.HTTPMaxRequestBytesDefault {
|
||||||
|
t.Fatalf("unexpected max request bytes: %d", got.MaxRequestBytes)
|
||||||
|
}
|
||||||
|
if got.MaxArtifactBytes != defaults.HTTPMaxArtifactBytesDefault {
|
||||||
|
t.Fatalf("unexpected max artifact bytes: %d", got.MaxArtifactBytes)
|
||||||
|
}
|
||||||
|
if got.MaxResponseBytes != defaults.HTTPMaxResponseBytesDefault {
|
||||||
|
t.Fatalf("unexpected max response bytes: %d", got.MaxResponseBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadConfigMissingExplicitPathReturnsError(t *testing.T) {
|
func TestLoadConfigMissingExplicitPathReturnsError(t *testing.T) {
|
||||||
tmp := t.TempDir()
|
tmp := t.TempDir()
|
||||||
missing := filepath.Join(tmp, "missing.yml")
|
missing := filepath.Join(tmp, "missing.yml")
|
||||||
@@ -92,6 +107,10 @@ profile_dir: ./profiles
|
|||||||
schema_dir: ./schemas
|
schema_dir: ./schemas
|
||||||
server:
|
server:
|
||||||
addr: 127.0.0.1:9090
|
addr: 127.0.0.1:9090
|
||||||
|
artifact_root: ./artifacts
|
||||||
|
max_request_bytes: 1024
|
||||||
|
max_artifact_bytes: 2048
|
||||||
|
max_response_bytes: 4096
|
||||||
defaults:
|
defaults:
|
||||||
render_format: json
|
render_format: json
|
||||||
`)
|
`)
|
||||||
@@ -113,11 +132,61 @@ defaults:
|
|||||||
if got.ServerAddr != "127.0.0.1:9090" {
|
if got.ServerAddr != "127.0.0.1:9090" {
|
||||||
t.Fatalf("unexpected server.addr: %q", got.ServerAddr)
|
t.Fatalf("unexpected server.addr: %q", got.ServerAddr)
|
||||||
}
|
}
|
||||||
|
if got.ArtifactRoot != filepath.Clean("./artifacts") {
|
||||||
|
t.Fatalf("unexpected server.artifact_root: %q", got.ArtifactRoot)
|
||||||
|
}
|
||||||
|
if got.MaxRequestBytes != 1024 {
|
||||||
|
t.Fatalf("unexpected server.max_request_bytes: %d", got.MaxRequestBytes)
|
||||||
|
}
|
||||||
|
if got.MaxArtifactBytes != 2048 {
|
||||||
|
t.Fatalf("unexpected server.max_artifact_bytes: %d", got.MaxArtifactBytes)
|
||||||
|
}
|
||||||
|
if got.MaxResponseBytes != 4096 {
|
||||||
|
t.Fatalf("unexpected server.max_response_bytes: %d", got.MaxResponseBytes)
|
||||||
|
}
|
||||||
if got.DefaultRenderFormat != renderformat.PreparedRunFormatJSON {
|
if got.DefaultRenderFormat != renderformat.PreparedRunFormatJSON {
|
||||||
t.Fatalf("unexpected defaults.render_format: %q", got.DefaultRenderFormat)
|
t.Fatalf("unexpected defaults.render_format: %q", got.DefaultRenderFormat)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadConfigAcceptsZeroHTTPSizeLimits(t *testing.T) {
|
||||||
|
path := writeConfigFile(t, "config.yml", `
|
||||||
|
server:
|
||||||
|
max_request_bytes: 0
|
||||||
|
max_artifact_bytes: 0
|
||||||
|
max_response_bytes: 0
|
||||||
|
`)
|
||||||
|
|
||||||
|
got, err := LoadConfig(path, true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if got.MaxRequestBytes != 0 || got.MaxArtifactBytes != 0 || got.MaxResponseBytes != 0 {
|
||||||
|
t.Fatalf("expected zero limits to be preserved, got request=%d artifact=%d response=%d", got.MaxRequestBytes, got.MaxArtifactBytes, got.MaxResponseBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadConfigRejectsNegativeHTTPSizeLimits(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
body string
|
||||||
|
}{
|
||||||
|
{name: "request", body: "server:\n max_request_bytes: -1\n"},
|
||||||
|
{name: "artifact", body: "server:\n max_artifact_bytes: -1\n"},
|
||||||
|
{name: "response", body: "server:\n max_response_bytes: -1\n"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
path := writeConfigFile(t, "config.yml", tc.body)
|
||||||
|
_, err := LoadConfig(path, true)
|
||||||
|
if !errors.Is(err, ErrInvalidConfig) {
|
||||||
|
t.Fatalf("expected ErrInvalidConfig, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadConfigEmptyFileResolvesToBuiltInDefaults(t *testing.T) {
|
func TestLoadConfigEmptyFileResolvesToBuiltInDefaults(t *testing.T) {
|
||||||
path := writeConfigFile(t, "config.yml", "")
|
path := writeConfigFile(t, "config.yml", "")
|
||||||
|
|
||||||
@@ -185,15 +254,26 @@ func TestApplyCLIOverridesAppliesPrecedence(t *testing.T) {
|
|||||||
ProfileDir: "/from/config/profiles",
|
ProfileDir: "/from/config/profiles",
|
||||||
SchemaDir: "/from/config/schemas",
|
SchemaDir: "/from/config/schemas",
|
||||||
ServerAddr: ":1234",
|
ServerAddr: ":1234",
|
||||||
|
ArtifactRoot: "/from/config/artifacts",
|
||||||
|
MaxRequestBytes: 111,
|
||||||
|
MaxArtifactBytes: 222,
|
||||||
|
MaxResponseBytes: 333,
|
||||||
DefaultRenderFormat: renderformat.PreparedRunFormatJSON,
|
DefaultRenderFormat: renderformat.PreparedRunFormatJSON,
|
||||||
}
|
}
|
||||||
|
maxRequestBytes := int64(0)
|
||||||
|
maxArtifactBytes := int64(444)
|
||||||
|
maxResponseBytes := int64(555)
|
||||||
|
|
||||||
got, err := ApplyCLIOverrides(base, CLIOverrides{
|
got, err := ApplyCLIOverrides(base, CLIOverrides{
|
||||||
PromptDir: "./prompts-cli",
|
PromptDir: "./prompts-cli",
|
||||||
ProfileDir: "./profiles-cli",
|
ProfileDir: "./profiles-cli",
|
||||||
SchemaDir: "./schemas-cli",
|
SchemaDir: "./schemas-cli",
|
||||||
ServerAddr: ":8081",
|
ServerAddr: ":8081",
|
||||||
RenderFormat: "text",
|
ArtifactRoot: "./artifacts-cli",
|
||||||
|
MaxRequestBytes: &maxRequestBytes,
|
||||||
|
MaxArtifactBytes: &maxArtifactBytes,
|
||||||
|
MaxResponseBytes: &maxResponseBytes,
|
||||||
|
RenderFormat: "text",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -211,11 +291,45 @@ func TestApplyCLIOverridesAppliesPrecedence(t *testing.T) {
|
|||||||
if got.ServerAddr != ":8081" {
|
if got.ServerAddr != ":8081" {
|
||||||
t.Fatalf("unexpected server addr: %q", got.ServerAddr)
|
t.Fatalf("unexpected server addr: %q", got.ServerAddr)
|
||||||
}
|
}
|
||||||
|
if got.ArtifactRoot != filepath.Clean("./artifacts-cli") {
|
||||||
|
t.Fatalf("unexpected artifact root: %q", got.ArtifactRoot)
|
||||||
|
}
|
||||||
|
if got.MaxRequestBytes != 0 {
|
||||||
|
t.Fatalf("unexpected max request bytes: %d", got.MaxRequestBytes)
|
||||||
|
}
|
||||||
|
if got.MaxArtifactBytes != 444 {
|
||||||
|
t.Fatalf("unexpected max artifact bytes: %d", got.MaxArtifactBytes)
|
||||||
|
}
|
||||||
|
if got.MaxResponseBytes != 555 {
|
||||||
|
t.Fatalf("unexpected max response bytes: %d", got.MaxResponseBytes)
|
||||||
|
}
|
||||||
if got.DefaultRenderFormat != renderformat.PreparedRunFormatText {
|
if got.DefaultRenderFormat != renderformat.PreparedRunFormatText {
|
||||||
t.Fatalf("unexpected render format: %q", got.DefaultRenderFormat)
|
t.Fatalf("unexpected render format: %q", got.DefaultRenderFormat)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestApplyCLIOverridesRejectsNegativeHTTPSizeLimits(t *testing.T) {
|
||||||
|
negative := int64(-1)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
overrides CLIOverrides
|
||||||
|
}{
|
||||||
|
{name: "request", overrides: CLIOverrides{MaxRequestBytes: &negative}},
|
||||||
|
{name: "artifact", overrides: CLIOverrides{MaxArtifactBytes: &negative}},
|
||||||
|
{name: "response", overrides: CLIOverrides{MaxResponseBytes: &negative}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := ApplyCLIOverrides(BuiltInDefaults(), tc.overrides)
|
||||||
|
if !errors.Is(err, ErrInvalidConfig) {
|
||||||
|
t.Fatalf("expected ErrInvalidConfig, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestApplyCLIOverridesInvalidRenderFormatReturnsError(t *testing.T) {
|
func TestApplyCLIOverridesInvalidRenderFormatReturnsError(t *testing.T) {
|
||||||
_, err := ApplyCLIOverrides(BuiltInDefaults(), CLIOverrides{RenderFormat: "yaml"})
|
_, err := ApplyCLIOverrides(BuiltInDefaults(), CLIOverrides{RenderFormat: "yaml"})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
|
|||||||
@@ -7,13 +7,16 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
HTTPAddrDefault = ":8080"
|
HTTPAddrDefault = ":8080"
|
||||||
SchemaDirDefault = "."
|
SchemaDirDefault = "."
|
||||||
OutputArtifactName = "output"
|
OutputArtifactName = "output"
|
||||||
ContentTypeTextPlain = "text/plain"
|
ContentTypeTextPlain = "text/plain"
|
||||||
ContentTypeTextMarkdown = "text/markdown"
|
ContentTypeTextMarkdown = "text/markdown"
|
||||||
ContentTypeApplicationJSON = "application/json"
|
ContentTypeApplicationJSON = "application/json"
|
||||||
OpenAIChatCompletionsPath = "/chat/completions"
|
OpenAIChatCompletionsPath = "/chat/completions"
|
||||||
|
HTTPMaxRequestBytesDefault = 16 * 1024 * 1024
|
||||||
|
HTTPMaxArtifactBytesDefault = 16 * 1024 * 1024
|
||||||
|
HTTPMaxResponseBytesDefault = 16 * 1024 * 1024
|
||||||
|
|
||||||
ExecutionDefaultTemperature = 0.0
|
ExecutionDefaultTemperature = 0.0
|
||||||
ExecutionDefaultMaxTokens = 0
|
ExecutionDefaultMaxTokens = 0
|
||||||
|
|||||||
@@ -40,14 +40,33 @@ const (
|
|||||||
ValidationSkipped ValidationStatus = "skipped"
|
ValidationSkipped ValidationStatus = "skipped"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// CacheControlType defines provider cache behavior for prompt content.
|
||||||
|
type CacheControlType string
|
||||||
|
|
||||||
|
const (
|
||||||
|
CacheControlEphemeral CacheControlType = "ephemeral"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// SessionIDMaxLength is OpenRouter's documented maximum session_id length.
|
||||||
|
SessionIDMaxLength = 256
|
||||||
|
)
|
||||||
|
|
||||||
|
// CacheControl describes provider cache metadata attached to prompt content.
|
||||||
|
type CacheControl struct {
|
||||||
|
Type CacheControlType `yaml:"type" json:"type"`
|
||||||
|
TTL string `yaml:"ttl,omitempty" json:"ttl,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
// RunRequest represents a request to generate a single artifact.
|
// RunRequest represents a request to generate a single artifact.
|
||||||
type RunRequest struct {
|
type RunRequest struct {
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion string
|
||||||
ProfileID string
|
ProfileID string
|
||||||
|
APIKey string `json:"-" yaml:"-"`
|
||||||
Inputs map[string]ArtifactRef
|
Inputs map[string]ArtifactRef
|
||||||
Vars map[string]string
|
Vars map[string]string
|
||||||
Execution *ExecutionTarget
|
Execution *ExecutionTargetOverride
|
||||||
Validation *OutputContract
|
Validation *OutputContract
|
||||||
Metadata map[string]string
|
Metadata map[string]string
|
||||||
}
|
}
|
||||||
@@ -71,25 +90,26 @@ type RunResult struct {
|
|||||||
StartTime time.Time
|
StartTime time.Time
|
||||||
EndTime time.Time
|
EndTime time.Time
|
||||||
Duration time.Duration
|
Duration time.Duration
|
||||||
Error error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
|
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
|
||||||
// It must never include resolved API key values, model output, or validation data.
|
// It must never include resolved API key values, model output, or validation data.
|
||||||
type PreparedRun struct {
|
type PreparedRun struct {
|
||||||
PromptID string `json:"prompt_id"`
|
PromptID string `json:"prompt_id"`
|
||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
PromptVersion string `json:"prompt_version,omitempty"`
|
||||||
PromptHash string `json:"prompt_hash,omitempty"`
|
PromptHash string `json:"prompt_hash,omitempty"`
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||||
OutputContract OutputContract `json:"output_contract"`
|
TargetPresence ExecutionTargetPresence `json:"-"`
|
||||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
OutputContract OutputContract `json:"output_contract"`
|
||||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||||
Messages []RenderedMessage `json:"messages"`
|
SessionID string `json:"session_id,omitempty"`
|
||||||
StartTime time.Time `json:"start_time,omitempty"`
|
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||||
EndTime time.Time `json:"end_time,omitempty"`
|
Messages []RenderedMessage `json:"messages"`
|
||||||
DurationMS int64 `json:"duration_ms,omitempty"`
|
StartTime time.Time `json:"start_time,omitempty"`
|
||||||
|
EndTime time.Time `json:"end_time,omitempty"`
|
||||||
|
DurationMS int64 `json:"duration_ms,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArtifactRef represents a reference to an input artifact.
|
// ArtifactRef represents a reference to an input artifact.
|
||||||
@@ -115,6 +135,7 @@ type PromptDefinition struct {
|
|||||||
Version string `yaml:"version"`
|
Version string `yaml:"version"`
|
||||||
DefaultProfile string `yaml:"default_profile"`
|
DefaultProfile string `yaml:"default_profile"`
|
||||||
Description string `yaml:"description"`
|
Description string `yaml:"description"`
|
||||||
|
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
|
||||||
Inputs []PromptInput `yaml:"inputs"`
|
Inputs []PromptInput `yaml:"inputs"`
|
||||||
Templates []PromptMessageTemplate `yaml:"templates"`
|
Templates []PromptMessageTemplate `yaml:"templates"`
|
||||||
OutputFormat OutputFormat `yaml:"output_format"`
|
OutputFormat OutputFormat `yaml:"output_format"`
|
||||||
@@ -131,38 +152,65 @@ type PromptInput struct {
|
|||||||
|
|
||||||
// PromptMessageTemplate defines a template for a chat message.
|
// PromptMessageTemplate defines a template for a chat message.
|
||||||
type PromptMessageTemplate struct {
|
type PromptMessageTemplate struct {
|
||||||
Role string `yaml:"role"`
|
Role string `yaml:"role"`
|
||||||
Content string `yaml:"content"`
|
Content string `yaml:"content"`
|
||||||
ContentFile string `yaml:"content_file"`
|
ContentFile string `yaml:"content_file"`
|
||||||
|
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecutionProfile describes how and where to execute a model.
|
// ExecutionProfile describes how and where to execute a model.
|
||||||
type ExecutionProfile struct {
|
type ExecutionProfile struct {
|
||||||
ID string `yaml:"id"`
|
ID string `yaml:"id"`
|
||||||
Endpoint string `yaml:"endpoint"`
|
Endpoint string `yaml:"endpoint"`
|
||||||
Model string `yaml:"model"`
|
Model string `yaml:"model"`
|
||||||
Temperature float64 `yaml:"temperature"`
|
Temperature float64 `yaml:"temperature"`
|
||||||
MaxTokens int `yaml:"max_tokens"`
|
MaxTokens int `yaml:"max_tokens"`
|
||||||
TopP float64 `yaml:"top_p"`
|
TopP float64 `yaml:"top_p"`
|
||||||
TimeoutSeconds int `yaml:"timeout_seconds"`
|
TimeoutSeconds int `yaml:"timeout_seconds"`
|
||||||
ServiceTier string `yaml:"service_tier"`
|
ServiceTier string `yaml:"service_tier"`
|
||||||
ReasoningEffort string `yaml:"reasoning_effort"`
|
ReasoningEffort string `yaml:"reasoning_effort"`
|
||||||
APIKeyEnv string `yaml:"api_key_env"`
|
APIKeyEnv string `yaml:"api_key_env"`
|
||||||
ExtraParams map[string]string `yaml:"extra_params"`
|
APIKeyRequired bool `yaml:"-" json:"-"`
|
||||||
|
ExtraParams map[string]any `yaml:"extra_params"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecutionTargetOverride represents per-request runtime setting overrides.
|
||||||
|
type ExecutionTargetOverride struct {
|
||||||
|
Endpoint string `json:"endpoint,omitempty"`
|
||||||
|
Model string `json:"model,omitempty"`
|
||||||
|
Temperature *float64 `json:"temperature,omitempty"`
|
||||||
|
MaxTokens *int `json:"max_tokens,omitempty"`
|
||||||
|
TopP *float64 `json:"top_p,omitempty"`
|
||||||
|
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||||
|
ServiceTier string `json:"service_tier,omitempty"`
|
||||||
|
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||||
|
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||||
|
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecutionTargetPresence tracks which effective runtime fields came from an
|
||||||
|
// explicit request override even when the resolved value is a zero value.
|
||||||
|
type ExecutionTargetPresence struct {
|
||||||
|
Temperature bool
|
||||||
|
MaxTokens bool
|
||||||
|
TopP bool
|
||||||
|
TimeoutSeconds bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecutionTarget represents effective model runtime settings for a run.
|
// ExecutionTarget represents effective model runtime settings for a run.
|
||||||
type ExecutionTarget struct {
|
type ExecutionTarget struct {
|
||||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||||
Model string `yaml:"model" json:"model"`
|
Model string `yaml:"model" json:"model"`
|
||||||
Temperature float64 `yaml:"temperature" json:"temperature"`
|
Temperature float64 `yaml:"temperature" json:"temperature"`
|
||||||
MaxTokens int `yaml:"max_tokens" json:"max_tokens"`
|
MaxTokens int `yaml:"max_tokens" json:"max_tokens"`
|
||||||
TopP float64 `yaml:"top_p" json:"top_p"`
|
TopP float64 `yaml:"top_p" json:"top_p"`
|
||||||
TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"`
|
TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"`
|
||||||
ServiceTier string `yaml:"service_tier" json:"service_tier"`
|
ServiceTier string `yaml:"service_tier" json:"service_tier"`
|
||||||
ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"`
|
ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"`
|
||||||
APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"`
|
APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"`
|
||||||
ExtraParams map[string]string `yaml:"extra_params" json:"extra_params"`
|
APIKey string `yaml:"-" json:"-"`
|
||||||
|
APIKeyRequired bool `yaml:"-" json:"-"`
|
||||||
|
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// OutputContract defines the requirements for the output artifact.
|
// OutputContract defines the requirements for the output artifact.
|
||||||
@@ -175,19 +223,22 @@ type OutputContract struct {
|
|||||||
|
|
||||||
// RenderedPrompt represents the prompt after template application.
|
// RenderedPrompt represents the prompt after template application.
|
||||||
type RenderedPrompt struct {
|
type RenderedPrompt struct {
|
||||||
Messages []RenderedMessage `json:"messages"`
|
SessionID string `json:"session_id,omitempty"`
|
||||||
|
Messages []RenderedMessage `json:"messages"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// RenderedMessage is a single message in a rendered prompt.
|
// RenderedMessage is a single message in a rendered prompt.
|
||||||
type RenderedMessage struct {
|
type RenderedMessage struct {
|
||||||
Role string `json:"role"`
|
Role string `json:"role"`
|
||||||
Content string `json:"content"`
|
Content string `json:"content"`
|
||||||
|
CacheControl *CacheControl `json:"cache_control,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// GenerateRequest is the internal request passed to the LLM client.
|
// GenerateRequest is the internal request passed to the LLM client.
|
||||||
type GenerateRequest struct {
|
type GenerateRequest struct {
|
||||||
Prompt RenderedPrompt
|
Prompt RenderedPrompt
|
||||||
Target ExecutionTarget
|
Target ExecutionTarget
|
||||||
|
TargetPresence ExecutionTargetPresence
|
||||||
StructuredOutput *StructuredOutputSpec
|
StructuredOutput *StructuredOutputSpec
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -222,6 +273,8 @@ type TokenUsage struct {
|
|||||||
PromptTokens int
|
PromptTokens int
|
||||||
CompletionTokens int
|
CompletionTokens int
|
||||||
TotalTokens int
|
TotalTokens int
|
||||||
|
CachedTokens int
|
||||||
|
CacheWriteTokens int
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidationResult represents the outcome of an output validation.
|
// ValidationResult represents the outcome of an output validation.
|
||||||
@@ -233,23 +286,3 @@ type ValidationResult struct {
|
|||||||
RepairAttempts int
|
RepairAttempts int
|
||||||
IsValid bool
|
IsValid bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// RunMetadata contains auditing information for a run.
|
|
||||||
type RunMetadata struct {
|
|
||||||
RunID string
|
|
||||||
PromptID string
|
|
||||||
PromptVersion string
|
|
||||||
PromptHash string
|
|
||||||
RenderedPromptHash string
|
|
||||||
SelectedProfileID string
|
|
||||||
InputHashes map[string]string
|
|
||||||
ModelEndpoint string
|
|
||||||
ModelName string
|
|
||||||
Params ExecutionTarget
|
|
||||||
Timestamp time.Time
|
|
||||||
Duration time.Duration
|
|
||||||
Usage TokenUsage
|
|
||||||
ValidationMode ValidationMode
|
|
||||||
ValidationStatus ValidationStatus
|
|
||||||
RepairAttempts int
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
|
|||||||
Endpoint: "http://llm/v1",
|
Endpoint: "http://llm/v1",
|
||||||
Model: "gpt-test",
|
Model: "gpt-test",
|
||||||
APIKeyEnv: envName,
|
APIKeyEnv: envName,
|
||||||
|
APIKey: secret,
|
||||||
},
|
},
|
||||||
InputHashes: map[string]string{"transcript": "hash-1"},
|
InputHashes: map[string]string{"transcript": "hash-1"},
|
||||||
RenderedPromptHash: "rendered-hash",
|
RenderedPromptHash: "rendered-hash",
|
||||||
@@ -53,3 +54,88 @@ func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPreparedRunJSONIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
||||||
|
prepared := PreparedRun{
|
||||||
|
PromptID: "prompt.id",
|
||||||
|
SelectedProfileID: "local-fast",
|
||||||
|
EffectiveModelParams: ExecutionTarget{
|
||||||
|
Endpoint: "http://llm/v1",
|
||||||
|
Model: "gpt-test",
|
||||||
|
},
|
||||||
|
RenderedPromptHash: "rendered-hash",
|
||||||
|
Messages: []RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "You are helpful.",
|
||||||
|
CacheControl: &CacheControl{
|
||||||
|
Type: CacheControlEphemeral,
|
||||||
|
TTL: "1h",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{Role: "user", Content: "Summarize this."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := json.Marshal(prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var decoded struct {
|
||||||
|
Messages []map[string]any `json:"messages"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(b, &decoded); err != nil {
|
||||||
|
t.Fatalf("unmarshal failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(decoded.Messages) != 2 {
|
||||||
|
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
|
||||||
|
}
|
||||||
|
|
||||||
|
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected cache_control on first message, got %#v", decoded.Messages[0])
|
||||||
|
}
|
||||||
|
if cacheControl["type"] != string(CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
||||||
|
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
||||||
|
}
|
||||||
|
if _, ok := decoded.Messages[1]["cache_control"]; ok {
|
||||||
|
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparedRunJSONIncludesSessionIDOnlyWhenPresent(t *testing.T) {
|
||||||
|
prepared := PreparedRun{
|
||||||
|
PromptID: "prompt.id",
|
||||||
|
SelectedProfileID: "local-fast",
|
||||||
|
EffectiveModelParams: ExecutionTarget{
|
||||||
|
Endpoint: "http://llm/v1",
|
||||||
|
Model: "gpt-test",
|
||||||
|
},
|
||||||
|
SessionID: "session-123",
|
||||||
|
RenderedPromptHash: "rendered-hash",
|
||||||
|
Messages: []RenderedMessage{{Role: "user", Content: "Summarize this."}},
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := json.Marshal(prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var decoded map[string]any
|
||||||
|
if err := json.Unmarshal(b, &decoded); err != nil {
|
||||||
|
t.Fatalf("unmarshal failed: %v", err)
|
||||||
|
}
|
||||||
|
if decoded["session_id"] != "session-123" {
|
||||||
|
t.Fatalf("expected session_id in prepared run JSON, got %#v", decoded["session_id"])
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared.SessionID = ""
|
||||||
|
b, err = json.Marshal(prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal failed: %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(b), "session_id") {
|
||||||
|
t.Fatalf("expected empty session_id to be omitted, got %s", b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,7 +2,10 @@ package filecatalog
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -23,7 +26,7 @@ func FindYAMLFiles(ctx context.Context, root string) ([]string, error) {
|
|||||||
if d.IsDir() {
|
if d.IsDir() {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if !isYAMLFile(d.Name()) {
|
if !IsYAMLFile(d.Name()) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
files = append(files, path)
|
files = append(files, path)
|
||||||
@@ -33,15 +36,100 @@ func FindYAMLFiles(ctx context.Context, root string) ([]string, error) {
|
|||||||
return files, err
|
return files, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FindFSYAMLFiles returns sorted paths for .yaml and .yml files under root in fsys.
|
||||||
|
func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, error) {
|
||||||
|
cleanRoot := CleanFSRoot(root)
|
||||||
|
var files []string
|
||||||
|
err := fs.WalkDir(fsys, cleanRoot, func(name string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
if d.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !IsYAMLFile(d.Name()) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
files = append(files, name)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
sort.Strings(files)
|
||||||
|
return files, err
|
||||||
|
}
|
||||||
|
|
||||||
// RelativePath computes a clean relative path from root to path.
|
// RelativePath computes a clean relative path from root to path.
|
||||||
func RelativePath(root string, path string) string {
|
func RelativePath(root string, filePath string) string {
|
||||||
rel, err := filepath.Rel(root, path)
|
rel, err := filepath.Rel(root, filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return filepath.Clean(path)
|
return filepath.Clean(filePath)
|
||||||
}
|
}
|
||||||
return filepath.Clean(rel)
|
return filepath.Clean(rel)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CleanFSRoot normalizes a root path for use with fs.FS.
|
||||||
|
func CleanFSRoot(root string) string {
|
||||||
|
root = strings.TrimSpace(root)
|
||||||
|
if root == "" || root == "." {
|
||||||
|
return "."
|
||||||
|
}
|
||||||
|
return path.Clean(root)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DisplayPath returns name relative to root for messages about fs.FS paths.
|
||||||
|
func DisplayPath(root string, name string) string {
|
||||||
|
cleanRoot := CleanFSRoot(root)
|
||||||
|
cleanName := path.Clean(name)
|
||||||
|
if cleanRoot == "." {
|
||||||
|
return cleanName
|
||||||
|
}
|
||||||
|
prefix := strings.TrimSuffix(cleanRoot, "/") + "/"
|
||||||
|
if strings.HasPrefix(cleanName, prefix) {
|
||||||
|
return strings.TrimPrefix(cleanName, prefix)
|
||||||
|
}
|
||||||
|
return cleanName
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolveFSPath resolves userPath from baseDir and keeps it inside root.
|
||||||
|
func ResolveFSPath(root string, baseDir string, userPath string) (string, string, error) {
|
||||||
|
cleanRoot := CleanFSRoot(root)
|
||||||
|
cleanBase := path.Clean(strings.TrimSpace(baseDir))
|
||||||
|
if cleanBase == "" {
|
||||||
|
cleanBase = cleanRoot
|
||||||
|
}
|
||||||
|
if !containsFSPath(cleanRoot, cleanBase) {
|
||||||
|
return "", "", fmt.Errorf("base path %q is outside source root %q", cleanBase, cleanRoot)
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanUserPath := strings.TrimSpace(userPath)
|
||||||
|
if cleanUserPath == "" {
|
||||||
|
return "", "", fmt.Errorf("path is required")
|
||||||
|
}
|
||||||
|
cleanUserPath = path.Clean(cleanUserPath)
|
||||||
|
if path.IsAbs(cleanUserPath) {
|
||||||
|
return "", "", fmt.Errorf("path %q must be relative", userPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved := path.Clean(path.Join(cleanBase, cleanUserPath))
|
||||||
|
if !containsFSPath(cleanRoot, resolved) {
|
||||||
|
return "", "", fmt.Errorf("path %q escapes source root %q", userPath, cleanRoot)
|
||||||
|
}
|
||||||
|
return resolved, DisplayPath(cleanRoot, resolved), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func containsFSPath(root string, name string) bool {
|
||||||
|
root = CleanFSRoot(root)
|
||||||
|
name = path.Clean(name)
|
||||||
|
if root == "." {
|
||||||
|
return name == "." || (name != ".." && !strings.HasPrefix(name, "../"))
|
||||||
|
}
|
||||||
|
return name == root || strings.HasPrefix(name, strings.TrimSuffix(root, "/")+"/")
|
||||||
|
}
|
||||||
|
|
||||||
// Stem strips .yaml or .yml from a file name.
|
// Stem strips .yaml or .yml from a file name.
|
||||||
func Stem(name string) string {
|
func Stem(name string) string {
|
||||||
name = strings.TrimSuffix(name, ".yaml")
|
name = strings.TrimSuffix(name, ".yaml")
|
||||||
@@ -49,6 +137,6 @@ func Stem(name string) string {
|
|||||||
return name
|
return name
|
||||||
}
|
}
|
||||||
|
|
||||||
func isYAMLFile(name string) bool {
|
func IsYAMLFile(name string) bool {
|
||||||
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
|
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,9 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestFindYAMLFilesNestedSortedAndFiltered(t *testing.T) {
|
func TestFindYAMLFilesNestedSortedAndFiltered(t *testing.T) {
|
||||||
@@ -43,6 +45,42 @@ func TestFindYAMLFilesHonorsContextCancellation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFindFSYAMLFilesNestedSortedAndFiltered(t *testing.T) {
|
||||||
|
fsys := fstest.MapFS{
|
||||||
|
"prompts/z/prompt.yml": &fstest.MapFile{Data: []byte("id: z")},
|
||||||
|
"prompts/a/profile.yaml": &fstest.MapFile{Data: []byte("id: a")},
|
||||||
|
"prompts/a/ignore.txt": &fstest.MapFile{Data: []byte("not yaml")},
|
||||||
|
"prompts/b/ignore.yaml.bak": &fstest.MapFile{Data: []byte("not yaml")},
|
||||||
|
"other/ignored.yaml": &fstest.MapFile{Data: []byte("id: ignored")},
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := FindFSYAMLFiles(context.Background(), fsys, " prompts ")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []string{
|
||||||
|
"prompts/a/profile.yaml",
|
||||||
|
"prompts/z/prompt.yml",
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("expected sorted YAML files %v, got %v", want, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFindFSYAMLFilesHonorsContextCancellation(t *testing.T) {
|
||||||
|
fsys := fstest.MapFS{
|
||||||
|
"one.yaml": &fstest.MapFile{Data: []byte("id: one")},
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
_, err := FindFSYAMLFiles(ctx, fsys, ".")
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("expected context.Canceled, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRelativePathNested(t *testing.T) {
|
func TestRelativePathNested(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
path := filepath.Join(root, "nested", "profiles", "local.yaml")
|
path := filepath.Join(root, "nested", "profiles", "local.yaml")
|
||||||
@@ -53,6 +91,133 @@ func TestRelativePathNested(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCleanFSRoot(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
root string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "empty", root: "", want: "."},
|
||||||
|
{name: "dot", root: ".", want: "."},
|
||||||
|
{name: "trimmed", root: " prompts/../profiles ", want: "profiles"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if got := CleanFSRoot(tc.root); got != tc.want {
|
||||||
|
t.Fatalf("expected %q, got %q", tc.want, got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDisplayPath(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
root string
|
||||||
|
path string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "root dot", root: ".", path: "profiles/local.yaml", want: "profiles/local.yaml"},
|
||||||
|
{name: "nested root", root: "profiles", path: "profiles/local.yaml", want: "local.yaml"},
|
||||||
|
{name: "outside root", root: "profiles", path: "other/local.yaml", want: "other/local.yaml"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if got := DisplayPath(tc.root, tc.path); got != tc.want {
|
||||||
|
t.Fatalf("expected %q, got %q", tc.want, got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveFSPath(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
root string
|
||||||
|
baseDir string
|
||||||
|
userPath string
|
||||||
|
wantPath string
|
||||||
|
wantDisplay string
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "sibling inside root",
|
||||||
|
root: "prompts",
|
||||||
|
baseDir: "prompts/nested",
|
||||||
|
userPath: "./messages/user.tmpl",
|
||||||
|
wantPath: "prompts/nested/messages/user.tmpl",
|
||||||
|
wantDisplay: "nested/messages/user.tmpl",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "parent inside root",
|
||||||
|
root: "prompts",
|
||||||
|
baseDir: "prompts/nested",
|
||||||
|
userPath: "../shared/user.tmpl",
|
||||||
|
wantPath: "prompts/shared/user.tmpl",
|
||||||
|
wantDisplay: "shared/user.tmpl",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "escape rejected",
|
||||||
|
root: "prompts",
|
||||||
|
baseDir: "prompts/nested",
|
||||||
|
userPath: "../../outside.tmpl",
|
||||||
|
wantErr: "escapes source root",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "absolute path rejected",
|
||||||
|
root: "prompts",
|
||||||
|
baseDir: "prompts/nested",
|
||||||
|
userPath: "/outside.tmpl",
|
||||||
|
wantErr: "must be relative",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty path rejected",
|
||||||
|
root: "prompts",
|
||||||
|
baseDir: "prompts/nested",
|
||||||
|
userPath: " ",
|
||||||
|
wantErr: "path is required",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "dot root allows normal relative path",
|
||||||
|
root: ".",
|
||||||
|
baseDir: ".",
|
||||||
|
userPath: "schemas/events.schema.json",
|
||||||
|
wantPath: "schemas/events.schema.json",
|
||||||
|
wantDisplay: "schemas/events.schema.json",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "dot root rejects parent escape",
|
||||||
|
root: ".",
|
||||||
|
baseDir: ".",
|
||||||
|
userPath: "../outside.tmpl",
|
||||||
|
wantErr: "escapes source root",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
gotPath, gotDisplay, err := ResolveFSPath(tc.root, tc.baseDir, tc.userPath)
|
||||||
|
if tc.wantErr != "" {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("expected error containing %q", tc.wantErr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), tc.wantErr) {
|
||||||
|
t.Fatalf("expected error to contain %q, got %v", tc.wantErr, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if gotPath != tc.wantPath || gotDisplay != tc.wantDisplay {
|
||||||
|
t.Fatalf("expected path/display %q/%q, got %q/%q", tc.wantPath, tc.wantDisplay, gotPath, gotDisplay)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestStemStripsYAMLExtensions(t *testing.T) {
|
func TestStemStripsYAMLExtensions(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -73,6 +238,27 @@ func TestStemStripsYAMLExtensions(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestIsYAMLFile(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
in string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{name: "yaml", in: "prompt.yaml", want: true},
|
||||||
|
{name: "yml", in: "profile.yml", want: true},
|
||||||
|
{name: "backup", in: "profile.yaml.bak", want: false},
|
||||||
|
{name: "uppercase", in: "profile.YAML", want: false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if got := IsYAMLFile(tc.in); got != tc.want {
|
||||||
|
t.Fatalf("expected %v, got %v", tc.want, got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func mustWriteFile(t *testing.T, path string, content string) {
|
func mustWriteFile(t *testing.T, path string, content string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
|||||||
@@ -96,6 +96,9 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
|||||||
if prepared.PromptHash != "" {
|
if prepared.PromptHash != "" {
|
||||||
fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash)
|
fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash)
|
||||||
}
|
}
|
||||||
|
if prepared.SessionID != "" {
|
||||||
|
fmt.Fprintf(&b, "session_id: %s\n", prepared.SessionID)
|
||||||
|
}
|
||||||
fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash)
|
fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash)
|
||||||
|
|
||||||
target := prepared.EffectiveModelParams
|
target := prepared.EffectiveModelParams
|
||||||
@@ -123,7 +126,11 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
|||||||
}
|
}
|
||||||
sort.Strings(keys)
|
sort.Strings(keys)
|
||||||
for _, k := range keys {
|
for _, k := range keys {
|
||||||
fmt.Fprintf(&b, " %s: %s\n", k, target.ExtraParams[k])
|
renderedValue, err := formatExtraParamTextValue(target.ExtraParams[k])
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to format extra_params.%s: %w", k, err)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, " %s: %s\n", k, renderedValue)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -151,6 +158,13 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
|||||||
messages := byRole[role]
|
messages := byRole[role]
|
||||||
for i, msg := range messages {
|
for i, msg := range messages {
|
||||||
fmt.Fprintf(&b, " - message: %d\n", i+1)
|
fmt.Fprintf(&b, " - message: %d\n", i+1)
|
||||||
|
if msg.CacheControl != nil {
|
||||||
|
fmt.Fprintf(&b, " cache_control: %s", msg.CacheControl.Type)
|
||||||
|
if msg.CacheControl.TTL != "" {
|
||||||
|
fmt.Fprintf(&b, " ttl=%s", msg.CacheControl.TTL)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(&b)
|
||||||
|
}
|
||||||
fmt.Fprintln(&b, " content: |")
|
fmt.Fprintln(&b, " content: |")
|
||||||
content := msg.Content
|
content := msg.Content
|
||||||
if content == "" {
|
if content == "" {
|
||||||
@@ -165,3 +179,15 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
|||||||
|
|
||||||
return b.Bytes(), nil
|
return b.Bytes(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func formatExtraParamTextValue(value any) (string, error) {
|
||||||
|
if s, ok := value.(string); ok {
|
||||||
|
return s, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return string(b), nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -49,6 +49,36 @@ func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTextFormatterRendersExtraParamsDeterministically(t *testing.T) {
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
prepared.EffectiveModelParams.ExtraParams = map[string]any{
|
||||||
|
"z_string": "enabled",
|
||||||
|
"b_number": 42,
|
||||||
|
"a_object": map[string]any{
|
||||||
|
"nested": "value",
|
||||||
|
"count": 2,
|
||||||
|
},
|
||||||
|
"c_array": []any{"first", 3, false},
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
s := string(out)
|
||||||
|
|
||||||
|
want := strings.Join([]string{
|
||||||
|
" extra_params:",
|
||||||
|
" a_object: {\"count\":2,\"nested\":\"value\"}",
|
||||||
|
" b_number: 42",
|
||||||
|
" c_array: [\"first\",3,false]",
|
||||||
|
" z_string: enabled",
|
||||||
|
}, "\n")
|
||||||
|
if !strings.Contains(s, want) {
|
||||||
|
t.Fatalf("expected deterministic extra_params block %q, got:\n%s", want, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
||||||
const secret = "super-secret-api-key"
|
const secret = "super-secret-api-key"
|
||||||
t.Setenv("SCRIPTORIUM_API_KEY", secret)
|
t.Setenv("SCRIPTORIUM_API_KEY", secret)
|
||||||
@@ -62,8 +92,94 @@ func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTextFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) {
|
||||||
|
const directKey = "direct-format-key"
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
prepared.EffectiveModelParams.APIKey = directKey
|
||||||
|
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(out), directKey) {
|
||||||
|
t.Fatalf("text output should not include direct api key value: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTextFormatterIncludesMessageCacheControlBeforeContent(t *testing.T) {
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
prepared.Messages = []domain.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "System guidance.",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
TTL: "1h",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{Role: "user", Content: "Summarize the transcript."},
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
s := string(out)
|
||||||
|
if !strings.Contains(s, " system:\n - message: 1\n cache_control: ephemeral ttl=1h\n content: |") {
|
||||||
|
t.Fatalf("expected system message cache control before content, got:\n%s", s)
|
||||||
|
}
|
||||||
|
if strings.Count(s, "cache_control:") != 1 {
|
||||||
|
t.Fatalf("expected exactly one cache_control line, got:\n%s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTextFormatterIncludesSessionIDWhenPresent(t *testing.T) {
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
prepared.SessionID = "session-123"
|
||||||
|
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(out), "session_id: session-123\n") {
|
||||||
|
t.Fatalf("expected session_id in text output, got:\n%s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTextFormatterOmitsEmptyCacheControlTTL(t *testing.T) {
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
prepared.Messages = []domain.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "System guidance.",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
s := string(out)
|
||||||
|
if !strings.Contains(s, " cache_control: ephemeral\n") {
|
||||||
|
t.Fatalf("expected cache_control line without ttl, got:\n%s", s)
|
||||||
|
}
|
||||||
|
if strings.Contains(s, "ttl=") {
|
||||||
|
t.Fatalf("expected empty ttl to be omitted, got:\n%s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
||||||
prepared := samplePreparedRun()
|
prepared := samplePreparedRun()
|
||||||
|
prepared.SessionID = "session-123"
|
||||||
|
prepared.EffectiveModelParams.ExtraParams = map[string]any{
|
||||||
|
"number": 42,
|
||||||
|
"nested": map[string]any{
|
||||||
|
"enabled": true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -87,9 +203,24 @@ func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
|||||||
if decoded["rendered_prompt_hash"] != "rendered-hash" {
|
if decoded["rendered_prompt_hash"] != "rendered-hash" {
|
||||||
t.Fatalf("expected rendered_prompt_hash in json output, got %#v", decoded["rendered_prompt_hash"])
|
t.Fatalf("expected rendered_prompt_hash in json output, got %#v", decoded["rendered_prompt_hash"])
|
||||||
}
|
}
|
||||||
if _, ok := decoded["effective_model_params"]; !ok {
|
if decoded["session_id"] != "session-123" {
|
||||||
|
t.Fatalf("expected session_id in json output, got %#v", decoded["session_id"])
|
||||||
|
}
|
||||||
|
modelParams, ok := decoded["effective_model_params"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
t.Fatalf("expected effective_model_params in json output, got %#v", decoded)
|
t.Fatalf("expected effective_model_params in json output, got %#v", decoded)
|
||||||
}
|
}
|
||||||
|
extraParams, ok := modelParams["extra_params"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected extra_params in json output, got %#v", modelParams["extra_params"])
|
||||||
|
}
|
||||||
|
if extraParams["number"] != float64(42) {
|
||||||
|
t.Fatalf("unexpected numeric extra param in json output: %#v", extraParams["number"])
|
||||||
|
}
|
||||||
|
nested, ok := extraParams["nested"].(map[string]any)
|
||||||
|
if !ok || nested["enabled"] != true {
|
||||||
|
t.Fatalf("unexpected nested extra param in json output: %#v", extraParams["nested"])
|
||||||
|
}
|
||||||
if _, ok := decoded["input_hashes"]; !ok {
|
if _, ok := decoded["input_hashes"]; !ok {
|
||||||
t.Fatalf("expected input_hashes in json output, got %#v", decoded)
|
t.Fatalf("expected input_hashes in json output, got %#v", decoded)
|
||||||
}
|
}
|
||||||
@@ -98,6 +229,47 @@ func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestJSONFormatterIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
prepared.Messages = []domain.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "System guidance.",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
TTL: "1h",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{Role: "user", Content: "Summarize the transcript."},
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var decoded struct {
|
||||||
|
Messages []map[string]any `json:"messages"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(out, &decoded); err != nil {
|
||||||
|
t.Fatalf("expected valid json output, got %v", err)
|
||||||
|
}
|
||||||
|
if len(decoded.Messages) != 2 {
|
||||||
|
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
|
||||||
|
}
|
||||||
|
|
||||||
|
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected first message cache_control, got %#v", decoded.Messages[0])
|
||||||
|
}
|
||||||
|
if cacheControl["type"] != string(domain.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
||||||
|
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
||||||
|
}
|
||||||
|
if _, ok := decoded.Messages[1]["cache_control"]; ok {
|
||||||
|
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
||||||
const secret = "super-secret-api-key"
|
const secret = "super-secret-api-key"
|
||||||
t.Setenv("SCRIPTORIUM_API_KEY", secret)
|
t.Setenv("SCRIPTORIUM_API_KEY", secret)
|
||||||
@@ -111,6 +283,20 @@ func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestJSONFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) {
|
||||||
|
const directKey = "direct-format-key"
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
prepared.EffectiveModelParams.APIKey = directKey
|
||||||
|
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(out), directKey) {
|
||||||
|
t.Fatalf("json output should not include direct api key value: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParsePreparedRunOutputFormatRecognizesSupportedNames(t *testing.T) {
|
func TestParsePreparedRunOutputFormatRecognizesSupportedNames(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
@@ -35,7 +36,6 @@ type OpenAICompatibleConfig struct {
|
|||||||
type OpenAICompatibleClient struct {
|
type OpenAICompatibleClient struct {
|
||||||
baseURL string
|
baseURL string
|
||||||
defaultModel string
|
defaultModel string
|
||||||
timeout time.Duration
|
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -54,10 +54,11 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
|
|||||||
|
|
||||||
var client *http.Client
|
var client *http.Client
|
||||||
if cfg.HTTPClient != nil {
|
if cfg.HTTPClient != nil {
|
||||||
client = cfg.HTTPClient
|
cloned := *cfg.HTTPClient
|
||||||
if client.Timeout == 0 {
|
if cloned.Timeout == 0 {
|
||||||
client.Timeout = timeout
|
cloned.Timeout = timeout
|
||||||
}
|
}
|
||||||
|
client = &cloned
|
||||||
} else {
|
} else {
|
||||||
client = &http.Client{Timeout: timeout}
|
client = &http.Client{Timeout: timeout}
|
||||||
}
|
}
|
||||||
@@ -65,7 +66,6 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
|
|||||||
return &OpenAICompatibleClient{
|
return &OpenAICompatibleClient{
|
||||||
baseURL: strings.TrimRight(baseURL, "/"),
|
baseURL: strings.TrimRight(baseURL, "/"),
|
||||||
defaultModel: cfg.Model,
|
defaultModel: cfg.Model,
|
||||||
timeout: timeout,
|
|
||||||
httpClient: client,
|
httpClient: client,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@@ -89,7 +89,12 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
payload, err := json.Marshal(wireReq)
|
wirePayload, err := openAIChatRequestPayload(wireReq)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
payload, err := json.Marshal(wirePayload)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
|
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
|
||||||
}
|
}
|
||||||
@@ -99,7 +104,9 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
|
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
|
||||||
}
|
}
|
||||||
httpReq.Header.Set("Content-Type", "application/json")
|
httpReq.Header.Set("Content-Type", "application/json")
|
||||||
if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" {
|
if apiKey := strings.TrimSpace(req.Target.APIKey); apiKey != "" {
|
||||||
|
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
||||||
|
} else if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" {
|
||||||
apiKey := strings.TrimSpace(os.Getenv(envName))
|
apiKey := strings.TrimSpace(os.Getenv(envName))
|
||||||
if apiKey == "" {
|
if apiKey == "" {
|
||||||
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
|
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
|
||||||
@@ -107,17 +114,20 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
effectiveTimeout := c.timeout
|
|
||||||
if req.Target.TimeoutSeconds > 0 {
|
|
||||||
effectiveTimeout = time.Duration(req.Target.TimeoutSeconds) * time.Second
|
|
||||||
}
|
|
||||||
|
|
||||||
httpClient := c.httpClient
|
httpClient := c.httpClient
|
||||||
if httpClient == nil {
|
if httpClient == nil {
|
||||||
httpClient = &http.Client{Timeout: effectiveTimeout}
|
httpClient = &http.Client{Timeout: defaults.LLMRequestTimeoutDefault}
|
||||||
} else if httpClient.Timeout != effectiveTimeout {
|
}
|
||||||
|
if req.Target.TimeoutSeconds > 0 {
|
||||||
|
effectiveTimeout := time.Duration(req.Target.TimeoutSeconds) * time.Second
|
||||||
|
if httpClient.Timeout != effectiveTimeout {
|
||||||
|
cloned := *httpClient
|
||||||
|
cloned.Timeout = effectiveTimeout
|
||||||
|
httpClient = &cloned
|
||||||
|
}
|
||||||
|
} else if req.TargetPresence.TimeoutSeconds && httpClient.Timeout != 0 {
|
||||||
cloned := *httpClient
|
cloned := *httpClient
|
||||||
cloned.Timeout = effectiveTimeout
|
cloned.Timeout = 0
|
||||||
httpClient = &cloned
|
httpClient = &cloned
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -128,8 +138,8 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
defer httpResp.Body.Close()
|
defer httpResp.Body.Close()
|
||||||
|
|
||||||
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
|
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
|
||||||
body, _ := io.ReadAll(io.LimitReader(httpResp.Body, 4096))
|
_, _ = io.Copy(io.Discard, io.LimitReader(httpResp.Body, 4096))
|
||||||
return nil, fmt.Errorf("%w: status=%d body=%q", ErrUnexpectedStatus, httpResp.StatusCode, strings.TrimSpace(string(body)))
|
return nil, fmt.Errorf("%w: status=%d", ErrUnexpectedStatus, httpResp.StatusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
var wireResp openAIChatResponse
|
var wireResp openAIChatResponse
|
||||||
@@ -151,6 +161,8 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
PromptTokens: wireResp.Usage.PromptTokens,
|
PromptTokens: wireResp.Usage.PromptTokens,
|
||||||
CompletionTokens: wireResp.Usage.CompletionTokens,
|
CompletionTokens: wireResp.Usage.CompletionTokens,
|
||||||
TotalTokens: wireResp.Usage.TotalTokens,
|
TotalTokens: wireResp.Usage.TotalTokens,
|
||||||
|
CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens,
|
||||||
|
CacheWriteTokens: wireResp.Usage.CacheWriteTokens,
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@@ -167,27 +179,36 @@ func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultMod
|
|||||||
wireReq := openAIChatRequest{
|
wireReq := openAIChatRequest{
|
||||||
Model: model,
|
Model: model,
|
||||||
}
|
}
|
||||||
|
if sessionID := strings.TrimSpace(req.Prompt.SessionID); sessionID != "" {
|
||||||
wireReq.Messages = make([]openAIChatMessage, 0, len(req.Prompt.Messages))
|
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
|
||||||
for _, msg := range req.Prompt.Messages {
|
return openAIChatRequest{}, fmt.Errorf("session_id length %d exceeds maximum %d", n, domain.SessionIDMaxLength)
|
||||||
wireReq.Messages = append(wireReq.Messages, openAIChatMessage{
|
}
|
||||||
Role: msg.Role,
|
wireReq.SessionID = sessionID
|
||||||
Content: msg.Content,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if req.Target.Temperature != 0 {
|
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
|
||||||
|
for _, msg := range req.Prompt.Messages {
|
||||||
|
wireReq.Messages = append(wireReq.Messages, openAIChatRequestMessageFromRenderedMessage(msg))
|
||||||
|
}
|
||||||
|
|
||||||
|
if req.Target.Temperature != 0 || req.TargetPresence.Temperature {
|
||||||
wireReq.Temperature = &req.Target.Temperature
|
wireReq.Temperature = &req.Target.Temperature
|
||||||
}
|
}
|
||||||
if req.Target.MaxTokens != 0 {
|
if req.Target.MaxTokens != 0 || req.TargetPresence.MaxTokens {
|
||||||
wireReq.MaxTokens = &req.Target.MaxTokens
|
wireReq.MaxTokens = &req.Target.MaxTokens
|
||||||
}
|
}
|
||||||
if req.Target.TopP != 0 {
|
if req.Target.TopP != 0 || req.TargetPresence.TopP {
|
||||||
wireReq.TopP = &req.Target.TopP
|
wireReq.TopP = &req.Target.TopP
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(req.Target.ServiceTier) != "" {
|
if strings.TrimSpace(req.Target.ServiceTier) != "" {
|
||||||
wireReq.ServiceTier = req.Target.ServiceTier
|
wireReq.ServiceTier = req.Target.ServiceTier
|
||||||
}
|
}
|
||||||
|
if strings.TrimSpace(req.Target.ReasoningEffort) != "" {
|
||||||
|
wireReq.ReasoningEffort = req.Target.ReasoningEffort
|
||||||
|
}
|
||||||
|
if len(req.Target.ExtraParams) > 0 {
|
||||||
|
wireReq.ExtraParams = req.Target.ExtraParams
|
||||||
|
}
|
||||||
if req.StructuredOutput != nil {
|
if req.StructuredOutput != nil {
|
||||||
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
|
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -200,28 +221,106 @@ func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultMod
|
|||||||
}
|
}
|
||||||
|
|
||||||
type openAIChatRequest struct {
|
type openAIChatRequest struct {
|
||||||
Model string `json:"model"`
|
Model string `json:"model"`
|
||||||
Messages []openAIChatMessage `json:"messages"`
|
SessionID string `json:"session_id,omitempty"`
|
||||||
Temperature *float64 `json:"temperature,omitempty"`
|
Messages []openAIChatRequestMessage `json:"messages"`
|
||||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
Temperature *float64 `json:"temperature,omitempty"`
|
||||||
TopP *float64 `json:"top_p,omitempty"`
|
MaxTokens *int `json:"max_tokens,omitempty"`
|
||||||
ServiceTier string `json:"service_tier,omitempty"`
|
TopP *float64 `json:"top_p,omitempty"`
|
||||||
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"`
|
ServiceTier string `json:"service_tier,omitempty"`
|
||||||
|
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||||
|
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"`
|
||||||
|
ExtraParams map[string]any `json:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type openAIChatMessage struct {
|
func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
||||||
|
out := map[string]any{
|
||||||
|
"model": req.Model,
|
||||||
|
"messages": req.Messages,
|
||||||
|
}
|
||||||
|
if req.SessionID != "" {
|
||||||
|
out["session_id"] = req.SessionID
|
||||||
|
}
|
||||||
|
if req.Temperature != nil {
|
||||||
|
out["temperature"] = *req.Temperature
|
||||||
|
}
|
||||||
|
if req.MaxTokens != nil {
|
||||||
|
out["max_tokens"] = *req.MaxTokens
|
||||||
|
}
|
||||||
|
if req.TopP != nil {
|
||||||
|
out["top_p"] = *req.TopP
|
||||||
|
}
|
||||||
|
if req.ServiceTier != "" {
|
||||||
|
out["service_tier"] = req.ServiceTier
|
||||||
|
}
|
||||||
|
if req.ReasoningEffort != "" {
|
||||||
|
out["reasoning_effort"] = req.ReasoningEffort
|
||||||
|
}
|
||||||
|
if req.ResponseFormat != nil {
|
||||||
|
out["response_format"] = req.ResponseFormat
|
||||||
|
}
|
||||||
|
|
||||||
|
for key, value := range req.ExtraParams {
|
||||||
|
if key == "" {
|
||||||
|
return nil, errors.New("extra_params key must not be empty")
|
||||||
|
}
|
||||||
|
if _, reserved := reservedOpenAIChatRequestFields[key]; reserved {
|
||||||
|
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
|
||||||
|
}
|
||||||
|
if _, err := json.Marshal(value); err != nil {
|
||||||
|
return nil, fmt.Errorf("extra_params.%s must be JSON-serializable: %w", key, err)
|
||||||
|
}
|
||||||
|
out[key] = value
|
||||||
|
}
|
||||||
|
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var reservedOpenAIChatRequestFields = map[string]struct{}{
|
||||||
|
"model": {},
|
||||||
|
"session_id": {},
|
||||||
|
"messages": {},
|
||||||
|
"temperature": {},
|
||||||
|
"max_tokens": {},
|
||||||
|
"top_p": {},
|
||||||
|
"service_tier": {},
|
||||||
|
"reasoning_effort": {},
|
||||||
|
"response_format": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
type openAIChatRequestMessage struct {
|
||||||
|
Role string `json:"role"`
|
||||||
|
Content any `json:"content"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type openAIChatTextContentBlock struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
Text string `json:"text"`
|
||||||
|
CacheControl *openAICacheControl `json:"cache_control,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type openAICacheControl struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
TTL string `json:"ttl,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type openAIChatResponseMessage struct {
|
||||||
Role string `json:"role"`
|
Role string `json:"role"`
|
||||||
Content string `json:"content"`
|
Content string `json:"content"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type openAIChatResponse struct {
|
type openAIChatResponse struct {
|
||||||
Choices []struct {
|
Choices []struct {
|
||||||
Message openAIChatMessage `json:"message"`
|
Message openAIChatResponseMessage `json:"message"`
|
||||||
} `json:"choices"`
|
} `json:"choices"`
|
||||||
Usage struct {
|
Usage struct {
|
||||||
PromptTokens int `json:"prompt_tokens"`
|
PromptTokens int `json:"prompt_tokens"`
|
||||||
CompletionTokens int `json:"completion_tokens"`
|
CompletionTokens int `json:"completion_tokens"`
|
||||||
TotalTokens int `json:"total_tokens"`
|
TotalTokens int `json:"total_tokens"`
|
||||||
|
PromptTokensDetails struct {
|
||||||
|
CachedTokens int `json:"cached_tokens"`
|
||||||
|
} `json:"prompt_tokens_details"`
|
||||||
|
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||||
} `json:"usage"`
|
} `json:"usage"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -236,6 +335,28 @@ type openAIJSONSchemaEnvelope struct {
|
|||||||
Schema any `json:"schema"`
|
Schema any `json:"schema"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func openAIChatRequestMessageFromRenderedMessage(msg domain.RenderedMessage) openAIChatRequestMessage {
|
||||||
|
wireMsg := openAIChatRequestMessage{
|
||||||
|
Role: msg.Role,
|
||||||
|
Content: msg.Content,
|
||||||
|
}
|
||||||
|
if msg.CacheControl == nil {
|
||||||
|
return wireMsg
|
||||||
|
}
|
||||||
|
|
||||||
|
wireMsg.Content = []openAIChatTextContentBlock{
|
||||||
|
{
|
||||||
|
Type: "text",
|
||||||
|
Text: msg.Content,
|
||||||
|
CacheControl: &openAICacheControl{
|
||||||
|
Type: string(msg.CacheControl.Type),
|
||||||
|
TTL: msg.CacheControl.TTL,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return wireMsg
|
||||||
|
}
|
||||||
|
|
||||||
func toOpenAIResponseFormat(spec *domain.StructuredOutputSpec) (*openAIResponseFormat, error) {
|
func toOpenAIResponseFormat(spec *domain.StructuredOutputSpec) (*openAIResponseFormat, error) {
|
||||||
if spec == nil {
|
if spec == nil {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"math"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -13,6 +14,61 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestNewOpenAICompatibleClientDoesNotMutateSuppliedZeroTimeoutClient(t *testing.T) {
|
||||||
|
transport := http.DefaultTransport
|
||||||
|
supplied := &http.Client{Transport: transport}
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
|
HTTPClient: supplied,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected constructor error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if supplied.Timeout != 0 {
|
||||||
|
t.Fatalf("expected supplied client timeout to remain zero, got %v", supplied.Timeout)
|
||||||
|
}
|
||||||
|
if client.httpClient == supplied {
|
||||||
|
t.Fatal("expected constructed client to use a cloned HTTP client")
|
||||||
|
}
|
||||||
|
if client.httpClient.Timeout <= 0 {
|
||||||
|
t.Fatalf("expected constructed client to use a positive default timeout, got %v", client.httpClient.Timeout)
|
||||||
|
}
|
||||||
|
if client.httpClient.Transport != transport {
|
||||||
|
t.Fatal("expected cloned client to preserve the supplied transport")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewOpenAICompatibleClientDoesNotMutateSuppliedNonzeroTimeoutClient(t *testing.T) {
|
||||||
|
transport := http.DefaultTransport
|
||||||
|
suppliedTimeout := 37 * time.Second
|
||||||
|
supplied := &http.Client{
|
||||||
|
Timeout: suppliedTimeout,
|
||||||
|
Transport: transport,
|
||||||
|
}
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
|
Timeout: 2 * time.Second,
|
||||||
|
HTTPClient: supplied,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected constructor error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if supplied.Timeout != suppliedTimeout {
|
||||||
|
t.Fatalf("expected supplied client timeout to remain %v, got %v", suppliedTimeout, supplied.Timeout)
|
||||||
|
}
|
||||||
|
if client.httpClient == supplied {
|
||||||
|
t.Fatal("expected constructed client to use a cloned HTTP client")
|
||||||
|
}
|
||||||
|
if client.httpClient.Timeout != suppliedTimeout {
|
||||||
|
t.Fatalf("expected cloned client timeout %v, got %v", suppliedTimeout, client.httpClient.Timeout)
|
||||||
|
}
|
||||||
|
if client.httpClient.Transport != transport {
|
||||||
|
t.Fatal("expected cloned client to preserve the supplied transport")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
|
func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
|
||||||
type observedRequest struct {
|
type observedRequest struct {
|
||||||
Authorization string
|
Authorization string
|
||||||
@@ -89,6 +145,9 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
|
|||||||
if resp.Usage.PromptTokens != 11 || resp.Usage.CompletionTokens != 22 || resp.Usage.TotalTokens != 33 {
|
if resp.Usage.PromptTokens != 11 || resp.Usage.CompletionTokens != 22 || resp.Usage.TotalTokens != 33 {
|
||||||
t.Fatalf("unexpected usage: %+v", resp.Usage)
|
t.Fatalf("unexpected usage: %+v", resp.Usage)
|
||||||
}
|
}
|
||||||
|
if resp.Usage.CachedTokens != 0 || resp.Usage.CacheWriteTokens != 0 {
|
||||||
|
t.Fatalf("expected absent cache usage fields to remain zero, got %+v", resp.Usage)
|
||||||
|
}
|
||||||
|
|
||||||
if obs.Authorization != "Bearer secret-key" {
|
if obs.Authorization != "Bearer secret-key" {
|
||||||
t.Fatalf("unexpected Authorization header: %q", obs.Authorization)
|
t.Fatalf("unexpected Authorization header: %q", obs.Authorization)
|
||||||
@@ -144,6 +203,277 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientDirectAPIKeyPreferredOverEnv(t *testing.T) {
|
||||||
|
const directKey = "direct-llm-key"
|
||||||
|
t.Setenv("SCRIPTORIUM_TEST_API_KEY", "env-key")
|
||||||
|
|
||||||
|
var gotAuth string
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
gotAuth = r.Header.Get("Authorization")
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{
|
||||||
|
Model: "model",
|
||||||
|
APIKeyEnv: "SCRIPTORIUM_TEST_API_KEY",
|
||||||
|
APIKey: directKey,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if gotAuth != "Bearer "+directKey {
|
||||||
|
t.Fatalf("unexpected Authorization header: %q", gotAuth)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientSerializesCacheControlledMessageAsContentBlock(t *testing.T) {
|
||||||
|
var observedBody map[string]any
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
defer r.Body.Close()
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||||
|
t.Fatalf("failed to decode request body: %v", err)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "Stable instructions.",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
TTL: "1h",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{Role: "user", Content: "Dynamic request."},
|
||||||
|
}},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, forbidden := range []string{"cache_control", "extra_params"} {
|
||||||
|
if _, exists := observedBody[forbidden]; exists {
|
||||||
|
t.Fatalf("expected top-level %s to be omitted, got %#v", forbidden, observedBody[forbidden])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
msgs, ok := observedBody["messages"].([]any)
|
||||||
|
if !ok || len(msgs) != 2 {
|
||||||
|
t.Fatalf("unexpected messages payload: %#v", observedBody["messages"])
|
||||||
|
}
|
||||||
|
msg0 := msgs[0].(map[string]any)
|
||||||
|
if msg0["role"] != "system" {
|
||||||
|
t.Fatalf("unexpected first message role: %#v", msg0["role"])
|
||||||
|
}
|
||||||
|
contentBlocks, ok := msg0["content"].([]any)
|
||||||
|
if !ok || len(contentBlocks) != 1 {
|
||||||
|
t.Fatalf("expected first message content block array, got %#v", msg0["content"])
|
||||||
|
}
|
||||||
|
block := contentBlocks[0].(map[string]any)
|
||||||
|
if block["type"] != "text" || block["text"] != "Stable instructions." {
|
||||||
|
t.Fatalf("unexpected text content block: %#v", block)
|
||||||
|
}
|
||||||
|
cacheControl, ok := block["cache_control"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected cache_control on content block, got %#v", block)
|
||||||
|
}
|
||||||
|
if cacheControl["type"] != string(domain.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
||||||
|
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
||||||
|
}
|
||||||
|
|
||||||
|
msg1 := msgs[1].(map[string]any)
|
||||||
|
if msg1["role"] != "user" || msg1["content"] != "Dynamic request." {
|
||||||
|
t.Fatalf("expected uncached message to keep string content, got %#v", msg1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientOmitsEmptyCacheControlTTL(t *testing.T) {
|
||||||
|
var observedBody map[string]any
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
defer r.Body.Close()
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||||
|
t.Fatalf("failed to decode request body: %v", err)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "Stable instructions.",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
msgs := observedBody["messages"].([]any)
|
||||||
|
msg0 := msgs[0].(map[string]any)
|
||||||
|
contentBlocks := msg0["content"].([]any)
|
||||||
|
block := contentBlocks[0].(map[string]any)
|
||||||
|
cacheControl := block["cache_control"].(map[string]any)
|
||||||
|
if cacheControl["type"] != string(domain.CacheControlEphemeral) {
|
||||||
|
t.Fatalf("unexpected cache_control type: %#v", cacheControl)
|
||||||
|
}
|
||||||
|
if _, exists := cacheControl["ttl"]; exists {
|
||||||
|
t.Fatalf("expected empty ttl to be omitted, got %#v", cacheControl)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientSerializesSessionID(t *testing.T) {
|
||||||
|
var observedBody map[string]any
|
||||||
|
var observedSessionHeader string
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
observedSessionHeader = r.Header.Get("x-session-id")
|
||||||
|
defer r.Body.Close()
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||||
|
t.Fatalf("failed to decode request body: %v", err)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{
|
||||||
|
SessionID: " session-123 ",
|
||||||
|
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
|
||||||
|
},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if observedBody["session_id"] != "session-123" {
|
||||||
|
t.Fatalf("expected top-level session_id, got %#v", observedBody["session_id"])
|
||||||
|
}
|
||||||
|
if observedSessionHeader != "" {
|
||||||
|
t.Fatalf("did not expect x-session-id header, got %q", observedSessionHeader)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientOmitsEmptySessionID(t *testing.T) {
|
||||||
|
var observedBody map[string]any
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
defer r.Body.Close()
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||||
|
t.Fatalf("failed to decode request body: %v", err)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{
|
||||||
|
SessionID: " ",
|
||||||
|
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
|
||||||
|
},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if _, exists := observedBody["session_id"]; exists {
|
||||||
|
t.Fatalf("expected empty session_id to be omitted, got %#v", observedBody["session_id"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientRejectsTooLongSessionID(t *testing.T) {
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
|
BaseURL: "http://example.com/v1",
|
||||||
|
Model: "model",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{
|
||||||
|
SessionID: strings.Repeat("x", domain.SessionIDMaxLength+1),
|
||||||
|
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid request error")
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientParsesCacheUsage(t *testing.T) {
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = w.Write([]byte(`{
|
||||||
|
"choices": [{"message": {"role": "assistant", "content": "ok"}}],
|
||||||
|
"usage": {
|
||||||
|
"prompt_tokens": 100,
|
||||||
|
"completion_tokens": 20,
|
||||||
|
"total_tokens": 120,
|
||||||
|
"prompt_tokens_details": {"cached_tokens": 80},
|
||||||
|
"cache_write_tokens": 60
|
||||||
|
}
|
||||||
|
}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "model"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if resp.Usage.PromptTokens != 100 || resp.Usage.CompletionTokens != 20 || resp.Usage.TotalTokens != 120 {
|
||||||
|
t.Fatalf("unexpected base usage fields: %+v", resp.Usage)
|
||||||
|
}
|
||||||
|
if resp.Usage.CachedTokens != 80 || resp.Usage.CacheWriteTokens != 60 {
|
||||||
|
t.Fatalf("unexpected cache usage fields: %+v", resp.Usage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *testing.T) {
|
func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *testing.T) {
|
||||||
var observedBody map[string]any
|
var observedBody map[string]any
|
||||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -175,7 +505,7 @@ func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *test
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOpenAICompatibleClientOmitsReasoningEffortAndExtraParams(t *testing.T) {
|
func TestOpenAICompatibleClientSerializesReasoningEffortAndExtraParams(t *testing.T) {
|
||||||
var observedBody map[string]any
|
var observedBody map[string]any
|
||||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
defer r.Body.Close()
|
defer r.Body.Close()
|
||||||
@@ -196,19 +526,286 @@ func TestOpenAICompatibleClientOmitsReasoningEffortAndExtraParams(t *testing.T)
|
|||||||
Target: domain.ExecutionTarget{
|
Target: domain.ExecutionTarget{
|
||||||
Model: "model",
|
Model: "model",
|
||||||
ReasoningEffort: "high",
|
ReasoningEffort: "high",
|
||||||
ExtraParams: map[string]string{
|
ExtraParams: map[string]any{
|
||||||
"provider_option": "on",
|
"string_value": "on",
|
||||||
|
"number_value": 42,
|
||||||
|
"boolean_value": true,
|
||||||
|
"object_value": map[string]any{"nested": "value", "count": 2},
|
||||||
|
"array_value": []any{"first", 3, false},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
|
if observedBody["reasoning_effort"] != "high" {
|
||||||
|
t.Fatalf("expected reasoning_effort high, got %#v", observedBody["reasoning_effort"])
|
||||||
|
}
|
||||||
|
if observedBody["string_value"] != "on" {
|
||||||
|
t.Fatalf("unexpected string extra param: %#v", observedBody["string_value"])
|
||||||
|
}
|
||||||
|
if observedBody["number_value"] != float64(42) {
|
||||||
|
t.Fatalf("unexpected number extra param: %#v", observedBody["number_value"])
|
||||||
|
}
|
||||||
|
if observedBody["boolean_value"] != true {
|
||||||
|
t.Fatalf("unexpected boolean extra param: %#v", observedBody["boolean_value"])
|
||||||
|
}
|
||||||
|
objectValue, ok := observedBody["object_value"].(map[string]any)
|
||||||
|
if !ok || objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
|
||||||
|
t.Fatalf("unexpected object extra param: %#v", observedBody["object_value"])
|
||||||
|
}
|
||||||
|
if _, exists := observedBody["extra_params"]; exists {
|
||||||
|
t.Fatalf("expected extra_params wrapper omitted, got %#v", observedBody["extra_params"])
|
||||||
|
}
|
||||||
|
arrayValue, ok := observedBody["array_value"].([]any)
|
||||||
|
if !ok || len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
|
||||||
|
t.Fatalf("unexpected array extra param: %#v", observedBody["array_value"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientOmitsReasoningEffortWhenUnset(t *testing.T) {
|
||||||
|
var observedBody map[string]any
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
defer r.Body.Close()
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||||
|
t.Fatalf("failed to decode request body: %v", err)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
if _, exists := observedBody["reasoning_effort"]; exists {
|
if _, exists := observedBody["reasoning_effort"]; exists {
|
||||||
t.Fatalf("expected reasoning_effort omitted, got %#v", observedBody["reasoning_effort"])
|
t.Fatalf("expected reasoning_effort omitted, got %#v", observedBody["reasoning_effort"])
|
||||||
}
|
}
|
||||||
if _, exists := observedBody["extra_params"]; exists {
|
if _, exists := observedBody["extra_params"]; exists {
|
||||||
t.Fatalf("expected extra_params omitted, got %#v", observedBody["extra_params"])
|
t.Fatalf("expected extra_params wrapper omitted, got %#v", observedBody["extra_params"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientSerializesExplicitZeroNumericOverrides(t *testing.T) {
|
||||||
|
var observedBody map[string]any
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
defer r.Body.Close()
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||||
|
t.Fatalf("failed to decode request body: %v", err)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model"},
|
||||||
|
TargetPresence: domain.ExecutionTargetPresence{
|
||||||
|
Temperature: true,
|
||||||
|
MaxTokens: true,
|
||||||
|
TopP: true,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if observedBody["temperature"] != float64(0) {
|
||||||
|
t.Fatalf("expected explicit zero temperature, got %#v", observedBody["temperature"])
|
||||||
|
}
|
||||||
|
if observedBody["max_tokens"] != float64(0) {
|
||||||
|
t.Fatalf("expected explicit zero max_tokens, got %#v", observedBody["max_tokens"])
|
||||||
|
}
|
||||||
|
if observedBody["top_p"] != float64(0) {
|
||||||
|
t.Fatalf("expected explicit zero top_p, got %#v", observedBody["top_p"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientOmitsImplicitZeroNumericFields(t *testing.T) {
|
||||||
|
var observedBody map[string]any
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
defer r.Body.Close()
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||||
|
t.Fatalf("failed to decode request body: %v", err)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
for _, field := range []string{"temperature", "max_tokens", "top_p"} {
|
||||||
|
if _, exists := observedBody[field]; exists {
|
||||||
|
t.Fatalf("expected implicit zero field %q to be omitted, got body %#v", field, observedBody)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientExplicitZeroTimeoutDisablesClientTimeout(t *testing.T) {
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
|
BaseURL: ts.URL + "/v1",
|
||||||
|
Timeout: time.Nanosecond,
|
||||||
|
HTTPClient: &http.Client{Timeout: time.Nanosecond},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model", TimeoutSeconds: 0},
|
||||||
|
TargetPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected explicit zero timeout to disable client timeout, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientOmittedTimeoutUsesClientTimeout(t *testing.T) {
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
|
BaseURL: ts.URL + "/v1",
|
||||||
|
Timeout: time.Nanosecond,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model", TimeoutSeconds: 0},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected omitted timeout to use client timeout")
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrRequestFailed) {
|
||||||
|
t.Fatalf("expected ErrRequestFailed, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientSuppliedHTTPClientTimeoutOverridesConfigTimeout(t *testing.T) {
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
time.Sleep(25 * time.Millisecond)
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
|
BaseURL: ts.URL + "/v1",
|
||||||
|
Model: "model",
|
||||||
|
Timeout: 5 * time.Millisecond,
|
||||||
|
HTTPClient: &http.Client{Timeout: 100 * time.Millisecond},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected supplied client timeout to allow the request, got %v", err)
|
||||||
|
}
|
||||||
|
if resp.Content != "ok" {
|
||||||
|
t.Fatalf("unexpected response content: %q", resp.Content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
extraParams map[string]any
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "empty key", extraParams: map[string]any{"": "empty"}, want: "key must not be empty"},
|
||||||
|
{name: "unserializable value", extraParams: map[string]any{"bad": math.Inf(1)}, want: "JSON-serializable"},
|
||||||
|
}
|
||||||
|
for _, key := range []string{
|
||||||
|
"model",
|
||||||
|
"session_id",
|
||||||
|
"messages",
|
||||||
|
"temperature",
|
||||||
|
"max_tokens",
|
||||||
|
"top_p",
|
||||||
|
"service_tier",
|
||||||
|
"reasoning_effort",
|
||||||
|
"response_format",
|
||||||
|
} {
|
||||||
|
tests = append(tests, struct {
|
||||||
|
name string
|
||||||
|
extraParams map[string]any
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
name: "reserved key " + key,
|
||||||
|
extraParams: map[string]any{key: "collision"},
|
||||||
|
want: "reserved request field",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
called := false
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
called = true
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model", ExtraParams: tc.extraParams},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid request error")
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), tc.want) {
|
||||||
|
t.Fatalf("expected error to contain %q, got %v", tc.want, err)
|
||||||
|
}
|
||||||
|
if called {
|
||||||
|
t.Fatal("provider should not be called for invalid extra_params")
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -332,9 +929,10 @@ func TestOpenAICompatibleClientEndpointOverride(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestOpenAICompatibleClientNon2xxError(t *testing.T) {
|
func TestOpenAICompatibleClientNon2xxError(t *testing.T) {
|
||||||
|
const sensitiveBody = `provider-secret-fragment request_payload_details`
|
||||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
w.WriteHeader(http.StatusBadRequest)
|
w.WriteHeader(http.StatusBadRequest)
|
||||||
_, _ = w.Write([]byte(`{"error":"bad request payload"}`))
|
_, _ = w.Write([]byte(`{"error":"` + sensitiveBody + `"}`))
|
||||||
}))
|
}))
|
||||||
defer ts.Close()
|
defer ts.Close()
|
||||||
|
|
||||||
@@ -352,8 +950,11 @@ func TestOpenAICompatibleClientNon2xxError(t *testing.T) {
|
|||||||
if !errors.Is(err, ErrUnexpectedStatus) {
|
if !errors.Is(err, ErrUnexpectedStatus) {
|
||||||
t.Fatalf("expected ErrUnexpectedStatus, got %v", err)
|
t.Fatalf("expected ErrUnexpectedStatus, got %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "400") || !strings.Contains(err.Error(), "bad request payload") {
|
if !strings.Contains(err.Error(), "status=400") {
|
||||||
t.Fatalf("expected status/body details, got %v", err)
|
t.Fatalf("expected status detail, got %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(err.Error(), sensitiveBody) {
|
||||||
|
t.Fatalf("expected provider response body to be redacted, got %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -436,9 +1037,10 @@ func TestOpenAICompatibleClientRequestTimeoutOverride(t *testing.T) {
|
|||||||
defer ts.Close()
|
defer ts.Close()
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: ts.URL + "/v1",
|
BaseURL: ts.URL + "/v1",
|
||||||
Model: "m",
|
Model: "m",
|
||||||
Timeout: 50 * time.Millisecond,
|
Timeout: 5 * time.Millisecond,
|
||||||
|
HTTPClient: &http.Client{Timeout: 50 * time.Millisecond},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
9
internal/profile/builtin/assets/aion-labs/aion-2.yml
Normal file
9
internal/profile/builtin/assets/aion-labs/aion-2.yml
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: aion-2
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: aion-labs/aion-2.0
|
||||||
|
temperature: 0.72
|
||||||
|
reasoning_effort: high
|
||||||
|
top_p: 0.95
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
id: claude-fable-latest
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "~anthropic/claude-fable-latest"
|
||||||
|
reasoning_effort: high
|
||||||
|
timeout_seconds: 600
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
id: claude-haiku-latest
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "~anthropic/claude-haiku-latest"
|
||||||
|
reasoning_effort: medium
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
id: claude-opus-latest
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "~anthropic/claude-opus-latest"
|
||||||
|
reasoning_effort: high
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
id: claude-sonnet-latest
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "~anthropic/claude-sonnet-latest"
|
||||||
|
reasoning_effort: high
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
id: deepseek-3-2
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: deepseek/deepseek-v3.2
|
||||||
|
reasoning_effort: high
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
id: deepseek-4-flash
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: deepseek/deepseek-v4-flash
|
||||||
|
#reasoning_effort: medium
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
id: deepseek-4-pro
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: deepseek/deepseek-v4-pro
|
||||||
|
reasoning_effort: high
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
id: gemini-2-flash-lite
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "google/gemini-2.5-flash-lite"
|
||||||
|
#temperature: 0.15
|
||||||
|
reasoning_effort: high
|
||||||
|
#top_p: 0.98
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
id: gemini-2-flash
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "google/gemini-2.5-flash"
|
||||||
|
#temperature: 0.15
|
||||||
|
reasoning_effort: high
|
||||||
|
#top_p: 0.98
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
9
internal/profile/builtin/assets/google/gemini-2-pro.yml
Normal file
9
internal/profile/builtin/assets/google/gemini-2-pro.yml
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: gemini-2-pro
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "google/gemini-2.5-pro"
|
||||||
|
#temperature: 0.15
|
||||||
|
reasoning_effort: high
|
||||||
|
#top_p: 0.98
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
id: gemini-3-flash-lite
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "google/gemini-3.1-flash-lite"
|
||||||
|
#temperature: 0.15
|
||||||
|
reasoning_effort: high
|
||||||
|
#top_p: 0.98
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
id: gemini-flash-latest
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "~google/gemini-flash-latest"
|
||||||
|
#temperature: 0.15
|
||||||
|
reasoning_effort: high
|
||||||
|
#top_p: 0.98
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
id: gemini-pro-latest
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "~google/gemini-pro-latest"
|
||||||
|
#temperature: 0.15
|
||||||
|
reasoning_effort: high
|
||||||
|
#top_p: 0.98
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
9
internal/profile/builtin/assets/google/gemma-4-31b.yml
Normal file
9
internal/profile/builtin/assets/google/gemma-4-31b.yml
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: gemma-4-31b
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: google/gemma-4-31b-it:exacto
|
||||||
|
temperature: 0.15
|
||||||
|
reasoning_effort: high
|
||||||
|
top_p: 0.98
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
9
internal/profile/builtin/assets/minimax/minimax-m2.yml
Normal file
9
internal/profile/builtin/assets/minimax/minimax-m2.yml
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: minimax-m2
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: minimax/minimax-m2.5
|
||||||
|
temperature: 0.5
|
||||||
|
reasoning_effort: high
|
||||||
|
top_p: 0.95
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
9
internal/profile/builtin/assets/minimax/minimax-m3.yml
Normal file
9
internal/profile/builtin/assets/minimax/minimax-m3.yml
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: minimax-m3
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: minimax/minimax-m3
|
||||||
|
#temperature: 0.5
|
||||||
|
reasoning_effort: high
|
||||||
|
#top_p: 0.95
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
id: mistral-large-2512
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: mistralai/mistral-large-2512
|
||||||
|
temperature: 0.15
|
||||||
|
top_p: 0.98
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
id: mistral-medium-3-5
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: mistralai/mistral-medium-3-5
|
||||||
|
temperature: 0.15
|
||||||
|
reasoning_effort: high
|
||||||
|
top_p: 0.98
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
id: mistral-small-3
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: mistralai/mistral-small-3.2-24b-instruct
|
||||||
|
temperature: 0.05
|
||||||
|
top_p: 1.0
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
id: mistral-small-4
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: mistralai/mistral-small-2603
|
||||||
|
temperature: 0.1
|
||||||
|
reasoning_effort: high
|
||||||
|
top_p: 0.98
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
id: nemotron-3-ultra
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: nvidia/nemotron-3-ultra-550b-a55b
|
||||||
|
reasoning_effort: high
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
7
internal/profile/builtin/assets/openai/gpt-5-mini.yml
Normal file
7
internal/profile/builtin/assets/openai/gpt-5-mini.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
id: gpt-5-mini
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "openai/gpt-5.4-mini"
|
||||||
|
reasoning_effort: high
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
7
internal/profile/builtin/assets/openai/gpt-5-nano.yml
Normal file
7
internal/profile/builtin/assets/openai/gpt-5-nano.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
id: gpt-5-nano
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "openai/gpt-5.4-nano"
|
||||||
|
reasoning_effort: high
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
31
internal/profile/builtin/repository.go
Normal file
31
internal/profile/builtin/repository.go
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
package builtin
|
||||||
|
|
||||||
|
import (
|
||||||
|
"embed"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||||
|
)
|
||||||
|
|
||||||
|
const assetRoot = "assets"
|
||||||
|
|
||||||
|
//go:embed assets/**/*.yml
|
||||||
|
var assets embed.FS
|
||||||
|
|
||||||
|
func NewRepository() profile.Repository {
|
||||||
|
return profile.NewFSRepository(assets, assetRoot)
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewRepositoryWithPrimary(primary profile.Repository) profile.Repository {
|
||||||
|
if primary == nil {
|
||||||
|
return NewRepository()
|
||||||
|
}
|
||||||
|
return profile.NewOverlayRepository(primary, NewRepository())
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewRepositoryWithDirectory(dir string) profile.Repository {
|
||||||
|
if strings.TrimSpace(dir) == "" {
|
||||||
|
return NewRepository()
|
||||||
|
}
|
||||||
|
return NewRepositoryWithPrimary(profile.NewFilesystemRepository(dir))
|
||||||
|
}
|
||||||
127
internal/profile/builtin/repository_test.go
Normal file
127
internal/profile/builtin/repository_test.go
Normal file
@@ -0,0 +1,127 @@
|
|||||||
|
package builtin
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io/fs"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
|
||||||
|
repo := NewRepository()
|
||||||
|
ids := loadBuiltInProfileIDs(t)
|
||||||
|
if len(ids) == 0 {
|
||||||
|
t.Fatal("expected built-in profiles")
|
||||||
|
}
|
||||||
|
|
||||||
|
for id := range ids {
|
||||||
|
t.Run(id, func(t *testing.T) {
|
||||||
|
p, err := repo.GetProfile(context.Background(), id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected built-in profile %q to load, got %v", id, err)
|
||||||
|
}
|
||||||
|
if p.ID != id {
|
||||||
|
t.Fatalf("expected profile id %q, got %q", id, p.ID)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuiltInProfilesDoNotContainDuplicateIDsOrRawAPIKeys(t *testing.T) {
|
||||||
|
loadBuiltInProfileIDs(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadBuiltInProfileIDs(t *testing.T) map[string]string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
ids := map[string]string{}
|
||||||
|
err := fs.WalkDir(assets, assetRoot, func(name string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if d.IsDir() || !strings.HasSuffix(name, ".yml") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := assets.ReadFile(name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to read built-in profile %s: %v", name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var raw map[string]any
|
||||||
|
if err := yaml.Unmarshal(data, &raw); err != nil {
|
||||||
|
t.Fatalf("failed to decode built-in profile %s: %v", name, err)
|
||||||
|
}
|
||||||
|
if _, ok := raw["api_key"]; ok {
|
||||||
|
t.Fatalf("built-in profile %s contains raw api_key", name)
|
||||||
|
}
|
||||||
|
id, ok := raw["id"].(string)
|
||||||
|
if !ok || strings.TrimSpace(id) == "" {
|
||||||
|
t.Fatalf("built-in profile %s has missing id", name)
|
||||||
|
}
|
||||||
|
if previous, ok := ids[id]; ok {
|
||||||
|
t.Fatalf("duplicate built-in profile id %q in %s and %s", id, previous, name)
|
||||||
|
}
|
||||||
|
ids[id] = name
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to walk built-in profiles: %v", err)
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRepositoryWithPrimaryUsesPrimaryBeforeBuiltIns(t *testing.T) {
|
||||||
|
repo := NewRepositoryWithPrimary(staticProfileRepo{
|
||||||
|
profiles: map[string]string{"mistral-small-3": "custom-model"},
|
||||||
|
})
|
||||||
|
|
||||||
|
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected profile to load, got %v", err)
|
||||||
|
}
|
||||||
|
if p.Model != "custom-model" {
|
||||||
|
t.Fatalf("expected primary profile to override built-in, got %+v", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRepositoryWithPrimaryFallsBackToBuiltIns(t *testing.T) {
|
||||||
|
repo := NewRepositoryWithPrimary(staticProfileRepo{})
|
||||||
|
|
||||||
|
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected built-in profile to load, got %v", err)
|
||||||
|
}
|
||||||
|
if p.ID != "mistral-small-3" {
|
||||||
|
t.Fatalf("unexpected profile: %+v", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRepositoryWithPrimaryDoesNotFallBackAfterPrimaryError(t *testing.T) {
|
||||||
|
repo := NewRepositoryWithPrimary(staticProfileRepo{err: profile.ErrInvalidProfile})
|
||||||
|
|
||||||
|
_, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
||||||
|
if !errors.Is(err, profile.ErrInvalidProfile) {
|
||||||
|
t.Fatalf("expected primary error, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type staticProfileRepo struct {
|
||||||
|
profiles map[string]string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
if r.err != nil {
|
||||||
|
return nil, r.err
|
||||||
|
}
|
||||||
|
if model, ok := r.profiles[id]; ok {
|
||||||
|
return &domain.ExecutionProfile{ID: id, Endpoint: "http://primary/v1", Model: model}, nil
|
||||||
|
}
|
||||||
|
return nil, profile.ErrProfileNotFound
|
||||||
|
}
|
||||||
@@ -5,8 +5,9 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
@@ -30,11 +31,56 @@ func NewFilesystemRepository(dir string) Repository {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
return loadProfile(ctx, os.DirFS(r.dir), ".", id)
|
||||||
|
}
|
||||||
|
|
||||||
|
type fsRepository struct {
|
||||||
|
fsys fs.FS
|
||||||
|
root string
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewFSRepository(fsys fs.FS, root string) Repository {
|
||||||
|
return &fsRepository{fsys: fsys, root: root}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *fsRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
return loadProfile(ctx, r.fsys, r.root, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
type overlayRepository struct {
|
||||||
|
primary Repository
|
||||||
|
fallback Repository
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewOverlayRepository(primary, fallback Repository) Repository {
|
||||||
|
return &overlayRepository{primary: primary, fallback: fallback}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *overlayRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
if r.primary != nil {
|
||||||
|
prof, err := r.primary.GetProfile(ctx, id)
|
||||||
|
if err == nil {
|
||||||
|
return prof, nil
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrProfileNotFound) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if r.fallback == nil {
|
||||||
|
return nil, ErrProfileNotFound
|
||||||
|
}
|
||||||
|
return r.fallback.GetProfile(ctx, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*domain.ExecutionProfile, error) {
|
||||||
if strings.TrimSpace(id) == "" {
|
if strings.TrimSpace(id) == "" {
|
||||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
|
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
|
||||||
}
|
}
|
||||||
|
if fsys == nil {
|
||||||
|
return nil, fmt.Errorf("failed to read profile directory: filesystem is nil")
|
||||||
|
}
|
||||||
|
|
||||||
files, err := filecatalog.FindYAMLFiles(ctx, r.dir)
|
files, err := filecatalog.FindFSYAMLFiles(ctx, fsys, root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read profile directory: %w", err)
|
return nil, fmt.Errorf("failed to read profile directory: %w", err)
|
||||||
}
|
}
|
||||||
@@ -47,9 +93,9 @@ func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*doma
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
relPath := filecatalog.RelativePath(r.dir, fullPath)
|
relPath := filecatalog.DisplayPath(root, fullPath)
|
||||||
fileMatch := filecatalog.Stem(filepath.Base(fullPath)) == id
|
fileMatch := filecatalog.Stem(path.Base(fullPath)) == id
|
||||||
data, err := os.ReadFile(fullPath)
|
data, err := fs.ReadFile(fsys, fullPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
|
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,11 +2,15 @@ package profile
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestFilesystemRepository_GetProfile(t *testing.T) {
|
func TestFilesystemRepository_GetProfile(t *testing.T) {
|
||||||
@@ -85,6 +89,63 @@ temperature: 0.1
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("valid profile with JSON-compatible extra params", func(t *testing.T) {
|
||||||
|
writeProfileTestFile(t, filepath.Join(tmpDir, "json-extra-params.yaml"), `
|
||||||
|
id: json-extra-params
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: nested-model
|
||||||
|
extra_params:
|
||||||
|
string_value: enabled
|
||||||
|
number_value: 42
|
||||||
|
boolean_value: true
|
||||||
|
object_value:
|
||||||
|
nested: value
|
||||||
|
count: 2
|
||||||
|
array_value:
|
||||||
|
- first
|
||||||
|
- 3
|
||||||
|
- false
|
||||||
|
`)
|
||||||
|
|
||||||
|
p, err := repo.GetProfile(ctx, "json-extra-params")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var got map[string]any
|
||||||
|
encoded, err := json.Marshal(p.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected extra_params to marshal as JSON, got %v", err)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(encoded, &got); err != nil {
|
||||||
|
t.Fatalf("expected extra_params JSON to decode, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got["string_value"] != "enabled" {
|
||||||
|
t.Fatalf("unexpected string extra param: %#v", got["string_value"])
|
||||||
|
}
|
||||||
|
if got["number_value"] != float64(42) {
|
||||||
|
t.Fatalf("unexpected number extra param: %#v", got["number_value"])
|
||||||
|
}
|
||||||
|
if got["boolean_value"] != true {
|
||||||
|
t.Fatalf("unexpected boolean extra param: %#v", got["boolean_value"])
|
||||||
|
}
|
||||||
|
objectValue, ok := got["object_value"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected object extra param, got %#v", got["object_value"])
|
||||||
|
}
|
||||||
|
if objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
|
||||||
|
t.Fatalf("unexpected object extra param: %#v", objectValue)
|
||||||
|
}
|
||||||
|
arrayValue, ok := got["array_value"].([]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected array extra param, got %#v", got["array_value"])
|
||||||
|
}
|
||||||
|
if len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
|
||||||
|
t.Fatalf("unexpected array extra param: %#v", arrayValue)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
|
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
|
||||||
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
|
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
|
||||||
id: duplicate-profile
|
id: duplicate-profile
|
||||||
@@ -203,3 +264,216 @@ func writeProfileTestFile(t *testing.T, path string, content string) {
|
|||||||
t.Fatalf("failed to write profile test file %q: %v", path, err)
|
t.Fatalf("failed to write profile test file %q: %v", path, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFSRepository(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
t.Run("loads valid profiles from nested directories", func(t *testing.T) {
|
||||||
|
repo := NewFSRepository(fstest.MapFS{
|
||||||
|
"profiles/provider/nested.yaml": profileMapFile(`
|
||||||
|
id: nested-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: nested-model
|
||||||
|
temperature: 0.1
|
||||||
|
`),
|
||||||
|
}, "profiles")
|
||||||
|
|
||||||
|
p, err := repo.GetProfile(ctx, "nested-profile")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if p.ID != "nested-profile" || p.Model != "nested-model" {
|
||||||
|
t.Fatalf("unexpected profile: %+v", p)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("rejects unknown YAML fields", func(t *testing.T) {
|
||||||
|
repo := NewFSRepository(fstest.MapFS{
|
||||||
|
"profiles/unknown.yaml": profileMapFile(`
|
||||||
|
id: unknown-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
unknown: value
|
||||||
|
`),
|
||||||
|
}, "profiles")
|
||||||
|
|
||||||
|
_, err := repo.GetProfile(ctx, "unknown-profile")
|
||||||
|
if !errors.Is(err, ErrInvalidYAML) {
|
||||||
|
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("rejects raw api_key in selected profile", func(t *testing.T) {
|
||||||
|
repo := NewFSRepository(fstest.MapFS{
|
||||||
|
"profiles/raw.yaml": profileMapFile(`
|
||||||
|
id: raw-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
api_key: secret
|
||||||
|
`),
|
||||||
|
}, "profiles")
|
||||||
|
|
||||||
|
_, err := repo.GetProfile(ctx, "raw-profile")
|
||||||
|
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||||
|
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("ignores raw api_key in non-selected profiles", func(t *testing.T) {
|
||||||
|
repo := NewFSRepository(fstest.MapFS{
|
||||||
|
"profiles/raw.yaml": profileMapFile(`
|
||||||
|
id: raw-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
api_key: secret
|
||||||
|
`),
|
||||||
|
"profiles/valid.yaml": profileMapFile(`
|
||||||
|
id: valid-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
`),
|
||||||
|
}, "profiles")
|
||||||
|
|
||||||
|
p, err := repo.GetProfile(ctx, "valid-profile")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if p.ID != "valid-profile" {
|
||||||
|
t.Fatalf("unexpected profile: %+v", p)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("rejects duplicate IDs within one source", func(t *testing.T) {
|
||||||
|
repo := NewFSRepository(fstest.MapFS{
|
||||||
|
"profiles/a.yaml": profileMapFile(`
|
||||||
|
id: duplicate-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: first
|
||||||
|
`),
|
||||||
|
"profiles/nested/b.yaml": profileMapFile(`
|
||||||
|
id: duplicate-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: second
|
||||||
|
`),
|
||||||
|
}, "profiles")
|
||||||
|
|
||||||
|
_, err := repo.GetProfile(ctx, "duplicate-profile")
|
||||||
|
if !errors.Is(err, ErrInvalidProfile) {
|
||||||
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"duplicate execution profile id", "a.yaml", "nested/b.yaml"} {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Fatalf("expected error to contain %q, got %v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOverlayRepository(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"}
|
||||||
|
fallbackProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://fallback", Model: "fallback"}
|
||||||
|
|
||||||
|
t.Run("returns primary matches before fallback matches", func(t *testing.T) {
|
||||||
|
repo := NewOverlayRepository(
|
||||||
|
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": primaryProfile}},
|
||||||
|
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
|
||||||
|
)
|
||||||
|
|
||||||
|
p, err := repo.GetProfile(ctx, "shared")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if p.Model != "primary" {
|
||||||
|
t.Fatalf("expected primary profile, got %+v", p)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("falls back on primary not found", func(t *testing.T) {
|
||||||
|
repo := NewOverlayRepository(
|
||||||
|
staticProfileRepo{},
|
||||||
|
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
|
||||||
|
)
|
||||||
|
|
||||||
|
p, err := repo.GetProfile(ctx, "shared")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if p.Model != "fallback" {
|
||||||
|
t.Fatalf("expected fallback profile, got %+v", p)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("does not fall back after primary load errors", func(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
err error
|
||||||
|
}{
|
||||||
|
{name: "invalid yaml", err: ErrInvalidYAML},
|
||||||
|
{name: "invalid profile", err: ErrInvalidProfile},
|
||||||
|
{name: "raw api key", err: ErrRawAPIKeyNotAllowed},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
repo := NewOverlayRepository(
|
||||||
|
staticProfileRepo{err: tc.err},
|
||||||
|
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
|
||||||
|
)
|
||||||
|
|
||||||
|
_, err := repo.GetProfile(ctx, "shared")
|
||||||
|
if !errors.Is(err, tc.err) {
|
||||||
|
t.Fatalf("expected %v, got %v", tc.err, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("returns not found when both sources miss", func(t *testing.T) {
|
||||||
|
repo := NewOverlayRepository(staticProfileRepo{}, staticProfileRepo{})
|
||||||
|
|
||||||
|
_, err := repo.GetProfile(ctx, "missing")
|
||||||
|
if !errors.Is(err, ErrProfileNotFound) {
|
||||||
|
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("nil primary uses fallback", func(t *testing.T) {
|
||||||
|
repo := NewOverlayRepository(nil, staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}})
|
||||||
|
|
||||||
|
p, err := repo.GetProfile(ctx, "shared")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if p.Model != "fallback" {
|
||||||
|
t.Fatalf("expected fallback profile, got %+v", p)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("nil fallback returns not found after primary miss", func(t *testing.T) {
|
||||||
|
repo := NewOverlayRepository(staticProfileRepo{}, nil)
|
||||||
|
|
||||||
|
_, err := repo.GetProfile(ctx, "missing")
|
||||||
|
if !errors.Is(err, ErrProfileNotFound) {
|
||||||
|
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func profileMapFile(content string) *fstest.MapFile {
|
||||||
|
return &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))}
|
||||||
|
}
|
||||||
|
|
||||||
|
type staticProfileRepo struct {
|
||||||
|
profiles map[string]*domain.ExecutionProfile
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
if r.err != nil {
|
||||||
|
return nil, r.err
|
||||||
|
}
|
||||||
|
if p, ok := r.profiles[id]; ok {
|
||||||
|
cp := *p
|
||||||
|
return &cp, nil
|
||||||
|
}
|
||||||
|
return nil, ErrProfileNotFound
|
||||||
|
}
|
||||||
|
|||||||
@@ -6,7 +6,9 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
"strings"
|
||||||
"text/template"
|
"text/template"
|
||||||
|
"unicode/utf8"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -50,6 +52,11 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
sessionID, err := renderSessionID(definition.SessionID, funcs, vars)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
var renderedMessages []domain.RenderedMessage
|
var renderedMessages []domain.RenderedMessage
|
||||||
|
|
||||||
for i, tmplMsg := range definition.Templates {
|
for i, tmplMsg := range definition.Templates {
|
||||||
@@ -75,12 +82,44 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
}
|
}
|
||||||
|
|
||||||
renderedMessages = append(renderedMessages, domain.RenderedMessage{
|
renderedMessages = append(renderedMessages, domain.RenderedMessage{
|
||||||
Role: tmplMsg.Role,
|
Role: tmplMsg.Role,
|
||||||
Content: buf.String(),
|
Content: buf.String(),
|
||||||
|
CacheControl: cloneCacheControl(tmplMsg.CacheControl),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
return &domain.RenderedPrompt{
|
return &domain.RenderedPrompt{
|
||||||
Messages: renderedMessages,
|
SessionID: sessionID,
|
||||||
|
Messages: renderedMessages,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
|
||||||
|
if strings.TrimSpace(raw) == "" {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
tmpl, err := template.New("session_id").Funcs(funcs).Option("missingkey=error").Parse(raw)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
if err := tmpl.Execute(&buf, vars); err != nil {
|
||||||
|
return "", fmt.Errorf("%w: session_id: %w", ErrRenderFailure, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionID := strings.TrimSpace(buf.String())
|
||||||
|
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
|
||||||
|
return "", fmt.Errorf("%w: session_id length %d exceeds maximum %d", ErrRenderFailure, n, domain.SessionIDMaxLength)
|
||||||
|
}
|
||||||
|
return sessionID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneCacheControl(in *domain.CacheControl) *domain.CacheControl {
|
||||||
|
if in == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := *in
|
||||||
|
return &out
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package prompt
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
@@ -78,6 +79,66 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("copying cache control to rendered messages", func(t *testing.T) {
|
||||||
|
def := &domain.PromptDefinition{
|
||||||
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "You are concise.",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
TTL: "1h",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{Role: "user", Content: "Analyze this: {{input \"transcript\"}}"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := renderer.Render(ctx, def, inputs, vars)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(res.Messages) != 2 {
|
||||||
|
t.Fatalf("expected 2 messages, got %d", len(res.Messages))
|
||||||
|
}
|
||||||
|
if res.Messages[0].CacheControl == nil {
|
||||||
|
t.Fatal("expected rendered cache control")
|
||||||
|
}
|
||||||
|
if res.Messages[0].CacheControl.Type != domain.CacheControlEphemeral {
|
||||||
|
t.Fatalf("unexpected cache control type: %q", res.Messages[0].CacheControl.Type)
|
||||||
|
}
|
||||||
|
if res.Messages[0].CacheControl.TTL != "1h" {
|
||||||
|
t.Fatalf("unexpected cache control ttl: %q", res.Messages[0].CacheControl.TTL)
|
||||||
|
}
|
||||||
|
if res.Messages[1].CacheControl != nil {
|
||||||
|
t.Fatalf("expected no cache control on second message, got %#v", res.Messages[1].CacheControl)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("rendered cache control does not alias source template", func(t *testing.T) {
|
||||||
|
source := &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}
|
||||||
|
def := &domain.PromptDefinition{
|
||||||
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{Role: "system", Content: "You are concise.", CacheControl: source},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := renderer.Render(ctx, def, inputs, vars)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if res.Messages[0].CacheControl == source {
|
||||||
|
t.Fatal("expected rendered cache control to be cloned")
|
||||||
|
}
|
||||||
|
|
||||||
|
res.Messages[0].CacheControl.TTL = ""
|
||||||
|
if source.TTL != "1h" {
|
||||||
|
t.Fatalf("source cache control was mutated, ttl=%q", source.TTL)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("accessing vars", func(t *testing.T) {
|
t.Run("accessing vars", func(t *testing.T) {
|
||||||
def := &domain.PromptDefinition{
|
def := &domain.PromptDefinition{
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
@@ -95,6 +156,78 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("rendering session id from vars", func(t *testing.T) {
|
||||||
|
def := &domain.PromptDefinition{
|
||||||
|
SessionID: " {{ .session_id }} ",
|
||||||
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{Role: "system", Content: "Speak in a {{.tone}} tone."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := renderer.Render(ctx, def, inputs, map[string]string{
|
||||||
|
"tone": "concise",
|
||||||
|
"session_id": "agent-session-123",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if res.SessionID != "agent-session-123" {
|
||||||
|
t.Fatalf("unexpected session id: %q", res.SessionID)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("empty rendered session id is omitted", func(t *testing.T) {
|
||||||
|
def := &domain.PromptDefinition{
|
||||||
|
SessionID: " ",
|
||||||
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{Role: "system", Content: "Speak in a {{.tone}} tone."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := renderer.Render(ctx, def, inputs, vars)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if res.SessionID != "" {
|
||||||
|
t.Fatalf("expected empty session id, got %q", res.SessionID)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("missing session id var fails rendering", func(t *testing.T) {
|
||||||
|
def := &domain.PromptDefinition{
|
||||||
|
SessionID: "{{ .session_id }}",
|
||||||
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{Role: "system", Content: "Speak in a {{.tone}} tone."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := renderer.Render(ctx, def, inputs, vars)
|
||||||
|
if !errors.Is(err, ErrRenderFailure) {
|
||||||
|
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("too long rendered session id fails rendering", func(t *testing.T) {
|
||||||
|
def := &domain.PromptDefinition{
|
||||||
|
SessionID: "{{ .session_id }}",
|
||||||
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{Role: "system", Content: "Speak in a {{.tone}} tone."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := renderer.Render(ctx, def, inputs, map[string]string{
|
||||||
|
"tone": "concise",
|
||||||
|
"session_id": strings.Repeat("x", domain.SessionIDMaxLength+1),
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrRenderFailure) {
|
||||||
|
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("inserting required input artifact", func(t *testing.T) {
|
t.Run("inserting required input artifact", func(t *testing.T) {
|
||||||
def := &domain.PromptDefinition{
|
def := &domain.PromptDefinition{
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
|
|||||||
@@ -5,7 +5,9 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -24,11 +26,17 @@ type filesystemRepository struct {
|
|||||||
dir string
|
dir string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type fsRepository struct {
|
||||||
|
fsys fs.FS
|
||||||
|
root string
|
||||||
|
}
|
||||||
|
|
||||||
type promptDefinitionFile struct {
|
type promptDefinitionFile struct {
|
||||||
ID string `yaml:"id"`
|
ID string `yaml:"id"`
|
||||||
Version string `yaml:"version"`
|
Version string `yaml:"version"`
|
||||||
DefaultProfile *string `yaml:"default_profile"`
|
DefaultProfile *string `yaml:"default_profile"`
|
||||||
Description string `yaml:"description"`
|
Description string `yaml:"description"`
|
||||||
|
SessionID string `yaml:"session_id"`
|
||||||
Inputs []promptInputFile `yaml:"inputs"`
|
Inputs []promptInputFile `yaml:"inputs"`
|
||||||
Messages []promptMessageFile `yaml:"messages"`
|
Messages []promptMessageFile `yaml:"messages"`
|
||||||
Output promptOutputContractFile `yaml:"output"`
|
Output promptOutputContractFile `yaml:"output"`
|
||||||
@@ -42,9 +50,15 @@ type promptInputFile struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type promptMessageFile struct {
|
type promptMessageFile struct {
|
||||||
Role string `yaml:"role"`
|
Role string `yaml:"role"`
|
||||||
Content string `yaml:"content"`
|
Content string `yaml:"content"`
|
||||||
ContentFile string `yaml:"content_file"`
|
ContentFile string `yaml:"content_file"`
|
||||||
|
CacheControl *cacheControlFile `yaml:"cache_control"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type cacheControlFile struct {
|
||||||
|
Type string `yaml:"type"`
|
||||||
|
TTL string `yaml:"ttl"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type promptOutputContractFile struct {
|
type promptOutputContractFile struct {
|
||||||
@@ -58,6 +72,10 @@ func NewFilesystemRepository(dir string) Repository {
|
|||||||
return &filesystemRepository{dir: dir}
|
return &filesystemRepository{dir: dir}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func NewFSRepository(fsys fs.FS, root string) Repository {
|
||||||
|
return &fsRepository{fsys: fsys, root: root}
|
||||||
|
}
|
||||||
|
|
||||||
func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
||||||
if strings.TrimSpace(id) == "" {
|
if strings.TrimSpace(id) == "" {
|
||||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
|
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
|
||||||
@@ -125,6 +143,10 @@ func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id strin
|
|||||||
return nil, ErrPromptDefinitionNotFound
|
return nil, ErrPromptDefinitionNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *fsRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
||||||
|
return loadPromptDefinition(ctx, r.fsys, r.root, id, version)
|
||||||
|
}
|
||||||
|
|
||||||
type promptDefinitionMatch struct {
|
type promptDefinitionMatch struct {
|
||||||
def *domain.PromptDefinition
|
def *domain.PromptDefinition
|
||||||
path string
|
path string
|
||||||
@@ -159,7 +181,152 @@ func promptDefinitionFileHasID(path string, id string) bool {
|
|||||||
return strings.TrimSpace(raw.ID) == id
|
return strings.TrimSpace(raw.ID) == id
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func loadPromptDefinition(ctx context.Context, fsys fs.FS, root string, id string, version string) (*domain.PromptDefinition, error) {
|
||||||
|
if strings.TrimSpace(id) == "" {
|
||||||
|
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
|
||||||
|
}
|
||||||
|
if fsys == nil {
|
||||||
|
return nil, fmt.Errorf("failed to read prompt definition directory: filesystem is nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
files, err := filecatalog.FindFSYAMLFiles(ctx, fsys, root)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
|
||||||
|
}
|
||||||
|
cleanRoot := filecatalog.CleanFSRoot(root)
|
||||||
|
rootInfo, err := fs.Stat(fsys, cleanRoot)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var matches []promptDefinitionMatch
|
||||||
|
for _, fullPath := range files {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
relPath := filecatalog.DisplayPath(root, fullPath)
|
||||||
|
fileMatch := filecatalog.Stem(path.Base(fullPath)) == id
|
||||||
|
data, err := fs.ReadFile(fsys, fullPath)
|
||||||
|
if err != nil {
|
||||||
|
if fileMatch {
|
||||||
|
return nil, fmt.Errorf("%w: %s: failed to read prompt definition file: %v", ErrInvalidYAML, relPath, err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
raw, err := decodePromptDefinition(data)
|
||||||
|
if err != nil {
|
||||||
|
if fileMatch || promptDefinitionDataHasID(data, id) {
|
||||||
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
def, err := normalizePromptDefinitionFromFS(raw, fsys, root, fullPath, rootInfo.IsDir())
|
||||||
|
if err != nil {
|
||||||
|
if fileMatch || strings.TrimSpace(raw.ID) == id {
|
||||||
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, relPath, err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if def.ID != id {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if version != "" && def.Version != version {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
matches = append(matches, promptDefinitionMatch{
|
||||||
|
def: def,
|
||||||
|
path: relPath,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(matches) > 1 {
|
||||||
|
paths := make([]string, 0, len(matches))
|
||||||
|
for _, match := range matches {
|
||||||
|
paths = append(paths, match.path)
|
||||||
|
}
|
||||||
|
if version != "" {
|
||||||
|
return nil, fmt.Errorf("%w: duplicate prompt definition id %q version %q found in: %s", ErrInvalidPromptDefinition, id, version, strings.Join(paths, ", "))
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("%w: duplicate prompt definition id %q found in: %s", ErrInvalidPromptDefinition, id, strings.Join(paths, ", "))
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(matches) == 1 {
|
||||||
|
return matches[0].def, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, ErrPromptDefinitionNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodePromptDefinition(data []byte) (*promptDefinitionFile, error) {
|
||||||
|
var raw promptDefinitionFile
|
||||||
|
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||||
|
decoder.KnownFields(true)
|
||||||
|
if err := decoder.Decode(&raw); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &raw, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func promptDefinitionDataHasID(data []byte, id string) bool {
|
||||||
|
var raw struct {
|
||||||
|
ID string `yaml:"id"`
|
||||||
|
}
|
||||||
|
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&raw); err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(raw.ID) == id
|
||||||
|
}
|
||||||
|
|
||||||
func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*domain.PromptDefinition, error) {
|
func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*domain.PromptDefinition, error) {
|
||||||
|
promptDir := filepath.Dir(sourcePath)
|
||||||
|
return normalizePromptDefinitionWithContent(raw, func(contentFile string) (string, string, error) {
|
||||||
|
resolvedPath := strings.TrimSpace(contentFile)
|
||||||
|
if !filepath.IsAbs(resolvedPath) {
|
||||||
|
resolvedPath = filepath.Join(promptDir, resolvedPath)
|
||||||
|
}
|
||||||
|
resolvedPath = filepath.Clean(resolvedPath)
|
||||||
|
|
||||||
|
body, err := os.ReadFile(resolvedPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
return string(body), resolvedPath, nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizePromptDefinitionFromFS(raw *promptDefinitionFile, fsys fs.FS, root string, sourcePath string, rootIsDir bool) (*domain.PromptDefinition, error) {
|
||||||
|
promptDir := path.Dir(sourcePath)
|
||||||
|
return normalizePromptDefinitionWithContent(raw, func(contentFile string) (string, string, error) {
|
||||||
|
var resolvedPath string
|
||||||
|
if rootIsDir {
|
||||||
|
var err error
|
||||||
|
resolvedPath, _, err = filecatalog.ResolveFSPath(root, promptDir, contentFile)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
resolvedPath = strings.TrimSpace(contentFile)
|
||||||
|
if !path.IsAbs(resolvedPath) {
|
||||||
|
resolvedPath = path.Join(promptDir, resolvedPath)
|
||||||
|
}
|
||||||
|
resolvedPath = strings.TrimPrefix(path.Clean(resolvedPath), "/")
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := fs.ReadFile(fsys, resolvedPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
return string(body), resolvedPath, nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizePromptDefinitionWithContent(raw *promptDefinitionFile, readContentFile func(string) (string, string, error)) (*domain.PromptDefinition, error) {
|
||||||
if raw == nil {
|
if raw == nil {
|
||||||
return nil, errors.New("prompt definition is nil")
|
return nil, errors.New("prompt definition is nil")
|
||||||
}
|
}
|
||||||
@@ -199,7 +366,6 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
|
|||||||
}
|
}
|
||||||
|
|
||||||
templates := make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
|
templates := make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
|
||||||
promptDir := filepath.Dir(sourcePath)
|
|
||||||
for i, msg := range raw.Messages {
|
for i, msg := range raw.Messages {
|
||||||
role := strings.TrimSpace(msg.Role)
|
role := strings.TrimSpace(msg.Role)
|
||||||
if role == "" {
|
if role == "" {
|
||||||
@@ -212,27 +378,27 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
|
|||||||
return nil, fmt.Errorf("message %d (%s) must set exactly one of content or content_file", i, role)
|
return nil, fmt.Errorf("message %d (%s) must set exactly one of content or content_file", i, role)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
cacheControl, err := normalizeCacheControl(msg.CacheControl)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("message %d (%s) cache_control: %w", i, role, err)
|
||||||
|
}
|
||||||
|
|
||||||
templateContent := msg.Content
|
templateContent := msg.Content
|
||||||
resolvedContentFile := ""
|
resolvedContentFile := ""
|
||||||
if hasContentFile {
|
if hasContentFile {
|
||||||
resolvedPath := strings.TrimSpace(msg.ContentFile)
|
body, resolvedPath, err := readContentFile(msg.ContentFile)
|
||||||
if !filepath.IsAbs(resolvedPath) {
|
|
||||||
resolvedPath = filepath.Join(promptDir, resolvedPath)
|
|
||||||
}
|
|
||||||
resolvedPath = filepath.Clean(resolvedPath)
|
|
||||||
|
|
||||||
body, err := os.ReadFile(resolvedPath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("prompt %q message %d (%s): failed to read content_file %q: %w", id, i, role, msg.ContentFile, err)
|
return nil, fmt.Errorf("prompt %q message %d (%s): failed to read content_file %q: %w", id, i, role, msg.ContentFile, err)
|
||||||
}
|
}
|
||||||
templateContent = string(body)
|
templateContent = body
|
||||||
resolvedContentFile = resolvedPath
|
resolvedContentFile = resolvedPath
|
||||||
}
|
}
|
||||||
|
|
||||||
templates = append(templates, domain.PromptMessageTemplate{
|
templates = append(templates, domain.PromptMessageTemplate{
|
||||||
Role: role,
|
Role: role,
|
||||||
Content: templateContent,
|
Content: templateContent,
|
||||||
ContentFile: resolvedContentFile,
|
ContentFile: resolvedContentFile,
|
||||||
|
CacheControl: cacheControl,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -262,6 +428,7 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
|
|||||||
Version: version,
|
Version: version,
|
||||||
DefaultProfile: defaultProfile,
|
DefaultProfile: defaultProfile,
|
||||||
Description: strings.TrimSpace(raw.Description),
|
Description: strings.TrimSpace(raw.Description),
|
||||||
|
SessionID: strings.TrimSpace(raw.SessionID),
|
||||||
Inputs: inputs,
|
Inputs: inputs,
|
||||||
Templates: templates,
|
Templates: templates,
|
||||||
OutputFormat: raw.Output.Format,
|
OutputFormat: raw.Output.Format,
|
||||||
@@ -274,6 +441,30 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func normalizeCacheControl(raw *cacheControlFile) (*domain.CacheControl, error) {
|
||||||
|
if raw == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
cacheType := strings.TrimSpace(raw.Type)
|
||||||
|
if cacheType == "" {
|
||||||
|
return nil, errors.New("type is required")
|
||||||
|
}
|
||||||
|
if domain.CacheControlType(cacheType) != domain.CacheControlEphemeral {
|
||||||
|
return nil, fmt.Errorf("unsupported type %q", cacheType)
|
||||||
|
}
|
||||||
|
|
||||||
|
ttl := strings.TrimSpace(raw.TTL)
|
||||||
|
if ttl != "" && ttl != "1h" {
|
||||||
|
return nil, fmt.Errorf("unsupported ttl %q", ttl)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlType(cacheType),
|
||||||
|
TTL: ttl,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func isValidOutputFormat(f domain.OutputFormat) bool {
|
func isValidOutputFormat(f domain.OutputFormat) bool {
|
||||||
switch f {
|
switch f {
|
||||||
case domain.FormatText, domain.FormatMarkdown, domain.FormatJSON:
|
case domain.FormatText, domain.FormatMarkdown, domain.FormatJSON:
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
)
|
)
|
||||||
@@ -68,6 +69,44 @@ func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("valid cache control with ttl", func(t *testing.T) {
|
||||||
|
p, err := repo.GetPromptDefinition(ctx, "valid-cache-control-ttl", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if len(p.Templates) != 2 {
|
||||||
|
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
|
||||||
|
}
|
||||||
|
assertCacheControl(t, p.Templates[0].CacheControl, domain.CacheControlEphemeral, "1h")
|
||||||
|
if p.Templates[1].CacheControl != nil {
|
||||||
|
t.Fatalf("expected second message cache control to be nil, got %#v", p.Templates[1].CacheControl)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("valid cache control without ttl", func(t *testing.T) {
|
||||||
|
p, err := repo.GetPromptDefinition(ctx, "valid-cache-control-without-ttl", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if len(p.Templates) != 2 {
|
||||||
|
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
|
||||||
|
}
|
||||||
|
assertCacheControl(t, p.Templates[0].CacheControl, domain.CacheControlEphemeral, "")
|
||||||
|
if p.Templates[1].CacheControl != nil {
|
||||||
|
t.Fatalf("expected second message cache control to be nil, got %#v", p.Templates[1].CacheControl)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("valid session id template", func(t *testing.T) {
|
||||||
|
p, err := repo.GetPromptDefinition(ctx, "valid-session-id", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if p.SessionID != "{{ .session_id }}" {
|
||||||
|
t.Fatalf("expected trimmed session_id template, got %q", p.SessionID)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("valid nested file-backed prompt resolves content file relative to nested YAML", func(t *testing.T) {
|
t.Run("valid nested file-backed prompt resolves content file relative to nested YAML", func(t *testing.T) {
|
||||||
nestedDir := filepath.Join(tmpDir, "dnd", "recap")
|
nestedDir := filepath.Join(tmpDir, "dnd", "recap")
|
||||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
||||||
@@ -258,6 +297,10 @@ output:
|
|||||||
{name: "invalid validation mode", id: "invalid_validation_mode", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"invalid validation mode"}},
|
{name: "invalid validation mode", id: "invalid_validation_mode", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"invalid validation mode"}},
|
||||||
{name: "json_schema without schema_path", id: "json_schema_without_schema_path", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"schema_path"}},
|
{name: "json_schema without schema_path", id: "json_schema_without_schema_path", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"schema_path"}},
|
||||||
{name: "unknown input field", id: "unknown_input_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unknown_input_setting not found"}},
|
{name: "unknown input field", id: "unknown_input_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unknown_input_setting not found"}},
|
||||||
|
{name: "empty cache control type", id: "empty_cache_control_type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "type is required"}},
|
||||||
|
{name: "unsupported cache control type", id: "unsupported_cache_control_type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported type"}},
|
||||||
|
{name: "unsupported cache control ttl", id: "unsupported_cache_control_ttl", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported ttl"}},
|
||||||
|
{name: "unknown cache control field", id: "unknown_cache_control_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unexpected not found"}},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
@@ -282,6 +325,173 @@ output:
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFSRepositoryGetPromptDefinition(t *testing.T) {
|
||||||
|
repo := NewFSRepository(fstest.MapFS{
|
||||||
|
"prompts/nested/prompt.yaml": &fstest.MapFile{Data: []byte(`
|
||||||
|
id: fs-prompt
|
||||||
|
version: "1.0.0"
|
||||||
|
inputs:
|
||||||
|
- name: transcript
|
||||||
|
required: true
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: ./messages/user.tmpl
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
|
`)},
|
||||||
|
"prompts/nested/messages/user.tmpl": &fstest.MapFile{Data: []byte(`Summarize {{input "transcript"}}.`)},
|
||||||
|
}, "prompts")
|
||||||
|
|
||||||
|
got, err := repo.GetPromptDefinition(context.Background(), "fs-prompt", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if got.ID != "fs-prompt" {
|
||||||
|
t.Fatalf("unexpected prompt id: %q", got.ID)
|
||||||
|
}
|
||||||
|
if len(got.Templates) != 1 || !strings.Contains(got.Templates[0].Content, `{{input "transcript"}}`) {
|
||||||
|
t.Fatalf("expected content_file body to be loaded, got %+v", got.Templates)
|
||||||
|
}
|
||||||
|
if got.Templates[0].ContentFile != "prompts/nested/messages/user.tmpl" {
|
||||||
|
t.Fatalf("unexpected content file path: %q", got.Templates[0].ContentFile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFSRepositoryContentFileContainment(t *testing.T) {
|
||||||
|
t.Run("nested prompt can reference file inside root", func(t *testing.T) {
|
||||||
|
repo := NewFSRepository(fstest.MapFS{
|
||||||
|
"prompts/nested/prompt.yaml": &fstest.MapFile{Data: []byte(`
|
||||||
|
id: fs-contained-prompt
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: ../shared/user.tmpl
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
|
`)},
|
||||||
|
"prompts/shared/user.tmpl": &fstest.MapFile{Data: []byte(`Inside root.`)},
|
||||||
|
}, "prompts")
|
||||||
|
|
||||||
|
got, err := repo.GetPromptDefinition(context.Background(), "fs-contained-prompt", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if len(got.Templates) != 1 || got.Templates[0].Content != "Inside root." {
|
||||||
|
t.Fatalf("expected contained content file, got %+v", got.Templates)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
contentFile string
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{name: "parent escape rejected", contentFile: "../outside.tmpl", wantErr: "escapes source root"},
|
||||||
|
{name: "absolute path rejected", contentFile: "/outside.tmpl", wantErr: "must be relative"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
repo := NewFSRepository(fstest.MapFS{
|
||||||
|
"prompts/prompt.yaml": &fstest.MapFile{Data: []byte(`
|
||||||
|
id: fs-escaped-prompt
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: ` + tc.contentFile + `
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
|
`)},
|
||||||
|
"outside.tmpl": &fstest.MapFile{Data: []byte(`Outside root.`)},
|
||||||
|
}, "prompts")
|
||||||
|
|
||||||
|
_, err := repo.GetPromptDefinition(context.Background(), "fs-escaped-prompt", "")
|
||||||
|
if !errors.Is(err, ErrInvalidPromptDefinition) {
|
||||||
|
t.Fatalf("expected ErrInvalidPromptDefinition, got %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), tc.wantErr) {
|
||||||
|
t.Fatalf("expected error to contain %q, got %v", tc.wantErr, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFSRepositoryRejectsDuplicatePromptIDs(t *testing.T) {
|
||||||
|
repo := NewFSRepository(fstest.MapFS{
|
||||||
|
"one.yaml": &fstest.MapFile{Data: []byte(`
|
||||||
|
id: duplicate-fs-prompt
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: First.
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
repair_attempts: 0
|
||||||
|
`)},
|
||||||
|
"nested/two.yaml": &fstest.MapFile{Data: []byte(`
|
||||||
|
id: duplicate-fs-prompt
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: Second.
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
repair_attempts: 0
|
||||||
|
`)},
|
||||||
|
}, ".")
|
||||||
|
|
||||||
|
_, err := repo.GetPromptDefinition(context.Background(), "duplicate-fs-prompt", "")
|
||||||
|
if !errors.Is(err, ErrInvalidPromptDefinition) {
|
||||||
|
t.Fatalf("expected ErrInvalidPromptDefinition, got %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "one.yaml") || !strings.Contains(err.Error(), "nested/two.yaml") {
|
||||||
|
t.Fatalf("expected duplicate paths in error, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFSRepositoryRejectsUnknownYAMLFields(t *testing.T) {
|
||||||
|
repo := NewFSRepository(fstest.MapFS{
|
||||||
|
"not_named_like_id.yaml": &fstest.MapFile{Data: []byte(`
|
||||||
|
id: strict-fs-prompt
|
||||||
|
version: "1.0.0"
|
||||||
|
unknown: true
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: Invalid.
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
repair_attempts: 0
|
||||||
|
`)},
|
||||||
|
}, ".")
|
||||||
|
|
||||||
|
_, err := repo.GetPromptDefinition(context.Background(), "strict-fs-prompt", "")
|
||||||
|
if !errors.Is(err, ErrInvalidYAML) {
|
||||||
|
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertCacheControl(t *testing.T, got *domain.CacheControl, wantType domain.CacheControlType, wantTTL string) {
|
||||||
|
t.Helper()
|
||||||
|
if got == nil {
|
||||||
|
t.Fatal("expected cache control, got nil")
|
||||||
|
}
|
||||||
|
if got.Type != wantType {
|
||||||
|
t.Fatalf("unexpected cache control type: got %q want %q", got.Type, wantType)
|
||||||
|
}
|
||||||
|
if got.TTL != wantTTL {
|
||||||
|
t.Fatalf("unexpected cache control ttl: got %q want %q", got.TTL, wantTTL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func writePromptTestFile(t *testing.T, path string, content string) {
|
func writePromptTestFile(t *testing.T, path string, content string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil {
|
if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil {
|
||||||
|
|||||||
10
internal/promptdef/testdata/empty_cache_control_type.yaml
vendored
Normal file
10
internal/promptdef/testdata/empty_cache_control_type.yaml
vendored
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
id: empty-cache-control-type
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: "Use cached instructions."
|
||||||
|
cache_control: {}
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
12
internal/promptdef/testdata/unknown_cache_control_field.yaml
vendored
Normal file
12
internal/promptdef/testdata/unknown_cache_control_field.yaml
vendored
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
id: unknown-cache-control-field
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: "Use cached instructions."
|
||||||
|
cache_control:
|
||||||
|
type: ephemeral
|
||||||
|
unexpected: true
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
12
internal/promptdef/testdata/unsupported_cache_control_ttl.yaml
vendored
Normal file
12
internal/promptdef/testdata/unsupported_cache_control_ttl.yaml
vendored
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
id: unsupported-cache-control-ttl
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: "Use cached instructions."
|
||||||
|
cache_control:
|
||||||
|
type: ephemeral
|
||||||
|
ttl: 5m
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
11
internal/promptdef/testdata/unsupported_cache_control_type.yaml
vendored
Normal file
11
internal/promptdef/testdata/unsupported_cache_control_type.yaml
vendored
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
id: unsupported-cache-control-type
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: "Use cached instructions."
|
||||||
|
cache_control:
|
||||||
|
type: persistent
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
14
internal/promptdef/testdata/valid_cache_control_ttl.yaml
vendored
Normal file
14
internal/promptdef/testdata/valid_cache_control_ttl.yaml
vendored
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
id: valid-cache-control-ttl
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: "Use cached instructions."
|
||||||
|
cache_control:
|
||||||
|
type: ephemeral
|
||||||
|
ttl: 1h
|
||||||
|
- role: user
|
||||||
|
content: "Summarize the input."
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
13
internal/promptdef/testdata/valid_cache_control_without_ttl.yaml
vendored
Normal file
13
internal/promptdef/testdata/valid_cache_control_without_ttl.yaml
vendored
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
id: valid-cache-control-without-ttl
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: "Use cached instructions."
|
||||||
|
cache_control:
|
||||||
|
type: ephemeral
|
||||||
|
- role: user
|
||||||
|
content: "Summarize the input."
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
10
internal/promptdef/testdata/valid_session_id.yaml
vendored
Normal file
10
internal/promptdef/testdata/valid_session_id.yaml
vendored
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
id: valid-session-id
|
||||||
|
version: "1.0.0"
|
||||||
|
session_id: " {{ .session_id }} "
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: Hello.
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 0
|
||||||
@@ -27,7 +27,9 @@ var (
|
|||||||
ErrInvalidRequest = errors.New("invalid run request")
|
ErrInvalidRequest = errors.New("invalid run request")
|
||||||
ErrProfileRequired = errors.New("profile selection is required")
|
ErrProfileRequired = errors.New("profile selection is required")
|
||||||
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
||||||
ErrProfileLoad = errors.New("failed to load prompt definition")
|
ErrAPIKeyRequired = errors.New("api key is required")
|
||||||
|
ErrPromptLoad = errors.New("failed to load prompt definition")
|
||||||
|
ErrProfileLoad = errors.New("failed to load execution profile")
|
||||||
ErrArtifactLoad = errors.New("failed to load artifact")
|
ErrArtifactLoad = errors.New("failed to load artifact")
|
||||||
ErrPromptRender = errors.New("failed to render prompt")
|
ErrPromptRender = errors.New("failed to render prompt")
|
||||||
ErrLLMGenerate = errors.New("failed to generate output")
|
ErrLLMGenerate = errors.New("failed to generate output")
|
||||||
@@ -90,11 +92,15 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
}
|
}
|
||||||
|
|
||||||
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: prepared.Messages},
|
Prompt: domain.RenderedPrompt{SessionID: prepared.SessionID, Messages: prepared.Messages},
|
||||||
Target: prepared.EffectiveModelParams,
|
Target: prepared.EffectiveModelParams,
|
||||||
|
TargetPresence: prepared.TargetPresence,
|
||||||
StructuredOutput: prepared.StructuredOutput,
|
StructuredOutput: prepared.StructuredOutput,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if errors.Is(err, llm.ErrInvalidRequest) {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err)
|
return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -167,11 +173,11 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
|||||||
|
|
||||||
def, err := r.promptDefs.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
|
def, err := r.promptDefs.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||||
}
|
}
|
||||||
promptDefinitionHash, err := hashPromptDefinition(def)
|
promptDefinitionHash, err := hashPromptDefinition(def)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrProfileLoad, err)
|
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
selectedProfileID := strings.TrimSpace(req.ProfileID)
|
selectedProfileID := strings.TrimSpace(req.ProfileID)
|
||||||
@@ -187,14 +193,18 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
|||||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
effectiveModel := resolveExecutionTarget(execProfile, req.Execution)
|
effectiveModel, targetPresence, err := resolveExecutionTarget(execProfile, req.Execution)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
effectiveModel.APIKey = req.APIKey
|
||||||
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
|
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
|
||||||
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(effectiveModel.Model) == "" {
|
if strings.TrimSpace(effectiveModel.Model) == "" {
|
||||||
return nil, fmt.Errorf("%w: execution model is required", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: execution model is required", ErrInvalidRequest)
|
||||||
}
|
}
|
||||||
if err := validateAPIKeyEnv(effectiveModel.APIKeyEnv); err != nil {
|
if err := validateAPIKey(effectiveModel.APIKeyEnv, effectiveModel.APIKey, effectiveModel.APIKeyRequired); err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -230,9 +240,11 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
|||||||
PromptHash: promptDefinitionHash,
|
PromptHash: promptDefinitionHash,
|
||||||
SelectedProfileID: selectedProfileID,
|
SelectedProfileID: selectedProfileID,
|
||||||
EffectiveModelParams: effectiveModel,
|
EffectiveModelParams: effectiveModel,
|
||||||
|
TargetPresence: targetPresence,
|
||||||
OutputContract: effectiveContract,
|
OutputContract: effectiveContract,
|
||||||
StructuredOutput: structuredOutput,
|
StructuredOutput: structuredOutput,
|
||||||
InputHashes: inputHashes,
|
InputHashes: inputHashes,
|
||||||
|
SessionID: renderedPrompt.SessionID,
|
||||||
RenderedPromptHash: hashRenderedPrompt(*renderedPrompt),
|
RenderedPromptHash: hashRenderedPrompt(*renderedPrompt),
|
||||||
Messages: renderedPrompt.Messages,
|
Messages: renderedPrompt.Messages,
|
||||||
StartTime: start,
|
StartTime: start,
|
||||||
@@ -353,28 +365,90 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution
|
|||||||
if strings.TrimSpace(override.APIKeyEnv) != "" {
|
if strings.TrimSpace(override.APIKeyEnv) != "" {
|
||||||
out.APIKeyEnv = override.APIKeyEnv
|
out.APIKeyEnv = override.APIKeyEnv
|
||||||
}
|
}
|
||||||
|
if override.APIKeyRequired {
|
||||||
|
out.APIKeyRequired = true
|
||||||
|
}
|
||||||
if len(override.ExtraParams) > 0 {
|
if len(override.ExtraParams) > 0 {
|
||||||
cp := make(map[string]string, len(override.ExtraParams))
|
out.ExtraParams = copyExtraParams(override.ExtraParams)
|
||||||
for k, v := range override.ExtraParams {
|
|
||||||
cp[k] = v
|
|
||||||
}
|
|
||||||
out.ExtraParams = cp
|
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTarget) domain.ExecutionTarget {
|
func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
|
||||||
|
out := base
|
||||||
|
var presence domain.ExecutionTargetPresence
|
||||||
|
if override.Endpoint != "" {
|
||||||
|
out.Endpoint = override.Endpoint
|
||||||
|
}
|
||||||
|
if override.Model != "" {
|
||||||
|
out.Model = override.Model
|
||||||
|
}
|
||||||
|
if override.Temperature != nil {
|
||||||
|
if *override.Temperature < 0 || *override.Temperature > 2 {
|
||||||
|
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("temperature must be between 0 and 2")
|
||||||
|
}
|
||||||
|
out.Temperature = *override.Temperature
|
||||||
|
presence.Temperature = true
|
||||||
|
}
|
||||||
|
if override.MaxTokens != nil {
|
||||||
|
if *override.MaxTokens < 0 {
|
||||||
|
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("max_tokens must be greater than or equal to 0")
|
||||||
|
}
|
||||||
|
out.MaxTokens = *override.MaxTokens
|
||||||
|
presence.MaxTokens = true
|
||||||
|
}
|
||||||
|
if override.TopP != nil {
|
||||||
|
if *override.TopP < 0 || *override.TopP > 1 {
|
||||||
|
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("top_p must be between 0 and 1")
|
||||||
|
}
|
||||||
|
out.TopP = *override.TopP
|
||||||
|
presence.TopP = true
|
||||||
|
}
|
||||||
|
if override.TimeoutSeconds != nil {
|
||||||
|
if *override.TimeoutSeconds < 0 {
|
||||||
|
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("timeout_seconds must be greater than or equal to 0")
|
||||||
|
}
|
||||||
|
out.TimeoutSeconds = *override.TimeoutSeconds
|
||||||
|
presence.TimeoutSeconds = true
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(override.ServiceTier) != "" {
|
||||||
|
out.ServiceTier = override.ServiceTier
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(override.ReasoningEffort) != "" {
|
||||||
|
out.ReasoningEffort = override.ReasoningEffort
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(override.APIKeyEnv) != "" {
|
||||||
|
out.APIKeyEnv = override.APIKeyEnv
|
||||||
|
}
|
||||||
|
if len(override.ExtraParams) > 0 {
|
||||||
|
out.ExtraParams = copyExtraParams(override.ExtraParams)
|
||||||
|
}
|
||||||
|
return out, presence, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
|
||||||
out := defaults.ExecutionTargetDefault()
|
out := defaults.ExecutionTargetDefault()
|
||||||
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
|
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
|
||||||
|
var presence domain.ExecutionTargetPresence
|
||||||
if override != nil {
|
if override != nil {
|
||||||
out = mergeExecutionTarget(out, *override)
|
var err error
|
||||||
|
out, presence, err = mergeExecutionTargetOverride(out, *override)
|
||||||
|
if err != nil {
|
||||||
|
return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return out
|
return out, presence, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateAPIKeyEnv(apiKeyEnv string) error {
|
func validateAPIKey(apiKeyEnv string, apiKey string, apiKeyRequired bool) error {
|
||||||
|
if strings.TrimSpace(apiKey) != "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
envName := strings.TrimSpace(apiKeyEnv)
|
envName := strings.TrimSpace(apiKeyEnv)
|
||||||
if envName == "" {
|
if envName == "" {
|
||||||
|
if apiKeyRequired {
|
||||||
|
return ErrAPIKeyRequired
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(os.Getenv(envName)) == "" {
|
if strings.TrimSpace(os.Getenv(envName)) == "" {
|
||||||
@@ -387,13 +461,6 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
|
|||||||
if p == nil {
|
if p == nil {
|
||||||
return domain.ExecutionTarget{}
|
return domain.ExecutionTarget{}
|
||||||
}
|
}
|
||||||
cp := map[string]string(nil)
|
|
||||||
if len(p.ExtraParams) > 0 {
|
|
||||||
cp = make(map[string]string, len(p.ExtraParams))
|
|
||||||
for k, v := range p.ExtraParams {
|
|
||||||
cp[k] = v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return domain.ExecutionTarget{
|
return domain.ExecutionTarget{
|
||||||
Endpoint: p.Endpoint,
|
Endpoint: p.Endpoint,
|
||||||
Model: p.Model,
|
Model: p.Model,
|
||||||
@@ -404,10 +471,22 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
|
|||||||
ServiceTier: p.ServiceTier,
|
ServiceTier: p.ServiceTier,
|
||||||
ReasoningEffort: p.ReasoningEffort,
|
ReasoningEffort: p.ReasoningEffort,
|
||||||
APIKeyEnv: p.APIKeyEnv,
|
APIKeyEnv: p.APIKeyEnv,
|
||||||
ExtraParams: cp,
|
APIKeyRequired: p.APIKeyRequired,
|
||||||
|
ExtraParams: copyExtraParams(p.ExtraParams),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func copyExtraParams(src map[string]any) map[string]any {
|
||||||
|
if len(src) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cp := make(map[string]any, len(src))
|
||||||
|
for k, v := range src {
|
||||||
|
cp[k] = v
|
||||||
|
}
|
||||||
|
return cp
|
||||||
|
}
|
||||||
|
|
||||||
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract {
|
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract {
|
||||||
contract := def.Validation
|
contract := def.Validation
|
||||||
if contract.Format == "" {
|
if contract.Format == "" {
|
||||||
@@ -424,10 +503,23 @@ func resolveOutputContract(def *domain.PromptDefinition, override *domain.Output
|
|||||||
|
|
||||||
func hashRenderedPrompt(p domain.RenderedPrompt) string {
|
func hashRenderedPrompt(p domain.RenderedPrompt) string {
|
||||||
var b strings.Builder
|
var b strings.Builder
|
||||||
|
if p.SessionID != "" {
|
||||||
|
b.WriteString("session_id=")
|
||||||
|
b.WriteString(p.SessionID)
|
||||||
|
b.WriteString("\n---\n")
|
||||||
|
}
|
||||||
for _, msg := range p.Messages {
|
for _, msg := range p.Messages {
|
||||||
b.WriteString(msg.Role)
|
b.WriteString(msg.Role)
|
||||||
b.WriteByte('\n')
|
b.WriteByte('\n')
|
||||||
b.WriteString(msg.Content)
|
b.WriteString(msg.Content)
|
||||||
|
if msg.CacheControl != nil {
|
||||||
|
b.WriteString("\ncache_control.type=")
|
||||||
|
b.WriteString(string(msg.CacheControl.Type))
|
||||||
|
if msg.CacheControl.TTL != "" {
|
||||||
|
b.WriteString("\ncache_control.ttl=")
|
||||||
|
b.WriteString(msg.CacheControl.TTL)
|
||||||
|
}
|
||||||
|
}
|
||||||
b.WriteString("\n---\n")
|
b.WriteString("\n---\n")
|
||||||
}
|
}
|
||||||
h := sha256.Sum256([]byte(b.String()))
|
h := sha256.Sum256([]byte(b.String()))
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||||
@@ -160,7 +161,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
|
|||||||
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
|
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
|
||||||
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
|
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
|
||||||
}}
|
}}
|
||||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||||
llmClient := &fakeLLM{forbid: true}
|
llmClient := &fakeLLM{forbid: true}
|
||||||
|
|
||||||
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
||||||
@@ -172,7 +173,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
|
|||||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
||||||
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
|
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
|
||||||
},
|
},
|
||||||
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
|
Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -198,6 +199,9 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
|
|||||||
if len(prepared.Messages) != 2 {
|
if len(prepared.Messages) != 2 {
|
||||||
t.Fatalf("expected two messages, got %d", len(prepared.Messages))
|
t.Fatalf("expected two messages, got %d", len(prepared.Messages))
|
||||||
}
|
}
|
||||||
|
if prepared.SessionID != "session-123" {
|
||||||
|
t.Fatalf("expected prepared session id, got %q", prepared.SessionID)
|
||||||
|
}
|
||||||
if llmClient.calls != 0 {
|
if llmClient.calls != 0 {
|
||||||
t.Fatalf("prepare should not call llm, calls=%d", llmClient.calls)
|
t.Fatalf("prepare should not call llm, calls=%d", llmClient.calls)
|
||||||
}
|
}
|
||||||
@@ -249,6 +253,17 @@ func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerPreparePromptLoadFailure(t *testing.T) {
|
||||||
|
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
||||||
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p"})
|
||||||
|
if !errors.Is(err, ErrPromptLoad) {
|
||||||
|
t.Fatalf("expected ErrPromptLoad, got %v", err)
|
||||||
|
}
|
||||||
|
if errors.Is(err, ErrProfileLoad) {
|
||||||
|
t.Fatalf("did not expect ErrProfileLoad, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
|
func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
|
||||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
@@ -269,11 +284,11 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
|
|||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Inputs: singleInputRef(),
|
Inputs: singleInputRef(),
|
||||||
Execution: &domain.ExecutionTarget{
|
Execution: &domain.ExecutionTargetOverride{
|
||||||
Endpoint: "http://override/v1",
|
Endpoint: "http://override/v1",
|
||||||
Model: "override-model",
|
Model: "override-model",
|
||||||
Temperature: 0.7,
|
Temperature: float64Ptr(0.7),
|
||||||
TimeoutSeconds: 30,
|
TimeoutSeconds: intPtr(30),
|
||||||
ServiceTier: "flex",
|
ServiceTier: "flex",
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -291,6 +306,143 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerPrepareRequestNumericOverridePresence(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
override *domain.ExecutionTargetOverride
|
||||||
|
wantTemperature float64
|
||||||
|
wantMaxTokens int
|
||||||
|
wantTopP float64
|
||||||
|
wantTimeoutSecs int
|
||||||
|
wantPresence domain.ExecutionTargetPresence
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "omitted preserves profile values",
|
||||||
|
override: &domain.ExecutionTargetOverride{},
|
||||||
|
wantTemperature: 0.7,
|
||||||
|
wantMaxTokens: 321,
|
||||||
|
wantTopP: 0.8,
|
||||||
|
wantTimeoutSecs: 45,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "explicit zero temperature",
|
||||||
|
override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(0)},
|
||||||
|
wantTemperature: 0,
|
||||||
|
wantMaxTokens: 321,
|
||||||
|
wantTopP: 0.8,
|
||||||
|
wantTimeoutSecs: 45,
|
||||||
|
wantPresence: domain.ExecutionTargetPresence{Temperature: true},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "explicit zero max tokens",
|
||||||
|
override: &domain.ExecutionTargetOverride{MaxTokens: intPtr(0)},
|
||||||
|
wantTemperature: 0.7,
|
||||||
|
wantMaxTokens: 0,
|
||||||
|
wantTopP: 0.8,
|
||||||
|
wantTimeoutSecs: 45,
|
||||||
|
wantPresence: domain.ExecutionTargetPresence{MaxTokens: true},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "explicit zero top p",
|
||||||
|
override: &domain.ExecutionTargetOverride{TopP: float64Ptr(0)},
|
||||||
|
wantTemperature: 0.7,
|
||||||
|
wantMaxTokens: 321,
|
||||||
|
wantTopP: 0,
|
||||||
|
wantTimeoutSecs: 45,
|
||||||
|
wantPresence: domain.ExecutionTargetPresence{TopP: true},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "explicit zero timeout",
|
||||||
|
override: &domain.ExecutionTargetOverride{TimeoutSeconds: intPtr(0)},
|
||||||
|
wantTemperature: 0.7,
|
||||||
|
wantMaxTokens: 321,
|
||||||
|
wantTopP: 0.8,
|
||||||
|
wantTimeoutSecs: 0,
|
||||||
|
wantPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"exec": {
|
||||||
|
ID: "exec",
|
||||||
|
Endpoint: "http://profile/v1",
|
||||||
|
Model: "profile-model",
|
||||||
|
Temperature: 0.7,
|
||||||
|
MaxTokens: 321,
|
||||||
|
TopP: 0.8,
|
||||||
|
TimeoutSeconds: 45,
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
&fakeLLM{forbid: true},
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
|
||||||
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
Execution: tc.override,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
got := prepared.EffectiveModelParams
|
||||||
|
if got.Temperature != tc.wantTemperature ||
|
||||||
|
got.MaxTokens != tc.wantMaxTokens ||
|
||||||
|
got.TopP != tc.wantTopP ||
|
||||||
|
got.TimeoutSeconds != tc.wantTimeoutSecs {
|
||||||
|
t.Fatalf("unexpected effective numeric settings: %+v", got)
|
||||||
|
}
|
||||||
|
if prepared.TargetPresence != tc.wantPresence {
|
||||||
|
t.Fatalf("unexpected target presence: got %+v want %+v", prepared.TargetPresence, tc.wantPresence)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPrepareInvalidRequestNumericOverridesFail(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
override *domain.ExecutionTargetOverride
|
||||||
|
}{
|
||||||
|
{name: "temperature below range", override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(-0.1)}},
|
||||||
|
{name: "temperature above range", override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(2.1)}},
|
||||||
|
{name: "max tokens below range", override: &domain.ExecutionTargetOverride{MaxTokens: intPtr(-1)}},
|
||||||
|
{name: "top p below range", override: &domain.ExecutionTargetOverride{TopP: float64Ptr(-0.1)}},
|
||||||
|
{name: "top p above range", override: &domain.ExecutionTargetOverride{TopP: float64Ptr(1.1)}},
|
||||||
|
{name: "timeout below range", override: &domain.ExecutionTargetOverride{TimeoutSeconds: intPtr(-1)}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
&fakeLLM{forbid: true},
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
|
||||||
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
Execution: tc.override,
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
|
func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
|
||||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
@@ -577,6 +729,100 @@ func TestDeriveStructuredSchemaName(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHashRenderedPromptIncludesCacheControlWhenPresent(t *testing.T) {
|
||||||
|
uncached := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
|
{Role: "system", Content: "sys"},
|
||||||
|
{Role: "user", Content: "usr"},
|
||||||
|
}}
|
||||||
|
wantLegacyHash := hashString("system\nsys\n---\nuser\nusr\n---\n")
|
||||||
|
if got := hashRenderedPrompt(uncached); got != wantLegacyHash {
|
||||||
|
t.Fatalf("expected no-cache hash to preserve legacy input, got %q want %q", got, wantLegacyHash)
|
||||||
|
}
|
||||||
|
|
||||||
|
withCache := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "sys",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
TTL: "1h",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{Role: "user", Content: "usr"},
|
||||||
|
}}
|
||||||
|
alsoWithCache := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "sys",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
TTL: "1h",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{Role: "user", Content: "usr"},
|
||||||
|
}}
|
||||||
|
withoutTTL := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "sys",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{Role: "user", Content: "usr"},
|
||||||
|
}}
|
||||||
|
|
||||||
|
cachedHash := hashRenderedPrompt(withCache)
|
||||||
|
if cachedHash == hashRenderedPrompt(uncached) {
|
||||||
|
t.Fatal("expected cache control to change rendered prompt hash")
|
||||||
|
}
|
||||||
|
if cachedHash != hashRenderedPrompt(alsoWithCache) {
|
||||||
|
t.Fatal("expected identical cache control metadata to produce stable hash")
|
||||||
|
}
|
||||||
|
if cachedHash == hashRenderedPrompt(withoutTTL) {
|
||||||
|
t.Fatal("expected ttl changes to affect rendered prompt hash")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHashRenderedPromptIncludesSessionIDWhenPresent(t *testing.T) {
|
||||||
|
withoutSession := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
|
{Role: "system", Content: "sys"},
|
||||||
|
{Role: "user", Content: "usr"},
|
||||||
|
}}
|
||||||
|
withSession := domain.RenderedPrompt{
|
||||||
|
SessionID: "session-123",
|
||||||
|
Messages: []domain.RenderedMessage{
|
||||||
|
{Role: "system", Content: "sys"},
|
||||||
|
{Role: "user", Content: "usr"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
alsoWithSession := domain.RenderedPrompt{
|
||||||
|
SessionID: "session-123",
|
||||||
|
Messages: []domain.RenderedMessage{
|
||||||
|
{Role: "system", Content: "sys"},
|
||||||
|
{Role: "user", Content: "usr"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
otherSession := domain.RenderedPrompt{
|
||||||
|
SessionID: "session-456",
|
||||||
|
Messages: []domain.RenderedMessage{
|
||||||
|
{Role: "system", Content: "sys"},
|
||||||
|
{Role: "user", Content: "usr"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionHash := hashRenderedPrompt(withSession)
|
||||||
|
if sessionHash == hashRenderedPrompt(withoutSession) {
|
||||||
|
t.Fatal("expected session_id to change rendered prompt hash")
|
||||||
|
}
|
||||||
|
if sessionHash != hashRenderedPrompt(alsoWithSession) {
|
||||||
|
t.Fatal("expected identical session_id to produce stable hash")
|
||||||
|
}
|
||||||
|
if sessionHash == hashRenderedPrompt(otherSession) {
|
||||||
|
t.Fatal("expected session_id value changes to affect rendered prompt hash")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerRunSuccessful(t *testing.T) {
|
func TestRunnerRunSuccessful(t *testing.T) {
|
||||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
|
||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}
|
||||||
@@ -584,7 +830,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
|||||||
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
|
"a://t": {Body: []byte("transcript"), Hash: hashString("transcript")},
|
||||||
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
|
"a://g": {Body: []byte("glossary"), Hash: hashString("glossary")},
|
||||||
}}
|
}}
|
||||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
|
||||||
|
|
||||||
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
||||||
@@ -596,7 +842,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
|||||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
||||||
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
|
"glossary": {Type: domain.ArtifactRefFile, URI: "a://g"},
|
||||||
},
|
},
|
||||||
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
|
Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -631,6 +877,48 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
|||||||
if llmClient.lastReq.Target.TimeoutSeconds != 90 {
|
if llmClient.lastReq.Target.TimeoutSeconds != 90 {
|
||||||
t.Fatalf("expected timeout propagation, got %d", llmClient.lastReq.Target.TimeoutSeconds)
|
t.Fatalf("expected timeout propagation, got %d", llmClient.lastReq.Target.TimeoutSeconds)
|
||||||
}
|
}
|
||||||
|
if !llmClient.lastReq.TargetPresence.Temperature || !llmClient.lastReq.TargetPresence.TimeoutSeconds {
|
||||||
|
t.Fatalf("expected numeric override presence to be sent to llm, got %+v", llmClient.lastReq.TargetPresence)
|
||||||
|
}
|
||||||
|
if llmClient.lastReq.Prompt.SessionID != "session-123" {
|
||||||
|
t.Fatalf("expected session id to be sent to llm, got %q", llmClient.lastReq.Prompt.SessionID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunPassesExtraParamsToGenerateRequestTarget(t *testing.T) {
|
||||||
|
extraParams := map[string]any{
|
||||||
|
"string_value": "enabled",
|
||||||
|
"number_value": 42,
|
||||||
|
"boolean_value": true,
|
||||||
|
"object_value": map[string]any{"nested": "value"},
|
||||||
|
"array_value": []any{"first", 3, false},
|
||||||
|
}
|
||||||
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||||
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"exec": {
|
||||||
|
ID: "exec",
|
||||||
|
Endpoint: "http://profile/v1",
|
||||||
|
Model: "profile-model",
|
||||||
|
ExtraParams: extraParams,
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
|
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||||
|
|
||||||
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(res.EffectiveModelParams.ExtraParams, extraParams) {
|
||||||
|
t.Fatalf("expected run result extra_params to match profile values, got %#v", res.EffectiveModelParams.ExtraParams)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(llmClient.lastReq.Target.ExtraParams, extraParams) {
|
||||||
|
t.Fatalf("expected generate request extra_params to match profile values, got %#v", llmClient.lastReq.Target.ExtraParams)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T) {
|
func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T) {
|
||||||
@@ -649,7 +937,7 @@ func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T)
|
|||||||
Inputs: map[string]domain.ArtifactRef{
|
Inputs: map[string]domain.ArtifactRef{
|
||||||
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
"transcript": {Type: domain.ArtifactRefFile, URI: "a://t"},
|
||||||
},
|
},
|
||||||
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90},
|
Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)},
|
||||||
}
|
}
|
||||||
|
|
||||||
prepared, err := runner.Prepare(context.Background(), req)
|
prepared, err := runner.Prepare(context.Background(), req)
|
||||||
@@ -777,11 +1065,11 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T)
|
|||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Inputs: singleInputRef(),
|
Inputs: singleInputRef(),
|
||||||
Execution: &domain.ExecutionTarget{
|
Execution: &domain.ExecutionTargetOverride{
|
||||||
Endpoint: "http://override/v1",
|
Endpoint: "http://override/v1",
|
||||||
Model: "override-model",
|
Model: "override-model",
|
||||||
Temperature: 0.7,
|
Temperature: float64Ptr(0.7),
|
||||||
TimeoutSeconds: 30,
|
TimeoutSeconds: intPtr(30),
|
||||||
ServiceTier: "flex",
|
ServiceTier: "flex",
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -902,6 +1190,75 @@ func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
|
||||||
|
const directKey = "direct-runner-key"
|
||||||
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||||
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "SCRIPTORIUM_MISSING_KEY"},
|
||||||
|
}}
|
||||||
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
|
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||||
|
|
||||||
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
APIKey: directKey,
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if llmClient.lastReq.Target.APIKey != directKey {
|
||||||
|
t.Fatalf("expected direct API key to reach LLM request")
|
||||||
|
}
|
||||||
|
if llmClient.lastReq.Target.APIKeyEnv != "SCRIPTORIUM_MISSING_KEY" {
|
||||||
|
t.Fatalf("expected api_key_env name to remain on target, got %q", llmClient.lastReq.Target.APIKeyEnv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPrepareAPIKeyRequiredFailsWithoutDirectKey(t *testing.T) {
|
||||||
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||||
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
|
||||||
|
}}
|
||||||
|
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||||
|
|
||||||
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrAPIKeyRequired) {
|
||||||
|
t.Fatalf("expected ErrAPIKeyRequired, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunAPIKeyRequiredSucceedsWithDirectKey(t *testing.T) {
|
||||||
|
const directKey = "direct-required-key"
|
||||||
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||||
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
|
||||||
|
}}
|
||||||
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
|
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||||
|
|
||||||
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
APIKey: directKey,
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if llmClient.lastReq.Target.APIKey != directKey {
|
||||||
|
t.Fatalf("expected direct API key to reach LLM request")
|
||||||
|
}
|
||||||
|
if !llmClient.lastReq.Target.APIKeyRequired {
|
||||||
|
t.Fatalf("expected APIKeyRequired to be carried to target")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
|
func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
|
||||||
const envName = "SCRIPTORIUM_RUNTIME_API_KEY"
|
const envName = "SCRIPTORIUM_RUNTIME_API_KEY"
|
||||||
t.Setenv(envName, "runtime-secret")
|
t.Setenv(envName, "runtime-secret")
|
||||||
@@ -916,7 +1273,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
|
|||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Inputs: singleInputRef(),
|
Inputs: singleInputRef(),
|
||||||
Execution: &domain.ExecutionTarget{APIKeyEnv: envName},
|
Execution: &domain.ExecutionTargetOverride{APIKeyEnv: envName},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -941,7 +1298,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) {
|
|||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Inputs: singleInputRef(),
|
Inputs: singleInputRef(),
|
||||||
Execution: &domain.ExecutionTarget{APIKeyEnv: runtimeEnv},
|
Execution: &domain.ExecutionTargetOverride{APIKeyEnv: runtimeEnv},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -977,8 +1334,11 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
|
|||||||
func TestRunnerRunPromptLoadFailure(t *testing.T) {
|
func TestRunnerRunPromptLoadFailure(t *testing.T) {
|
||||||
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
|
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
|
||||||
if !errors.Is(err, ErrProfileLoad) {
|
if !errors.Is(err, ErrPromptLoad) {
|
||||||
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
t.Fatalf("expected ErrPromptLoad, got %v", err)
|
||||||
|
}
|
||||||
|
if errors.Is(err, ErrProfileLoad) {
|
||||||
|
t.Fatalf("did not expect ErrProfileLoad, got %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1040,6 +1400,28 @@ func TestRunnerRunLLMFailure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
&fakeLLM{err: llm.ErrInvalidRequest},
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
|
}
|
||||||
|
if errors.Is(err, ErrLLMGenerate) {
|
||||||
|
t.Fatalf("did not expect ErrLLMGenerate, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerRunValidationStillWorks(t *testing.T) {
|
func TestRunnerRunValidationStillWorks(t *testing.T) {
|
||||||
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
|
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
@@ -1082,7 +1464,7 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
|
|||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Inputs: singleInputRef(),
|
Inputs: singleInputRef(),
|
||||||
Execution: &domain.ExecutionTarget{Endpoint: "http://override/v1", Model: "override-model", TimeoutSeconds: 22},
|
Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "override-model", TimeoutSeconds: intPtr(22)},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -1169,7 +1551,8 @@ func TestExecutionProfileToTargetPopulatesAllFieldsAndCopiesExtraParams(t *testi
|
|||||||
ServiceTier: "priority",
|
ServiceTier: "priority",
|
||||||
ReasoningEffort: "medium",
|
ReasoningEffort: "medium",
|
||||||
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
||||||
ExtraParams: map[string]string{
|
APIKeyRequired: true,
|
||||||
|
ExtraParams: map[string]any{
|
||||||
"provider_option": "on",
|
"provider_option": "on",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -1183,7 +1566,8 @@ func TestExecutionProfileToTargetPopulatesAllFieldsAndCopiesExtraParams(t *testi
|
|||||||
target.TimeoutSeconds != src.TimeoutSeconds ||
|
target.TimeoutSeconds != src.TimeoutSeconds ||
|
||||||
target.ServiceTier != src.ServiceTier ||
|
target.ServiceTier != src.ServiceTier ||
|
||||||
target.ReasoningEffort != src.ReasoningEffort ||
|
target.ReasoningEffort != src.ReasoningEffort ||
|
||||||
target.APIKeyEnv != src.APIKeyEnv {
|
target.APIKeyEnv != src.APIKeyEnv ||
|
||||||
|
target.APIKeyRequired != src.APIKeyRequired {
|
||||||
t.Fatalf("expected all profile fields to populate target, got %+v", target)
|
t.Fatalf("expected all profile fields to populate target, got %+v", target)
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(target.ExtraParams, src.ExtraParams) {
|
if !reflect.DeepEqual(target.ExtraParams, src.ExtraParams) {
|
||||||
@@ -1208,12 +1592,19 @@ func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testin
|
|||||||
ServiceTier: "priority",
|
ServiceTier: "priority",
|
||||||
ReasoningEffort: "low",
|
ReasoningEffort: "low",
|
||||||
APIKeyEnv: "PROFILE_KEY",
|
APIKeyEnv: "PROFILE_KEY",
|
||||||
ExtraParams: map[string]string{
|
APIKeyRequired: true,
|
||||||
|
ExtraParams: map[string]any{
|
||||||
"profile_option": "enabled",
|
"profile_option": "enabled",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
target := resolveExecutionTarget(profileValue, nil)
|
target, presence, err := resolveExecutionTarget(profileValue, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if presence != (domain.ExecutionTargetPresence{}) {
|
||||||
|
t.Fatalf("expected no request override presence, got %+v", presence)
|
||||||
|
}
|
||||||
if target.Endpoint != profileValue.Endpoint ||
|
if target.Endpoint != profileValue.Endpoint ||
|
||||||
target.Model != profileValue.Model ||
|
target.Model != profileValue.Model ||
|
||||||
target.Temperature != profileValue.Temperature ||
|
target.Temperature != profileValue.Temperature ||
|
||||||
@@ -1222,7 +1613,8 @@ func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testin
|
|||||||
target.TimeoutSeconds != profileValue.TimeoutSeconds ||
|
target.TimeoutSeconds != profileValue.TimeoutSeconds ||
|
||||||
target.ServiceTier != profileValue.ServiceTier ||
|
target.ServiceTier != profileValue.ServiceTier ||
|
||||||
target.ReasoningEffort != profileValue.ReasoningEffort ||
|
target.ReasoningEffort != profileValue.ReasoningEffort ||
|
||||||
target.APIKeyEnv != profileValue.APIKeyEnv {
|
target.APIKeyEnv != profileValue.APIKeyEnv ||
|
||||||
|
target.APIKeyRequired != profileValue.APIKeyRequired {
|
||||||
t.Fatalf("expected profile values to populate target, got %+v", target)
|
t.Fatalf("expected profile values to populate target, got %+v", target)
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(target.ExtraParams, profileValue.ExtraParams) {
|
if !reflect.DeepEqual(target.ExtraParams, profileValue.ExtraParams) {
|
||||||
@@ -1242,32 +1634,38 @@ func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFiel
|
|||||||
ServiceTier: "priority",
|
ServiceTier: "priority",
|
||||||
ReasoningEffort: "medium",
|
ReasoningEffort: "medium",
|
||||||
APIKeyEnv: "PROFILE_KEY",
|
APIKeyEnv: "PROFILE_KEY",
|
||||||
ExtraParams: map[string]string{
|
ExtraParams: map[string]any{
|
||||||
"profile_only": "yes",
|
"profile_only": "yes",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
override := &domain.ExecutionTarget{
|
override := &domain.ExecutionTargetOverride{
|
||||||
Endpoint: "http://override/v1",
|
Endpoint: "http://override/v1",
|
||||||
Model: "override-model",
|
Model: "override-model",
|
||||||
Temperature: 0.9,
|
Temperature: float64Ptr(0.9),
|
||||||
MaxTokens: 111,
|
MaxTokens: intPtr(111),
|
||||||
TopP: 0.5,
|
TopP: float64Ptr(0.5),
|
||||||
TimeoutSeconds: 30,
|
TimeoutSeconds: intPtr(30),
|
||||||
ServiceTier: "flex",
|
ServiceTier: "flex",
|
||||||
ReasoningEffort: "high",
|
ReasoningEffort: "high",
|
||||||
APIKeyEnv: "RUNTIME_KEY",
|
APIKeyEnv: "RUNTIME_KEY",
|
||||||
ExtraParams: map[string]string{
|
ExtraParams: map[string]any{
|
||||||
"runtime_only": "yes",
|
"runtime_only": "yes",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
target := resolveExecutionTarget(profileValue, override)
|
target, presence, err := resolveExecutionTarget(profileValue, override)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if presence != (domain.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}) {
|
||||||
|
t.Fatalf("unexpected override presence: %+v", presence)
|
||||||
|
}
|
||||||
if target.Endpoint != override.Endpoint ||
|
if target.Endpoint != override.Endpoint ||
|
||||||
target.Model != override.Model ||
|
target.Model != override.Model ||
|
||||||
target.Temperature != override.Temperature ||
|
target.Temperature != *override.Temperature ||
|
||||||
target.MaxTokens != override.MaxTokens ||
|
target.MaxTokens != *override.MaxTokens ||
|
||||||
target.TopP != override.TopP ||
|
target.TopP != *override.TopP ||
|
||||||
target.TimeoutSeconds != override.TimeoutSeconds ||
|
target.TimeoutSeconds != *override.TimeoutSeconds ||
|
||||||
target.ServiceTier != override.ServiceTier ||
|
target.ServiceTier != override.ServiceTier ||
|
||||||
target.ReasoningEffort != override.ReasoningEffort ||
|
target.ReasoningEffort != override.ReasoningEffort ||
|
||||||
target.APIKeyEnv != override.APIKeyEnv {
|
target.APIKeyEnv != override.APIKeyEnv {
|
||||||
@@ -1311,12 +1709,12 @@ func TestMergeExecutionTargetEmptyStringOverridesDoNotErase(t *testing.T) {
|
|||||||
|
|
||||||
func TestMergeExecutionTargetEmptyExtraParamsDoesNotErase(t *testing.T) {
|
func TestMergeExecutionTargetEmptyExtraParamsDoesNotErase(t *testing.T) {
|
||||||
base := domain.ExecutionTarget{
|
base := domain.ExecutionTarget{
|
||||||
ExtraParams: map[string]string{
|
ExtraParams: map[string]any{
|
||||||
"keep": "value",
|
"keep": "value",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
override := domain.ExecutionTarget{
|
override := domain.ExecutionTarget{
|
||||||
ExtraParams: map[string]string{},
|
ExtraParams: map[string]any{},
|
||||||
}
|
}
|
||||||
|
|
||||||
merged := mergeExecutionTarget(base, override)
|
merged := mergeExecutionTarget(base, override)
|
||||||
@@ -1392,6 +1790,14 @@ func singleInputRef() map[string]domain.ArtifactRef {
|
|||||||
return map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}}
|
return map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func float64Ptr(v float64) *float64 {
|
||||||
|
return &v
|
||||||
|
}
|
||||||
|
|
||||||
|
func intPtr(v int) *int {
|
||||||
|
return &v
|
||||||
|
}
|
||||||
|
|
||||||
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
|
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
|
||||||
return NewRunner(
|
return NewRunner(
|
||||||
promptRepo,
|
promptRepo,
|
||||||
|
|||||||
@@ -5,11 +5,14 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/filecatalog"
|
||||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -18,11 +21,30 @@ type StandardValidator struct {
|
|||||||
schemaBaseDir string
|
schemaBaseDir string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type FSValidator struct {
|
||||||
|
fsys fs.FS
|
||||||
|
root string
|
||||||
|
}
|
||||||
|
|
||||||
func NewStandardValidator(schemaBaseDir string) Validator {
|
func NewStandardValidator(schemaBaseDir string) Validator {
|
||||||
return &StandardValidator{schemaBaseDir: schemaBaseDir}
|
return &StandardValidator{schemaBaseDir: schemaBaseDir}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func NewFSValidator(fsys fs.FS, root string) Validator {
|
||||||
|
return &FSValidator{fsys: fsys, root: root}
|
||||||
|
}
|
||||||
|
|
||||||
func (v *StandardValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
|
func (v *StandardValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
|
||||||
|
return validateArtifact(ctx, artifact, contract, v.validateJSONSchema)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *FSValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
|
||||||
|
return validateArtifact(ctx, artifact, contract, v.validateJSONSchema)
|
||||||
|
}
|
||||||
|
|
||||||
|
type schemaValidatorFunc func(instance any, schemaPath string) ([]string, error)
|
||||||
|
|
||||||
|
func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, validateSchema schemaValidatorFunc) (domain.ValidationResult, error) {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return domain.ValidationResult{}, ctx.Err()
|
return domain.ValidationResult{}, ctx.Err()
|
||||||
@@ -74,21 +96,14 @@ func (v *StandardValidator) Validate(ctx context.Context, artifact *domain.Artif
|
|||||||
return res, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
schemaPath, err := v.resolveSchemaPath(contract.SchemaPath)
|
validationErrors, err := validateSchema(instance, contract.SchemaPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return domain.ValidationResult{}, err
|
return domain.ValidationResult{}, err
|
||||||
}
|
}
|
||||||
|
if len(validationErrors) > 0 {
|
||||||
compiler := jsonschema.NewCompiler()
|
|
||||||
schema, err := compiler.Compile(schemaPath)
|
|
||||||
if err != nil {
|
|
||||||
return domain.ValidationResult{}, fmt.Errorf("failed to compile JSON schema %q: %w", schemaPath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := schema.Validate(instance); err != nil {
|
|
||||||
res.Status = domain.ValidationFailed
|
res.Status = domain.ValidationFailed
|
||||||
res.IsValid = false
|
res.IsValid = false
|
||||||
res.Errors = []string{fmt.Sprintf("json schema validation failed: %v", err)}
|
res.Errors = validationErrors
|
||||||
return res, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -100,6 +115,46 @@ func (v *StandardValidator) Validate(ctx context.Context, artifact *domain.Artif
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (v *StandardValidator) validateJSONSchema(instance any, schemaPath string) ([]string, error) {
|
||||||
|
resolvedSchemaPath, err := v.resolveSchemaPath(schemaPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
compiler := jsonschema.NewCompiler()
|
||||||
|
schema, err := compiler.Compile(resolvedSchemaPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := schema.Validate(instance); err != nil {
|
||||||
|
return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *FSValidator) validateJSONSchema(instance any, schemaPath string) ([]string, error) {
|
||||||
|
schemaName, schemaDoc, err := v.loadSchemaDocument(schemaPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
resourceURL := fsSchemaResourceURL(schemaName)
|
||||||
|
compiler := jsonschema.NewCompiler()
|
||||||
|
if err := compiler.AddResource(resourceURL, schemaDoc); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
|
||||||
|
}
|
||||||
|
schema, err := compiler.Compile(resourceURL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := schema.Validate(instance); err != nil {
|
||||||
|
return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
func parseJSON(body []byte) (any, error) {
|
func parseJSON(body []byte) (any, error) {
|
||||||
var v any
|
var v any
|
||||||
if err := json.Unmarshal(body, &v); err != nil {
|
if err := json.Unmarshal(body, &v); err != nil {
|
||||||
@@ -132,6 +187,20 @@ func (v *StandardValidator) LoadSchemaDocument(ctx context.Context, schemaPath s
|
|||||||
return doc, nil
|
return doc, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (v *FSValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
_, doc, err := v.loadSchemaDocument(schemaPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return doc, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error) {
|
func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error) {
|
||||||
if strings.TrimSpace(schemaPath) == "" {
|
if strings.TrimSpace(schemaPath) == "" {
|
||||||
return "", errors.New("schema path is required for json_schema validation")
|
return "", errors.New("schema path is required for json_schema validation")
|
||||||
@@ -149,3 +218,75 @@ func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error)
|
|||||||
|
|
||||||
return resolved, nil
|
return resolved, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (v *FSValidator) loadSchemaDocument(schemaPath string) (string, any, error) {
|
||||||
|
resolved, err := v.resolveSchemaPath(schemaPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
raw, err := fs.ReadFile(v.fsys, resolved)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var doc any
|
||||||
|
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||||
|
return "", nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
|
||||||
|
}
|
||||||
|
return resolved, doc, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *FSValidator) resolveSchemaPath(schemaPath string) (string, error) {
|
||||||
|
if strings.TrimSpace(schemaPath) == "" {
|
||||||
|
return "", errors.New("schema path is required for json_schema validation")
|
||||||
|
}
|
||||||
|
if v.fsys == nil {
|
||||||
|
return "", errors.New("schema filesystem is nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanRoot := filecatalog.CleanFSRoot(v.root)
|
||||||
|
rootInfo, err := fs.Stat(v.fsys, cleanRoot)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to access schema source %q: %w", cleanRoot, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var resolved string
|
||||||
|
if rootInfo.IsDir() {
|
||||||
|
resolvedPath, _, err := filecatalog.ResolveFSPath(cleanRoot, cleanRoot, schemaPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
resolved = resolvedPath
|
||||||
|
} else {
|
||||||
|
cleanSchemaPath, err := cleanSchemaFSPath(schemaPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if cleanSchemaPath != path.Base(cleanRoot) {
|
||||||
|
return "", fmt.Errorf("schema path %q does not match schema file %q", cleanSchemaPath, path.Base(cleanRoot))
|
||||||
|
}
|
||||||
|
resolved = cleanRoot
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := fs.Stat(v.fsys, resolved); err != nil {
|
||||||
|
return "", fmt.Errorf("failed to access schema file %q: %w", resolved, err)
|
||||||
|
}
|
||||||
|
return resolved, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanSchemaFSPath(schemaPath string) (string, error) {
|
||||||
|
cleaned := strings.TrimSpace(schemaPath)
|
||||||
|
if cleaned == "" {
|
||||||
|
return "", errors.New("schema path is required for json_schema validation")
|
||||||
|
}
|
||||||
|
cleaned = path.Clean(cleaned)
|
||||||
|
if path.IsAbs(cleaned) {
|
||||||
|
return "", fmt.Errorf("schema path %q must be relative", schemaPath)
|
||||||
|
}
|
||||||
|
return cleaned, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func fsSchemaResourceURL(schemaName string) string {
|
||||||
|
return "scriptorium-schema:///" + strings.TrimPrefix(path.Clean(schemaName), "/")
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,7 +4,9 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
)
|
)
|
||||||
@@ -250,3 +252,132 @@ func TestStandardValidatorLoadSchemaDocumentInvalidJSON(t *testing.T) {
|
|||||||
t.Fatal("expected decode error")
|
t.Fatal("expected decode error")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFSValidatorJSONSchemaSuccess(t *testing.T) {
|
||||||
|
v := NewFSValidator(fstest.MapFS{
|
||||||
|
"schemas/events.schema.json": &fstest.MapFile{Data: []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["events"],
|
||||||
|
"properties": {
|
||||||
|
"events": {"type": "array"}
|
||||||
|
}
|
||||||
|
}`)},
|
||||||
|
}, "schemas")
|
||||||
|
|
||||||
|
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
|
||||||
|
ValidationMode: domain.ValidationJSONSchema,
|
||||||
|
SchemaPath: "events.schema.json",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if res.Status != domain.ValidationPassed || !res.IsValid {
|
||||||
|
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFSValidatorJSONSchemaPathContainment(t *testing.T) {
|
||||||
|
t.Run("nested schema inside root succeeds", func(t *testing.T) {
|
||||||
|
v := NewFSValidator(fstest.MapFS{
|
||||||
|
"schemas/nested/events.schema.json": &fstest.MapFile{Data: []byte(`{
|
||||||
|
"type": "object",
|
||||||
|
"required": ["events"],
|
||||||
|
"properties": {
|
||||||
|
"events": {"type": "array"}
|
||||||
|
}
|
||||||
|
}`)},
|
||||||
|
}, "schemas")
|
||||||
|
|
||||||
|
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
|
||||||
|
ValidationMode: domain.ValidationJSONSchema,
|
||||||
|
SchemaPath: "nested/events.schema.json",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if res.Status != domain.ValidationPassed || !res.IsValid {
|
||||||
|
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
schemaPath string
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{name: "parent escape rejected", schemaPath: "../outside.schema.json", wantErr: "escapes source root"},
|
||||||
|
{name: "absolute path rejected", schemaPath: "/outside.schema.json", wantErr: "must be relative"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
v := NewFSValidator(fstest.MapFS{
|
||||||
|
"schemas/events.schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
|
||||||
|
"outside.schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
|
||||||
|
"schemas/outside.schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
|
||||||
|
}, "schemas")
|
||||||
|
|
||||||
|
_, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
|
||||||
|
ValidationMode: domain.ValidationJSONSchema,
|
||||||
|
SchemaPath: tc.schemaPath,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected schema path error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), tc.wantErr) {
|
||||||
|
t.Fatalf("expected error to contain %q, got %v", tc.wantErr, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFSValidatorSingleSchemaFileUsesBaseName(t *testing.T) {
|
||||||
|
v := NewFSValidator(fstest.MapFS{
|
||||||
|
"events.schema.json": &fstest.MapFile{Data: []byte(`{
|
||||||
|
"type": "object",
|
||||||
|
"required": ["events"],
|
||||||
|
"properties": {
|
||||||
|
"events": {"type": "array"}
|
||||||
|
}
|
||||||
|
}`)},
|
||||||
|
}, "events.schema.json")
|
||||||
|
|
||||||
|
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
|
||||||
|
ValidationMode: domain.ValidationJSONSchema,
|
||||||
|
SchemaPath: "events.schema.json",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if res.Status != domain.ValidationPassed || !res.IsValid {
|
||||||
|
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
|
||||||
|
ValidationMode: domain.ValidationJSONSchema,
|
||||||
|
SchemaPath: "other.schema.json",
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected schema path mismatch error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFSValidatorLoadSchemaDocument(t *testing.T) {
|
||||||
|
v := NewFSValidator(fstest.MapFS{
|
||||||
|
"schemas/schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
|
||||||
|
}, "schemas")
|
||||||
|
loader, ok := v.(SchemaDocumentLoader)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("fs validator must implement SchemaDocumentLoader")
|
||||||
|
}
|
||||||
|
|
||||||
|
doc, err := loader.LoadSchemaDocument(context.Background(), "schema.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
obj, ok := doc.(map[string]any)
|
||||||
|
if !ok || obj["type"] != "object" {
|
||||||
|
t.Fatalf("unexpected schema document: %#v", doc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
218
json_copy.go
Normal file
218
json_copy.go
Normal file
@@ -0,0 +1,218 @@
|
|||||||
|
package scriptorium
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"reflect"
|
||||||
|
"strconv"
|
||||||
|
)
|
||||||
|
|
||||||
|
const maxSafeJSONInteger = 1<<53 - 1
|
||||||
|
|
||||||
|
type jsonVisit struct {
|
||||||
|
typ reflect.Type
|
||||||
|
ptr uintptr
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyPublicJSONMap(src map[string]any) (map[string]any, error) {
|
||||||
|
if src == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
copied, err := copyPublicJSONValue(reflect.ValueOf(src), "extra_params", make(map[jsonVisit]struct{}))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out, ok := copied.(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("extra_params: expected object")
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
|
||||||
|
if !value.IsValid() {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if value.Kind() == reflect.Interface {
|
||||||
|
if value.IsNil() {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return copyPublicJSONValue(value.Elem(), path, seen)
|
||||||
|
}
|
||||||
|
if !value.CanInterface() {
|
||||||
|
return nil, fmt.Errorf("%s: value cannot be copied", path)
|
||||||
|
}
|
||||||
|
if number, ok := value.Interface().(json.Number); ok {
|
||||||
|
f, err := strconv.ParseFloat(number.String(), 64)
|
||||||
|
if err != nil || math.IsNaN(f) || math.IsInf(f, 0) {
|
||||||
|
return nil, fmt.Errorf("%s: invalid JSON number", path)
|
||||||
|
}
|
||||||
|
return number, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
switch value.Kind() {
|
||||||
|
case reflect.Bool, reflect.String:
|
||||||
|
return value.Interface(), nil
|
||||||
|
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||||
|
if value.Int() < -maxSafeJSONInteger || value.Int() > maxSafeJSONInteger {
|
||||||
|
return nil, fmt.Errorf("%s: integer is outside the JSON-safe range", path)
|
||||||
|
}
|
||||||
|
return value.Interface(), nil
|
||||||
|
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
|
||||||
|
if value.Uint() > maxSafeJSONInteger {
|
||||||
|
return nil, fmt.Errorf("%s: integer is outside the JSON-safe range", path)
|
||||||
|
}
|
||||||
|
return value.Interface(), nil
|
||||||
|
case reflect.Float32, reflect.Float64:
|
||||||
|
f := value.Convert(reflect.TypeOf(float64(0))).Float()
|
||||||
|
if math.IsNaN(f) || math.IsInf(f, 0) {
|
||||||
|
return nil, fmt.Errorf("%s: floating-point value must be finite", path)
|
||||||
|
}
|
||||||
|
return value.Interface(), nil
|
||||||
|
case reflect.Pointer:
|
||||||
|
if value.IsNil() {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
visit := jsonVisit{typ: value.Type(), ptr: value.Pointer()}
|
||||||
|
if _, ok := seen[visit]; ok {
|
||||||
|
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||||
|
}
|
||||||
|
seen[visit] = struct{}{}
|
||||||
|
defer delete(seen, visit)
|
||||||
|
return copyPublicJSONValue(value.Elem(), path, seen)
|
||||||
|
case reflect.Map:
|
||||||
|
return copyPublicJSONMapValue(value, path, seen)
|
||||||
|
case reflect.Slice:
|
||||||
|
if value.IsNil() {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return copyPublicJSONSequenceValue(value, path, seen)
|
||||||
|
case reflect.Array:
|
||||||
|
return copyPublicJSONSequenceValue(value, path, seen)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
|
||||||
|
if value.IsNil() {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if value.Type().Key().Kind() != reflect.String {
|
||||||
|
return nil, fmt.Errorf("%s: map key type %s is not supported", path, value.Type().Key())
|
||||||
|
}
|
||||||
|
|
||||||
|
visit := jsonVisit{typ: value.Type(), ptr: value.Pointer()}
|
||||||
|
if _, ok := seen[visit]; ok {
|
||||||
|
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||||
|
}
|
||||||
|
seen[visit] = struct{}{}
|
||||||
|
defer delete(seen, visit)
|
||||||
|
|
||||||
|
type entry struct {
|
||||||
|
key reflect.Value
|
||||||
|
name string
|
||||||
|
value any
|
||||||
|
}
|
||||||
|
entries := make([]entry, 0, value.Len())
|
||||||
|
preserveType := true
|
||||||
|
elemType := value.Type().Elem()
|
||||||
|
iter := value.MapRange()
|
||||||
|
for iter.Next() {
|
||||||
|
key := iter.Key()
|
||||||
|
name := key.String()
|
||||||
|
copied, err := copyPublicJSONValue(iter.Value(), path+"."+name, seen)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
entries = append(entries, entry{key: key, name: name, value: copied})
|
||||||
|
if copied == nil {
|
||||||
|
if !canAssignNil(elemType) {
|
||||||
|
preserveType = false
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !reflect.TypeOf(copied).AssignableTo(elemType) {
|
||||||
|
preserveType = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if preserveType {
|
||||||
|
out := reflect.MakeMapWithSize(value.Type(), len(entries))
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.value == nil {
|
||||||
|
out.SetMapIndex(entry.key, reflect.Zero(elemType))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out.SetMapIndex(entry.key, reflect.ValueOf(entry.value))
|
||||||
|
}
|
||||||
|
return out.Interface(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make(map[string]any, len(entries))
|
||||||
|
for _, entry := range entries {
|
||||||
|
out[entry.name] = entry.value
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyPublicJSONSequenceValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
|
||||||
|
var visit jsonVisit
|
||||||
|
if value.Kind() == reflect.Slice {
|
||||||
|
visit = jsonVisit{typ: value.Type(), ptr: value.Pointer()}
|
||||||
|
if _, ok := seen[visit]; ok {
|
||||||
|
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||||
|
}
|
||||||
|
seen[visit] = struct{}{}
|
||||||
|
defer delete(seen, visit)
|
||||||
|
}
|
||||||
|
|
||||||
|
values := make([]any, value.Len())
|
||||||
|
preserveType := true
|
||||||
|
elemType := value.Type().Elem()
|
||||||
|
for i := 0; i < value.Len(); i++ {
|
||||||
|
copied, err := copyPublicJSONValue(value.Index(i), fmt.Sprintf("%s[%d]", path, i), seen)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
values[i] = copied
|
||||||
|
if copied == nil {
|
||||||
|
if !canAssignNil(elemType) {
|
||||||
|
preserveType = false
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !reflect.TypeOf(copied).AssignableTo(elemType) {
|
||||||
|
preserveType = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if preserveType {
|
||||||
|
out := reflect.New(value.Type()).Elem()
|
||||||
|
if value.Kind() == reflect.Slice {
|
||||||
|
out = reflect.MakeSlice(value.Type(), value.Len(), value.Len())
|
||||||
|
}
|
||||||
|
for i, copied := range values {
|
||||||
|
if copied == nil {
|
||||||
|
out.Index(i).Set(reflect.Zero(elemType))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out.Index(i).Set(reflect.ValueOf(copied))
|
||||||
|
}
|
||||||
|
return out.Interface(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([]any, len(values))
|
||||||
|
copy(out, values)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func canAssignNil(typ reflect.Type) bool {
|
||||||
|
switch typ.Kind() {
|
||||||
|
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
23
llm_adapter.go
Normal file
23
llm_adapter.go
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
package scriptorium
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
type publicLLMClientAdapter struct {
|
||||||
|
client LLMClient
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a publicLLMClientAdapter) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
|
resp, err := a.client.Generate(ctx, fromDomainGenerateRequest(req))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if resp == nil {
|
||||||
|
return nil, fmt.Errorf("%w: llm client returned nil response", ErrLLMGenerate)
|
||||||
|
}
|
||||||
|
return toDomainGenerateResponse(resp), nil
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user