Compare commits

5 Commits

51 changed files with 2168 additions and 102 deletions

View File

@@ -11,6 +11,7 @@ func toDomainRunRequest(req RunRequest) domain.RunRequest {
PromptID: req.PromptID,
PromptVersion: req.PromptVersion,
ProfileID: req.ProfileID,
APIKey: req.APIKey,
Inputs: toDomainArtifactRefMap(req.Inputs),
Vars: copyStringMap(req.Vars),
Execution: toDomainExecutionTargetOverride(req.Execution),
@@ -72,6 +73,7 @@ func fromDomainGenerateRequest(req domain.GenerateRequest) GenerateRequest {
Target: fromDomainExecutionTarget(req.Target),
TargetPresence: fromDomainExecutionTargetPresence(req.TargetPresence),
StructuredOutput: fromDomainStructuredOutputSpec(req.StructuredOutput),
APIKey: req.Target.APIKey,
}
}

View File

@@ -29,8 +29,9 @@ Integration references:
- `run` and `render` require:
- `--prompt`
- at least one `--input`
- an effective `prompt_dir` and `profile_dir` (from flags or config)
- `serve` requires an effective `prompt_dir` and `profile_dir` (from flags or config).
- an effective `prompt_dir` from flags or config
- `serve` requires an effective `prompt_dir` from flags or config.
- `profile_dir` is optional. If omitted, only built-in profiles are available; if provided, custom profiles override built-ins with the same ID.
- Positional arguments are rejected.
- Prompt cache control is configured in prompt YAML (`messages[].cache_control`), not with CLI flags.
- Provider-specific `reasoning_effort` and `extra_params` are configured in profile YAML or HTTP model overrides, not with CLI flags.
@@ -41,7 +42,7 @@ Integration references:
- `--config <path>`: app config file path.
- `--prompt-dir <dir>`: prompt definition directory.
- `--profile-dir <dir>`: profile definition directory.
- `--profile-dir <dir>`: custom profile definition directory.
- `--schema-dir <dir>`: schema base directory for `json_schema` validation.
- `--prompt <id>`: prompt ID to execute. Required.
- `--prompt-id <id>`: deprecated alias for `--prompt`.
@@ -79,7 +80,7 @@ Notes:
- `--config <path>`: app config file path.
- `--addr <listen-address>`: HTTP listen address.
- `--prompt-dir <dir>`: prompt definition directory.
- `--profile-dir <dir>`: profile definition directory.
- `--profile-dir <dir>`: custom profile definition directory.
- `--schema-dir <dir>`: schema base directory for `json_schema` validation.
Notes:

View File

@@ -21,10 +21,9 @@ When `--config <path>` is provided, that file is required.
```yaml
prompt_dir: ./examples/prompts
profile_dir: ./examples/profiles
```
This is enough to use `run` and `render` when prompt/profile files are valid.
This is enough to use `run` and `render` when prompts select built-in profiles.
## Production-Oriented App Config
@@ -45,7 +44,7 @@ defaults:
Top-level fields:
- `prompt_dir` (optional): default prompt definition directory.
- `profile_dir` (optional): default profile definition directory.
- `profile_dir` (optional): default custom profile definition directory.
- `schema_dir` (optional): base directory for schema files used by `json_schema` validation.
- `server.addr` (optional): default listen address for `serve`.
- `defaults.render_format` (optional): default `render` output format (`text` or `json`).
@@ -173,7 +172,7 @@ Repair behavior boundary:
## Profile Definition Files
Execution profiles are YAML files anywhere under `profile_dir`, including nested subdirectories.
Scriptorium includes built-in execution profiles. Custom execution profiles are YAML files anywhere under `profile_dir`, including nested subdirectories.
Subdirectories are organizational only. Callers still select profiles by the YAML `id`, not by file path. For example, `profiles/local/local-quality.yaml` may still declare `id: local-quality`, and callers use `--profile local-quality`.
@@ -212,6 +211,9 @@ Field reference:
Profile rules:
- `profile_dir` is optional. If omitted, only built-in profiles are available.
- If `profile_dir` is set, custom profiles from that directory override built-in profiles with the same `id`.
- Duplicate IDs within the custom profile directory are invalid. Matching IDs across custom and built-in profiles are valid override behavior.
- Profile decoding is strict; unknown YAML fields are rejected.
- Raw `api_key` is rejected; use `api_key_env`.
- If `api_key_env` is set, that environment variable must be set when preparing/running.
@@ -250,7 +252,7 @@ Supported artifact reference types for request inputs are `file` and `inline`.
- App config: `examples/config.yml`
- Prompt examples: `examples/prompts/`
- Profile examples: `examples/profiles/`
- Custom profile examples: `examples/profiles/`
- Schema examples: `examples/schemas/`
- Input fixtures: `examples/fixtures/`
- Render example script: `examples/render-markdown-summary.sh`

View File

@@ -21,7 +21,17 @@ if err != nil {
}
```
`PromptDir` and `ProfileDir` are required. `SchemaDir` defaults to the built-in schema directory. `Timeout` and `HTTPClient` configure the default OpenAI-compatible client used by `Run` when no custom LLM client is supplied.
`PromptDir` is required unless an explicit prompt source option is supplied. `ProfileDir` is optional; omit it to use built-in profiles only, or set it to overlay custom profiles above built-ins. `SchemaDir` defaults to the built-in schema directory. `Timeout` and `HTTPClient` configure the default OpenAI-compatible client used by `Run` when no custom LLM client is supplied.
## Asset Sources
Directory fields on `Config` remain the compatibility path. Explicit source options override the matching directory field:
- `WithPromptFS(fsys, root)` and `WithPromptFile(path)`
- `WithProfileFS(fsys, root)` and `WithProfileFile(path)`
- `WithSchemaFS(fsys, root)` and `WithSchemaFile(path)`
Prompt and profile sources load standard Scriptorium YAML with the same strict validation as directory sources. Prompt `content_file` paths resolve relative to the prompt file in the same source. Profile options overlay custom profiles above built-ins. Schema `fs.FS` sources preserve prompt `schema_path` semantics; schema file options expose the file by its base name.
## Prepare A Prompt
@@ -54,6 +64,7 @@ Input helpers:
```go
result, err := engine.Run(ctx, scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
APIKey: apiKey,
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
@@ -67,6 +78,8 @@ _ = result.Artifact
`RunResult` includes the run ID, output artifact, raw output, validation result, prompt/profile/model metadata, effective model parameters, input hashes, token/cache usage, and timing fields. Validation content failures return a successful `RunResult` with failed validation status. Runtime validation errors return `ErrValidation`.
For the public Go API, pass provider credentials with `RunRequest.APIKey`. The value is request-scoped, uses `json:"-"`, is preferred over profile `api_key_env` by the default OpenAI-compatible client, and is not included in `PreparedRun` or `RunResult` JSON. Do not store raw keys in config, prompt files, or profile YAML.
## Inject An LLM Client
Use `WithLLMClient` for tests or custom model integrations:
@@ -84,7 +97,7 @@ func (fakeLLM) Generate(ctx context.Context, req scriptorium.GenerateRequest) (*
engine, err := scriptorium.NewEngine(cfg, scriptorium.WithLLMClient(fakeLLM{}))
```
The injected client receives the rendered prompt, effective execution target, target presence metadata for explicit numeric overrides, and structured-output spec. `WithLLMClient(nil)` returns `ErrInvalidConfig`.
The injected client receives the rendered prompt, effective execution target, target presence metadata for explicit numeric overrides, structured-output spec, and request API key when provided. `GenerateRequest.APIKey` also uses `json:"-"`; custom and fake clients should avoid logging or serializing it. `WithLLMClient(nil)` returns `ErrInvalidConfig`.
## Request Overrides

View File

@@ -122,7 +122,12 @@ Structured output is currently `json_schema` only, serialized as:
## Authentication Header
If `Target.APIKeyEnv` is set:
If `Target.APIKey` is set:
- set `Authorization: Bearer <value>`
- do not read `Target.APIKeyEnv`
If `Target.APIKey` is empty and `Target.APIKeyEnv` is set:
- resolve environment variable value at request time
- set `Authorization: Bearer <value>`
@@ -131,7 +136,7 @@ If the environment variable is unset/empty:
- request fails before HTTP call (`ErrInvalidRequest`)
If `Target.APIKeyEnv` is empty:
If both `Target.APIKey` and `Target.APIKeyEnv` are empty:
- no `Authorization` header is sent

View File

@@ -9,12 +9,13 @@ This document describes implemented adapter/repository boundaries and their curr
- `internal/adapter/cli`: CLI command parsing, app wiring, stdout/stderr handling, exit codes.
- `internal/adapter/http`: HTTP request/response mapping for `POST /v1/runs`.
- root package `scriptorium`: public Go library facade for preparing and running prompt requests.
- `internal/promptdef`: filesystem prompt-definition repository.
- `internal/profile`: filesystem execution-profile repository.
- `internal/promptdef`: filesystem and `fs.FS` prompt-definition repositories.
- `internal/profile`: filesystem, `fs.FS`, and overlay execution-profile repositories.
- `internal/profile/builtin`: embedded built-in execution-profile repository.
- `internal/artifact`: input artifact reader.
- `internal/prompt`: Go-template renderer.
- `internal/llm`: OpenAI-compatible LLM client implementation.
- `internal/validate`: output validator.
- `internal/validate`: filesystem and `fs.FS` output validators.
- `internal/format`: prepared-run formatters for `render` output.
## Inputs And Outputs
@@ -36,12 +37,22 @@ Public library facade:
- Input: typed `scriptorium.RunRequest` values.
- Output: typed `PreparedRun` and `RunResult` values plus public sentinel errors.
- Custom LLM behavior is injected with `WithLLMClient`; otherwise the default OpenAI-compatible client is used.
- `RunRequest.APIKey` is a request-scoped Go value only; it is converted into internal execution state for LLM generation and stripped from public result types.
- Prompt, profile, and schema source options can use directories, single files, or `fs.FS` roots. Explicit source options override the matching `Config` directory field.
- Public types are facade types converted at the package boundary; internal domain types remain internal.
Filesystem repositories:
Prompt/profile repositories:
- Input: prompt/profile YAML files under configured directories.
- Input: prompt/profile YAML files under configured directories or `fs.FS` roots.
- Output: normalized domain definitions/profiles or typed errors.
- Single-file public sources are represented as `fs.FS` roots containing one YAML file; lookup still uses YAML `id` values.
Profile repository composition:
- Built-in profiles are embedded and loaded through the same profile validation rules as filesystem profiles.
- When no custom profile directory is configured, the runner receives the built-in profile repository.
- When a custom profile directory/file/`fs.FS` source is configured, the runner receives an overlay repository with custom profiles as primary and built-ins as fallback.
- Overlay lookup falls back only after custom profile-not-found errors; custom load/validation/raw-key errors are returned directly.
Artifact reader:
@@ -52,11 +63,13 @@ LLM adapter:
- Input: `domain.GenerateRequest`.
- Output: `domain.GenerateResponse`.
- Direct API-key values are preferred when present; otherwise `api_key_env` is resolved from the process environment.
Validator:
- Input: artifact body + output contract.
- Output: validation result or runtime validation error.
- Schema documents may be loaded from a directory, single file, or `fs.FS` root in the public package. CLI and HTTP continue to use directory-backed schema loading.
## Boundaries
@@ -69,7 +82,7 @@ Validator:
Primary app settings consumed by adapters:
- `prompt_dir`
- `profile_dir`
- `profile_dir` (optional custom profile source)
- `schema_dir`
- `server.addr`
- `defaults.render_format`
@@ -93,7 +106,9 @@ Strict decoding and input checks:
- config/prompt/profile loaders reject unknown YAML fields.
- prompt/profile repositories scan nested subdirectories recursively.
- prompt/profile lookup uses YAML `id` values; subdirectory paths are organizational only.
- prompt `content_file` paths resolve relative to the prompt YAML file within the same source.
- duplicate prompt/profile IDs are invalid and fail instead of using first-match behavior.
- duplicate profile IDs across custom and built-in sources are allowed; the custom source overrides the built-in profile.
- HTTP DTO decoder rejects unknown JSON fields.
- raw API key payload fields are rejected by strict decoding in profile/http paths.
@@ -114,6 +129,7 @@ LLM adapter:
- compatible cache usage response fields are parsed into domain token usage.
- non-2xx responses map to request failure errors.
- malformed responses (including missing/empty first choice content) are errors.
- direct API-key values are never serialized in provider request bodies.
Validator:

View File

@@ -104,9 +104,10 @@ Validation content failures are not run errors:
- selected profile values
- request overrides
- request numeric overrides are presence-aware, so omitted values preserve the current effective value and explicit zero values override it
6. verify required `api_key_env` environment variable:
- missing/empty env value returns an invalid request with `ErrAPIKeyEnvMissing`
- only the environment-variable name is retained; secret value is never returned
6. verify credentials when the effective target names `api_key_env`:
- a request-scoped direct API key satisfies the credential requirement
- otherwise a missing/empty env value returns an invalid request with `ErrAPIKeyEnvMissing`
- only the environment-variable name is returned in public output; secret values are never returned
7. resolve output contract and structured-output schema payload when `json_schema` mode is active.
8. read input artifacts.
9. render prompt messages, including any normalized message cache-control metadata.
@@ -122,7 +123,8 @@ Runtime target notes:
- The OpenAI-compatible client serializes non-empty `reasoning_effort` as a top-level provider request field.
- The OpenAI-compatible client flattens `extra_params` into provider-specific top-level JSON request fields.
- Empty `extra_params` keys, reserved outbound field names, and values that cannot be JSON-encoded fail before the provider request.
- Resolved API-key values are never stored in `PreparedRun`, `RunResult`, logs, or HTTP responses.
- Resolved API-key values are never serialized in prepared/run output, public results, logs, or HTTP responses.
- Public direct API-key values are carried only far enough to call the configured LLM client and are excluded from JSON/YAML serialization.
## Run Flow

View File

@@ -33,23 +33,23 @@ Relevant links:
- [Configuration reference](config.md)
- [CLI reference](cli.md)
## Missing Prompt/Profile Directory Settings
## Missing Prompt Directory Settings
Symptom:
- CLI parse errors saying prompt directory or profile directory is required.
- CLI parse errors saying prompt directory is required.
Likely cause:
- Neither CLI flags nor config provide effective `prompt_dir` / `profile_dir`.
- Neither CLI flags nor config provide an effective `prompt_dir`.
Diagnostic step:
- Run the failing command with explicit `--prompt-dir` and `--profile-dir` once to verify.
- Run the failing command with explicit `--prompt-dir` once to verify.
Safe fix:
- Set `prompt_dir` and `profile_dir` in config, or always pass both flags.
- Set `prompt_dir` in config, or always pass `--prompt-dir`.
Relevant links:

138
engine.go
View File

@@ -4,7 +4,10 @@ import (
"context"
"errors"
"fmt"
"io/fs"
"net/http"
"os"
"path/filepath"
"strings"
"time"
@@ -12,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile/builtin"
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
@@ -52,6 +56,12 @@ type Option func(*engineOptions) error
type engineOptions struct {
llmClient llm.Client
promptDefs promptdef.Repository
profiles profile.Repository
validator validate.Validator
promptSource bool
profileSource bool
validatorSource bool
}
// WithLLMClient injects a custom LLM client for execution.
@@ -65,16 +75,87 @@ func WithLLMClient(client LLMClient) Option {
}
}
func WithPromptFS(fsys fs.FS, root string) Option {
return func(options *engineOptions) error {
if fsys == nil {
return ErrInvalidConfig
}
if strings.TrimSpace(root) == "" {
return ErrInvalidConfig
}
options.promptDefs = promptdef.NewFSRepository(fsys, root)
options.promptSource = true
return nil
}
}
func WithPromptFile(path string) Option {
return func(options *engineOptions) error {
fsys, root, err := fileSource(path)
if err != nil {
return err
}
options.promptDefs = promptdef.NewFSRepository(fsys, root)
options.promptSource = true
return nil
}
}
func WithProfileFS(fsys fs.FS, root string) Option {
return func(options *engineOptions) error {
if fsys == nil {
return ErrInvalidConfig
}
if strings.TrimSpace(root) == "" {
return ErrInvalidConfig
}
options.profiles = profile.NewFSRepository(fsys, root)
options.profileSource = true
return nil
}
}
func WithProfileFile(path string) Option {
return func(options *engineOptions) error {
fsys, root, err := fileSource(path)
if err != nil {
return err
}
options.profiles = profile.NewFSRepository(fsys, root)
options.profileSource = true
return nil
}
}
func WithSchemaFS(fsys fs.FS, root string) Option {
return func(options *engineOptions) error {
if fsys == nil {
return ErrInvalidConfig
}
if strings.TrimSpace(root) == "" {
return ErrInvalidConfig
}
options.validator = validate.NewFSValidator(fsys, root)
options.validatorSource = true
return nil
}
}
func WithSchemaFile(path string) Option {
return func(options *engineOptions) error {
fsys, root, err := fileSource(path)
if err != nil {
return err
}
options.validator = validate.NewFSValidator(fsys, root)
options.validatorSource = true
return nil
}
}
// NewEngine constructs an Engine using the same default internal components as
// the CLI and HTTP adapters.
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
if strings.TrimSpace(cfg.PromptDir) == "" {
return nil, fmt.Errorf("%w: prompt directory is required", ErrInvalidConfig)
}
if strings.TrimSpace(cfg.ProfileDir) == "" {
return nil, fmt.Errorf("%w: profile directory is required", ErrInvalidConfig)
}
var options engineOptions
for _, opt := range opts {
if opt == nil {
@@ -85,10 +166,27 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
}
}
promptDefs := options.promptDefs
if !options.promptSource {
if strings.TrimSpace(cfg.PromptDir) == "" {
return nil, fmt.Errorf("%w: prompt directory is required", ErrInvalidConfig)
}
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
}
profiles := builtin.NewRepositoryWithDirectory(cfg.ProfileDir)
if options.profileSource {
profiles = builtin.NewRepositoryWithPrimary(options.profiles)
}
validator := options.validator
if !options.validatorSource {
schemaDir := cfg.SchemaDir
if strings.TrimSpace(schemaDir) == "" {
schemaDir = defaults.SchemaDirDefault
}
validator = validate.NewStandardValidator(schemaDir)
}
llmClient := options.llmClient
if llmClient == nil {
@@ -104,16 +202,36 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
return &Engine{
runner: usecase.NewRunner(
promptdef.NewFilesystemRepository(cfg.PromptDir),
profile.NewFilesystemRepository(cfg.ProfileDir),
promptDefs,
profiles,
artifactadapter.NewCompositeReader(),
prompt.NewGoRenderer(),
llmClient,
validate.NewStandardValidator(schemaDir),
validator,
),
}, nil
}
func fileSource(name string) (fs.FS, string, error) {
cleanName := strings.TrimSpace(name)
if cleanName == "" {
return nil, "", ErrInvalidConfig
}
dir := filepath.Dir(cleanName)
base := filepath.Base(cleanName)
if base == "." || base == string(filepath.Separator) || strings.TrimSpace(base) == "" {
return nil, "", ErrInvalidConfig
}
info, err := os.Stat(cleanName)
if err != nil {
return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, cleanName, err)
}
if info.IsDir() {
return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, cleanName)
}
return os.DirFS(dir), filepath.ToSlash(base), nil
}
// Prepare resolves a prompt request without calling an LLM.
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
if e == nil || e.runner == nil {

View File

@@ -4,11 +4,14 @@ import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/scriptorium"
)
@@ -20,10 +23,10 @@ func TestNewEngineRejectsMissingPromptDir(t *testing.T) {
}
}
func TestNewEngineRejectsMissingProfileDir(t *testing.T) {
func TestNewEngineAcceptsMissingProfileDir(t *testing.T) {
_, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: "./examples/prompts"})
if !errors.Is(err, scriptorium.ErrInvalidConfig) {
t.Fatalf("expected ErrInvalidConfig, got %v", err)
if err != nil {
t.Fatalf("expected missing profile dir to use built-ins, got %v", err)
}
}
@@ -205,6 +208,7 @@ func TestRunSucceedsWithInjectedLLMClient(t *testing.T) {
}
func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) {
const directKey = "direct-injected-key"
fake := &fakeLLMClient{
response: &scriptorium.GenerateResponse{Content: "ok"},
}
@@ -214,6 +218,7 @@ func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) {
_, err := engine.Run(context.Background(), scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
APIKey: directKey,
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
"glossary": scriptorium.Inline("gate: A guarded passage."),
@@ -244,6 +249,159 @@ func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) {
if req.StructuredOutput != nil {
t.Fatalf("did not expect structured output for markdown prompt: %+v", req.StructuredOutput)
}
if req.APIKey != directKey {
t.Fatalf("expected direct key on injected generate request")
}
payload, err := json.Marshal(req)
if err != nil {
t.Fatalf("expected generate request to marshal, got %v", err)
}
if strings.Contains(string(payload), directKey) {
t.Fatalf("generate request JSON leaked direct API key: %s", payload)
}
}
func TestRunUsesDirectAPIKeyWithDefaultLLMClient(t *testing.T) {
const directKey = "direct-public-key"
const missingEnv = "SCRIPTORIUM_PUBLIC_DIRECT_MISSING"
t.Setenv(missingEnv, "")
var gotAuth string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotAuth = r.Header.Get("Authorization")
if r.URL.Path != "/v1/chat/completions" {
t.Errorf("unexpected path: %s", r.URL.Path)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"choices": [{"message": {"role": "assistant", "content": "# Summary\n\nDone."}}],
"usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7}
}`))
}))
defer server.Close()
profileDir := t.TempDir()
writePublicProfileFileWithAPIKeyEnv(t, profileDir, "direct-auth", server.URL+"/v1", "test-model", missingEnv)
engine, err := scriptorium.NewEngine(scriptorium.Config{
PromptDir: "./examples/prompts",
ProfileDir: profileDir,
SchemaDir: "./examples/schemas",
})
if err != nil {
t.Fatalf("expected engine construction to succeed, got %v", err)
}
result, err := engine.Run(context.Background(), scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
ProfileID: "direct-auth",
APIKey: directKey,
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
"glossary": scriptorium.Inline("gate: A guarded passage."),
},
})
if err != nil {
t.Fatalf("expected run with direct API key to succeed, got %v", err)
}
if gotAuth != "Bearer "+directKey {
t.Fatalf("unexpected Authorization header: %q", gotAuth)
}
if result.Usage.TotalTokens != 7 {
t.Fatalf("unexpected usage: %+v", result.Usage)
}
payload, err := json.Marshal(result)
if err != nil {
t.Fatalf("expected run result to marshal, got %v", err)
}
if strings.Contains(string(payload), directKey) {
t.Fatalf("run result JSON leaked direct API key: %s", payload)
}
}
func TestPrepareDirectAPIKeyBypassesMissingEnvWithoutLeakingOrHashing(t *testing.T) {
const missingEnv = "SCRIPTORIUM_PUBLIC_PREPARE_MISSING"
const firstKey = "first-direct-key"
const secondKey = "second-direct-key"
t.Setenv(missingEnv, "")
profileDir := t.TempDir()
writePublicProfileFileWithAPIKeyEnv(t, profileDir, "direct-prepare", "http://localhost:8000/v1", "test-model", missingEnv)
engine, err := scriptorium.NewEngine(scriptorium.Config{
PromptDir: "./examples/prompts",
ProfileDir: profileDir,
SchemaDir: "./examples/schemas",
})
if err != nil {
t.Fatalf("expected engine construction to succeed, got %v", err)
}
baseReq := scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
ProfileID: "direct-prepare",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
"glossary": scriptorium.Inline("gate: A guarded passage."),
},
}
firstReq := baseReq
firstReq.APIKey = firstKey
firstPrepared, err := engine.Prepare(context.Background(), firstReq)
if err != nil {
t.Fatalf("expected prepare with direct API key to succeed, got %v", err)
}
secondReq := baseReq
secondReq.APIKey = secondKey
secondPrepared, err := engine.Prepare(context.Background(), secondReq)
if err != nil {
t.Fatalf("expected prepare with alternate direct API key to succeed, got %v", err)
}
if firstPrepared.PromptHash != secondPrepared.PromptHash {
t.Fatalf("direct API keys changed prompt hash: %q vs %q", firstPrepared.PromptHash, secondPrepared.PromptHash)
}
if firstPrepared.RenderedPromptHash != secondPrepared.RenderedPromptHash {
t.Fatalf("direct API keys changed rendered prompt hash: %q vs %q", firstPrepared.RenderedPromptHash, secondPrepared.RenderedPromptHash)
}
payload, err := json.Marshal(firstPrepared)
if err != nil {
t.Fatalf("expected prepared run to marshal, got %v", err)
}
if strings.Contains(string(payload), firstKey) {
t.Fatalf("prepared run JSON leaked direct API key: %s", payload)
}
}
func TestMissingCredentialsFailClearlyWhenProfileRequiresAuth(t *testing.T) {
const missingEnv = "SCRIPTORIUM_PUBLIC_AUTH_MISSING"
t.Setenv(missingEnv, "")
profileDir := t.TempDir()
writePublicProfileFileWithAPIKeyEnv(t, profileDir, "requires-auth", "http://localhost:8000/v1", "test-model", missingEnv)
engine, err := scriptorium.NewEngine(scriptorium.Config{
PromptDir: "./examples/prompts",
ProfileDir: profileDir,
SchemaDir: "./examples/schemas",
})
if err != nil {
t.Fatalf("expected engine construction to succeed, got %v", err)
}
_, err = engine.Prepare(context.Background(), scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
ProfileID: "requires-auth",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
"glossary": scriptorium.Inline("gate: A guarded passage."),
},
})
if !errors.Is(err, scriptorium.ErrInvalidRequest) {
t.Fatalf("expected invalid request for missing credentials, got %v", err)
}
if err == nil || !strings.Contains(err.Error(), missingEnv) {
t.Fatalf("expected missing env name in error, got %v", err)
}
}
func TestRunValidationFailureReturnsResult(t *testing.T) {
@@ -435,6 +593,373 @@ unknown_field: true
}
}
func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
t.Setenv("OPENROUTER_API_KEY", "test-key")
engine, err := scriptorium.NewEngine(scriptorium.Config{
PromptDir: "./examples/prompts",
SchemaDir: "./examples/schemas",
})
if err != nil {
t.Fatalf("expected engine construction to succeed, got %v", err)
}
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
ProfileID: "mistral-small-3",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
"glossary": scriptorium.Inline("gate: A guarded passage."),
},
})
if err != nil {
t.Fatalf("expected built-in profile prepare to succeed, got %v", err)
}
if prepared.SelectedProfileID != "mistral-small-3" {
t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID)
}
if prepared.EffectiveModelParams.Model != "mistralai/mistral-small-3.2-24b-instruct" {
t.Fatalf("unexpected built-in model: %q", prepared.EffectiveModelParams.Model)
}
}
func TestPromptDefaultProfileCanUseBuiltInProfile(t *testing.T) {
t.Setenv("OPENROUTER_API_KEY", "test-key")
promptDir := t.TempDir()
writePublicPromptFile(t, promptDir, "prompt.builtin.default", "mistral-small-3")
engine, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: promptDir})
if err != nil {
t.Fatalf("expected engine construction to succeed, got %v", err)
}
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
PromptID: "prompt.builtin.default",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
},
})
if err != nil {
t.Fatalf("expected built-in default profile prepare to succeed, got %v", err)
}
if prepared.SelectedProfileID != "mistral-small-3" {
t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID)
}
}
func TestCustomProfileOverridesBuiltInProfile(t *testing.T) {
t.Setenv("OPENROUTER_API_KEY", "test-key")
profileDir := t.TempDir()
writePublicProfileFile(t, profileDir, "mistral-small-3", "http://localhost:8000/v1", "custom-model")
engine, err := scriptorium.NewEngine(scriptorium.Config{
PromptDir: "./examples/prompts",
ProfileDir: profileDir,
SchemaDir: "./examples/schemas",
})
if err != nil {
t.Fatalf("expected engine construction to succeed, got %v", err)
}
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
ProfileID: "mistral-small-3",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
"glossary": scriptorium.Inline("gate: A guarded passage."),
},
})
if err != nil {
t.Fatalf("expected custom profile prepare to succeed, got %v", err)
}
if prepared.EffectiveModelParams.Model != "custom-model" {
t.Fatalf("expected custom profile to override built-in, got %q", prepared.EffectiveModelParams.Model)
}
}
func TestMalformedCustomProfileDoesNotFallbackToBuiltIn(t *testing.T) {
t.Setenv("OPENROUTER_API_KEY", "test-key")
profileDir := t.TempDir()
if err := os.WriteFile(filepath.Join(profileDir, "mistral-small-3.yml"), []byte(`
id: mistral-small-3
endpoint: http://localhost:8000/v1
model: custom-model
unexpected: true
`), 0o644); err != nil {
t.Fatal(err)
}
engine, err := scriptorium.NewEngine(scriptorium.Config{
PromptDir: "./examples/prompts",
ProfileDir: profileDir,
SchemaDir: "./examples/schemas",
})
if err != nil {
t.Fatalf("expected engine construction to succeed, got %v", err)
}
_, err = engine.Prepare(context.Background(), scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
ProfileID: "mistral-small-3",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
"glossary": scriptorium.Inline("gate: A guarded passage."),
},
})
if !errors.Is(err, scriptorium.ErrProfileLoad) {
t.Fatalf("expected custom profile load error, got %v", err)
}
}
func TestPrepareWorksWithPromptFSAndRelativeContentFile(t *testing.T) {
promptFS := fstest.MapFS{
"assets/prompts/fs-summary.yaml": &fstest.MapFile{Data: []byte(`
id: fs.summary
version: "1.0.0"
default_profile: local-fast
inputs:
- name: transcript
required: true
messages:
- role: user
content_file: ./messages/summary.tmpl
output:
format: text
validation_mode: none
repair_attempts: 0
`)},
"assets/prompts/messages/summary.tmpl": &fstest.MapFile{Data: []byte(`Summarize {{input "transcript"}} from prompt fs.`)},
}
engine, err := scriptorium.NewEngine(scriptorium.Config{
PromptDir: t.TempDir(),
ProfileDir: "./examples/profiles",
SchemaDir: "./examples/schemas",
}, scriptorium.WithPromptFS(promptFS, "assets/prompts"))
if err != nil {
t.Fatalf("expected engine construction to succeed, got %v", err)
}
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
PromptID: "fs.summary",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
},
})
if err != nil {
t.Fatalf("expected prepare to succeed, got %v", err)
}
if len(prepared.Messages) != 1 || !strings.Contains(prepared.Messages[0].Content, "prompt fs") {
t.Fatalf("expected content_file body from prompt fs, got %+v", prepared.Messages)
}
}
func TestPrepareWorksWithPromptFile(t *testing.T) {
promptDir := t.TempDir()
promptPath := filepath.Join(promptDir, "single.yaml")
if err := os.WriteFile(promptPath, []byte(`
id: single.file.prompt
version: "1.0.0"
default_profile: local-fast
inputs:
- name: transcript
required: true
messages:
- role: user
content: "Summarize {{input \"transcript\"}} from file."
output:
format: text
validation_mode: none
repair_attempts: 0
`), 0o644); err != nil {
t.Fatal(err)
}
engine, err := scriptorium.NewEngine(scriptorium.Config{
ProfileDir: "./examples/profiles",
SchemaDir: "./examples/schemas",
}, scriptorium.WithPromptFile(promptPath))
if err != nil {
t.Fatalf("expected engine construction to succeed, got %v", err)
}
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
PromptID: "single.file.prompt",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
},
})
if err != nil {
t.Fatalf("expected prepare to succeed, got %v", err)
}
if prepared.PromptID != "single.file.prompt" {
t.Fatalf("unexpected prompt id: %q", prepared.PromptID)
}
}
func TestPrepareWorksWithProfileFSOverBuiltIns(t *testing.T) {
profileFS := fstest.MapFS{
"profiles/mistral-small-3.yaml": &fstest.MapFile{Data: []byte(`
id: mistral-small-3
endpoint: http://profile-fs/v1
model: profile-fs-model
`)},
}
engine, err := scriptorium.NewEngine(scriptorium.Config{
PromptDir: "./examples/prompts",
SchemaDir: "./examples/schemas",
}, scriptorium.WithProfileFS(profileFS, "profiles"))
if err != nil {
t.Fatalf("expected engine construction to succeed, got %v", err)
}
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
ProfileID: "mistral-small-3",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
"glossary": scriptorium.Inline("gate: A guarded passage."),
},
})
if err != nil {
t.Fatalf("expected prepare to succeed, got %v", err)
}
if prepared.EffectiveModelParams.Model != "profile-fs-model" {
t.Fatalf("expected profile fs to override built-in, got %q", prepared.EffectiveModelParams.Model)
}
}
func TestPrepareWorksWithProfileFileOverBuiltIns(t *testing.T) {
profileDir := t.TempDir()
profilePath := filepath.Join(profileDir, "mistral-small-3.yaml")
if err := os.WriteFile(profilePath, []byte(`
id: mistral-small-3
endpoint: http://profile-file/v1
model: profile-file-model
`), 0o644); err != nil {
t.Fatal(err)
}
engine, err := scriptorium.NewEngine(scriptorium.Config{
PromptDir: "./examples/prompts",
SchemaDir: "./examples/schemas",
}, scriptorium.WithProfileFile(profilePath))
if err != nil {
t.Fatalf("expected engine construction to succeed, got %v", err)
}
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
ProfileID: "mistral-small-3",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
"glossary": scriptorium.Inline("gate: A guarded passage."),
},
})
if err != nil {
t.Fatalf("expected prepare to succeed, got %v", err)
}
if prepared.EffectiveModelParams.Model != "profile-file-model" {
t.Fatalf("expected profile file to override built-in, got %q", prepared.EffectiveModelParams.Model)
}
}
func TestRunStructuredOutputWorksWithSchemaFS(t *testing.T) {
fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: `{"events":[]}`}}
engine, err := scriptorium.NewEngine(scriptorium.Config{
PromptDir: t.TempDir(),
ProfileDir: "./examples/profiles",
SchemaDir: t.TempDir(),
},
scriptorium.WithPromptFS(publicStructuredPromptFS("schema.fs.prompt", "events.schema.json"), "prompts"),
scriptorium.WithSchemaFS(publicSchemaFS(), "schemas"),
scriptorium.WithLLMClient(fake),
)
if err != nil {
t.Fatalf("expected engine construction to succeed, got %v", err)
}
result, err := engine.Run(context.Background(), scriptorium.RunRequest{
PromptID: "schema.fs.prompt",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
},
})
if err != nil {
t.Fatalf("expected run to succeed, got %v", err)
}
if result.Validation.Status != scriptorium.ValidationPassed || !result.Validation.IsValid {
t.Fatalf("expected schema validation to pass, got %+v", result.Validation)
}
if len(fake.requests) != 1 || fake.requests[0].StructuredOutput == nil {
t.Fatalf("expected structured output request, got %+v", fake.requests)
}
}
func TestRunStructuredOutputWorksWithSchemaFile(t *testing.T) {
schemaDir := t.TempDir()
schemaPath := filepath.Join(schemaDir, "events.schema.json")
if err := os.WriteFile(schemaPath, []byte(publicSchemaJSON()), 0o644); err != nil {
t.Fatal(err)
}
fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: `{"events":[]}`}}
engine, err := scriptorium.NewEngine(scriptorium.Config{
ProfileDir: "./examples/profiles",
},
scriptorium.WithPromptFS(publicStructuredPromptFS("schema.file.prompt", "events.schema.json"), "prompts"),
scriptorium.WithSchemaFile(schemaPath),
scriptorium.WithLLMClient(fake),
)
if err != nil {
t.Fatalf("expected engine construction to succeed, got %v", err)
}
result, err := engine.Run(context.Background(), scriptorium.RunRequest{
PromptID: "schema.file.prompt",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
},
})
if err != nil {
t.Fatalf("expected run to succeed, got %v", err)
}
if result.Validation.Status != scriptorium.ValidationPassed || !result.Validation.IsValid {
t.Fatalf("expected schema validation to pass, got %+v", result.Validation)
}
}
func TestSourceOptionsRejectInvalidInputs(t *testing.T) {
missingFile := filepath.Join(t.TempDir(), "missing.yaml")
directoryPath := t.TempDir()
tests := []struct {
name string
opt scriptorium.Option
}{
{name: "prompt fs nil", opt: scriptorium.WithPromptFS(nil, "prompts")},
{name: "prompt fs empty root", opt: scriptorium.WithPromptFS(fstest.MapFS{}, "")},
{name: "prompt file empty", opt: scriptorium.WithPromptFile("")},
{name: "prompt file missing", opt: scriptorium.WithPromptFile(missingFile)},
{name: "prompt file directory", opt: scriptorium.WithPromptFile(directoryPath)},
{name: "profile fs nil", opt: scriptorium.WithProfileFS(nil, "profiles")},
{name: "profile fs empty root", opt: scriptorium.WithProfileFS(fstest.MapFS{}, "")},
{name: "profile file empty", opt: scriptorium.WithProfileFile("")},
{name: "schema fs nil", opt: scriptorium.WithSchemaFS(nil, "schemas")},
{name: "schema fs empty root", opt: scriptorium.WithSchemaFS(fstest.MapFS{}, "")},
{name: "schema file empty", opt: scriptorium.WithSchemaFile("")},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: "./examples/prompts"}, tc.opt)
if !errors.Is(err, scriptorium.ErrInvalidConfig) {
t.Fatalf("expected ErrInvalidConfig, got %v", err)
}
})
}
}
func TestExtraParamsTypedNestedValuesAreCopiedAcrossPublicBoundary(t *testing.T) {
fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}
engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake))
@@ -544,6 +1069,87 @@ func exampleConfig(schemaDir string) scriptorium.Config {
}
}
func writePublicPromptFile(t *testing.T, dir, id, defaultProfile string) {
t.Helper()
data := `id: ` + id + `
version: "1.0.0"
default_profile: ` + defaultProfile + `
inputs:
- name: transcript
required: true
messages:
- role: user
content: "Summarize: {{input \"transcript\"}}"
output:
format: text
validation_mode: none
repair_attempts: 0
`
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil {
t.Fatalf("failed to write prompt fixture: %v", err)
}
}
func writePublicProfileFile(t *testing.T, dir, id, endpoint, model string) {
t.Helper()
data := `id: ` + id + `
endpoint: ` + endpoint + `
model: ` + model + `
`
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil {
t.Fatalf("failed to write profile fixture: %v", err)
}
}
func writePublicProfileFileWithAPIKeyEnv(t *testing.T, dir, id, endpoint, model, apiKeyEnv string) {
t.Helper()
data := `id: ` + id + `
endpoint: ` + endpoint + `
model: ` + model + `
api_key_env: ` + apiKeyEnv + `
`
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil {
t.Fatalf("failed to write profile fixture: %v", err)
}
}
func publicStructuredPromptFS(id string, schemaPath string) fstest.MapFS {
return fstest.MapFS{
"prompts/prompt.yaml": &fstest.MapFile{Data: []byte(`id: ` + id + `
version: "1.0.0"
default_profile: local-fast
inputs:
- name: transcript
required: true
messages:
- role: user
content: "Extract events from {{input \"transcript\"}}."
output:
format: json
validation_mode: json_schema
schema_path: ` + schemaPath + `
repair_attempts: 0
`)},
}
}
func publicSchemaFS() fstest.MapFS {
return fstest.MapFS{
"schemas/events.schema.json": &fstest.MapFile{Data: []byte(publicSchemaJSON())},
}
}
func publicSchemaJSON() string {
return `{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"required": ["events"],
"properties": {
"events": {"type": "array"}
}
}`
}
type fakeLLMClient struct {
response *scriptorium.GenerateResponse
err error

View File

@@ -19,7 +19,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile/builtin"
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
@@ -34,7 +34,6 @@ const (
const (
errPromptDirRequired = "prompt directory is required; provide --prompt-dir or config.yml prompt_dir"
errProfileDirRequired = "profile directory is required; provide --profile-dir or config.yml profile_dir"
)
type runConfig struct {
@@ -305,12 +304,14 @@ func parseServeArgs(args []string) (*serveConfig, error) {
cfg.schemaDir = settings.schemaDir
cfg.addr = settings.serverAddr
if err := validateRequiredLibraryDirs(cfg.promptDir, cfg.profileDir); err != nil {
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
return nil, err
}
cfg.promptDir = filepath.Clean(cfg.promptDir)
if strings.TrimSpace(cfg.profileDir) != "" {
cfg.profileDir = filepath.Clean(cfg.profileDir)
}
cfg.schemaDir = filepath.Clean(cfg.schemaDir)
return cfg, nil
}
@@ -353,7 +354,7 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
cfg.schemaDir = settings.schemaDir
cfg.defaultRenderFormat = settings.defaultRenderFormat
if err := validateRequiredLibraryDirs(cfg.promptDir, cfg.profileDir); err != nil {
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
return err
}
if strings.TrimSpace(cfg.promptID) == "" {
@@ -363,7 +364,9 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
return errors.New("at least one --input is required")
}
cfg.promptDir = filepath.Clean(cfg.promptDir)
if strings.TrimSpace(cfg.profileDir) != "" {
cfg.profileDir = filepath.Clean(cfg.profileDir)
}
if cfg.outputPath != "" {
cfg.outputPath = filepath.Clean(cfg.outputPath)
}
@@ -467,20 +470,17 @@ func resolveCommonSettings(fs *flag.FlagSet, configPath string, overrides appcon
}, nil
}
func validateRequiredLibraryDirs(promptDir, profileDir string) error {
func validateRequiredLibraryDirs(promptDir string) error {
if strings.TrimSpace(promptDir) == "" {
return errors.New(errPromptDirRequired)
}
if strings.TrimSpace(profileDir) == "" {
return errors.New(errProfileDirRequired)
}
return nil
}
func newRunner(promptDir, profileDir, schemaDir string, llmClient llm.Client) *usecase.Runner {
return usecase.NewRunner(
promptdef.NewFilesystemRepository(promptDir),
profile.NewFilesystemRepository(profileDir),
builtin.NewRepositoryWithDirectory(profileDir),
artifactadapter.NewCompositeReader(),
prompt.NewGoRenderer(),
llmClient,

View File

@@ -74,12 +74,12 @@ func TestParseRunArgsRequiredFlags(t *testing.T) {
t.Fatalf("expected clear prompt-dir guidance, got %v", err)
}
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p", "--input", "a=b"})
if err == nil {
t.Fatal("expected missing --profile-dir error")
cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p", "--input", "a=b"})
if err != nil {
t.Fatalf("expected missing --profile-dir to be accepted, got %v", err)
}
if !strings.Contains(err.Error(), "profile directory is required") {
t.Fatalf("expected clear profile-dir guidance, got %v", err)
if cfg.profileDir != "" {
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
}
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--input", "a=b"})
@@ -161,17 +161,12 @@ func TestParseServeArgsRequiredFlags(t *testing.T) {
t.Fatalf("expected clear prompt-dir guidance, got %v", err)
}
_, err = parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts"})
if err == nil {
t.Fatal("expected missing --profile-dir error")
}
if !strings.Contains(err.Error(), "profile directory is required") {
t.Fatalf("expected clear profile-dir guidance, got %v", err)
}
cfg, err := parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles"})
cfg, err := parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts"})
if err != nil {
t.Fatalf("expected valid serve args, got %v", err)
t.Fatalf("expected missing --profile-dir to be accepted, got %v", err)
}
if cfg.profileDir != "" {
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
}
if cfg.addr != defaults.HTTPAddrDefault {
t.Fatalf("expected default addr %s, got %q", defaults.HTTPAddrDefault, cfg.addr)
@@ -565,21 +560,21 @@ profile_dir: ./profiles
}
}
func TestParseRunArgsFailsClearlyWhenNoEffectiveProfileDir(t *testing.T) {
func TestParseRunArgsAcceptsMissingEffectiveProfileDir(t *testing.T) {
configPath := writeAppConfigFile(t, `
prompt_dir: ./prompts
`)
_, err := parseRunArgs([]string{
cfg, err := parseRunArgs([]string{
"--config", configPath,
"--prompt", "p",
"--input", "a=b",
})
if err == nil {
t.Fatal("expected missing profile_dir error")
if err != nil {
t.Fatalf("expected missing profile_dir to be accepted, got %v", err)
}
if !strings.Contains(err.Error(), "profile directory is required") || !strings.Contains(err.Error(), "config.yml profile_dir") {
t.Fatalf("expected clear profile_dir guidance, got %v", err)
if cfg.profileDir != "" {
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
}
}
@@ -601,21 +596,21 @@ profile_dir: ./profiles
}
}
func TestParseRenderArgsFailsClearlyWhenNoEffectiveProfileDir(t *testing.T) {
func TestParseRenderArgsAcceptsMissingEffectiveProfileDir(t *testing.T) {
configPath := writeAppConfigFile(t, `
prompt_dir: ./prompts
`)
_, err := parseRenderArgs([]string{
cfg, err := parseRenderArgs([]string{
"--config", configPath,
"--prompt", "p",
"--input", "a=b",
})
if err == nil {
t.Fatal("expected missing profile_dir error")
if err != nil {
t.Fatalf("expected missing profile_dir to be accepted, got %v", err)
}
if !strings.Contains(err.Error(), "profile directory is required") || !strings.Contains(err.Error(), "config.yml profile_dir") {
t.Fatalf("expected clear profile_dir guidance, got %v", err)
if cfg.profileDir != "" {
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
}
}
@@ -929,6 +924,29 @@ func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
}
}
func TestRenderCommandUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
t.Setenv("OPENROUTER_API_KEY", "test-key")
lib := newCLITestLibrary(t)
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
writePromptFile(t, lib.promptDir, "prompt.builtin", "mistral-small-3")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--prompt-dir", lib.promptDir,
"--prompt", "prompt.builtin",
"--input", "transcript=" + inputPath,
})
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
if !strings.Contains(stdout, "selected_profile_id: mistral-small-3") {
t.Fatalf("expected built-in selected profile, got %q", stdout)
}
if !strings.Contains(stdout, "model: mistralai/mistral-small-3.2-24b-instruct") {
t.Fatalf("expected built-in model, got %q", stdout)
}
}
func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
lib := newCLITestLibrary(t)
inputPath := lib.writeInputFile(t, "transcript.md", "hello")

View File

@@ -63,6 +63,7 @@ type RunRequest struct {
PromptID string
PromptVersion string
ProfileID string
APIKey string `json:"-" yaml:"-"`
Inputs map[string]ArtifactRef
Vars map[string]string
Execution *ExecutionTargetOverride
@@ -207,6 +208,7 @@ type ExecutionTarget struct {
ServiceTier string `yaml:"service_tier" json:"service_tier"`
ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"`
APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"`
APIKey string `yaml:"-" json:"-"`
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
}

View File

@@ -20,6 +20,7 @@ func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
Endpoint: "http://llm/v1",
Model: "gpt-test",
APIKeyEnv: envName,
APIKey: secret,
},
InputHashes: map[string]string{"transcript": "hash-1"},
RenderedPromptHash: "rendered-hash",

View File

@@ -92,6 +92,20 @@ func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
}
}
func TestTextFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) {
const directKey = "direct-format-key"
prepared := samplePreparedRun()
prepared.EffectiveModelParams.APIKey = directKey
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if strings.Contains(string(out), directKey) {
t.Fatalf("text output should not include direct api key value: %s", out)
}
}
func TestTextFormatterIncludesMessageCacheControlBeforeContent(t *testing.T) {
prepared := samplePreparedRun()
prepared.Messages = []domain.RenderedMessage{
@@ -269,6 +283,20 @@ func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
}
}
func TestJSONFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) {
const directKey = "direct-format-key"
prepared := samplePreparedRun()
prepared.EffectiveModelParams.APIKey = directKey
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if strings.Contains(string(out), directKey) {
t.Fatalf("json output should not include direct api key value: %s", out)
}
}
func TestParsePreparedRunOutputFormatRecognizesSupportedNames(t *testing.T) {
tests := []struct {
name string

View File

@@ -105,7 +105,9 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
}
httpReq.Header.Set("Content-Type", "application/json")
if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" {
if apiKey := strings.TrimSpace(req.Target.APIKey); apiKey != "" {
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
} else if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" {
apiKey := strings.TrimSpace(os.Getenv(envName))
if apiKey == "" {
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)

View File

@@ -148,6 +148,38 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
}
}
func TestOpenAICompatibleClientDirectAPIKeyPreferredOverEnv(t *testing.T) {
const directKey = "direct-llm-key"
t.Setenv("SCRIPTORIUM_TEST_API_KEY", "env-key")
var gotAuth string
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotAuth = r.Header.Get("Authorization")
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{
Model: "model",
APIKeyEnv: "SCRIPTORIUM_TEST_API_KEY",
APIKey: directKey,
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if gotAuth != "Bearer "+directKey {
t.Fatalf("unexpected Authorization header: %q", gotAuth)
}
}
func TestOpenAICompatibleClientSerializesCacheControlledMessageAsContentBlock(t *testing.T) {
var observedBody map[string]any
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

View File

@@ -0,0 +1,9 @@
id: aion-2
endpoint: https://openrouter.ai/api/v1
model: aion-labs/aion-2.0
temperature: 0.72
reasoning_effort: high
top_p: 0.95
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,7 @@
id: claude-fable-latest
endpoint: https://openrouter.ai/api/v1
model: "~anthropic/claude-fable-latest"
reasoning_effort: high
timeout_seconds: 600
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,7 @@
id: claude-haiku-latest
endpoint: https://openrouter.ai/api/v1
model: "~anthropic/claude-haiku-latest"
reasoning_effort: medium
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,7 @@
id: claude-opus-latest
endpoint: https://openrouter.ai/api/v1
model: "~anthropic/claude-opus-latest"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,7 @@
id: claude-sonnet-latest
endpoint: https://openrouter.ai/api/v1
model: "~anthropic/claude-sonnet-latest"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,7 @@
id: deepseek-3-2
endpoint: https://openrouter.ai/api/v1
model: deepseek/deepseek-v3.2
reasoning_effort: high
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,7 @@
id: deepseek-4-pro
endpoint: https://openrouter.ai/api/v1
model: deepseek/deepseek-v4-pro
reasoning_effort: high
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,9 @@
id: gemini-2-flash-lite
endpoint: https://openrouter.ai/api/v1
model: "google/gemini-2.5-flash-lite"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,9 @@
id: gemini-2-flash
endpoint: https://openrouter.ai/api/v1
model: "google/gemini-2.5-flash"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,9 @@
id: gemini-2-pro
endpoint: https://openrouter.ai/api/v1
model: "google/gemini-2.5-pro"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,9 @@
id: gemini-3-flash-lite
endpoint: https://openrouter.ai/api/v1
model: "google/gemini-3.1-flash-lite"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,9 @@
id: gemini-flash-latest
endpoint: https://openrouter.ai/api/v1
model: "~google/gemini-flash-latest"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,9 @@
id: gemini-pro-latest
endpoint: https://openrouter.ai/api/v1
model: "~google/gemini-pro-latest"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,9 @@
id: gemma-4-31b
endpoint: https://openrouter.ai/api/v1
model: google/gemma-4-31b-it:exacto
temperature: 0.15
reasoning_effort: high
top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,9 @@
id: minimax-m2
endpoint: https://openrouter.ai/api/v1
model: minimax/minimax-m2.5
temperature: 0.5
reasoning_effort: high
top_p: 0.95
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,9 @@
id: minimax-m3
endpoint: https://openrouter.ai/api/v1
model: minimax/minimax-m3
#temperature: 0.5
reasoning_effort: high
#top_p: 0.95
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,7 @@
id: mistral-large-2512
endpoint: https://openrouter.ai/api/v1
model: mistralai/mistral-large-2512
temperature: 0.15
top_p: 0.98
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -0,0 +1,8 @@
id: mistral-medium-3-5
endpoint: https://openrouter.ai/api/v1
model: mistralai/mistral-medium-3-5
temperature: 0.15
reasoning_effort: high
top_p: 0.98
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -0,0 +1,7 @@
id: mistral-small-3
endpoint: https://openrouter.ai/api/v1
model: mistralai/mistral-small-3.2-24b-instruct
temperature: 0.05
top_p: 1.0
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -0,0 +1,8 @@
id: mistral-small-4
endpoint: https://openrouter.ai/api/v1
model: mistralai/mistral-small-2603
temperature: 0.1
reasoning_effort: high
top_p: 0.98
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -0,0 +1,7 @@
id: nemotron-3-ultra
endpoint: https://openrouter.ai/api/v1
model: nvidia/nemotron-3-ultra-550b-a55b
reasoning_effort: high
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,7 @@
id: gpt-5-mini
endpoint: https://openrouter.ai/api/v1
model: "openai/gpt-5.4-mini"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,7 @@
id: gpt-5-nano
endpoint: https://openrouter.ai/api/v1
model: "openai/gpt-5.4-nano"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -0,0 +1,31 @@
package builtin
import (
"embed"
"strings"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
)
const assetRoot = "assets"
//go:embed assets/**/*.yml
var assets embed.FS
func NewRepository() profile.Repository {
return profile.NewFSRepository(assets, assetRoot)
}
func NewRepositoryWithPrimary(primary profile.Repository) profile.Repository {
if primary == nil {
return NewRepository()
}
return profile.NewOverlayRepository(primary, NewRepository())
}
func NewRepositoryWithDirectory(dir string) profile.Repository {
if strings.TrimSpace(dir) == "" {
return NewRepository()
}
return NewRepositoryWithPrimary(profile.NewFilesystemRepository(dir))
}

View File

@@ -0,0 +1,127 @@
package builtin
import (
"context"
"errors"
"io/fs"
"strings"
"testing"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
"gopkg.in/yaml.v3"
)
func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
repo := NewRepository()
ids := loadBuiltInProfileIDs(t)
if len(ids) == 0 {
t.Fatal("expected built-in profiles")
}
for id := range ids {
t.Run(id, func(t *testing.T) {
p, err := repo.GetProfile(context.Background(), id)
if err != nil {
t.Fatalf("expected built-in profile %q to load, got %v", id, err)
}
if p.ID != id {
t.Fatalf("expected profile id %q, got %q", id, p.ID)
}
})
}
}
func TestBuiltInProfilesDoNotContainDuplicateIDsOrRawAPIKeys(t *testing.T) {
loadBuiltInProfileIDs(t)
}
func loadBuiltInProfileIDs(t *testing.T) map[string]string {
t.Helper()
ids := map[string]string{}
err := fs.WalkDir(assets, assetRoot, func(name string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() || !strings.HasSuffix(name, ".yml") {
return nil
}
data, err := assets.ReadFile(name)
if err != nil {
t.Fatalf("failed to read built-in profile %s: %v", name, err)
}
var raw map[string]any
if err := yaml.Unmarshal(data, &raw); err != nil {
t.Fatalf("failed to decode built-in profile %s: %v", name, err)
}
if _, ok := raw["api_key"]; ok {
t.Fatalf("built-in profile %s contains raw api_key", name)
}
id, ok := raw["id"].(string)
if !ok || strings.TrimSpace(id) == "" {
t.Fatalf("built-in profile %s has missing id", name)
}
if previous, ok := ids[id]; ok {
t.Fatalf("duplicate built-in profile id %q in %s and %s", id, previous, name)
}
ids[id] = name
return nil
})
if err != nil {
t.Fatalf("failed to walk built-in profiles: %v", err)
}
return ids
}
func TestRepositoryWithPrimaryUsesPrimaryBeforeBuiltIns(t *testing.T) {
repo := NewRepositoryWithPrimary(staticProfileRepo{
profiles: map[string]string{"mistral-small-3": "custom-model"},
})
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
if err != nil {
t.Fatalf("expected profile to load, got %v", err)
}
if p.Model != "custom-model" {
t.Fatalf("expected primary profile to override built-in, got %+v", p)
}
}
func TestRepositoryWithPrimaryFallsBackToBuiltIns(t *testing.T) {
repo := NewRepositoryWithPrimary(staticProfileRepo{})
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
if err != nil {
t.Fatalf("expected built-in profile to load, got %v", err)
}
if p.ID != "mistral-small-3" {
t.Fatalf("unexpected profile: %+v", p)
}
}
func TestRepositoryWithPrimaryDoesNotFallBackAfterPrimaryError(t *testing.T) {
repo := NewRepositoryWithPrimary(staticProfileRepo{err: profile.ErrInvalidProfile})
_, err := repo.GetProfile(context.Background(), "mistral-small-3")
if !errors.Is(err, profile.ErrInvalidProfile) {
t.Fatalf("expected primary error, got %v", err)
}
}
type staticProfileRepo struct {
profiles map[string]string
err error
}
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
if r.err != nil {
return nil, r.err
}
if model, ok := r.profiles[id]; ok {
return &domain.ExecutionProfile{ID: id, Endpoint: "http://primary/v1", Model: model}, nil
}
return nil, profile.ErrProfileNotFound
}

View File

@@ -5,12 +5,12 @@ import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"path"
"strings"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
"gitea.maximumdirect.net/eric/scriptorium/internal/filecatalog"
"gopkg.in/yaml.v3"
)
@@ -30,11 +30,56 @@ func NewFilesystemRepository(dir string) Repository {
}
func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
return loadProfile(ctx, os.DirFS(r.dir), ".", id)
}
type fsRepository struct {
fsys fs.FS
root string
}
func NewFSRepository(fsys fs.FS, root string) Repository {
return &fsRepository{fsys: fsys, root: root}
}
func (r *fsRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
return loadProfile(ctx, r.fsys, r.root, id)
}
type overlayRepository struct {
primary Repository
fallback Repository
}
func NewOverlayRepository(primary, fallback Repository) Repository {
return &overlayRepository{primary: primary, fallback: fallback}
}
func (r *overlayRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
if r.primary != nil {
prof, err := r.primary.GetProfile(ctx, id)
if err == nil {
return prof, nil
}
if !errors.Is(err, ErrProfileNotFound) {
return nil, err
}
}
if r.fallback == nil {
return nil, ErrProfileNotFound
}
return r.fallback.GetProfile(ctx, id)
}
func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*domain.ExecutionProfile, error) {
if strings.TrimSpace(id) == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
}
if fsys == nil {
return nil, fmt.Errorf("failed to read profile directory: filesystem is nil")
}
files, err := filecatalog.FindYAMLFiles(ctx, r.dir)
files, err := findProfileYAMLFiles(ctx, fsys, root)
if err != nil {
return nil, fmt.Errorf("failed to read profile directory: %w", err)
}
@@ -47,9 +92,9 @@ func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*doma
default:
}
relPath := filecatalog.RelativePath(r.dir, fullPath)
fileMatch := filecatalog.Stem(filepath.Base(fullPath)) == id
data, err := os.ReadFile(fullPath)
relPath := displayPath(root, fullPath)
fileMatch := profileFileStem(path.Base(fullPath)) == id
data, err := fs.ReadFile(fsys, fullPath)
if err != nil {
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
}
@@ -102,6 +147,61 @@ func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*doma
return nil, ErrProfileNotFound
}
func findProfileYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, error) {
cleanRoot := cleanFSRoot(root)
var files []string
err := fs.WalkDir(fsys, cleanRoot, func(name string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if d.IsDir() {
return nil
}
if !isProfileYAMLFile(d.Name()) {
return nil
}
files = append(files, name)
return nil
})
return files, err
}
func cleanFSRoot(root string) string {
root = strings.TrimSpace(root)
if root == "" || root == "." {
return "."
}
return path.Clean(root)
}
func displayPath(root string, name string) string {
cleanRoot := cleanFSRoot(root)
cleanName := path.Clean(name)
if cleanRoot == "." {
return cleanName
}
prefix := strings.TrimSuffix(cleanRoot, "/") + "/"
if strings.HasPrefix(cleanName, prefix) {
return strings.TrimPrefix(cleanName, prefix)
}
return cleanName
}
func profileFileStem(name string) string {
name = strings.TrimSuffix(name, ".yaml")
name = strings.TrimSuffix(name, ".yml")
return name
}
func isProfileYAMLFile(name string) bool {
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
}
type profileMatch struct {
profile *domain.ExecutionProfile
path string

View File

@@ -8,6 +8,9 @@ import (
"path/filepath"
"strings"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
)
func TestFilesystemRepository_GetProfile(t *testing.T) {
@@ -261,3 +264,216 @@ func writeProfileTestFile(t *testing.T, path string, content string) {
t.Fatalf("failed to write profile test file %q: %v", path, err)
}
}
func TestFSRepository(t *testing.T) {
ctx := context.Background()
t.Run("loads valid profiles from nested directories", func(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"profiles/provider/nested.yaml": profileMapFile(`
id: nested-profile
endpoint: http://localhost:8000/v1
model: nested-model
temperature: 0.1
`),
}, "profiles")
p, err := repo.GetProfile(ctx, "nested-profile")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.ID != "nested-profile" || p.Model != "nested-model" {
t.Fatalf("unexpected profile: %+v", p)
}
})
t.Run("rejects unknown YAML fields", func(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"profiles/unknown.yaml": profileMapFile(`
id: unknown-profile
endpoint: http://localhost:8000/v1
model: model
unknown: value
`),
}, "profiles")
_, err := repo.GetProfile(ctx, "unknown-profile")
if !errors.Is(err, ErrInvalidYAML) {
t.Fatalf("expected ErrInvalidYAML, got %v", err)
}
})
t.Run("rejects raw api_key in selected profile", func(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"profiles/raw.yaml": profileMapFile(`
id: raw-profile
endpoint: http://localhost:8000/v1
model: model
api_key: secret
`),
}, "profiles")
_, err := repo.GetProfile(ctx, "raw-profile")
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
}
})
t.Run("ignores raw api_key in non-selected profiles", func(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"profiles/raw.yaml": profileMapFile(`
id: raw-profile
endpoint: http://localhost:8000/v1
model: model
api_key: secret
`),
"profiles/valid.yaml": profileMapFile(`
id: valid-profile
endpoint: http://localhost:8000/v1
model: model
`),
}, "profiles")
p, err := repo.GetProfile(ctx, "valid-profile")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.ID != "valid-profile" {
t.Fatalf("unexpected profile: %+v", p)
}
})
t.Run("rejects duplicate IDs within one source", func(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"profiles/a.yaml": profileMapFile(`
id: duplicate-profile
endpoint: http://localhost:8000/v1
model: first
`),
"profiles/nested/b.yaml": profileMapFile(`
id: duplicate-profile
endpoint: http://localhost:8000/v1
model: second
`),
}, "profiles")
_, err := repo.GetProfile(ctx, "duplicate-profile")
if !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("expected ErrInvalidProfile, got %v", err)
}
for _, want := range []string{"duplicate execution profile id", "a.yaml", "nested/b.yaml"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("expected error to contain %q, got %v", want, err)
}
}
})
}
func TestOverlayRepository(t *testing.T) {
ctx := context.Background()
primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"}
fallbackProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://fallback", Model: "fallback"}
t.Run("returns primary matches before fallback matches", func(t *testing.T) {
repo := NewOverlayRepository(
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": primaryProfile}},
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
)
p, err := repo.GetProfile(ctx, "shared")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.Model != "primary" {
t.Fatalf("expected primary profile, got %+v", p)
}
})
t.Run("falls back on primary not found", func(t *testing.T) {
repo := NewOverlayRepository(
staticProfileRepo{},
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
)
p, err := repo.GetProfile(ctx, "shared")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.Model != "fallback" {
t.Fatalf("expected fallback profile, got %+v", p)
}
})
t.Run("does not fall back after primary load errors", func(t *testing.T) {
for _, tc := range []struct {
name string
err error
}{
{name: "invalid yaml", err: ErrInvalidYAML},
{name: "invalid profile", err: ErrInvalidProfile},
{name: "raw api key", err: ErrRawAPIKeyNotAllowed},
} {
t.Run(tc.name, func(t *testing.T) {
repo := NewOverlayRepository(
staticProfileRepo{err: tc.err},
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
)
_, err := repo.GetProfile(ctx, "shared")
if !errors.Is(err, tc.err) {
t.Fatalf("expected %v, got %v", tc.err, err)
}
})
}
})
t.Run("returns not found when both sources miss", func(t *testing.T) {
repo := NewOverlayRepository(staticProfileRepo{}, staticProfileRepo{})
_, err := repo.GetProfile(ctx, "missing")
if !errors.Is(err, ErrProfileNotFound) {
t.Fatalf("expected ErrProfileNotFound, got %v", err)
}
})
t.Run("nil primary uses fallback", func(t *testing.T) {
repo := NewOverlayRepository(nil, staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}})
p, err := repo.GetProfile(ctx, "shared")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if p.Model != "fallback" {
t.Fatalf("expected fallback profile, got %+v", p)
}
})
t.Run("nil fallback returns not found after primary miss", func(t *testing.T) {
repo := NewOverlayRepository(staticProfileRepo{}, nil)
_, err := repo.GetProfile(ctx, "missing")
if !errors.Is(err, ErrProfileNotFound) {
t.Fatalf("expected ErrProfileNotFound, got %v", err)
}
})
}
func profileMapFile(content string) *fstest.MapFile {
return &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))}
}
type staticProfileRepo struct {
profiles map[string]*domain.ExecutionProfile
err error
}
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
if r.err != nil {
return nil, r.err
}
if p, ok := r.profiles[id]; ok {
cp := *p
return &cp, nil
}
return nil, ErrProfileNotFound
}

View File

@@ -5,7 +5,9 @@ import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
@@ -24,6 +26,11 @@ type filesystemRepository struct {
dir string
}
type fsRepository struct {
fsys fs.FS
root string
}
type promptDefinitionFile struct {
ID string `yaml:"id"`
Version string `yaml:"version"`
@@ -65,6 +72,10 @@ func NewFilesystemRepository(dir string) Repository {
return &filesystemRepository{dir: dir}
}
func NewFSRepository(fsys fs.FS, root string) Repository {
return &fsRepository{fsys: fsys, root: root}
}
func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
if strings.TrimSpace(id) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
@@ -132,6 +143,10 @@ func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id strin
return nil, ErrPromptDefinitionNotFound
}
func (r *fsRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
return loadPromptDefinition(ctx, r.fsys, r.root, id, version)
}
type promptDefinitionMatch struct {
def *domain.PromptDefinition
path string
@@ -166,7 +181,187 @@ func promptDefinitionFileHasID(path string, id string) bool {
return strings.TrimSpace(raw.ID) == id
}
func loadPromptDefinition(ctx context.Context, fsys fs.FS, root string, id string, version string) (*domain.PromptDefinition, error) {
if strings.TrimSpace(id) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
}
if fsys == nil {
return nil, fmt.Errorf("failed to read prompt definition directory: filesystem is nil")
}
files, err := findPromptDefinitionYAMLFiles(ctx, fsys, root)
if err != nil {
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
}
var matches []promptDefinitionMatch
for _, fullPath := range files {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
relPath := displayPath(root, fullPath)
fileMatch := filecatalog.Stem(path.Base(fullPath)) == id
data, err := fs.ReadFile(fsys, fullPath)
if err != nil {
if fileMatch {
return nil, fmt.Errorf("%w: %s: failed to read prompt definition file: %v", ErrInvalidYAML, relPath, err)
}
continue
}
raw, err := decodePromptDefinition(data)
if err != nil {
if fileMatch || promptDefinitionDataHasID(data, id) {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
}
continue
}
def, err := normalizePromptDefinitionFromFS(raw, fsys, fullPath)
if err != nil {
if fileMatch || strings.TrimSpace(raw.ID) == id {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, relPath, err)
}
continue
}
if def.ID != id {
continue
}
if version != "" && def.Version != version {
continue
}
matches = append(matches, promptDefinitionMatch{
def: def,
path: relPath,
})
}
if len(matches) > 1 {
paths := make([]string, 0, len(matches))
for _, match := range matches {
paths = append(paths, match.path)
}
if version != "" {
return nil, fmt.Errorf("%w: duplicate prompt definition id %q version %q found in: %s", ErrInvalidPromptDefinition, id, version, strings.Join(paths, ", "))
}
return nil, fmt.Errorf("%w: duplicate prompt definition id %q found in: %s", ErrInvalidPromptDefinition, id, strings.Join(paths, ", "))
}
if len(matches) == 1 {
return matches[0].def, nil
}
return nil, ErrPromptDefinitionNotFound
}
func findPromptDefinitionYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, error) {
cleanRoot := cleanFSRoot(root)
var files []string
err := fs.WalkDir(fsys, cleanRoot, func(name string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if d.IsDir() {
return nil
}
if !isPromptDefinitionYAMLFile(d.Name()) {
return nil
}
files = append(files, name)
return nil
})
return files, err
}
func cleanFSRoot(root string) string {
root = strings.TrimSpace(root)
if root == "" || root == "." {
return "."
}
return path.Clean(root)
}
func displayPath(root string, name string) string {
cleanRoot := cleanFSRoot(root)
cleanName := path.Clean(name)
if cleanRoot == "." {
return cleanName
}
prefix := strings.TrimSuffix(cleanRoot, "/") + "/"
if strings.HasPrefix(cleanName, prefix) {
return strings.TrimPrefix(cleanName, prefix)
}
return cleanName
}
func isPromptDefinitionYAMLFile(name string) bool {
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
}
func decodePromptDefinition(data []byte) (*promptDefinitionFile, error) {
var raw promptDefinitionFile
decoder := yaml.NewDecoder(bytes.NewReader(data))
decoder.KnownFields(true)
if err := decoder.Decode(&raw); err != nil {
return nil, err
}
return &raw, nil
}
func promptDefinitionDataHasID(data []byte, id string) bool {
var raw struct {
ID string `yaml:"id"`
}
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&raw); err != nil {
return false
}
return strings.TrimSpace(raw.ID) == id
}
func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*domain.PromptDefinition, error) {
promptDir := filepath.Dir(sourcePath)
return normalizePromptDefinitionWithContent(raw, func(contentFile string) (string, string, error) {
resolvedPath := strings.TrimSpace(contentFile)
if !filepath.IsAbs(resolvedPath) {
resolvedPath = filepath.Join(promptDir, resolvedPath)
}
resolvedPath = filepath.Clean(resolvedPath)
body, err := os.ReadFile(resolvedPath)
if err != nil {
return "", "", err
}
return string(body), resolvedPath, nil
})
}
func normalizePromptDefinitionFromFS(raw *promptDefinitionFile, fsys fs.FS, sourcePath string) (*domain.PromptDefinition, error) {
promptDir := path.Dir(sourcePath)
return normalizePromptDefinitionWithContent(raw, func(contentFile string) (string, string, error) {
resolvedPath := strings.TrimSpace(contentFile)
if !path.IsAbs(resolvedPath) {
resolvedPath = path.Join(promptDir, resolvedPath)
}
resolvedPath = strings.TrimPrefix(path.Clean(resolvedPath), "/")
body, err := fs.ReadFile(fsys, resolvedPath)
if err != nil {
return "", "", err
}
return string(body), resolvedPath, nil
})
}
func normalizePromptDefinitionWithContent(raw *promptDefinitionFile, readContentFile func(string) (string, string, error)) (*domain.PromptDefinition, error) {
if raw == nil {
return nil, errors.New("prompt definition is nil")
}
@@ -206,7 +401,6 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
}
templates := make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
promptDir := filepath.Dir(sourcePath)
for i, msg := range raw.Messages {
role := strings.TrimSpace(msg.Role)
if role == "" {
@@ -227,17 +421,11 @@ func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*d
templateContent := msg.Content
resolvedContentFile := ""
if hasContentFile {
resolvedPath := strings.TrimSpace(msg.ContentFile)
if !filepath.IsAbs(resolvedPath) {
resolvedPath = filepath.Join(promptDir, resolvedPath)
}
resolvedPath = filepath.Clean(resolvedPath)
body, err := os.ReadFile(resolvedPath)
body, resolvedPath, err := readContentFile(msg.ContentFile)
if err != nil {
return nil, fmt.Errorf("prompt %q message %d (%s): failed to read content_file %q: %w", id, i, role, msg.ContentFile, err)
}
templateContent = string(body)
templateContent = body
resolvedContentFile = resolvedPath
}

View File

@@ -8,6 +8,7 @@ import (
"path/filepath"
"strings"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
)
@@ -324,6 +325,97 @@ output:
})
}
func TestFSRepositoryGetPromptDefinition(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"prompts/nested/prompt.yaml": &fstest.MapFile{Data: []byte(`
id: fs-prompt
version: "1.0.0"
inputs:
- name: transcript
required: true
messages:
- role: user
content_file: ./messages/user.tmpl
output:
format: markdown
validation_mode: basic
repair_attempts: 0
`)},
"prompts/nested/messages/user.tmpl": &fstest.MapFile{Data: []byte(`Summarize {{input "transcript"}}.`)},
}, "prompts")
got, err := repo.GetPromptDefinition(context.Background(), "fs-prompt", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if got.ID != "fs-prompt" {
t.Fatalf("unexpected prompt id: %q", got.ID)
}
if len(got.Templates) != 1 || !strings.Contains(got.Templates[0].Content, `{{input "transcript"}}`) {
t.Fatalf("expected content_file body to be loaded, got %+v", got.Templates)
}
if got.Templates[0].ContentFile != "prompts/nested/messages/user.tmpl" {
t.Fatalf("unexpected content file path: %q", got.Templates[0].ContentFile)
}
}
func TestFSRepositoryRejectsDuplicatePromptIDs(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"one.yaml": &fstest.MapFile{Data: []byte(`
id: duplicate-fs-prompt
version: "1.0.0"
messages:
- role: user
content: First.
output:
format: text
validation_mode: none
repair_attempts: 0
`)},
"nested/two.yaml": &fstest.MapFile{Data: []byte(`
id: duplicate-fs-prompt
version: "1.0.0"
messages:
- role: user
content: Second.
output:
format: text
validation_mode: none
repair_attempts: 0
`)},
}, ".")
_, err := repo.GetPromptDefinition(context.Background(), "duplicate-fs-prompt", "")
if !errors.Is(err, ErrInvalidPromptDefinition) {
t.Fatalf("expected ErrInvalidPromptDefinition, got %v", err)
}
if !strings.Contains(err.Error(), "one.yaml") || !strings.Contains(err.Error(), "nested/two.yaml") {
t.Fatalf("expected duplicate paths in error, got %v", err)
}
}
func TestFSRepositoryRejectsUnknownYAMLFields(t *testing.T) {
repo := NewFSRepository(fstest.MapFS{
"not_named_like_id.yaml": &fstest.MapFile{Data: []byte(`
id: strict-fs-prompt
version: "1.0.0"
unknown: true
messages:
- role: user
content: Invalid.
output:
format: text
validation_mode: none
repair_attempts: 0
`)},
}, ".")
_, err := repo.GetPromptDefinition(context.Background(), "strict-fs-prompt", "")
if !errors.Is(err, ErrInvalidYAML) {
t.Fatalf("expected ErrInvalidYAML, got %v", err)
}
}
func assertCacheControl(t *testing.T, got *domain.CacheControl, wantType domain.CacheControlType, wantTTL string) {
t.Helper()
if got == nil {

View File

@@ -195,13 +195,14 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
effectiveModel.APIKey = req.APIKey
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
}
if strings.TrimSpace(effectiveModel.Model) == "" {
return nil, fmt.Errorf("%w: execution model is required", ErrInvalidRequest)
}
if err := validateAPIKeyEnv(effectiveModel.APIKeyEnv); err != nil {
if err := validateAPIKey(effectiveModel.APIKeyEnv, effectiveModel.APIKey); err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
@@ -434,7 +435,10 @@ func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *dom
return out, presence, nil
}
func validateAPIKeyEnv(apiKeyEnv string) error {
func validateAPIKey(apiKeyEnv string, apiKey string) error {
if strings.TrimSpace(apiKey) != "" {
return nil
}
envName := strings.TrimSpace(apiKeyEnv)
if envName == "" {
return nil

View File

@@ -1179,6 +1179,32 @@ func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) {
}
}
func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
const directKey = "direct-runner-key"
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "SCRIPTORIUM_MISSING_KEY"},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
APIKey: directKey,
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if llmClient.lastReq.Target.APIKey != directKey {
t.Fatalf("expected direct API key to reach LLM request")
}
if llmClient.lastReq.Target.APIKeyEnv != "SCRIPTORIUM_MISSING_KEY" {
t.Fatalf("expected api_key_env name to remain on target, got %q", llmClient.lastReq.Target.APIKeyEnv)
}
}
func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
const envName = "SCRIPTORIUM_RUNTIME_API_KEY"
t.Setenv(envName, "runtime-secret")

View File

@@ -5,7 +5,9 @@ import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
@@ -18,10 +20,19 @@ type StandardValidator struct {
schemaBaseDir string
}
type FSValidator struct {
fsys fs.FS
root string
}
func NewStandardValidator(schemaBaseDir string) Validator {
return &StandardValidator{schemaBaseDir: schemaBaseDir}
}
func NewFSValidator(fsys fs.FS, root string) Validator {
return &FSValidator{fsys: fsys, root: root}
}
func (v *StandardValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
select {
case <-ctx.Done():
@@ -100,6 +111,88 @@ func (v *StandardValidator) Validate(ctx context.Context, artifact *domain.Artif
}
}
func (v *FSValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
select {
case <-ctx.Done():
return domain.ValidationResult{}, ctx.Err()
default:
}
res := domain.ValidationResult{
Mode: contract.ValidationMode,
SchemaPath: contract.SchemaPath,
RepairAttempts: contract.RepairAttempts,
}
if artifact == nil {
return domain.ValidationResult{}, errors.New("artifact is required for validation")
}
switch contract.ValidationMode {
case domain.ValidationNone:
res.Status = domain.ValidationSkipped
res.IsValid = true
return res, nil
case domain.ValidationBasic:
if strings.TrimSpace(string(artifact.Body)) == "" {
res.Status = domain.ValidationFailed
res.IsValid = false
res.Errors = []string{"output is empty"}
return res, nil
}
res.Status = domain.ValidationPassed
res.IsValid = true
return res, nil
case domain.ValidationJSON:
_, jsonErr := parseJSON(artifact.Body)
if jsonErr != nil {
res.Status = domain.ValidationFailed
res.IsValid = false
res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)}
return res, nil
}
res.Status = domain.ValidationPassed
res.IsValid = true
return res, nil
case domain.ValidationJSONSchema:
instance, jsonErr := parseJSON(artifact.Body)
if jsonErr != nil {
res.Status = domain.ValidationFailed
res.IsValid = false
res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)}
return res, nil
}
schemaName, schemaDoc, err := v.loadSchemaDocument(contract.SchemaPath)
if err != nil {
return domain.ValidationResult{}, err
}
resourceURL := fsSchemaResourceURL(schemaName)
compiler := jsonschema.NewCompiler()
if err := compiler.AddResource(resourceURL, schemaDoc); err != nil {
return domain.ValidationResult{}, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
}
schema, err := compiler.Compile(resourceURL)
if err != nil {
return domain.ValidationResult{}, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
}
if err := schema.Validate(instance); err != nil {
res.Status = domain.ValidationFailed
res.IsValid = false
res.Errors = []string{fmt.Sprintf("json schema validation failed: %v", err)}
return res, nil
}
res.Status = domain.ValidationPassed
res.IsValid = true
return res, nil
default:
return domain.ValidationResult{}, fmt.Errorf("unsupported validation mode: %q", contract.ValidationMode)
}
}
func parseJSON(body []byte) (any, error) {
var v any
if err := json.Unmarshal(body, &v); err != nil {
@@ -132,6 +225,20 @@ func (v *StandardValidator) LoadSchemaDocument(ctx context.Context, schemaPath s
return doc, nil
}
func (v *FSValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
_, doc, err := v.loadSchemaDocument(schemaPath)
if err != nil {
return nil, err
}
return doc, nil
}
func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error) {
if strings.TrimSpace(schemaPath) == "" {
return "", errors.New("schema path is required for json_schema validation")
@@ -149,3 +256,70 @@ func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error)
return resolved, nil
}
func (v *FSValidator) loadSchemaDocument(schemaPath string) (string, any, error) {
resolved, err := v.resolveSchemaPath(schemaPath)
if err != nil {
return "", nil, err
}
raw, err := fs.ReadFile(v.fsys, resolved)
if err != nil {
return "", nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err)
}
var doc any
if err := json.Unmarshal(raw, &doc); err != nil {
return "", nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
}
return resolved, doc, nil
}
func (v *FSValidator) resolveSchemaPath(schemaPath string) (string, error) {
if strings.TrimSpace(schemaPath) == "" {
return "", errors.New("schema path is required for json_schema validation")
}
if v.fsys == nil {
return "", errors.New("schema filesystem is nil")
}
cleanRoot := cleanFSRoot(v.root)
rootInfo, err := fs.Stat(v.fsys, cleanRoot)
if err != nil {
return "", fmt.Errorf("failed to access schema source %q: %w", cleanRoot, err)
}
cleanSchemaPath := cleanSchemaFSPath(schemaPath)
var resolved string
if rootInfo.IsDir() {
resolved = path.Join(cleanRoot, cleanSchemaPath)
} else {
if cleanSchemaPath != path.Base(cleanRoot) {
return "", fmt.Errorf("schema path %q does not match schema file %q", cleanSchemaPath, path.Base(cleanRoot))
}
resolved = cleanRoot
}
if _, err := fs.Stat(v.fsys, resolved); err != nil {
return "", fmt.Errorf("failed to access schema file %q: %w", resolved, err)
}
return resolved, nil
}
func cleanSchemaFSPath(schemaPath string) string {
cleaned := strings.TrimSpace(schemaPath)
cleaned = strings.TrimPrefix(path.Clean(cleaned), "/")
return cleaned
}
func cleanFSRoot(root string) string {
root = strings.TrimSpace(root)
if root == "" || root == "." {
return "."
}
return strings.TrimPrefix(path.Clean(root), "/")
}
func fsSchemaResourceURL(schemaName string) string {
return "scriptorium-schema:///" + strings.TrimPrefix(path.Clean(schemaName), "/")
}

View File

@@ -5,6 +5,7 @@ import (
"os"
"path/filepath"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
)
@@ -250,3 +251,77 @@ func TestStandardValidatorLoadSchemaDocumentInvalidJSON(t *testing.T) {
t.Fatal("expected decode error")
}
}
func TestFSValidatorJSONSchemaSuccess(t *testing.T) {
v := NewFSValidator(fstest.MapFS{
"schemas/events.schema.json": &fstest.MapFile{Data: []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"required": ["events"],
"properties": {
"events": {"type": "array"}
}
}`)},
}, "schemas")
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "events.schema.json",
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.Status != domain.ValidationPassed || !res.IsValid {
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
}
}
func TestFSValidatorSingleSchemaFileUsesBaseName(t *testing.T) {
v := NewFSValidator(fstest.MapFS{
"events.schema.json": &fstest.MapFile{Data: []byte(`{
"type": "object",
"required": ["events"],
"properties": {
"events": {"type": "array"}
}
}`)},
}, "events.schema.json")
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "events.schema.json",
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.Status != domain.ValidationPassed || !res.IsValid {
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
}
_, err = v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "other.schema.json",
})
if err == nil {
t.Fatal("expected schema path mismatch error")
}
}
func TestFSValidatorLoadSchemaDocument(t *testing.T) {
v := NewFSValidator(fstest.MapFS{
"schemas/schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
}, "schemas")
loader, ok := v.(SchemaDocumentLoader)
if !ok {
t.Fatal("fs validator must implement SchemaDocumentLoader")
}
doc, err := loader.LoadSchemaDocument(context.Background(), "schema.json")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
obj, ok := doc.(map[string]any)
if !ok || obj["type"] != "object" {
t.Fatalf("unexpected schema document: %#v", doc)
}
}

View File

@@ -60,6 +60,7 @@ type RunRequest struct {
PromptID string
PromptVersion string
ProfileID string
APIKey string `json:"-"`
Inputs map[string]ArtifactRef
Vars map[string]string
Execution *ExecutionTargetOverride
@@ -232,6 +233,7 @@ type GenerateRequest struct {
Target ExecutionTarget `json:"target"`
TargetPresence ExecutionTargetPresence `json:"target_presence"`
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
APIKey string `json:"-"`
}
// GenerateResponse is returned by an injected LLM client.