Compare commits
66 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f33af20524 | |||
| 85d14784ca | |||
| edca9fbbc1 | |||
| 6c2541c1ee | |||
| b9b98b2aec | |||
| 8936ca7c18 | |||
| 3e5f5c5198 | |||
| 68ebe9ee50 | |||
| 36c7c5a358 | |||
| 157209f097 | |||
| 06b37c2bae | |||
| 1a0f15e210 | |||
| 5f946a5a1f | |||
| 1806df9888 | |||
| c35ac5a0cb | |||
| 40f8a1c628 | |||
| 27d7ad5057 | |||
| 1f08a1a94c | |||
| 2f42bdde39 | |||
| 9a00f30c7b | |||
| f71d2bbb73 | |||
| 6f64947e42 | |||
| 7bb4cf35b9 | |||
| fb0b21c51d | |||
| 5a00ca81a2 | |||
| e13610481d | |||
| 309fe9b7ea | |||
| adfd08ffe2 | |||
| c7263ab2a8 | |||
| 532c31c09a | |||
| 68cd90c657 | |||
| 416438d80d | |||
| 9298d8ae73 | |||
| 4c7278febc | |||
| 096208532e | |||
| 50bd19b9d1 | |||
| 2fc7204bd5 | |||
| c0d4ea0d4e | |||
| 4d7e1327ad | |||
| 3c33b52b15 | |||
| 280916bf4a | |||
| 033bc93d3c | |||
| 45c2644b9d | |||
| a74c03bd9b | |||
| ad115a2259 | |||
| 3074b3165a | |||
| 8703793b0c | |||
| 4cb4943a57 | |||
| c6c747e94d | |||
| 2bbf13e739 | |||
| 5edb24a9c1 | |||
| 99d5e96316 | |||
| 144d840fbe | |||
| ed0c9f6370 | |||
| a0e905ce46 | |||
| 719243e90c | |||
| a9e1b7435c | |||
| eb6dfb19b0 | |||
| d86b65adad | |||
| 31faaf4259 | |||
| 9932153b97 | |||
| f0ca233c25 | |||
| ff31f8daf8 | |||
| c927b7819d | |||
| 6d1fb66dd7 | |||
| e0b1d6a0dc |
3
.gitignore
vendored
3
.gitignore
vendored
@@ -56,6 +56,8 @@ mono_crash.*
|
||||
[Dd]ebugPublic/
|
||||
[Rr]elease/
|
||||
[Rr]eleases/
|
||||
!docs/releases/
|
||||
!docs/releases/*.md
|
||||
x64/
|
||||
x86/
|
||||
[Ww][Ii][Nn]32/
|
||||
@@ -433,4 +435,3 @@ FodyWeavers.xsd
|
||||
|
||||
# JetBrains Rider
|
||||
*.sln.iml
|
||||
|
||||
|
||||
@@ -11,9 +11,16 @@ steps:
|
||||
version="$CI_COMMIT_TAG"
|
||||
dist="dist"
|
||||
pkg="gitea.maximumdirect.net/eric/scriptorium/cmd/scriptorium"
|
||||
notes="docs/releases/$version.md"
|
||||
|
||||
if [ ! -f "$notes" ]; then
|
||||
printf 'release notes not found: %s\n' "$notes" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rm -rf "$dist"
|
||||
mkdir -p "$dist"
|
||||
cp "$notes" "$dist/RELEASE_NOTES.md"
|
||||
|
||||
build_binary() {
|
||||
goos="$1"
|
||||
@@ -22,7 +29,7 @@ steps:
|
||||
output="$dist/scriptorium-$version-$goos-$goarch$suffix"
|
||||
|
||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
|
||||
go build -trimpath -ldflags "-s -w -X gitea.maximumdirect.net/eric/scriptorium/internal/buildinfo.Version=$version" \
|
||||
go build -trimpath -ldflags "-s -w" \
|
||||
-o "$output" "$pkg"
|
||||
}
|
||||
|
||||
@@ -38,6 +45,7 @@ steps:
|
||||
from_secret: GITEA_RELEASE_TOKEN
|
||||
files:
|
||||
- dist/scriptorium-*
|
||||
note: dist/RELEASE_NOTES.md
|
||||
checksum: sha256
|
||||
checksum-file: SHA256SUMS
|
||||
checksum-flatten: true
|
||||
|
||||
@@ -1,4 +1 @@
|
||||
Please carefully review the relevant documents in `docs/policy` before making any changes to this repository.
|
||||
- `development.md` defines the contributor workflow for this application.
|
||||
- `architecture.md` provides the canonical high-level architecture policy for this repository, and should be reviewed before writing or changing any code.
|
||||
- `documentation.md` provides the canonical documentation policy for this repository, and should be reviewed before writing or changing any documentation.
|
||||
Please review `docs/development.md` for initial orientation in this repository and follow its task-specific reading guide.
|
||||
|
||||
38
README.md
38
README.md
@@ -1,12 +1,15 @@
|
||||
# scriptorium
|
||||
# Scriptorium
|
||||
|
||||
Scriptorium is a narrow prompt-execution application for rendering prompt
|
||||
requests, running them against OpenAI-compatible chat-completions endpoints, and
|
||||
serving the same run workflow over HTTP.
|
||||
Scriptorium is a prompt-execution application with a command-line interface and
|
||||
an HTTP service. It prepares prompt requests, runs them against
|
||||
OpenAI-compatible model endpoints, and returns generated output with validation
|
||||
metadata.
|
||||
|
||||
It keeps prompt definitions, execution profiles, schemas, and input artifacts as
|
||||
separate files so prompts can be reviewed and reused without baking model
|
||||
runtime settings into application code.
|
||||
The application uses
|
||||
[Promptkit v0.9.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/)
|
||||
for prompt, profile, schema, preparation, generation, and validation behavior.
|
||||
Scriptorium owns executable configuration, CLI and HTTP mapping, process
|
||||
behavior, output presentation, and HTTP artifact-containment policy.
|
||||
|
||||
## Quickstart
|
||||
|
||||
@@ -21,7 +24,9 @@ go run ./cmd/scriptorium render \
|
||||
--format json
|
||||
```
|
||||
|
||||
This command renders the prepared prompt and effective runtime settings without calling an LLM.
|
||||
This renders the prepared prompt and effective runtime settings without calling
|
||||
a model. For complete invocation and output behavior, see the
|
||||
[CLI reference](docs/cli.md).
|
||||
|
||||
## Documentation
|
||||
|
||||
@@ -29,17 +34,18 @@ This command renders the prepared prompt and effective runtime settings without
|
||||
- [Configuration reference](docs/config.md)
|
||||
- [HTTP API reference](docs/api.md)
|
||||
- [Operations guide](docs/operations.md)
|
||||
- [Troubleshooting](docs/troubleshooting.md)
|
||||
- [Consumer integration overview](docs/consumers/api.md)
|
||||
- [Go library package](docs/consumers/pkg-scriptorium.md)
|
||||
- [Migration from the former Go package](docs/consumers/migrating-to-promptkit.md)
|
||||
- [Subprocess integration](docs/integrations/subprocess.md)
|
||||
- [OpenAI-compatible chat integration](docs/integrations/openai-compatible-chat.md)
|
||||
- [Architecture policy](docs/policy/architecture.md)
|
||||
- [Promptkit framework formats](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md)
|
||||
- [Promptkit Go consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md)
|
||||
|
||||
## Examples
|
||||
|
||||
- `examples/config.yml`
|
||||
- `examples/config.full.yml`
|
||||
- `examples/render-markdown-summary.sh`
|
||||
- `examples/http-run.json`
|
||||
- `examples/go-library/prepare`
|
||||
- [Minimal configuration](examples/config.yml) and
|
||||
[complete configuration](examples/config.full.yml)
|
||||
- [Prompt definitions](examples/prompts/), [execution profiles](examples/profiles/),
|
||||
[schemas](examples/schemas/), and [synthetic input fixtures](examples/fixtures/)
|
||||
- [Render script](examples/render-markdown-summary.sh)
|
||||
- [HTTP request](examples/http-run.json)
|
||||
|
||||
406
convert.go
406
convert.go
@@ -1,406 +0,0 @@
|
||||
package scriptorium
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
)
|
||||
|
||||
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
||||
execution, err := toDomainExecutionTargetOverride(req.Execution)
|
||||
if err != nil {
|
||||
return domain.RunRequest{}, err
|
||||
}
|
||||
return domain.RunRequest{
|
||||
PromptID: req.PromptID,
|
||||
PromptVersion: req.PromptVersion,
|
||||
ProfileID: req.ProfileID,
|
||||
APIKey: req.APIKey,
|
||||
Inputs: toDomainArtifactRefMap(req.Inputs),
|
||||
Vars: copyStringMap(req.Vars),
|
||||
Execution: execution,
|
||||
Validation: toDomainOutputContractPtr(req.Validation),
|
||||
Metadata: copyStringMap(req.Metadata),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
||||
if prepared == nil {
|
||||
return nil
|
||||
}
|
||||
return &PreparedRun{
|
||||
PromptID: prepared.PromptID,
|
||||
PromptVersion: prepared.PromptVersion,
|
||||
PromptHash: prepared.PromptHash,
|
||||
SelectedProfileID: prepared.SelectedProfileID,
|
||||
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
|
||||
OutputContract: fromDomainOutputContract(prepared.OutputContract),
|
||||
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
|
||||
InputHashes: copyStringMap(prepared.InputHashes),
|
||||
SessionID: prepared.SessionID,
|
||||
RenderedPromptHash: prepared.RenderedPromptHash,
|
||||
Messages: fromDomainRenderedMessages(prepared.Messages),
|
||||
StartTime: prepared.StartTime,
|
||||
EndTime: prepared.EndTime,
|
||||
DurationMS: prepared.DurationMS,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
return &RunResult{
|
||||
RunID: result.RunID,
|
||||
Artifact: fromDomainArtifact(result.Artifact),
|
||||
RawOutput: result.RawOutput,
|
||||
Validation: fromDomainValidationResult(result.Validation),
|
||||
PromptID: result.PromptID,
|
||||
PromptVersion: result.PromptVersion,
|
||||
PromptHash: result.PromptHash,
|
||||
RenderedPromptHash: result.RenderedPromptHash,
|
||||
SelectedProfileID: result.SelectedProfileID,
|
||||
ModelName: result.ModelName,
|
||||
Endpoint: result.Endpoint,
|
||||
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
|
||||
InputHashes: copyStringMap(result.InputHashes),
|
||||
Usage: fromDomainTokenUsage(result.Usage),
|
||||
StartTime: result.StartTime,
|
||||
EndTime: result.EndTime,
|
||||
Duration: result.Duration,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainGenerateRequest(req domain.GenerateRequest) GenerateRequest {
|
||||
return GenerateRequest{
|
||||
Prompt: fromDomainRenderedPrompt(req.Prompt),
|
||||
Target: fromDomainExecutionTarget(req.Target),
|
||||
TargetPresence: fromDomainExecutionTargetPresence(req.TargetPresence),
|
||||
StructuredOutput: fromDomainStructuredOutputSpec(req.StructuredOutput),
|
||||
APIKey: req.Target.APIKey,
|
||||
}
|
||||
}
|
||||
|
||||
func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse {
|
||||
if resp == nil {
|
||||
return nil
|
||||
}
|
||||
return &domain.GenerateResponse{
|
||||
Content: resp.Content,
|
||||
Usage: toDomainTokenUsage(resp.Usage),
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainRenderedPrompt(prompt domain.RenderedPrompt) RenderedPrompt {
|
||||
return RenderedPrompt{
|
||||
SessionID: prompt.SessionID,
|
||||
Messages: fromDomainRenderedMessages(prompt.Messages),
|
||||
}
|
||||
}
|
||||
|
||||
func toDomainArtifactRefMap(src map[string]ArtifactRef) map[string]domain.ArtifactRef {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]domain.ArtifactRef, len(src))
|
||||
for k, v := range src {
|
||||
out[k] = toDomainArtifactRef(v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func toDomainArtifactRef(ref ArtifactRef) domain.ArtifactRef {
|
||||
return domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefType(ref.Type),
|
||||
URI: ref.URI,
|
||||
Body: ref.Body,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainArtifact(artifact domain.Artifact) Artifact {
|
||||
return Artifact{
|
||||
Name: artifact.Name,
|
||||
ContentType: artifact.ContentType,
|
||||
Body: copyBytes(artifact.Body),
|
||||
URI: artifact.URI,
|
||||
Size: artifact.Size,
|
||||
Hash: artifact.Hash,
|
||||
}
|
||||
}
|
||||
|
||||
func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain.ExecutionTargetOverride, error) {
|
||||
if override == nil {
|
||||
return nil, nil
|
||||
}
|
||||
extraParams, err := copyPublicJSONMap(override.ExtraParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &domain.ExecutionTargetOverride{
|
||||
Endpoint: override.Endpoint,
|
||||
Model: override.Model,
|
||||
Temperature: copyFloat64Ptr(override.Temperature),
|
||||
MaxTokens: copyIntPtr(override.MaxTokens),
|
||||
TopP: copyFloat64Ptr(override.TopP),
|
||||
TimeoutSeconds: copyIntPtr(override.TimeoutSeconds),
|
||||
ServiceTier: override.ServiceTier,
|
||||
ReasoningEffort: override.ReasoningEffort,
|
||||
APIKeyEnv: override.APIKeyEnv,
|
||||
ExtraParams: extraParams,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
||||
return ExecutionTarget{
|
||||
Endpoint: target.Endpoint,
|
||||
Model: target.Model,
|
||||
Temperature: target.Temperature,
|
||||
MaxTokens: target.MaxTokens,
|
||||
TopP: target.TopP,
|
||||
TimeoutSeconds: target.TimeoutSeconds,
|
||||
ServiceTier: target.ServiceTier,
|
||||
ReasoningEffort: target.ReasoningEffort,
|
||||
APIKeyEnv: target.APIKeyEnv,
|
||||
ExtraParams: copyAnyMap(target.ExtraParams),
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
||||
return ExecutionTargetPresence{
|
||||
Temperature: presence.Temperature,
|
||||
MaxTokens: presence.MaxTokens,
|
||||
TopP: presence.TopP,
|
||||
TimeoutSeconds: presence.TimeoutSeconds,
|
||||
}
|
||||
}
|
||||
|
||||
func toDomainOutputContractPtr(contract *OutputContract) *domain.OutputContract {
|
||||
if contract == nil {
|
||||
return nil
|
||||
}
|
||||
out := toDomainOutputContract(*contract)
|
||||
return &out
|
||||
}
|
||||
|
||||
func toDomainOutputContract(contract OutputContract) domain.OutputContract {
|
||||
return domain.OutputContract{
|
||||
Format: domain.OutputFormat(contract.Format),
|
||||
ValidationMode: domain.ValidationMode(contract.ValidationMode),
|
||||
SchemaPath: contract.SchemaPath,
|
||||
RepairAttempts: contract.RepairAttempts,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainOutputContract(contract domain.OutputContract) OutputContract {
|
||||
return OutputContract{
|
||||
Format: OutputFormat(contract.Format),
|
||||
ValidationMode: ValidationMode(contract.ValidationMode),
|
||||
SchemaPath: contract.SchemaPath,
|
||||
RepairAttempts: contract.RepairAttempts,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainValidationResult(result domain.ValidationResult) ValidationResult {
|
||||
return ValidationResult{
|
||||
Status: ValidationStatus(result.Status),
|
||||
Mode: ValidationMode(result.Mode),
|
||||
Errors: copyStringSlice(result.Errors),
|
||||
SchemaPath: result.SchemaPath,
|
||||
RepairAttempts: result.RepairAttempts,
|
||||
IsValid: result.IsValid,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainTokenUsage(usage domain.TokenUsage) TokenUsage {
|
||||
return TokenUsage{
|
||||
PromptTokens: usage.PromptTokens,
|
||||
CompletionTokens: usage.CompletionTokens,
|
||||
TotalTokens: usage.TotalTokens,
|
||||
CachedTokens: usage.CachedTokens,
|
||||
CacheWriteTokens: usage.CacheWriteTokens,
|
||||
}
|
||||
}
|
||||
|
||||
func toDomainTokenUsage(usage TokenUsage) domain.TokenUsage {
|
||||
return domain.TokenUsage{
|
||||
PromptTokens: usage.PromptTokens,
|
||||
CompletionTokens: usage.CompletionTokens,
|
||||
TotalTokens: usage.TotalTokens,
|
||||
CachedTokens: usage.CachedTokens,
|
||||
CacheWriteTokens: usage.CacheWriteTokens,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainRenderedMessages(messages []domain.RenderedMessage) []RenderedMessage {
|
||||
if messages == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]RenderedMessage, len(messages))
|
||||
for i, msg := range messages {
|
||||
out[i] = RenderedMessage{
|
||||
Role: msg.Role,
|
||||
Content: msg.Content,
|
||||
CacheControl: fromDomainCacheControl(msg.CacheControl),
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl {
|
||||
if cacheControl == nil {
|
||||
return nil
|
||||
}
|
||||
return &CacheControl{
|
||||
Type: CacheControlType(cacheControl.Type),
|
||||
TTL: cacheControl.TTL,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainStructuredOutputSpec(spec *domain.StructuredOutputSpec) *StructuredOutputSpec {
|
||||
if spec == nil {
|
||||
return nil
|
||||
}
|
||||
out := &StructuredOutputSpec{
|
||||
Type: StructuredOutputType(spec.Type),
|
||||
}
|
||||
if spec.JSONSchema != nil {
|
||||
out.JSONSchema = &StructuredOutputJSONSpec{
|
||||
Name: spec.JSONSchema.Name,
|
||||
Strict: spec.JSONSchema.Strict,
|
||||
Schema: copyAny(spec.JSONSchema.Schema),
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func copyStringMap(src map[string]string) map[string]string {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(src))
|
||||
for k, v := range src {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func copyAnyMap(src map[string]any) map[string]any {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(src))
|
||||
for k, v := range src {
|
||||
out[k] = copyAny(v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func copyAny(value any) any {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
switch v := value.(type) {
|
||||
case map[string]any:
|
||||
return copyAnyMap(v)
|
||||
case []any:
|
||||
out := make([]any, len(v))
|
||||
for i, item := range v {
|
||||
out[i] = copyAny(item)
|
||||
}
|
||||
return out
|
||||
case []string:
|
||||
return copyStringSlice(v)
|
||||
case []byte:
|
||||
return copyBytes(v)
|
||||
default:
|
||||
return copyReflectValue(reflect.ValueOf(value)).Interface()
|
||||
}
|
||||
}
|
||||
|
||||
func copyReflectValue(value reflect.Value) reflect.Value {
|
||||
if !value.IsValid() {
|
||||
return value
|
||||
}
|
||||
|
||||
switch value.Kind() {
|
||||
case reflect.Interface:
|
||||
if value.IsNil() {
|
||||
return reflect.Zero(value.Type())
|
||||
}
|
||||
copied := copyReflectValue(value.Elem())
|
||||
if copied.IsValid() && copied.Type().AssignableTo(value.Type()) {
|
||||
return copied
|
||||
}
|
||||
out := reflect.New(value.Type()).Elem()
|
||||
out.Set(copied)
|
||||
return out
|
||||
case reflect.Pointer:
|
||||
if value.IsNil() {
|
||||
return reflect.Zero(value.Type())
|
||||
}
|
||||
out := reflect.New(value.Type().Elem())
|
||||
out.Elem().Set(copyReflectValue(value.Elem()))
|
||||
return out
|
||||
case reflect.Map:
|
||||
if value.IsNil() {
|
||||
return reflect.Zero(value.Type())
|
||||
}
|
||||
out := reflect.MakeMapWithSize(value.Type(), value.Len())
|
||||
iter := value.MapRange()
|
||||
for iter.Next() {
|
||||
out.SetMapIndex(copyReflectValue(iter.Key()), copyReflectValue(iter.Value()))
|
||||
}
|
||||
return out
|
||||
case reflect.Slice:
|
||||
if value.IsNil() {
|
||||
return reflect.Zero(value.Type())
|
||||
}
|
||||
out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap())
|
||||
for i := 0; i < value.Len(); i++ {
|
||||
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
||||
}
|
||||
return out
|
||||
case reflect.Array:
|
||||
out := reflect.New(value.Type()).Elem()
|
||||
for i := 0; i < value.Len(); i++ {
|
||||
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
||||
}
|
||||
return out
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
func copyStringSlice(src []string) []string {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, len(src))
|
||||
copy(out, src)
|
||||
return out
|
||||
}
|
||||
|
||||
func copyBytes(src []byte) []byte {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]byte, len(src))
|
||||
copy(out, src)
|
||||
return out
|
||||
}
|
||||
|
||||
func copyFloat64Ptr(src *float64) *float64 {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
v := *src
|
||||
return &v
|
||||
}
|
||||
|
||||
func copyIntPtr(src *int) *int {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
v := *src
|
||||
return &v
|
||||
}
|
||||
48
docs/adr/0001-adopt-canonical-documentation-ownership.md
Normal file
48
docs/adr/0001-adopt-canonical-documentation-ownership.md
Normal file
@@ -0,0 +1,48 @@
|
||||
# ADR 0001: Adopt Canonical Documentation Ownership
|
||||
|
||||
## Status
|
||||
|
||||
Accepted
|
||||
|
||||
## Date
|
||||
|
||||
2026-07-26
|
||||
|
||||
## Context
|
||||
|
||||
Scriptorium's documentation grew alongside its CLI, HTTP, public Go, and
|
||||
integration interfaces. As a result, several documents repeated mutable
|
||||
contracts such as flags, configuration fields, and status behavior. Those
|
||||
parallel definitions made it unclear which document to update when behavior
|
||||
changed and increased the risk of documentation drift.
|
||||
|
||||
## Decision
|
||||
|
||||
Assign each documentation topic one canonical owner, as defined in
|
||||
[`docs/policy/documentation.md`](../policy/documentation.md). Non-owning
|
||||
documents may provide short orientation and links, but do not redefine volatile
|
||||
contracts. Current behavior is documented outside `docs/roadmap/`; roadmaps own
|
||||
future work, sequencing, and implementation status.
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
- Keep broad reference material in several audience-specific documents. This
|
||||
would preserve local convenience but leave conflicting contract definitions
|
||||
likely.
|
||||
- Consolidate all documentation into one reference. This would reduce duplicate
|
||||
text but would not serve the distinct needs of users, operators, consumers,
|
||||
and contributors.
|
||||
|
||||
## Rationale
|
||||
|
||||
Canonical ownership retains audience-specific guidance while making the source
|
||||
of truth for each contract discoverable. It also makes documentation changes
|
||||
reviewable alongside the implementation change that requires them.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Changes to behavior must update the canonical owner in the same change.
|
||||
- Cross-cutting documentation links to the owner instead of copying its
|
||||
details.
|
||||
- Documentation restructuring followed a dedicated implementation roadmap;
|
||||
repository history, not this ADR, records its completion.
|
||||
288
docs/adr/0002-split-promptkit-from-scriptorium.md
Normal file
288
docs/adr/0002-split-promptkit-from-scriptorium.md
Normal file
@@ -0,0 +1,288 @@
|
||||
# ADR 0002: Split Promptkit From Scriptorium
|
||||
|
||||
## Status
|
||||
|
||||
Accepted
|
||||
|
||||
## Date
|
||||
|
||||
2026-07-26
|
||||
|
||||
## Context
|
||||
|
||||
Scriptorium currently combines two products in one Go module:
|
||||
|
||||
- a reusable prompt-execution framework with a public Go facade; and
|
||||
- a runnable application with CLI and HTTP interfaces.
|
||||
|
||||
Downstream Go projects increasingly import the framework directly and do not
|
||||
use the executable interfaces. Keeping both products in one module couples
|
||||
framework releases, dependencies, documentation, and public API evolution to
|
||||
application-specific transport concerns.
|
||||
|
||||
Promptkit will become the framework project, and Scriptorium will become a slim
|
||||
application that consumes it. This ADR records that end-state boundary. It does
|
||||
not assert that the split has been implemented; until then, the current
|
||||
repository structure and contracts remain authoritative.
|
||||
|
||||
## Decision
|
||||
|
||||
### Projects And Module Paths
|
||||
|
||||
Create a repository named `promptkit` alongside Scriptorium:
|
||||
|
||||
| Project | Repository and Go module path | Root Go package |
|
||||
| --- | --- | --- |
|
||||
| Promptkit | `gitea.maximumdirect.net/eric/promptkit` | `promptkit` |
|
||||
| Scriptorium | `gitea.maximumdirect.net/eric/scriptorium` | No reusable root facade after migration |
|
||||
|
||||
Promptkit will expose its supported public API from the module root. Its
|
||||
implementation packages will remain under `internal/` unless a real consumer
|
||||
extension point requires a public type or interface.
|
||||
|
||||
Scriptorium will import only Promptkit's supported public packages. It will not
|
||||
import Promptkit implementation packages or reproduce Promptkit orchestration.
|
||||
|
||||
### Product Responsibilities
|
||||
|
||||
Promptkit owns application-neutral framework behavior:
|
||||
|
||||
- the engine and its `Prepare` and `Run` workflow;
|
||||
- public request, result, profile, option, extension, and error APIs;
|
||||
- prompt-definition loading and rendering;
|
||||
- profile loading, overlays, and the embedded built-in profile registry;
|
||||
- schema loading and output validation;
|
||||
- provider-neutral model-client boundaries and the OpenAI-compatible client;
|
||||
- artifact types, artifact-reader injection, and general-purpose inline and
|
||||
caller-selected file readers;
|
||||
- execution-setting resolution and framework defaults; and
|
||||
- framework-level secret redaction and error classification.
|
||||
|
||||
Scriptorium owns executable and transport behavior:
|
||||
|
||||
- the `scriptorium` process and its `run`, `render`, and `serve` commands;
|
||||
- CLI parsing, streams, output files, formatting, exit codes, and process
|
||||
cancellation behavior;
|
||||
- application-configuration discovery and CLI-over-configuration precedence;
|
||||
- HTTP routing, strict request decoding, DTO mapping, response encoding,
|
||||
status codes, and transport limits;
|
||||
- HTTP artifact-root containment and deployment policy;
|
||||
- server construction, server defaults, and process logging; and
|
||||
- executable release artifacts.
|
||||
|
||||
The dependency direction is:
|
||||
|
||||
```text
|
||||
Scriptorium CLI and HTTP adapters
|
||||
|
|
||||
v
|
||||
Promptkit public API
|
||||
|
|
||||
v
|
||||
injected sources, readers, and model clients
|
||||
```
|
||||
|
||||
### Current Package Disposition
|
||||
|
||||
Implementation may reorganize files during extraction, but each current package
|
||||
has this target owner:
|
||||
|
||||
| Current package or file group | Target owner | Disposition |
|
||||
| --- | --- | --- |
|
||||
| Root `scriptorium` facade files and tests | Promptkit | Move and rename the public package to `promptkit`; Scriptorium retains no compatibility facade. |
|
||||
| `internal/domain`, `internal/usecase` | Promptkit | Move as internal engine implementation. |
|
||||
| `internal/promptdef`, `internal/prompt` | Promptkit | Move as internal prompt loading and rendering. |
|
||||
| `internal/profile`, `internal/profile/builtin` | Promptkit | Move with embedded built-in assets and registry tests. |
|
||||
| `internal/filecatalog` | Promptkit | Move as source-loading support. |
|
||||
| `internal/validate` | Promptkit | Move as schema and output-validation implementation. |
|
||||
| `internal/llm` | Promptkit | Move with the OpenAI-compatible integration. |
|
||||
| `internal/artifact` | Split | Move general inline/file reading to Promptkit; keep rooted, denied, and byte-limited HTTP file reading in Scriptorium behind a Promptkit reader interface. |
|
||||
| `internal/defaults` | Split | Move framework, execution, output-artifact, content-type, and model-client defaults to Promptkit; keep CLI, HTTP, and server defaults in Scriptorium. |
|
||||
| `internal/adapter/cli`, `internal/adapter/http` | Scriptorium | Keep and refactor to use Promptkit's public API. |
|
||||
| `internal/config` | Scriptorium | Keep application settings, discovery, validation, and CLI precedence. |
|
||||
| `internal/format` | Scriptorium | Keep prepared-run presentation, rewritten against Promptkit public values. |
|
||||
| `cmd/scriptorium` | Scriptorium | Keep as the process entrypoint. |
|
||||
|
||||
Tests move with the behavior they protect. Cross-boundary tests will live with
|
||||
the consuming side: Promptkit protects framework contracts, while Scriptorium
|
||||
protects adapter mapping, HTTP containment, and executable behavior.
|
||||
|
||||
### Public Boundary
|
||||
|
||||
Promptkit's initial facade will preserve the useful shape of the current
|
||||
Scriptorium Go API where that reduces extraction risk. It will expose only the
|
||||
capabilities required by Promptkit consumers and by Scriptorium:
|
||||
|
||||
- engine construction, preparation, and execution;
|
||||
- public request, result, profile, and error values;
|
||||
- prompt, profile, schema, artifact-reader, validator, and model-client source
|
||||
or injection options that have demonstrated consumers; and
|
||||
- enough stable error identity for Scriptorium to map CLI and HTTP outcomes.
|
||||
|
||||
Promptkit will not export its domain package, runner implementation,
|
||||
repositories, adapter DTOs, or general internal constructors merely to
|
||||
simplify the move.
|
||||
|
||||
Scriptorium's CLI and HTTP adapters will depend on a small consumer-facing
|
||||
`Prepare`/`Run` interface where test substitution is needed. That interface
|
||||
belongs at the consuming boundary rather than forcing adapter concepts into
|
||||
Promptkit.
|
||||
|
||||
### Artifact Reading And HTTP Containment
|
||||
|
||||
Promptkit will define the artifact-reader extension point used during
|
||||
preparation. Its ordinary file reader may read a path deliberately supplied by
|
||||
an in-process or CLI caller and does not claim to be a deployment sandbox.
|
||||
|
||||
Scriptorium will implement the HTTP-specific reader that:
|
||||
|
||||
- denies file references when no artifact root is configured;
|
||||
- applies the configured artifact byte limit;
|
||||
- enforces Scriptorium's documented lexical root-containment rule; and
|
||||
- maps reader failures to Scriptorium HTTP error responses.
|
||||
|
||||
Scriptorium will inject that reader through Promptkit's public construction
|
||||
boundary. Promptkit will not know about HTTP roots, status codes, request DTOs,
|
||||
or deployment policy.
|
||||
|
||||
### Configuration And Default Ownership
|
||||
|
||||
Configuration ownership follows the behavior configured, not the current file
|
||||
location:
|
||||
|
||||
| Configuration category | Owner |
|
||||
| --- | --- |
|
||||
| Application configuration discovery, configuration-file precedence, `prompt_dir`, `profile_dir`, and `schema_dir` | Scriptorium |
|
||||
| CLI flags and their mapping to application settings or request overrides | Scriptorium |
|
||||
| `server.*`, render-output settings, HTTP byte limits, and server defaults | Scriptorium |
|
||||
| Prompt-definition, profile, and output-contract file formats | Promptkit |
|
||||
| Prompt/profile source selection, overlays, schema behavior, and built-in profiles | Promptkit |
|
||||
| Execution settings, presence-aware request overrides, and execution defaults | Promptkit |
|
||||
| Built-in OpenAI-compatible client settings, timeout behavior, and provider wire mapping | Promptkit |
|
||||
| HTTP request and response fields, including their mapping to framework values | Scriptorium |
|
||||
|
||||
Scriptorium will translate its application settings and external request
|
||||
values into Promptkit construction options and requests. When an omitted
|
||||
Scriptorium setting means “use the framework default,” Scriptorium will omit
|
||||
the override rather than copy Promptkit's numeric default.
|
||||
|
||||
### Compatibility And Versioning
|
||||
|
||||
This migration is intentionally breaking:
|
||||
|
||||
- new Go consumers will import `gitea.maximumdirect.net/eric/promptkit`;
|
||||
- Scriptorium will not provide aliases, forwarding wrappers, or deprecated
|
||||
compatibility packages for its former Go facade;
|
||||
- existing consumers may remain pinned to the final framework-bearing
|
||||
Scriptorium tag until migrated; and
|
||||
- intermediate migration phases need not preserve source compatibility, but
|
||||
each merged phase must be internally buildable and tested.
|
||||
|
||||
Promptkit's first release will be `v0.1.0`. During the migration, incompatible
|
||||
Promptkit changes may advance its minor version until a stable `v1` contract is
|
||||
declared. The first slim Scriptorium release will advance the Scriptorium minor
|
||||
version beyond the final framework-bearing release. Normal semantic-versioning
|
||||
rules apply independently to both projects after the migration.
|
||||
|
||||
Promptkit must be tagged before Scriptorium or another consumer publishes a
|
||||
release that depends on it. Release branches must use tagged module
|
||||
dependencies, not local replacements or unpublished revisions.
|
||||
|
||||
### Local Development And Cross-Repository Coordination
|
||||
|
||||
For coordinated local work, place both repositories in a temporary Go
|
||||
workspace or use an uncommitted module replacement. `go.work`,
|
||||
`go.work.sum`, and local filesystem `replace` directives must not be committed
|
||||
to release branches.
|
||||
|
||||
Cross-repository changes follow this order:
|
||||
|
||||
1. land and tag the required Promptkit capability;
|
||||
2. update Scriptorium and other consumers to that tag;
|
||||
3. run each repository's own CI and smoke checks; and
|
||||
4. release consumers only after the Promptkit tag is available.
|
||||
|
||||
Migration coordination must confirm out-of-band repository creation, Promptkit
|
||||
tags, and downstream migrations before dependent work proceeds.
|
||||
Cross-repository changes are coordinated, not treated as atomic commits.
|
||||
|
||||
### Documentation And Maintained Assets
|
||||
|
||||
Each repository will maintain its own README, contributor guide, architecture,
|
||||
documentation, testing, release, and operations material appropriate to that
|
||||
project. Cross-project documents will link to the canonical owner rather than
|
||||
copy its contract.
|
||||
|
||||
Existing documentation and maintained assets have these target owners:
|
||||
|
||||
| Current material | Target owner |
|
||||
| --- | --- |
|
||||
| Current README and executable quickstart | Scriptorium; Promptkit creates its own framework orientation |
|
||||
| Public Go package and Go-consumer guidance | Promptkit |
|
||||
| Prompt, profile, schema, execution-setting, and framework credential reference | Promptkit |
|
||||
| OpenAI-compatible integration contract and framework internal documents | Promptkit |
|
||||
| CLI, HTTP API, subprocess, and Scriptorium operations contracts | Scriptorium |
|
||||
| Consumer interface overview | Scriptorium, revised to route Go consumers to Promptkit |
|
||||
| Application-configuration discovery, server settings, and adapter internals | Scriptorium |
|
||||
| Current internal overview and source documentation | Split into repository-local overviews; Promptkit owns framework sources and Scriptorium owns HTTP containment |
|
||||
| This ADR and cross-project migration records | Scriptorium |
|
||||
| `examples/go-library` | Promptkit |
|
||||
| `examples/config*.yml`, `examples/render-markdown-summary.sh`, and `examples/http-run.json` | Scriptorium |
|
||||
| Example prompts, profiles, schemas, and synthetic fixtures used by the executable examples | Scriptorium |
|
||||
| Embedded built-in profile assets | Promptkit |
|
||||
| Scriptorium release workflow and executable packaging | Scriptorium |
|
||||
| Repository-level license, ignore rules, agent guidance, and development policies | Each repository maintains its own applicable copy |
|
||||
|
||||
Promptkit will create or retain its own minimal framework examples and test
|
||||
fixtures rather than making either repository's tests depend on the other's
|
||||
working tree. Scriptorium's framework-format documentation will become a short
|
||||
version-appropriate link to Promptkit, while its maintained executable examples
|
||||
remain self-contained.
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
- Keep the current combined repository and improve package naming. This avoids
|
||||
migration work but retains release and ownership coupling between the
|
||||
framework and executable.
|
||||
- Add Promptkit as a wrapper around the Scriptorium public package. This gives
|
||||
consumers a new import path but leaves framework ownership and dependency
|
||||
direction inverted.
|
||||
- Extract Promptkit while retaining a Scriptorium compatibility facade. This
|
||||
reduces immediate consumer changes but creates a second public API surface
|
||||
and prolongs duplicate maintenance.
|
||||
- Move all artifact reading into Promptkit. This would place HTTP containment,
|
||||
byte limits, and deployment policy in the application-neutral framework.
|
||||
- Keep Promptkit and Scriptorium as separate modules in one repository. This
|
||||
separates imports but not repository permissions, release workflows,
|
||||
issue ownership, or independent project evolution.
|
||||
|
||||
## Rationale
|
||||
|
||||
A separate Promptkit project makes the reusable framework the direct owner of
|
||||
the API that downstream Go projects already consume. Keeping Scriptorium as a
|
||||
public-API consumer exercises the same boundary as other consumers and prevents
|
||||
its adapters from relying on framework internals.
|
||||
|
||||
The selected split keeps transport and deployment policy close to the
|
||||
Scriptorium interfaces that expose it, while allowing Promptkit to remain
|
||||
useful to in-process consumers with different IO and security requirements.
|
||||
Explicit package, configuration, documentation, and asset ownership reduces
|
||||
ambiguity during extraction and after release.
|
||||
|
||||
## Consequences
|
||||
|
||||
- All Go consumers of the framework must change their import path.
|
||||
- Promptkit and Scriptorium gain independent issue, release, CI, policy, and
|
||||
documentation lifecycles.
|
||||
- Scriptorium becomes a real downstream integration test of Promptkit's public
|
||||
facade.
|
||||
- Framework changes that affect Scriptorium require tagged, ordered
|
||||
cross-repository coordination.
|
||||
- Some current packages, especially artifact reading and defaults, must be
|
||||
separated by responsibility rather than moved intact.
|
||||
- Scriptorium's current configuration and documentation references must be
|
||||
split between application and framework owners.
|
||||
- Maintainers must inventory and migrate downstream consumers explicitly; no
|
||||
compatibility facade will hide incomplete migration.
|
||||
- Until the split is implemented, the current repository structure and
|
||||
contracts remain authoritative.
|
||||
@@ -0,0 +1,67 @@
|
||||
# ADR 0003: Use Maintainer-Run Validation and Tag-Only Releases for Promptkit
|
||||
|
||||
## Status
|
||||
|
||||
Accepted
|
||||
|
||||
## Date
|
||||
|
||||
2026-07-28
|
||||
|
||||
## Context
|
||||
|
||||
[ADR 0002](0002-split-promptkit-from-scriptorium.md) established Promptkit as
|
||||
an independent Go library with its own repository, version history, validation,
|
||||
and release coordination. It anticipated independent hosted CI for Promptkit
|
||||
alongside Scriptorium's existing executable build and CI policy.
|
||||
|
||||
Promptkit is presently a single-maintainer library. It does not produce a
|
||||
runnable command, so executable packaging and binary-release automation do not
|
||||
apply. Its validation and release model should be explicit before repository
|
||||
guidance relies on it.
|
||||
|
||||
## Decision
|
||||
|
||||
Promptkit will use maintainer-run validation rather than hosted CI at this
|
||||
stage. From a clean checkout, the maintainer will run the repository-documented
|
||||
test, vet, build, formatting, documentation-link, and repository-hygiene checks
|
||||
before changes are accepted and before a release tag is published.
|
||||
|
||||
Promptkit releases consist of source commits and semantic Go module tags. The
|
||||
project does not release runnable binaries or maintain binary-packaging
|
||||
automation.
|
||||
|
||||
Scriptorium's executable build, hosted CI, and binary-release policies are
|
||||
unaffected. The repository boundary, independent version history, release
|
||||
ordering, and other migration decisions accepted by ADR 0002 remain in force.
|
||||
Where ADR 0002 anticipated independent hosted CI for Promptkit, this later ADR
|
||||
controls Promptkit validation.
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
- Add hosted Promptkit CI now. This would provide automated remote enforcement,
|
||||
but its setup and maintenance are not proportionate to the present
|
||||
single-maintainer library and do not replace the maintainer's release
|
||||
responsibility.
|
||||
- Require local Git hooks. Hooks can provide fast feedback, but they are
|
||||
machine-local, can be bypassed, and are not a durable substitute for the
|
||||
documented clean-checkout validation procedure.
|
||||
|
||||
## Rationale
|
||||
|
||||
A documented maintainer-run procedure provides a clear acceptance and release
|
||||
gate with little operational overhead for the project's current contribution
|
||||
pattern. If maintenance load or contributor patterns change, a later ADR may
|
||||
introduce hosted CI without changing Promptkit's library or tag-based release
|
||||
model.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Promptkit repository guidance must define the complete local validation
|
||||
procedure and the checks required before accepting or tagging a change.
|
||||
- Release evidence is the maintainer's successful clean-checkout validation,
|
||||
not a hosted CI result.
|
||||
- Promptkit releases contain source and semantic Go module tags only.
|
||||
- A future move to hosted CI requires a later architectural decision.
|
||||
- Scriptorium continues to validate, build, package, and release its executable
|
||||
under its own policies.
|
||||
112
docs/adr/0004-definition-boundary.md
Normal file
112
docs/adr/0004-definition-boundary.md
Normal file
@@ -0,0 +1,112 @@
|
||||
# Preserve Definition Compatibility And A Simple Execution Model
|
||||
|
||||
## Status
|
||||
|
||||
Accepted.
|
||||
|
||||
## Date
|
||||
|
||||
2026-08-29.
|
||||
|
||||
## Context
|
||||
|
||||
Scriptorium is a user-facing application built on Promptkit. Promptkit is a Go
|
||||
library with both declarative framework formats and public APIs intended for
|
||||
applications that need to assemble specialized integrations or workflows.
|
||||
|
||||
Scriptorium needs a durable rule for deciding which upstream capabilities it
|
||||
must support. Treating every new Promptkit public API as an application feature
|
||||
would steadily expand Scriptorium into a workflow framework. Selecting an
|
||||
arbitrary subset of prompt or profile fields would instead make valid Promptkit
|
||||
definitions unexpectedly unusable through Scriptorium.
|
||||
|
||||
The intended product is narrower: Scriptorium provides simple executable
|
||||
interfaces that select and execute predefined prompts and return their results
|
||||
in a consistent, repeatable form. It is not intended to own conversations,
|
||||
modify prompts dynamically, coordinate follow-up messages, or manage complex
|
||||
multi-step workflows.
|
||||
|
||||
## Decision
|
||||
|
||||
Scriptorium will aim to support the complete set of features expressible in
|
||||
valid Promptkit prompt and profile definitions for the Promptkit version it
|
||||
selects. This includes the schema, backend, source, and request plumbing needed
|
||||
to prepare and execute those definitions without Scriptorium imposing a
|
||||
narrower field-level format contract.
|
||||
|
||||
Promptkit remains the canonical parser and validator for its definitions.
|
||||
Scriptorium will pass its configured framework sources and mapped request
|
||||
values through the supported Promptkit public facade rather than copying
|
||||
Promptkit formats, defaults, or validation rules.
|
||||
|
||||
Promptkit library features that are not required to use valid prompt or profile
|
||||
definitions are considered individually. They may be incorporated when they
|
||||
serve Scriptorium's simple execution model and fit its application-owned CLI,
|
||||
HTTP, configuration, presentation, security, and process boundaries. A public
|
||||
Promptkit declaration does not by itself require a corresponding Scriptorium
|
||||
interface.
|
||||
|
||||
Each execution request will remain self-contained from the caller's
|
||||
perspective. A request must not depend on conversation or workflow state
|
||||
retained from a completed request, and Scriptorium will not become a general
|
||||
workflow or conversation system. Application-managed prompt modification,
|
||||
appended or follow-up message flows, conversation state, prepared-handle
|
||||
coordination, multi-step orchestration, and similar higher-level workflows are
|
||||
outside its intended scope.
|
||||
|
||||
Scriptorium may, and when necessary must, maintain process-scoped operational
|
||||
state shared by concurrent requests. This includes the configured Promptkit
|
||||
engine and backend registry, capacity admission and queue accounting,
|
||||
in-flight request coordination, cancellation, and ordinary server lifecycle
|
||||
resources. The state must be scoped so configured backend limits apply across
|
||||
all relevant in-flight requests. But this is operational implementation state,
|
||||
not durable user workflow or conversation state.
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
### Expose Every Promptkit Public Feature
|
||||
|
||||
This would make the application surface track the library API closely, but it
|
||||
would blur the boundary between a reusable Go library and a simple executable
|
||||
consumer. Library-oriented lifecycle, source-construction, extension, and
|
||||
workflow primitives would add commands, wire contracts, configuration, and
|
||||
state without necessarily improving predefined prompt execution.
|
||||
|
||||
### Support A Curated Subset Of Definition Fields
|
||||
|
||||
This would keep the application small in the short term, but valid Promptkit
|
||||
definitions could fail or behave differently solely because they were invoked
|
||||
through Scriptorium. Maintaining a parallel field-level compatibility list
|
||||
would also duplicate upstream format ownership and create recurring drift.
|
||||
|
||||
### Expand Scriptorium Into A Workflow Service
|
||||
|
||||
Scriptorium could own conversations, prompt changes, follow-up messages,
|
||||
retries, checkpoints, and multi-step execution. That is a different product
|
||||
with durable state, lifecycle, recovery, privacy, and operational requirements
|
||||
that are not justified by Scriptorium's current purpose.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Promptkit upgrades require an audit of the complete prompt and profile format
|
||||
contracts, not only a source-compatibility build.
|
||||
- Scriptorium adapters must not add unconditional request requirements that
|
||||
reject a definition Promptkit can validly prepare or execute.
|
||||
- Application configuration may need new plumbing, such as custom backend
|
||||
registration, when it is necessary to use a valid profile feature.
|
||||
- Definition parsing, validation, precedence, inheritance, defaults, and
|
||||
built-in catalogs remain Promptkit responsibilities.
|
||||
- A running HTTP server must share its appropriately scoped Promptkit engine
|
||||
across requests so the engine's backend admission and concurrency limits
|
||||
cannot be bypassed by per-request construction.
|
||||
- Process-scoped counters, queues, in-flight request records, cancellation, and
|
||||
lifecycle resources are permitted implementation state. They must be
|
||||
concurrency-safe and do not create a durable resume or conversation
|
||||
contract.
|
||||
- Library-only source constructors, injected collaborators, opaque lifecycle
|
||||
handles, and request-composition primitives remain optional Scriptorium
|
||||
features rather than automatic requirements.
|
||||
- New application features must be evaluated against the simple,
|
||||
request-independent predefined-prompt execution model.
|
||||
- Work that implements this decision remains tracked in roadmaps until it is
|
||||
delivered and incorporated into current-state contracts.
|
||||
354
docs/api.md
354
docs/api.md
@@ -2,296 +2,164 @@
|
||||
|
||||
This is the canonical public HTTP contract for Scriptorium.
|
||||
|
||||
Implemented route:
|
||||
## Service And Route
|
||||
|
||||
- `POST /v1/runs`
|
||||
`POST /v1/runs` runs one prompt request and returns generated output,
|
||||
validation, and metadata. The service has no built-in authentication or
|
||||
authorization; deploy it behind appropriate network and authentication controls.
|
||||
|
||||
For CLI behavior, see [CLI reference](cli.md). For config and prompt/profile
|
||||
file formats, see [Configuration reference](config.md).
|
||||
The service address and HTTP limits are configured as described in the
|
||||
[configuration reference](config.md). `serve` invocation is defined in the
|
||||
[CLI reference](cli.md).
|
||||
|
||||
The maintained request-shape example is `examples/http-run.json`. It requires a
|
||||
running `serve` process with an artifact root that can read the referenced
|
||||
files, plus a reachable model endpoint for full execution.
|
||||
|
||||
## Base URL And Deployment
|
||||
|
||||
`scriptorium serve` listens on `server.addr` or `serve --addr`. The default is
|
||||
`:8080`.
|
||||
|
||||
The route path is always:
|
||||
|
||||
```text
|
||||
/v1/runs
|
||||
```
|
||||
|
||||
The HTTP adapter has no built-in authentication or authorization. Deploy it
|
||||
behind trusted network and authentication controls.
|
||||
|
||||
## Media Types
|
||||
|
||||
- Request body: JSON object.
|
||||
- Response body: JSON object.
|
||||
- Response `Content-Type`: `application/json`.
|
||||
|
||||
Requests are decoded as JSON regardless of the request `Content-Type` header.
|
||||
There are no shared query parameters.
|
||||
Requests and responses are JSON objects. Requests are decoded as JSON regardless
|
||||
of their `Content-Type`; successful JSON responses use
|
||||
`Content-Type: application/json`. There are no query parameters.
|
||||
|
||||
## Request Limits
|
||||
|
||||
HTTP limits are configured through `server.*` config fields or `serve` flags:
|
||||
The configured request-body limit includes inline artifact bodies. The artifact
|
||||
limit applies to HTTP `file` inputs. The response limit applies to the encoded
|
||||
response, including the artifact body and optional raw output. A limit of zero
|
||||
disables that limit.
|
||||
|
||||
- `server.max_request_bytes`: encoded JSON request body limit, including inline input bodies.
|
||||
- `server.max_artifact_bytes`: file artifact limit for HTTP `file` input references.
|
||||
- `server.max_response_bytes`: encoded JSON response limit, including artifact body and optional raw output.
|
||||
|
||||
Each limit defaults to `16777216` bytes. `0` disables that limit.
|
||||
A request body over its limit returns `413 request_too_large`; an oversized
|
||||
file input returns `413 artifact_too_large`; an oversized encoded response
|
||||
returns `413 response_too_large`.
|
||||
|
||||
## `POST /v1/runs`
|
||||
|
||||
Runs one prompt request and returns the generated artifact, validation result,
|
||||
and metadata.
|
||||
|
||||
### Request Body
|
||||
|
||||
The maintained [request example](../examples/http-run.json) is a complete
|
||||
copyable shape. At the HTTP adapter boundary, the smallest valid shape is:
|
||||
|
||||
```json
|
||||
{
|
||||
"prompt_id": "generic.markdown_summary",
|
||||
"profile_id": "local-fast",
|
||||
"prompt_version": "1.0.0",
|
||||
"inputs": {
|
||||
"transcript": {
|
||||
"type": "file",
|
||||
"uri": "./examples/fixtures/transcript.md"
|
||||
},
|
||||
"glossary": {
|
||||
"type": "inline",
|
||||
"body": "party:\n - Rin"
|
||||
}
|
||||
},
|
||||
"vars": {
|
||||
"session_date": "2026-05-04"
|
||||
},
|
||||
"model": {
|
||||
"endpoint": "http://localhost:8000/v1",
|
||||
"model": "gpt-4o-mini",
|
||||
"temperature": 0,
|
||||
"max_tokens": 800,
|
||||
"top_p": 1,
|
||||
"timeout_seconds": 120,
|
||||
"service_tier": "priority",
|
||||
"reasoning_effort": "medium",
|
||||
"api_key_env": "SCRIPTORIUM_API_KEY",
|
||||
"extra_params": {
|
||||
"provider_option": "enabled"
|
||||
}
|
||||
},
|
||||
"include_raw_output": false
|
||||
"prompt_id": "generic.markdown_summary"
|
||||
}
|
||||
```
|
||||
|
||||
Request fields:
|
||||
|
||||
| Field | Required | Description |
|
||||
| Field | Required | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `prompt_id` | yes | Prompt ID. Must not be blank. |
|
||||
| `prompt_id` | yes | Non-blank prompt ID. |
|
||||
| `prompt_version` | no | Prompt version filter. |
|
||||
| `profile_id` | no | Execution profile ID. If omitted, the prompt must define `default_profile`. |
|
||||
| `inputs` | yes | Object mapping prompt input names to input references. Must contain at least one entry. |
|
||||
| `vars` | no | Object mapping template variable names to string values. |
|
||||
| `model` | no | Runtime model override object. |
|
||||
| `include_raw_output` | no | When `true`, include `raw_model_output` in the response. |
|
||||
| `profile_id` | no | Execution-profile ID; otherwise the prompt must set `default_profile`. |
|
||||
| `session_id` | no | Optional direct, non-secret session identifier. |
|
||||
| `inputs` | no | Optional object mapping input names to references. Promptkit decides whether the selected definition needs them. |
|
||||
| `vars` | no | Object mapping template-variable names to strings. |
|
||||
| `model` | no | Runtime model-override object. |
|
||||
| `include_raw_output` | no | Include `raw_model_output` when true. |
|
||||
|
||||
Input reference fields:
|
||||
An input reference has a required `type` of `file` or `inline`. A `file`
|
||||
reference requires `uri`; an `inline` reference requires `body`.
|
||||
|
||||
| Field | Required | Description |
|
||||
| --- | --- | --- |
|
||||
| `type` | yes | `file` or `inline`. |
|
||||
| `uri` | for `file` | File URI/path. |
|
||||
| `body` | for `inline` | Inline artifact body. |
|
||||
HTTP file references require a configured artifact root. Relative paths resolve
|
||||
within that root. Absolute paths must be lexically within it; traversal outside
|
||||
it is rejected with `400 artifact_not_allowed`. This lexical check does not
|
||||
resolve symlinks: the operating system follows symlinks inside the root,
|
||||
including ones that target outside it. Keep the root narrow and inaccessible to
|
||||
untrusted writers.
|
||||
|
||||
HTTP `file` references require `server.artifact_root` or `serve
|
||||
--artifact-root`. Relative file URIs resolve against that root. Absolute file
|
||||
URIs are accepted only when lexically inside the root. Relative traversal and
|
||||
absolute paths outside the root return `400 artifact_not_allowed`.
|
||||
The optional `model` object accepts `endpoint`, `model`, `temperature`,
|
||||
`max_tokens`, `top_p`, `timeout_seconds`, `service_tier`,
|
||||
`reasoning_effort`, `api_key_env`, and `extra_params`. Numeric ranges and
|
||||
framework credential semantics are defined by the
|
||||
[Promptkit format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md).
|
||||
Explicit zero values for the numeric fields are overrides; zero
|
||||
`timeout_seconds` disables the per-generation deadline only, retaining the
|
||||
request context and configured transport cap. The timeout layers are defined in
|
||||
the [Promptkit outbound integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/integrations/openai-compatible-chat.md#timeout-and-cancellation).
|
||||
|
||||
The containment check is lexical and does not resolve symlinks. Symlinks inside
|
||||
the artifact root are followed by the operating system, including symlinks that
|
||||
point outside the root. Keep the artifact root narrow and not writable by
|
||||
untrusted users.
|
||||
Raw API-key values are not accepted. `api_key` and any other unknown model
|
||||
field cause `400 invalid_json`.
|
||||
|
||||
Model override fields:
|
||||
`model.reasoning_effort` is an optional JSON string with three states: omission
|
||||
inherits the selected profile, a non-empty string replaces its value, and an
|
||||
empty string explicitly clears it. JSON `null` is treated as omission.
|
||||
|
||||
| Field | Description |
|
||||
| --- | --- |
|
||||
| `endpoint` | Runtime endpoint override. |
|
||||
| `model` | Runtime model override. |
|
||||
| `temperature` | Number in range `0..2`. Explicit `0` is an override. |
|
||||
| `max_tokens` | Integer greater than or equal to `0`. Explicit `0` is an override. |
|
||||
| `top_p` | Number in range `0..1`. Explicit `0` is an override. |
|
||||
| `timeout_seconds` | Integer greater than or equal to `0`. Explicit `0` disables the outbound client timeout. |
|
||||
| `service_tier` | Provider-specific request tier. |
|
||||
| `reasoning_effort` | Provider-specific reasoning setting. |
|
||||
| `api_key_env` | Name of an environment variable containing the API key. |
|
||||
| `extra_params` | JSON-compatible provider-specific top-level request fields. |
|
||||
`session_id` is passed directly to Promptkit. A nonblank value replaces a
|
||||
definition-rendered session ID; omission or a blank value lets the definition
|
||||
provide one. Promptkit trims direct values and limits them to 256 Unicode code
|
||||
points. Session IDs are not credentials and may be included in prepared data,
|
||||
results, and provider-facing requests, so use stable non-sensitive identifiers.
|
||||
|
||||
Raw API-key values are not accepted in HTTP payloads. A field such as
|
||||
`api_key` is rejected as unknown JSON.
|
||||
### Strict JSON
|
||||
|
||||
`extra_params` keys must not be empty and must not collide with reserved
|
||||
outbound fields: `model`, `session_id`, `messages`, `temperature`,
|
||||
`max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, or
|
||||
`response_format`.
|
||||
|
||||
### Strict JSON Rules
|
||||
|
||||
Request decoding is strict:
|
||||
|
||||
- malformed JSON returns `400 invalid_json`
|
||||
- unknown request fields return `400 invalid_json`
|
||||
- unknown `inputs` item fields return `400 invalid_json`
|
||||
- unknown `model` fields return `400 invalid_json`
|
||||
- trailing JSON tokens after the request object return `400 invalid_json`
|
||||
- request bodies above the configured limit return `413 request_too_large`
|
||||
Request decoding rejects malformed JSON, unknown fields at every request level,
|
||||
and trailing JSON tokens with `400 invalid_json`. A blank `prompt_id` returns
|
||||
`400 invalid_request`. Omitted or empty `inputs` are passed to Promptkit, which
|
||||
reports any definition-required or template-referenced inputs.
|
||||
|
||||
### Success Response
|
||||
|
||||
Status: `200 OK`
|
||||
A completed run returns `200 OK`, including when generated content fails its
|
||||
validation contract. The response contains:
|
||||
|
||||
```json
|
||||
{
|
||||
"artifact": {
|
||||
"name": "output",
|
||||
"content_type": "text/markdown",
|
||||
"body": "Generated content",
|
||||
"size": 17,
|
||||
"hash": "..."
|
||||
},
|
||||
"validation": {
|
||||
"status": "passed",
|
||||
"mode": "basic",
|
||||
"repair_attempts": 0,
|
||||
"is_valid": true
|
||||
},
|
||||
"metadata": {
|
||||
"run_id": "...",
|
||||
"prompt_id": "generic.markdown_summary",
|
||||
"prompt_version": "1.0.0",
|
||||
"prompt_hash": "...",
|
||||
"rendered_prompt_hash": "...",
|
||||
"selected_profile_id": "local-fast",
|
||||
"model_name": "gpt-4o-mini",
|
||||
"endpoint": "http://localhost:8000/v1",
|
||||
"model_params": {
|
||||
"endpoint": "http://localhost:8000/v1",
|
||||
"model": "gpt-4o-mini",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 500,
|
||||
"top_p": 1,
|
||||
"timeout_seconds": 90
|
||||
},
|
||||
"input_hashes": {
|
||||
"transcript": "..."
|
||||
},
|
||||
"usage": {
|
||||
"prompt_tokens": 11,
|
||||
"completion_tokens": 22,
|
||||
"total_tokens": 33,
|
||||
"cached_tokens": 0,
|
||||
"cache_write_tokens": 0
|
||||
},
|
||||
"start_time": "2026-05-04T12:00:00Z",
|
||||
"end_time": "2026-05-04T12:00:01Z",
|
||||
"duration_ms": 1000,
|
||||
"validation_mode": "basic",
|
||||
"validation_status": "passed",
|
||||
"repair_attempts_used": 0
|
||||
}
|
||||
}
|
||||
```
|
||||
- `artifact`: `name`, `content_type`, `body`, `size`, `hash`, and
|
||||
optional `uri`;
|
||||
- `validation`: `status`, `mode`, `repair_attempts`, `is_valid`, plus
|
||||
optional `errors` and `schema_path`;
|
||||
- `metadata`: run, prompt, rendered-prompt, profile, optional backend identity, model, input-hash, usage,
|
||||
timing, validation, and repair-attempt metadata; and
|
||||
- optional `raw_model_output` when requested.
|
||||
|
||||
Response fields:
|
||||
`metadata.model_params` has `endpoint`, `model`, `temperature`,
|
||||
`max_tokens`, `top_p`, and `timeout_seconds`, plus optional
|
||||
`backend_id`, `service_tier`, `reasoning_effort`, `api_key_env`, and `extra_params`.
|
||||
`metadata.usage` always includes `prompt_tokens`, `completion_tokens`,
|
||||
`total_tokens`, `cached_tokens`, and `cache_write_tokens`; unavailable
|
||||
cache usage is reported as zero.
|
||||
|
||||
- `artifact`: generated output artifact.
|
||||
- `validation`: validation result for the generated artifact.
|
||||
- `metadata`: run and effective runtime metadata.
|
||||
- `raw_model_output`: omitted unless `include_raw_output` is `true`.
|
||||
When Promptkit resolves a direct or definition-rendered session ID,
|
||||
`metadata.session_id` contains that effective result value. It is omitted when
|
||||
no effective session ID exists.
|
||||
|
||||
`artifact.uri` is omitted when empty. `validation.errors` and
|
||||
`validation.schema_path` are omitted when empty. `model_params.service_tier`,
|
||||
`model_params.reasoning_effort`, `model_params.api_key_env`, and
|
||||
`model_params.extra_params` are omitted when empty.
|
||||
`metadata.selected_backend_id` and `metadata.model_params.backend_id` report
|
||||
the corresponding Promptkit result fields independently when present. Both are
|
||||
omitted for an endpoint-only profile; Scriptorium does not infer backend
|
||||
identity from an endpoint.
|
||||
|
||||
`metadata.usage.cached_tokens` and `metadata.usage.cache_write_tokens` are
|
||||
always present as numbers. They are `0` when the provider omits compatible cache
|
||||
usage fields or reports no cache activity.
|
||||
|
||||
### Validation Failure Response
|
||||
|
||||
Generated-content validation failures still return `200 OK`.
|
||||
|
||||
```json
|
||||
{
|
||||
"validation": {
|
||||
"status": "failed",
|
||||
"mode": "json",
|
||||
"errors": ["invalid JSON: ..."],
|
||||
"repair_attempts": 0,
|
||||
"is_valid": false
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The response still includes `artifact` and `metadata`.
|
||||
A validation failure has `validation.status: "failed"`, `is_valid: false`,
|
||||
and any available diagnostic errors, while still returning the artifact and
|
||||
metadata.
|
||||
|
||||
## Error Responses
|
||||
|
||||
Error body shape:
|
||||
Errors have this shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"error": {
|
||||
"code": "invalid_request",
|
||||
"message": "prompt_id is required"
|
||||
}
|
||||
}
|
||||
{"error":{"code":"invalid_request","message":"prompt_id is required"}}
|
||||
```
|
||||
|
||||
Current status/code mapping:
|
||||
Messages are concise and do not expose wrapped internal causes.
|
||||
|
||||
| Status | Code | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `400` | `invalid_json` | Malformed JSON, unknown JSON field, or trailing JSON token. |
|
||||
| `400` | `invalid_request` | Missing/invalid request fields or invalid runtime overrides. |
|
||||
| `400` | `profile_required` | No `profile_id` and prompt has no `default_profile`. |
|
||||
| `400` | `prompt_load_failed` | Prompt definition YAML/contract failed to load. |
|
||||
| `400` | `profile_load_failed` | Profile YAML/contract failed to load, including raw `api_key`. |
|
||||
| `400` | `artifact_not_allowed` | HTTP file refs are disabled or requested path is outside artifact root. |
|
||||
| `400` | `artifact_read_failed` | Input artifact could not be read or input ref was unsupported/invalid. |
|
||||
| `400` | `invalid_json` | Malformed JSON, unknown field, or trailing JSON. |
|
||||
| `400` | `invalid_request` | Missing or invalid request data or runtime override. |
|
||||
| `400` | `profile_required` | No profile ID and no prompt default profile. |
|
||||
| `400` | `prompt_load_failed` | Prompt definition failed to load. |
|
||||
| `400` | `profile_load_failed` | Profile failed to load. |
|
||||
| `400` | `artifact_not_allowed` | HTTP file input is disabled or outside the artifact root. |
|
||||
| `400` | `artifact_read_failed` | Input artifact is invalid or cannot be read. |
|
||||
| `400` | `prompt_render_failed` | Prompt template rendering failed. |
|
||||
| `400` | `api_key_env_missing` | Selected `api_key_env` variable is unset or empty. |
|
||||
| `404` | `not_found` | Route path is unknown. |
|
||||
| `404` | `prompt_not_found` | Prompt ID/version was not found. |
|
||||
| `404` | `profile_not_found` | Profile ID was not found. |
|
||||
| `405` | `method_not_allowed` | Method is not `POST` on `/v1/runs`. |
|
||||
| `413` | `request_too_large` | Encoded JSON request body exceeds configured request limit. |
|
||||
| `413` | `artifact_too_large` | HTTP file input artifact exceeds configured artifact limit. |
|
||||
| `413` | `response_too_large` | Encoded JSON response exceeds configured response limit. |
|
||||
| `500` | `validation_runtime_failed` | Validator runtime/schema loading failed. |
|
||||
| `500` | `internal_error` | Unclassified server error. |
|
||||
| `400` | `api_key_env_missing` | The selected credential environment variable is unset or empty. |
|
||||
| `404` | `not_found` | Route does not exist. |
|
||||
| `404` | `prompt_not_found` | Prompt ID or version does not exist. |
|
||||
| `404` | `profile_not_found` | Profile ID does not exist. |
|
||||
| `405` | `method_not_allowed` | The route does not accept the method. |
|
||||
| `413` | `request_too_large` | Encoded request exceeds its limit. |
|
||||
| `413` | `artifact_too_large` | File input exceeds its limit. |
|
||||
| `413` | `response_too_large` | Encoded response exceeds its limit. |
|
||||
| `500` | `validation_runtime_failed` | Schema or validator runtime failure. |
|
||||
| `500` | `internal_error` | Unclassified server failure. |
|
||||
| `502` | `llm_failed` | Outbound model request failed. |
|
||||
|
||||
HTTP error messages are intentionally concise and do not include sensitive
|
||||
internal causes.
|
||||
| `503` | `capacity_exceeded` | The selected model backend has no admission capacity. No retry timing is supplied. |
|
||||
|
||||
## Retry And Idempotency
|
||||
|
||||
Scriptorium does not provide idempotency keys, pagination, caching headers, or
|
||||
rate limiting.
|
||||
|
||||
Clients may retry transport failures or `5xx` responses when their surrounding
|
||||
workflow can tolerate another model call. A retry can generate different output
|
||||
and incur another provider request.
|
||||
|
||||
## Example File
|
||||
|
||||
- `examples/http-run.json`
|
||||
Scriptorium provides no idempotency keys, pagination, caching headers, or rate
|
||||
limits. Clients may retry transport failures or `5xx` responses only when
|
||||
their workflow tolerates another model call: a retry can produce different
|
||||
output and incur another provider request.
|
||||
|
||||
329
docs/cli.md
329
docs/cli.md
@@ -1,5 +1,10 @@
|
||||
# CLI Reference
|
||||
|
||||
This is the canonical contract for invoking Scriptorium. Configuration discovery,
|
||||
precedence, application source locations, and server settings are defined in the
|
||||
[configuration reference](config.md). The [HTTP API reference](api.md) owns
|
||||
service request and response behavior.
|
||||
|
||||
## Shortest Useful Command
|
||||
|
||||
```bash
|
||||
@@ -10,224 +15,190 @@ go run ./cmd/scriptorium render \
|
||||
--input glossary=./examples/fixtures/glossary.yml
|
||||
```
|
||||
|
||||
`render` prepares the prompt, loads input artifacts, resolves the execution
|
||||
profile, and prints the prepared request without calling an LLM.
|
||||
`render` prepares a request without calling an LLM.
|
||||
|
||||
## Command Overview
|
||||
## Commands
|
||||
|
||||
- `scriptorium run`: prepare a prompt, call the configured LLM, write generated output, and print a run summary.
|
||||
- `scriptorium render`: prepare a prompt only; write prepared-run output as `text` or `json`.
|
||||
- `scriptorium serve`: start the HTTP server for `POST /v1/runs`.
|
||||
- `scriptorium run`: prepare a prompt, call the configured LLM, and write the
|
||||
generated artifact.
|
||||
- `scriptorium render`: prepare a prompt and write prepared-run output.
|
||||
- `scriptorium serve`: start the HTTP server.
|
||||
- `scriptorium inspect prompt`: inspect one prompt definition without model execution.
|
||||
- `scriptorium inspect profile`: inspect one effective profile without model execution.
|
||||
|
||||
Canonical related references:
|
||||
All commands accept `--config <path>` and reject positional arguments. An
|
||||
effective `prompt_dir` is required for every command. Supply it through the
|
||||
configuration contract or the command's `--prompt-dir` flag.
|
||||
|
||||
- [Configuration reference](config.md)
|
||||
- [HTTP API reference](api.md)
|
||||
- [Subprocess integration](integrations/subprocess.md)
|
||||
## `scriptorium run`
|
||||
|
||||
## Common Rules
|
||||
|
||||
- `--config` is supported by `run`, `render`, and `serve`.
|
||||
- Positional arguments are rejected.
|
||||
- `run` and `render` require `--prompt`, at least one `--input`, and an effective `prompt_dir`.
|
||||
- `serve` requires an effective `prompt_dir`.
|
||||
- `profile_dir` is optional. Without it, only built-in profiles are available.
|
||||
- If `profile_dir` is set, custom profiles override built-in profiles with the same ID.
|
||||
- Prompt cache control, `session_id`, structured output, and provider-specific profile fields are configured in YAML, not with CLI flags.
|
||||
|
||||
Config precedence is:
|
||||
|
||||
1. built-in defaults
|
||||
2. config file values
|
||||
3. CLI flags
|
||||
|
||||
## Flag Reference
|
||||
|
||||
### `scriptorium run`
|
||||
|
||||
```bash
|
||||
```text
|
||||
scriptorium run [flags]
|
||||
```
|
||||
|
||||
Required through flags or config:
|
||||
Required flags:
|
||||
|
||||
- `--prompt-dir <dir>`: prompt definition directory.
|
||||
|
||||
Required as flags:
|
||||
|
||||
- `--prompt <id>`: prompt ID to execute.
|
||||
- `--input name=path`: input file mapping. Repeat or use comma-separated mappings.
|
||||
| Flag | Meaning |
|
||||
| --- | --- |
|
||||
| `--prompt <id>` | Prompt ID to execute. |
|
||||
|
||||
Optional flags:
|
||||
|
||||
- `--config <path>`: application config file.
|
||||
- `--profile-dir <dir>`: custom profile definition directory.
|
||||
- `--schema-dir <dir>`: schema base directory for `json_schema` validation.
|
||||
- `--profile <id>`: execution profile override. If omitted, the prompt `default_profile` is used.
|
||||
- `--var name=value`: template variable mapping. Repeat or use comma-separated mappings.
|
||||
- `--out <path>`: write generated artifact body to a file instead of stdout.
|
||||
- `--llm-base-url <url>`: runtime endpoint override.
|
||||
- `--model <name>`: runtime model override.
|
||||
- `--api-key-env <name>`: runtime API-key environment variable name override.
|
||||
- `--temperature <float>`: runtime temperature override.
|
||||
- `--max-tokens <int>`: runtime max tokens override.
|
||||
- `--top-p <float>`: runtime top-p override.
|
||||
- `--timeout <duration>`: runtime timeout override using Go duration syntax, such as `30s` or `2m`.
|
||||
| Flag | Meaning |
|
||||
| --- | --- |
|
||||
| `--config <path>` | Application configuration file. |
|
||||
| `--prompt-dir <dir>` | Prompt-definition directory override. |
|
||||
| `--profile-dir <dir>` | Custom profile-directory override. |
|
||||
| `--schema-dir <dir>` | Schema base-directory override. |
|
||||
| `--prompt-version <version>` | Optional prompt-definition version selector. |
|
||||
| `--profile <id>` | Execution-profile override. |
|
||||
| `--session-id <id>` | Optional direct session identifier. |
|
||||
| `--input name=path` | Optional input file mapping; repeat or use comma-separated mappings. |
|
||||
| `--var name=value` | Template-variable mapping; repeat or use comma-separated mappings. |
|
||||
| `--out <path>` | Write generated content to this file instead of stdout. |
|
||||
| `--llm-base-url <url>` | Runtime endpoint override. |
|
||||
| `--model <name>` | Runtime model override. |
|
||||
| `--api-key-env <name>` | Runtime API-key environment-variable name override. |
|
||||
| `--temperature <float>` | Runtime temperature override. |
|
||||
| `--max-tokens <int>` | Runtime maximum-token override. |
|
||||
| `--top-p <float>` | Runtime top-p override. |
|
||||
| `--reasoning-effort <value>` | Runtime reasoning-effort override. |
|
||||
| `--timeout <duration>` | Runtime timeout override using Go duration syntax. |
|
||||
|
||||
Deprecated aliases:
|
||||
Deprecated aliases: `--prompt-id` for `--prompt`, and `--profile-id` for
|
||||
`--profile`.
|
||||
|
||||
- `--prompt-id <id>`: alias for `--prompt`.
|
||||
- `--profile-id <id>`: alias for `--profile`.
|
||||
Omitted numeric runtime flags preserve the selected effective value; explicit
|
||||
zero values override it. `--timeout 0s` disables the per-generation deadline
|
||||
only; the caller context and configured transport cap remain active. CLI
|
||||
durations are converted to whole seconds by truncation toward zero, so any
|
||||
duration whose absolute value is below one second becomes an explicit
|
||||
zero-second override. The timeout layers are defined in the
|
||||
[Promptkit outbound integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/integrations/openai-compatible-chat.md#timeout-and-cancellation).
|
||||
|
||||
Runtime override notes:
|
||||
There is no raw API-key flag. Use `--api-key-env`.
|
||||
|
||||
- Omitted numeric override flags preserve the selected profile/default value.
|
||||
- Explicit zero values override the selected profile/default value.
|
||||
- `--timeout 0s` disables the outbound HTTP client timeout for that request.
|
||||
- There is no raw API-key flag; use `--api-key-env`.
|
||||
`--prompt-version` is passed directly to Promptkit. When it is omitted, the
|
||||
selected prompt ID must have exactly one available version. `--input` is
|
||||
optional at the CLI boundary: Promptkit decides whether the selected definition
|
||||
requires declared inputs or template-referenced values.
|
||||
|
||||
### `scriptorium render`
|
||||
`--session-id` supplies a direct, non-secret session identifier. A nonblank
|
||||
value replaces a definition-rendered session ID; an omitted or blank value lets
|
||||
the definition supply one. Promptkit trims direct values and limits them to 256
|
||||
Unicode code points. Use stable, non-sensitive identifiers because effective
|
||||
session IDs may appear in prepared output, run metadata, and provider-facing
|
||||
requests.
|
||||
|
||||
```bash
|
||||
`--reasoning-effort` is presence-aware: omitting it inherits the selected
|
||||
profile value, a nonblank value replaces that value, and
|
||||
`--reasoning-effort=` explicitly clears inherited reasoning. Promptkit treats
|
||||
nonblank values as provider-specific opaque strings.
|
||||
|
||||
## `scriptorium render`
|
||||
|
||||
```text
|
||||
scriptorium render [flags]
|
||||
```
|
||||
|
||||
Required through flags or config:
|
||||
`--prompt <id>` is required. The following optional flags are supported:
|
||||
`--config`, `--prompt-dir`, `--profile-dir`, `--prompt-version`, `--profile`,
|
||||
`--input`, `--var`, `--out`, `--llm-base-url`,
|
||||
`--model`, `--api-key-env`, `--temperature`, `--max-tokens`, `--top-p`,
|
||||
`--reasoning-effort`, `--session-id`, `--timeout`, and `--format text|json`. Their meanings match the corresponding
|
||||
`run` flags; `--format` selects prepared-run output and otherwise uses
|
||||
`defaults.render_format`.
|
||||
|
||||
- `--prompt-dir <dir>`: prompt definition directory.
|
||||
The same deprecated aliases and numeric/timeout behavior as `run` apply.
|
||||
The same session and reasoning inheritance, replacement, and clearing behavior
|
||||
also applies.
|
||||
`render` does not accept `--schema-dir`; configure `schema_dir` through the
|
||||
configuration file. It resolves profiles and schemas as part of preparation but
|
||||
does not call an LLM.
|
||||
|
||||
Required as flags:
|
||||
## `scriptorium serve`
|
||||
|
||||
- `--prompt <id>`: prompt ID to render.
|
||||
- `--input name=path`: input file mapping. Repeat or use comma-separated mappings.
|
||||
|
||||
Optional flags:
|
||||
|
||||
- `--config <path>`: application config file.
|
||||
- `--prompt-dir <dir>`: prompt definition directory.
|
||||
- `--profile-dir <dir>`: custom profile definition directory.
|
||||
- `--profile <id>`: execution profile override.
|
||||
- `--var name=value`: template variable mapping. Repeat or use comma-separated mappings.
|
||||
- `--out <path>`: write prepared-run output to a file instead of stdout.
|
||||
- `--llm-base-url <url>`: runtime endpoint override for the prepared request.
|
||||
- `--model <name>`: runtime model override for the prepared request.
|
||||
- `--api-key-env <name>`: runtime API-key environment variable name override.
|
||||
- `--temperature <float>`: runtime temperature override.
|
||||
- `--max-tokens <int>`: runtime max tokens override.
|
||||
- `--top-p <float>`: runtime top-p override.
|
||||
- `--timeout <duration>`: runtime timeout override using Go duration syntax.
|
||||
- `--format text|json`: prepared-run output format. Defaults to config `defaults.render_format`, then `text`.
|
||||
|
||||
Deprecated aliases:
|
||||
|
||||
- `--prompt-id <id>`: alias for `--prompt`.
|
||||
- `--profile-id <id>`: alias for `--profile`.
|
||||
|
||||
Notes:
|
||||
|
||||
- `render` resolves profiles, loads schemas for `json_schema` prompts, and validates `api_key_env`.
|
||||
- `render` does not accept `--schema-dir`; use config `schema_dir` for render-time schema lookup.
|
||||
- `render` does not call the LLM.
|
||||
|
||||
### `scriptorium serve`
|
||||
|
||||
```bash
|
||||
```text
|
||||
scriptorium serve [flags]
|
||||
```
|
||||
|
||||
Required through flags or config:
|
||||
|
||||
- `--prompt-dir <dir>`: prompt definition directory.
|
||||
|
||||
Optional flags:
|
||||
|
||||
- `--config <path>`: application config file.
|
||||
- `--addr <listen-address>`: HTTP listen address.
|
||||
- `--prompt-dir <dir>`: prompt definition directory.
|
||||
- `--profile-dir <dir>`: custom profile definition directory.
|
||||
- `--schema-dir <dir>`: schema base directory for `json_schema` validation.
|
||||
- `--artifact-root <dir>`: base directory for HTTP `file` input references.
|
||||
- `--max-request-bytes <n>`: maximum HTTP request body bytes; `0` disables the limit.
|
||||
- `--max-artifact-bytes <n>`: maximum HTTP file artifact bytes; `0` disables the limit.
|
||||
- `--max-response-bytes <n>`: maximum encoded HTTP response body bytes; `0` disables the limit.
|
||||
| Flag | Meaning |
|
||||
| --- | --- |
|
||||
| `--config <path>` | Application configuration file. |
|
||||
| `--addr <listen-address>` | HTTP listen-address override. |
|
||||
| `--prompt-dir <dir>` | Prompt-definition directory override. |
|
||||
| `--profile-dir <dir>` | Custom profile-directory override. |
|
||||
| `--schema-dir <dir>` | Schema base-directory override. |
|
||||
| `--artifact-root <dir>` | Root for HTTP `file` input references. |
|
||||
| `--max-request-bytes <n>` | Maximum encoded HTTP request-body bytes; `0` disables the limit. |
|
||||
| `--max-artifact-bytes <n>` | Maximum HTTP file-input artifact bytes; `0` disables the limit. |
|
||||
| `--max-response-bytes <n>` | Maximum encoded HTTP response bytes; `0` disables the limit. |
|
||||
|
||||
Notes:
|
||||
`serve` accepts no runtime model override flags. HTTP request fields, response
|
||||
schemas, and error codes are defined in the [HTTP API reference](api.md).
|
||||
|
||||
- `serve` does not accept runtime model override flags such as `--model` or `--llm-base-url`.
|
||||
- HTTP request fields and error codes are documented in the [HTTP API reference](api.md).
|
||||
- HTTP `file` input references are rejected unless an artifact root is configured.
|
||||
- HTTP size-limit flags affect only `serve`.
|
||||
## `scriptorium inspect prompt`
|
||||
|
||||
```text
|
||||
scriptorium inspect prompt --prompt ID [--prompt-version VERSION]
|
||||
[--config PATH] [--prompt-dir DIR] [--format text|json] [--out PATH]
|
||||
```
|
||||
|
||||
`--prompt` is required. Inspection uses normal configuration discovery and a
|
||||
`--prompt-dir` override, defaults to text regardless of `defaults.render_format`,
|
||||
and writes to stdout unless `--out` is supplied. It loads and normalizes the
|
||||
selected definition but does not resolve a profile, load a schema, render a
|
||||
template, reserve backend capacity, or call a model.
|
||||
|
||||
## `scriptorium inspect profile`
|
||||
|
||||
```text
|
||||
scriptorium inspect profile --profile ID
|
||||
[--config PATH] [--profile-dir DIR] [--format text|json] [--out PATH]
|
||||
```
|
||||
|
||||
`--profile` is required; built-in profiles need no prompt directory. Inspection
|
||||
resolves profile inheritance and backend defaults, but never reads a credential
|
||||
value, loads a prompt, reserves capacity, or calls a model. Unset provider
|
||||
controls are shown as zero values when Promptkit leaves them unspecified.
|
||||
|
||||
## Input And Variable Syntax
|
||||
|
||||
- `--input name=path` maps prompt input names to local file paths.
|
||||
- `--var name=value` maps prompt template variables to string values.
|
||||
- Both flags can be repeated.
|
||||
- Both flags also accept comma-separated mappings, such as `--input transcript=./t.md,glossary=./g.yml`.
|
||||
- Values may contain `=` after the first separator, such as `--var note=a=b=c`.
|
||||
- Empty names and empty values are rejected.
|
||||
`--input name=path` maps an input name to a local file; `--var name=value`
|
||||
maps a template variable to a string. Both flags can be repeated or contain
|
||||
comma-separated mappings. Values may contain `=` after the first separator.
|
||||
Empty names and values are rejected.
|
||||
|
||||
CLI `run` and `render` convert every `--input` mapping to a `file` artifact
|
||||
reference. HTTP also supports `inline` input references; see [HTTP API
|
||||
reference](api.md).
|
||||
CLI inputs are file references. HTTP inline inputs are defined by the
|
||||
[HTTP API reference](api.md).
|
||||
|
||||
## Output Behavior
|
||||
## Output And Exit Behavior
|
||||
|
||||
`run`:
|
||||
- `run` writes generated content to stdout, or to `--out` when supplied, and
|
||||
writes a concise summary to stderr. The summary includes `backend=<id>` when
|
||||
Promptkit selected a backend; endpoint-only profiles omit it.
|
||||
- `render` writes prepared-run output to stdout, or to `--out` when supplied,
|
||||
without a success summary. Text output includes `selected_backend_id` after
|
||||
`selected_profile_id` when Promptkit selected one; endpoint-only profiles
|
||||
omit it.
|
||||
- `serve` writes startup and server errors to stderr.
|
||||
|
||||
- Writes generated artifact content to stdout by default.
|
||||
- Writes generated artifact content to `--out` when provided.
|
||||
- Prints a success summary to stderr.
|
||||
- Prints errors to stderr on failure.
|
||||
Exit statuses:
|
||||
|
||||
`render`:
|
||||
| Status | Meaning |
|
||||
| --- | --- |
|
||||
| `0` | Success. |
|
||||
| `1` | Parse, configuration, loading, rendering, generation, output-write, or other runtime error. |
|
||||
| `2` | `run` generated and wrote output, but validation failed. |
|
||||
|
||||
- Writes prepared-run output to stdout by default.
|
||||
- Writes prepared-run output to `--out` when provided.
|
||||
- Does not print a success summary.
|
||||
A backend admission rejection is a runtime error and prints `run error: model
|
||||
backend capacity is exhausted`. The HTTP capacity response is defined in the
|
||||
[HTTP API reference](api.md).
|
||||
|
||||
`serve`:
|
||||
## Workflows And Examples
|
||||
|
||||
- Logs startup and server errors to stderr.
|
||||
|
||||
## Exit Codes
|
||||
|
||||
- `0`: success.
|
||||
- `1`: parse, config, load, render, generation, output-write, or runtime error.
|
||||
- `2`: `run` completed and wrote output, but validation status is `failed`.
|
||||
|
||||
## Common Workflows
|
||||
|
||||
Render prompt inputs and variables as JSON:
|
||||
|
||||
```bash
|
||||
go run ./cmd/scriptorium render \
|
||||
--config ./examples/config.yml \
|
||||
--prompt generic.markdown_summary \
|
||||
--input transcript=./examples/fixtures/transcript.md \
|
||||
--input glossary=./examples/fixtures/glossary.yml \
|
||||
--var session_date=2026-05-04 \
|
||||
--format json
|
||||
```
|
||||
|
||||
Run a prompt with an explicit profile and file output:
|
||||
|
||||
```bash
|
||||
go run ./cmd/scriptorium run \
|
||||
--config ./examples/config.yml \
|
||||
--prompt generic.markdown_summary \
|
||||
--profile local-fast \
|
||||
--input transcript=./examples/fixtures/transcript.md \
|
||||
--input glossary=./examples/fixtures/glossary.yml \
|
||||
--out ./summary.md
|
||||
```
|
||||
|
||||
Start the HTTP server with example config:
|
||||
|
||||
```bash
|
||||
go run ./cmd/scriptorium serve --config ./examples/config.yml
|
||||
```
|
||||
|
||||
Copyable maintained script:
|
||||
|
||||
- `examples/render-markdown-summary.sh`
|
||||
The [maintained render script](../examples/render-markdown-summary.sh) is a
|
||||
copyable render workflow. The [HTTP request example](../examples/http-run.json)
|
||||
is for a running `serve` process.
|
||||
|
||||
418
docs/config.md
418
docs/config.md
@@ -1,324 +1,126 @@
|
||||
# Configuration Reference
|
||||
|
||||
## Config Discovery And Precedence
|
||||
This is the canonical reference for Scriptorium application settings. Prompt,
|
||||
profile, schema, execution-setting, built-in profile, and framework credential
|
||||
semantics are defined by the
|
||||
[Promptkit v0.9.0 format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md).
|
||||
For command syntax, see the [CLI reference](cli.md); for HTTP request shapes and
|
||||
outcomes, see the [HTTP API reference](api.md).
|
||||
|
||||
## Discovery And Precedence
|
||||
|
||||
Application settings are resolved in this order:
|
||||
|
||||
1. built-in defaults
|
||||
2. `config.yml` values
|
||||
3. CLI overrides
|
||||
1. built-in Scriptorium defaults;
|
||||
2. a configuration file; then
|
||||
3. CLI overrides.
|
||||
|
||||
When `--config` is omitted, Scriptorium searches:
|
||||
When `--config` is omitted, Scriptorium searches
|
||||
`/usr/local/etc/scriptorium/config.yml` and then `/etc/scriptorium/config.yml`.
|
||||
If neither exists, it uses built-in defaults. An explicit `--config` path must
|
||||
exist and decode successfully.
|
||||
|
||||
1. `/usr/local/etc/scriptorium/config.yml`
|
||||
2. `/etc/scriptorium/config.yml`
|
||||
The maintained [minimal configuration](../examples/config.yml) and
|
||||
[complete configuration](../examples/config.full.yml) are copyable examples.
|
||||
|
||||
If neither file exists, Scriptorium uses built-in defaults. When
|
||||
`--config <path>` is provided, that file must exist and decode successfully.
|
||||
## Application Configuration File
|
||||
|
||||
## Minimal Working Config
|
||||
Configuration is strict YAML: unknown fields are rejected. Empty string values
|
||||
do not override a prior value. Raw API-key fields are not accepted.
|
||||
|
||||
| Field | Default | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `prompt_dir` | unset | Promptkit prompt-definition source directory. `run`, `render`, and `serve` require an effective value. |
|
||||
| `profile_dir` | unset | Optional custom Promptkit profile source directory overlaid on Promptkit built-ins. |
|
||||
| `schema_dir` | `.` | Promptkit schema source directory for relative schema paths. |
|
||||
| `server.addr` | `:8080` | Address used by `serve`. |
|
||||
| `server.artifact_root` | unset | Root that enables HTTP `file` input references. |
|
||||
| `server.max_request_bytes` | `16777216` | Maximum encoded HTTP request-body bytes; `0` disables the limit. |
|
||||
| `server.max_artifact_bytes` | `16777216` | Maximum HTTP file-input artifact bytes; `0` disables the limit. |
|
||||
| `server.max_response_bytes` | `16777216` | Maximum encoded HTTP response bytes; `0` disables the limit. |
|
||||
| `defaults.render_format` | `text` | Default prepared-run output format: `text` or `json`. |
|
||||
| `backends` | unset | Optional mapping of custom Promptkit backend IDs to engine-scoped connection and capacity settings. |
|
||||
|
||||
The size fields must be zero or greater. The [HTTP API](api.md) defines how
|
||||
each limit is enforced and reported. `server.artifact_root` configures an HTTP
|
||||
deployment boundary; see [operations](operations.md) for deployment handling.
|
||||
|
||||
## Custom Backends
|
||||
|
||||
Use `backends` when a profile selects an application-defined backend ID:
|
||||
|
||||
```yaml
|
||||
prompt_dir: ./examples/prompts
|
||||
backends:
|
||||
local-gpu:
|
||||
endpoint: http://localhost:11434/v1
|
||||
api_key_env: LOCAL_GPU_API_KEY
|
||||
extra_params:
|
||||
provider_option: enabled
|
||||
concurrency_limit: 2
|
||||
queue_capacity: 0
|
||||
```
|
||||
|
||||
This is enough for `run` and `render` when selected prompts use built-in
|
||||
profiles. Set `profile_dir` when prompts or requests use custom profiles.
|
||||
|
||||
The maintained repository example is `examples/config.yml`.
|
||||
|
||||
## Production-Oriented Config
|
||||
|
||||
```yaml
|
||||
prompt_dir: /opt/scriptorium/prompts
|
||||
profile_dir: /opt/scriptorium/profiles
|
||||
schema_dir: /opt/scriptorium/schemas
|
||||
|
||||
server:
|
||||
addr: 127.0.0.1:8080
|
||||
artifact_root: /var/lib/scriptorium/artifacts
|
||||
max_request_bytes: 16777216
|
||||
max_artifact_bytes: 16777216
|
||||
max_response_bytes: 16777216
|
||||
|
||||
defaults:
|
||||
render_format: text
|
||||
```
|
||||
|
||||
The maintained full example is `examples/config.full.yml`.
|
||||
|
||||
## App Config Reference
|
||||
|
||||
Top-level fields:
|
||||
|
||||
| Field | Default | Description |
|
||||
| --- | --- | --- |
|
||||
| `prompt_dir` | unset | Directory containing prompt definition YAML files. Required effectively by `run`, `render`, and `serve`. |
|
||||
| `profile_dir` | unset | Directory containing custom profile YAML files. Built-in profiles remain available when unset. |
|
||||
| `schema_dir` | `.` | Base directory for relative JSON Schema paths. |
|
||||
| `server` | `{}` | HTTP service settings used by `serve`. |
|
||||
| `defaults` | `{}` | Adapter defaults. |
|
||||
|
||||
`server` fields:
|
||||
|
||||
| Field | Default | Description |
|
||||
| --- | --- | --- |
|
||||
| `server.addr` | `:8080` | Listen address for `serve`. |
|
||||
| `server.artifact_root` | unset | Base directory for HTTP `file` input references. Without it, HTTP file refs are rejected. |
|
||||
| `server.max_request_bytes` | `16777216` | Maximum encoded HTTP request body bytes. `0` disables the limit. |
|
||||
| `server.max_artifact_bytes` | `16777216` | Maximum HTTP file artifact bytes. `0` disables the limit. |
|
||||
| `server.max_response_bytes` | `16777216` | Maximum encoded HTTP response bytes. `0` disables the limit. |
|
||||
|
||||
`defaults` fields:
|
||||
|
||||
| Field | Default | Description |
|
||||
| --- | --- | --- |
|
||||
| `defaults.render_format` | `text` | Default `render` output format: `text` or `json`. |
|
||||
|
||||
Config rules:
|
||||
|
||||
- YAML decoding is strict; unknown fields are rejected.
|
||||
- HTTP size limits must be greater than or equal to `0`.
|
||||
- Empty string config values are ignored.
|
||||
- Raw API key fields are not supported in app config.
|
||||
|
||||
## Prompt Definition Files
|
||||
|
||||
Prompt definitions are YAML files anywhere under `prompt_dir`. Nested
|
||||
directories are organizational; callers select prompts by YAML `id`, not file
|
||||
path.
|
||||
|
||||
Example:
|
||||
|
||||
```yaml
|
||||
id: generic.structured_events
|
||||
version: "1.0.0"
|
||||
default_profile: local-quality
|
||||
description: Produce structured event JSON from a transcript.
|
||||
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: text/markdown
|
||||
description: Source transcript content
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/yaml
|
||||
description: Optional glossary context
|
||||
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./generic.structured_events.system.md
|
||||
- role: user
|
||||
content_file: ./generic.structured_events.user.md
|
||||
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: structured_events.schema.json
|
||||
repair_attempts: 0
|
||||
```
|
||||
|
||||
Prompt fields:
|
||||
|
||||
| Field | Required | Description |
|
||||
| --- | --- | --- |
|
||||
| `id` | yes | Prompt identifier used by `--prompt` and HTTP `prompt_id`. |
|
||||
| `version` | yes | Prompt version. |
|
||||
| `default_profile` | no | Profile ID used when a request does not provide a profile. |
|
||||
| `description` | no | Human-readable description. |
|
||||
| `session_id` | no | Go-template string rendered from request vars and forwarded as provider `session_id` when non-empty. |
|
||||
| `inputs` | no | Named input declarations. |
|
||||
| `messages` | yes | Chat message templates. |
|
||||
| `output` | yes | Output format and validation contract. |
|
||||
|
||||
`inputs[]` fields:
|
||||
|
||||
- `name` (required)
|
||||
- `required` (optional boolean)
|
||||
- `content_type` (optional metadata)
|
||||
- `description` (optional)
|
||||
|
||||
`messages[]` fields:
|
||||
|
||||
- `role` (required)
|
||||
- exactly one of `content` or `content_file`
|
||||
- `cache_control` (optional)
|
||||
|
||||
Message rules:
|
||||
|
||||
- `content_file` resolves relative to the prompt YAML file location.
|
||||
- Repeated roles are allowed.
|
||||
- Prompt YAML decoding is strict.
|
||||
- Duplicate input names are invalid.
|
||||
- Duplicate prompt IDs are invalid for a requested ID/version.
|
||||
|
||||
`messages[].cache_control` fields:
|
||||
|
||||
| Field | Required | Supported values |
|
||||
| --- | --- | --- |
|
||||
| `type` | yes | `ephemeral` |
|
||||
| `ttl` | no | `1h` |
|
||||
|
||||
`session_id` behavior:
|
||||
|
||||
- Rendered with the same variable context as message templates.
|
||||
- Trimmed and omitted when empty.
|
||||
- Rejected when longer than 256 Unicode code points.
|
||||
- CLI callers pass variables with `--var`; HTTP callers use `vars`.
|
||||
|
||||
`output` fields:
|
||||
|
||||
| Field | Required | Supported values |
|
||||
| --- | --- | --- |
|
||||
| `format` | yes | `text`, `markdown`, `json` |
|
||||
| `validation_mode` | yes | `none`, `basic`, `json`, `json_schema` |
|
||||
| `schema_path` | only for `json_schema` | Relative to `schema_dir` unless absolute. |
|
||||
| `repair_attempts` | yes | Integer greater than or equal to `0`. |
|
||||
|
||||
Repair boundary:
|
||||
|
||||
- `repair_attempts` is part of the prompt contract.
|
||||
- The current CLI and HTTP wiring constructs the runner without a repairer, so normal `run` and `serve` execution does not perform repair attempts.
|
||||
|
||||
## Profile Definition Files
|
||||
|
||||
Execution profiles are YAML files anywhere under `profile_dir`. Nested
|
||||
directories are organizational; callers select profiles by YAML `id`, not file
|
||||
path.
|
||||
|
||||
Scriptorium also ships built-in profiles. Custom profiles override built-ins
|
||||
with the same ID.
|
||||
|
||||
Example:
|
||||
|
||||
```yaml
|
||||
id: local-fast
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: gpt-4o-mini
|
||||
temperature: 0.2
|
||||
max_tokens: 500
|
||||
top_p: 1.0
|
||||
timeout_seconds: 90
|
||||
api_key_env: SCRIPTORIUM_API_KEY
|
||||
service_tier: priority
|
||||
reasoning_effort: medium
|
||||
extra_params:
|
||||
provider_route: primary
|
||||
```
|
||||
|
||||
Profile fields:
|
||||
|
||||
| Field | Required | Description |
|
||||
| --- | --- | --- |
|
||||
| `id` | yes | Profile identifier. |
|
||||
| `endpoint` | yes | OpenAI-compatible base URL including `/v1`. |
|
||||
| `model` | yes | Provider model name. |
|
||||
| `temperature` | no | Range `0..2`. |
|
||||
| `max_tokens` | no | Integer greater than or equal to `0`. |
|
||||
| `top_p` | no | Range `0..1`. |
|
||||
| `timeout_seconds` | no | Integer greater than or equal to `0`. |
|
||||
| `service_tier` | no | Provider-specific request tier. |
|
||||
| `reasoning_effort` | no | Provider-specific reasoning setting. |
|
||||
| `api_key_env` | no | Environment variable name containing the API key. |
|
||||
| `extra_params` | no | JSON-compatible provider-specific top-level request fields. |
|
||||
|
||||
Execution defaults before profile/request overrides:
|
||||
|
||||
| Field | Default |
|
||||
| --- | --- |
|
||||
| `temperature` | `0.0` |
|
||||
| `max_tokens` | `0` |
|
||||
| `top_p` | `1.0` |
|
||||
| `timeout_seconds` | `600` |
|
||||
|
||||
Profile rules:
|
||||
|
||||
- Profile YAML decoding is strict.
|
||||
- Duplicate custom profile IDs are invalid.
|
||||
- Matching custom and built-in IDs are valid override behavior.
|
||||
- Raw `api_key` is rejected; use `api_key_env`.
|
||||
- If `api_key_env` is set, the named environment variable must be set before `run`, `render`, or HTTP execution can prepare the request.
|
||||
- Profile numeric fields merge by non-zero value. Request overrides are presence-aware, so explicit zero values are supported through CLI flags or HTTP model overrides.
|
||||
- `extra_params` keys must not be empty and must not collide with reserved outbound fields: `model`, `session_id`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, or `response_format`.
|
||||
|
||||
Built-in profile catalog:
|
||||
|
||||
| Provider | ID | Model | API key env |
|
||||
| --- | --- | --- | --- |
|
||||
| aion-labs | `aion-2` | `aion-labs/aion-2.0` | `OPENROUTER_API_KEY` |
|
||||
| anthropic | `claude-fable-latest` | `~anthropic/claude-fable-latest` | `OPENROUTER_API_KEY` |
|
||||
| anthropic | `claude-haiku-latest` | `~anthropic/claude-haiku-latest` | `OPENROUTER_API_KEY` |
|
||||
| anthropic | `claude-opus-latest` | `~anthropic/claude-opus-latest` | `OPENROUTER_API_KEY` |
|
||||
| anthropic | `claude-sonnet-latest` | `~anthropic/claude-sonnet-latest` | `OPENROUTER_API_KEY` |
|
||||
| deepseek | `deepseek-3-2` | `deepseek/deepseek-v3.2` | `OPENROUTER_API_KEY` |
|
||||
| deepseek | `deepseek-4-pro` | `deepseek/deepseek-v4-pro` | `OPENROUTER_API_KEY` |
|
||||
| google | `gemini-2-flash` | `google/gemini-2.5-flash` | `OPENROUTER_API_KEY` |
|
||||
| google | `gemini-2-flash-lite` | `google/gemini-2.5-flash-lite` | `OPENROUTER_API_KEY` |
|
||||
| google | `gemini-2-pro` | `google/gemini-2.5-pro` | `OPENROUTER_API_KEY` |
|
||||
| google | `gemini-3-flash-lite` | `google/gemini-3.1-flash-lite` | `OPENROUTER_API_KEY` |
|
||||
| google | `gemini-flash-latest` | `~google/gemini-flash-latest` | `OPENROUTER_API_KEY` |
|
||||
| google | `gemini-pro-latest` | `~google/gemini-pro-latest` | `OPENROUTER_API_KEY` |
|
||||
| google | `gemma-4-31b` | `google/gemma-4-31b-it:exacto` | `OPENROUTER_API_KEY` |
|
||||
| minimax | `minimax-m2` | `minimax/minimax-m2.5` | `OPENROUTER_API_KEY` |
|
||||
| minimax | `minimax-m3` | `minimax/minimax-m3` | `OPENROUTER_API_KEY` |
|
||||
| mistral | `mistral-large-2512` | `mistralai/mistral-large-2512` | `OPENROUTER_API_KEY` |
|
||||
| mistral | `mistral-medium-3-5` | `mistralai/mistral-medium-3-5` | `OPENROUTER_API_KEY` |
|
||||
| mistral | `mistral-small-3` | `mistralai/mistral-small-3.2-24b-instruct` | `OPENROUTER_API_KEY` |
|
||||
| mistral | `mistral-small-4` | `mistralai/mistral-small-2603` | `OPENROUTER_API_KEY` |
|
||||
| nvidia | `nemotron-3-ultra` | `nvidia/nemotron-3-ultra-550b-a55b` | `OPENROUTER_API_KEY` |
|
||||
| openai | `gpt-5-mini` | `openai/gpt-5.4-mini` | `OPENROUTER_API_KEY` |
|
||||
| openai | `gpt-5-nano` | `openai/gpt-5.4-nano` | `OPENROUTER_API_KEY` |
|
||||
|
||||
## Schema Behavior
|
||||
|
||||
Schemas are JSON files, typically under `schema_dir`.
|
||||
|
||||
Rules:
|
||||
|
||||
- `output.validation_mode: json_schema` requires `output.schema_path`.
|
||||
- Relative `schema_path` values resolve from `schema_dir`.
|
||||
- Absolute `schema_path` values are used directly.
|
||||
- Nested schemas must be referenced by relative path; schemas are not searched recursively by basename.
|
||||
- Missing or invalid schema documents are runtime validation errors.
|
||||
- Invalid generated JSON produces validation status `failed`, not a runtime error.
|
||||
|
||||
## Artifact References
|
||||
|
||||
Supported request input artifact reference types are:
|
||||
|
||||
- `file`
|
||||
- `inline`
|
||||
|
||||
CLI `run` and `render` create `file` references from `--input name=path`.
|
||||
|
||||
HTTP `file` references require `server.artifact_root` or `serve
|
||||
--artifact-root`. Relative file URIs resolve under that root. Absolute paths
|
||||
and relative traversal outside the root are rejected by lexical checks. Symlinks
|
||||
inside the root are followed by the operating system, including symlinks that
|
||||
point outside the root.
|
||||
|
||||
HTTP `inline` references do not require an artifact root.
|
||||
|
||||
## Secrets Handling
|
||||
|
||||
- Keep secret values in environment variables.
|
||||
- Store only environment-variable names in `api_key_env`.
|
||||
- Do not put raw API keys in config, prompts, profiles, CLI arguments, examples, or HTTP request bodies.
|
||||
|
||||
## Maintained Examples
|
||||
|
||||
- Minimal app config: `examples/config.yml`
|
||||
- Full app config: `examples/config.full.yml`
|
||||
- Prompt examples: `examples/prompts/`
|
||||
- Custom profile examples: `examples/profiles/`
|
||||
- Schema examples: `examples/schemas/`
|
||||
- Input fixtures: `examples/fixtures/`
|
||||
- Render script: `examples/render-markdown-summary.sh`
|
||||
- HTTP request-shape example: `examples/http-run.json`
|
||||
|
||||
## Integration References
|
||||
Each mapping key is the case-sensitive backend ID. `endpoint` is required;
|
||||
`api_key_env`, `extra_params`, `concurrency_limit`, and `queue_capacity` are
|
||||
optional. `concurrency_limit: 0` leaves the backend unlimited. Omitting
|
||||
`queue_capacity` lets Promptkit use its default for a limited backend, while
|
||||
an explicit `queue_capacity: 0` disables queueing.
|
||||
|
||||
Configuration strictly owns the YAML shape and rejects unknown fields. Promptkit
|
||||
validates backend IDs, endpoints, environment-variable names, extra parameters,
|
||||
and capacity relationships when Scriptorium constructs its engine. There are no
|
||||
backend command-line overrides. Store only an environment-variable name in
|
||||
`api_key_env`; raw API-key fields are not accepted.
|
||||
|
||||
## Framework Source Mapping
|
||||
|
||||
Scriptorium passes `prompt_dir`, `profile_dir`, and `schema_dir` to Promptkit
|
||||
when constructing its engine. Scriptorium does not redefine or independently
|
||||
parse those framework file formats.
|
||||
|
||||
- Prompt selection, versions, message templates, inputs, output contracts, and
|
||||
session IDs are Promptkit contracts.
|
||||
- Profile fields, numeric ranges, execution defaults, overlay precedence,
|
||||
built-in profiles, and credential rules are Promptkit contracts.
|
||||
- Schema path behavior and generated-content validation are Promptkit
|
||||
contracts.
|
||||
|
||||
See the
|
||||
[tagged Promptkit format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md)
|
||||
for all of those definitions. The files under
|
||||
[`examples/prompts`](../examples/prompts/),
|
||||
[`examples/profiles`](../examples/profiles/), and
|
||||
[`examples/schemas`](../examples/schemas/) are maintained Scriptorium
|
||||
application inputs using that tagged format.
|
||||
|
||||
## Credentials And Outbound Behavior
|
||||
|
||||
Scriptorium maps `--api-key-env` and HTTP `model.api_key_env` into Promptkit
|
||||
request overrides. Keep secret values in environment variables and store only
|
||||
their names in configuration or framework source files. Do not place raw keys
|
||||
in configuration, prompts, profiles, CLI arguments, examples, or HTTP
|
||||
payloads.
|
||||
|
||||
An optional `api_key_env` whose environment value is absent or empty can result
|
||||
in an unauthenticated provider request. A profile that declares credentials
|
||||
required still fails as an invalid request when no credential source is
|
||||
selected, and fails with `ErrAPIKeyEnvMissing` when its selected environment
|
||||
source is absent or empty. Scriptorium never reads or emits the environment
|
||||
value itself.
|
||||
|
||||
Unset optional provider controls are omitted from compatible provider requests.
|
||||
A positive Promptkit `repair_attempts` budget can add provider calls, latency,
|
||||
token use, and cost; see the tagged format reference for its permitted values
|
||||
and validation-mode requirements.
|
||||
|
||||
Promptkit's
|
||||
[OpenAI-compatible integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/integrations/openai-compatible-chat.md)
|
||||
defines outbound authentication, provider request mapping, transport limits,
|
||||
and timeout layering.
|
||||
|
||||
## Related References
|
||||
|
||||
- [CLI reference](cli.md)
|
||||
- [HTTP API reference](api.md)
|
||||
- [Outbound OpenAI-compatible contract](integrations/openai-compatible-chat.md)
|
||||
- [Operations guide](operations.md)
|
||||
- [Promptkit framework formats](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md)
|
||||
|
||||
@@ -1,122 +1,38 @@
|
||||
# Consumer Integration Overview
|
||||
|
||||
This guide is for applications that call Scriptorium from another codebase.
|
||||
Scriptorium exposes executable interfaces. Choose between a local subprocess
|
||||
and the HTTP service according to the boundary your application needs.
|
||||
|
||||
Scriptorium exposes three integration surfaces:
|
||||
|
||||
| Surface | Use when |
|
||||
| Interface | Use when |
|
||||
| --- | --- |
|
||||
| Go package | The consumer is Go, needs typed requests/results, or wants injected LLM clients for tests. |
|
||||
| CLI subprocess | The consumer wants process isolation or is not written in Go. |
|
||||
| HTTP API | The consumer needs a service boundary or remote access to `POST /v1/runs`. |
|
||||
| CLI subprocess | The consumer needs a synchronous local process boundary or prepared output. |
|
||||
| HTTP API | The consumer needs a service boundary or remote access. |
|
||||
|
||||
Canonical references:
|
||||
- CLI subprocess: [subprocess integration](../integrations/subprocess.md)
|
||||
- HTTP service: [HTTP API reference](../api.md)
|
||||
- Application configuration: [configuration reference](../config.md)
|
||||
|
||||
- Go package: [Package scriptorium](pkg-scriptorium.md)
|
||||
- CLI subprocess: [Subprocess integration](../integrations/subprocess.md)
|
||||
- HTTP: [HTTP API reference](../api.md)
|
||||
- File formats: [Configuration reference](../config.md)
|
||||
|
||||
## Required Deployment Inputs
|
||||
|
||||
Every integration needs operators to provide:
|
||||
|
||||
- prompt definitions;
|
||||
- profile definitions or built-in profile IDs;
|
||||
- schema files when prompts use `json_schema`;
|
||||
- input artifacts or inline input bodies;
|
||||
- API-key environment variables or direct per-request keys where supported.
|
||||
|
||||
Raw API keys do not belong in config, prompt files, profile YAML, CLI
|
||||
arguments, or HTTP request bodies.
|
||||
|
||||
## Recommended Workflow
|
||||
|
||||
Use the Go package when:
|
||||
|
||||
- the consumer is a Go application;
|
||||
- the application needs `context.Context` cancellation;
|
||||
- repeated calls should avoid subprocess startup;
|
||||
- tests need a fake LLM client;
|
||||
- direct per-request `RunRequest.APIKey` is required.
|
||||
|
||||
Use the CLI subprocess when:
|
||||
|
||||
- the consumer is not Go;
|
||||
- process isolation is useful;
|
||||
- stdout/stderr separation and exit codes are enough;
|
||||
- the consumer already manages local files and environment variables.
|
||||
|
||||
Use HTTP when:
|
||||
|
||||
- Scriptorium should run as a service;
|
||||
- multiple clients need a shared prompt/profile deployment;
|
||||
- clients can reach a trusted, protected HTTP boundary.
|
||||
|
||||
## Minimal Go Example
|
||||
|
||||
```go
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||
PromptDir: "./examples/prompts",
|
||||
ProfileDir: "./examples/profiles",
|
||||
SchemaDir: "./examples/schemas",
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
prepared, err := engine.Prepare(ctx, scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = prepared.Messages
|
||||
```
|
||||
|
||||
Run the maintained package example:
|
||||
|
||||
```bash
|
||||
go run ./examples/go-library/prepare
|
||||
```
|
||||
|
||||
## Subprocess Workflow
|
||||
|
||||
Invoke `scriptorium render` for preflight and `scriptorium run` for generation.
|
||||
Capture stdout and stderr separately. Treat exit code `2` from `run` as a
|
||||
completed generation with failed validation.
|
||||
|
||||
See [Subprocess integration](../integrations/subprocess.md) for the stable
|
||||
invocation contract.
|
||||
|
||||
## HTTP Workflow
|
||||
|
||||
Run `scriptorium serve` behind trusted controls and send JSON requests to
|
||||
`POST /v1/runs`.
|
||||
|
||||
Do not duplicate endpoint schemas in consumers. Use the [HTTP API
|
||||
reference](../api.md) as the authoritative contract.
|
||||
Go applications that need an in-process prompt framework should import
|
||||
Promptkit directly. The tagged
|
||||
[Promptkit Go consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md)
|
||||
owns that interface; Scriptorium does not provide a Go library package.
|
||||
Consumers arriving from the former Scriptorium Go API should follow the
|
||||
[migration guide](migrating-to-promptkit.md).
|
||||
|
||||
## Consumer Responsibilities
|
||||
|
||||
Consumers are responsible for:
|
||||
|
||||
- selecting prompt/profile IDs as deployment configuration;
|
||||
- supplying all required inputs and vars;
|
||||
- protecting generated artifacts and rendered prompts as sensitive data;
|
||||
- deciding whether to keep output when validation fails;
|
||||
- implementing retries only when another model call is acceptable.
|
||||
- selecting and deploying prompt, profile, and schema assets;
|
||||
- supplying required inputs and template variables;
|
||||
- supplying credentials through the chosen interface;
|
||||
- protecting rendered prompts and generated artifacts as potentially
|
||||
sensitive;
|
||||
- deciding whether validation-failed output is usable; and
|
||||
- retrying only when another model call is acceptable.
|
||||
|
||||
Scriptorium does not persist run state. Retrying a failed or timed-out request
|
||||
can produce different output and can incur another provider request.
|
||||
|
||||
## Status Behavior
|
||||
|
||||
- Go package methods return typed results or errors that support `errors.Is`.
|
||||
- CLI `run` exits `2` when generation succeeds but validation fails.
|
||||
- HTTP returns `200 OK` for generated-content validation failures and exposes the failed status in the response body.
|
||||
- Runtime validation failures are errors.
|
||||
Scriptorium does not persist run state. A retry can produce different output
|
||||
and can incur another provider request. CLI exits belong to the
|
||||
[CLI reference](../cli.md), HTTP status behavior belongs to the
|
||||
[HTTP API reference](../api.md), and framework semantics belong to
|
||||
[Promptkit v0.9.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md).
|
||||
|
||||
109
docs/consumers/migrating-to-promptkit.md
Normal file
109
docs/consumers/migrating-to-promptkit.md
Normal file
@@ -0,0 +1,109 @@
|
||||
# Migrate From Scriptorium To Promptkit
|
||||
|
||||
## Supported Migration Boundary
|
||||
|
||||
Scriptorium `v0.11.1` at
|
||||
`gitea.maximumdirect.net/eric/scriptorium` is the final release that provides
|
||||
the former in-process Go framework. Promptkit `v0.9.0` at
|
||||
`gitea.maximumdirect.net/eric/promptkit` is the destination for that framework
|
||||
API. Scriptorium `v0.12.0` and later provide the CLI and HTTP application only.
|
||||
|
||||
There is no Scriptorium compatibility facade, alias package, forwarding
|
||||
package, or deprecated wrapper. A consumer that cannot migrate may remain
|
||||
pinned to Scriptorium `v0.11.1`, but that framework-bearing line does not
|
||||
provide the slim application release.
|
||||
|
||||
## Update A Go Consumer
|
||||
|
||||
Start from a clean consumer checkout and review the pending diff before
|
||||
committing it. Add the published Promptkit module:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.9.0
|
||||
```
|
||||
|
||||
For an ordinary consumer that imports the former root package under its
|
||||
default name, replace the exact import and package qualifier, then format the
|
||||
changed Go files:
|
||||
|
||||
```sh
|
||||
git grep -l \
|
||||
'"gitea.maximumdirect.net/eric/scriptorium"' \
|
||||
-- '*.go' |
|
||||
while IFS= read -r go_file
|
||||
do
|
||||
perl -pi -e \
|
||||
's{"gitea.maximumdirect.net/eric/scriptorium"}{"gitea.maximumdirect.net/eric/promptkit"}g; s{\bscriptorium\.}{promptkit.}g' \
|
||||
"$go_file"
|
||||
gofmt -w "$go_file"
|
||||
done
|
||||
```
|
||||
|
||||
Inspect the resulting diff. Consumers that used an import alias should retain
|
||||
or deliberately rename that alias instead of applying the qualifier
|
||||
replacement mechanically.
|
||||
|
||||
Remove the now-unused Scriptorium requirement through module tidiness and run
|
||||
the consumer's complete tests:
|
||||
|
||||
```sh
|
||||
go mod tidy
|
||||
go test ./...
|
||||
```
|
||||
|
||||
Confirm that `go.mod` selects Promptkit `v0.9.0` and that no Go file imports
|
||||
the former Scriptorium package:
|
||||
|
||||
```sh
|
||||
test "$(
|
||||
go list -m -f '{{.Path}}@{{.Version}}' \
|
||||
gitea.maximumdirect.net/eric/promptkit
|
||||
)" = 'gitea.maximumdirect.net/eric/promptkit@v0.9.0'
|
||||
if git grep -n \
|
||||
'gitea.maximumdirect.net/eric/scriptorium' \
|
||||
-- '*.go'
|
||||
then
|
||||
printf '%s\n' 'a former Scriptorium Go import remains' >&2
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
## Compatibility And Additions
|
||||
|
||||
Promptkit preserves the established engine, request, result, profile,
|
||||
source-option, model-client, artifact, validation-value, and public-error
|
||||
shapes where practical. Exact declarations and current behavior belong to the
|
||||
tagged [Promptkit consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md)
|
||||
and Go source.
|
||||
|
||||
Promptkit also includes migration-relevant public contracts that were not in
|
||||
Scriptorium `v0.11.1`:
|
||||
|
||||
- [`WithArtifactReader`](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/engine.go#L182)
|
||||
and the
|
||||
[`ArtifactReader` declaration](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/types.go#L304)
|
||||
provide the artifact-reading extension described by the tagged
|
||||
[extension-interface guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md#extension-interfaces).
|
||||
- [`ErrProfileRequired` and `ErrAPIKeyEnvMissing`](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/engine.go#L46-L61)
|
||||
provide the specific identities described by the tagged
|
||||
[error guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md#errors).
|
||||
|
||||
Use those tagged owners for exact signatures, wrapping guarantees, and
|
||||
extension behavior.
|
||||
|
||||
## Verify Consumer Behavior
|
||||
|
||||
Source compatibility is only the first check. Exercise the behavior the
|
||||
consumer actually relies upon, especially:
|
||||
|
||||
- prompt, profile, and schema source selection;
|
||||
- direct and environment-based credentials;
|
||||
- caller, generation, and transport timeout layering;
|
||||
- output validation and validation-failure handling;
|
||||
- injected model-client and artifact-reader extensions; and
|
||||
- every `errors.Is` branch used for recovery or classification.
|
||||
|
||||
Also verify any serialized values, redaction expectations, filesystem policy,
|
||||
and provider integration behavior that crosses the consumer's own boundary.
|
||||
Promptkit owns the in-process framework contract; Scriptorium owns only its
|
||||
executable CLI and HTTP application interfaces.
|
||||
@@ -1,284 +0,0 @@
|
||||
# Package scriptorium
|
||||
|
||||
Import path:
|
||||
|
||||
```go
|
||||
import "gitea.maximumdirect.net/eric/scriptorium"
|
||||
```
|
||||
|
||||
The root package is the public Go facade for Scriptorium's prompt prepare/run
|
||||
workflow. It exposes typed requests, results, source options, injected LLM
|
||||
clients, and stable public errors while keeping `internal/*` packages private.
|
||||
|
||||
## Intended Use Cases
|
||||
|
||||
Use the package when a Go application needs:
|
||||
|
||||
- in-process prompt preparation or execution;
|
||||
- typed request/result structs;
|
||||
- direct `context.Context` cancellation;
|
||||
- injected/fake LLM clients for tests;
|
||||
- direct per-request `RunRequest.APIKey`.
|
||||
|
||||
Use [Subprocess integration](../integrations/subprocess.md) or the [HTTP API](../api.md)
|
||||
when a process or service boundary is preferred.
|
||||
|
||||
## Construct An Engine
|
||||
|
||||
```go
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||
PromptDir: "./examples/prompts",
|
||||
ProfileDir: "./examples/profiles",
|
||||
SchemaDir: "./examples/schemas",
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
`Config` fields:
|
||||
|
||||
| Field | Description |
|
||||
| --- | --- |
|
||||
| `PromptDir` | Prompt definition directory. Required unless `WithPromptFS` or `WithPromptFile` is used. |
|
||||
| `ProfileDir` | Optional custom profile directory overlaid above built-in profiles. |
|
||||
| `SchemaDir` | Schema directory. Defaults to `.` when empty. |
|
||||
| `Timeout` | Default timeout for the built-in OpenAI-compatible client. |
|
||||
| `HTTPClient` | Optional HTTP client for the built-in OpenAI-compatible client. |
|
||||
|
||||
`NewEngine` accepts `nil` options and ignores them. Invalid construction wraps
|
||||
`ErrInvalidConfig`.
|
||||
|
||||
## Source Options
|
||||
|
||||
Directory fields are the compatibility path. Explicit source options override
|
||||
the matching directory field.
|
||||
|
||||
Prompt sources:
|
||||
|
||||
- `WithPromptFS(fsys, root)`
|
||||
- `WithPromptFile(path)`
|
||||
|
||||
Profile sources:
|
||||
|
||||
- `WithProfileFS(fsys, root)`
|
||||
- `WithProfileFile(path)`
|
||||
- `WithProfiles(profiles...)`
|
||||
|
||||
Schema sources:
|
||||
|
||||
- `WithSchemaFS(fsys, root)`
|
||||
- `WithSchemaFile(path)`
|
||||
|
||||
LLM source:
|
||||
|
||||
- `WithLLMClient(client)`
|
||||
|
||||
Source behavior:
|
||||
|
||||
- Prompt and profile YAML use the same strict rules as directory loading.
|
||||
- Prompt `content_file` values resolve relative to the prompt file.
|
||||
- `fs.FS` roots are containment boundaries for prompt content files and schema paths.
|
||||
- File options expose the selected file by its base name.
|
||||
- Profile source precedence is in-memory profiles, then explicit profile file/FS/directory source, then built-ins.
|
||||
- `WithLLMClient(nil)` returns `ErrInvalidConfig`.
|
||||
|
||||
## In-Memory Profiles
|
||||
|
||||
Use `WithProfiles` when the application already has typed model settings:
|
||||
|
||||
```go
|
||||
profile := scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
ID: "app.default",
|
||||
Endpoint: "https://openrouter.ai/api/v1",
|
||||
Model: "mistralai/mistral-small-3.2-24b-instruct",
|
||||
APIKeyRequired: true,
|
||||
})
|
||||
|
||||
engine, err := scriptorium.NewEngine(cfg, scriptorium.WithProfiles(profile))
|
||||
```
|
||||
|
||||
`Profile` and `OpenAICompatibleProfileConfig` include:
|
||||
|
||||
- `ID`
|
||||
- `Endpoint`
|
||||
- `Model`
|
||||
- `Temperature`
|
||||
- `MaxTokens`
|
||||
- `TopP`
|
||||
- `TimeoutSeconds`
|
||||
- `ServiceTier`
|
||||
- `ReasoningEffort`
|
||||
- `APIKeyRequired`
|
||||
- `ExtraParams`
|
||||
|
||||
`WithProfiles` rejects duplicate IDs in one call. In-memory profiles do not
|
||||
store raw keys. When `APIKeyRequired` is true, pass the secret on each request
|
||||
with `RunRequest.APIKey`.
|
||||
|
||||
`ExtraParams` must be JSON-compatible: strings, booleans, finite numbers,
|
||||
objects with string keys, arrays/slices, and nil. Unsupported values, non-string
|
||||
map keys, non-finite floats, and cycles return `ErrInvalidConfig` for profiles
|
||||
or `ErrInvalidRequest` for request overrides.
|
||||
|
||||
## Prepare Workflow
|
||||
|
||||
`Prepare` resolves prompt/profile/input/schema state and renders messages
|
||||
without calling an LLM.
|
||||
|
||||
```go
|
||||
prepared, err := engine.Prepare(ctx, scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = prepared.EffectiveModelParams
|
||||
```
|
||||
|
||||
`PreparedRun` includes prompt ID/version/hash, selected profile, effective
|
||||
model params, output contract, structured-output metadata, input hashes,
|
||||
rendered prompt hash, rendered messages, and timing fields. It does not include
|
||||
raw API-key values, model output, validation results, or internal target
|
||||
presence metadata.
|
||||
|
||||
## Run Workflow
|
||||
|
||||
`Run` calls `Prepare`, invokes the configured LLM client, builds the output
|
||||
artifact, and validates the output.
|
||||
|
||||
```go
|
||||
result, err := engine.Run(ctx, scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
APIKey: apiKey,
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = result.Artifact
|
||||
```
|
||||
|
||||
`RunResult` includes run ID, output artifact, raw output, validation result,
|
||||
prompt/profile/model metadata, effective model params, input hashes, usage, and
|
||||
timing fields.
|
||||
|
||||
Generated-content validation failures return a successful `RunResult` with
|
||||
`Validation.Status == ValidationFailed`. Runtime/schema validation errors
|
||||
return an error that matches `ErrValidation`.
|
||||
|
||||
## Inputs
|
||||
|
||||
Input helpers:
|
||||
|
||||
- `File(path)`: file-backed artifact reference.
|
||||
- `Inline(body)`: inline artifact body.
|
||||
- `InlineWithURI(uri, body)`: inline artifact body with URI metadata.
|
||||
|
||||
Input map keys must match the prompt's expected input names.
|
||||
|
||||
## Injected LLM Clients
|
||||
|
||||
Use `WithLLMClient` for tests or custom model integrations:
|
||||
|
||||
```go
|
||||
type fakeLLM struct{}
|
||||
|
||||
func (fakeLLM) Generate(ctx context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
|
||||
return &scriptorium.GenerateResponse{
|
||||
Content: "generated text",
|
||||
Usage: scriptorium.TokenUsage{TotalTokens: 12},
|
||||
}, nil
|
||||
}
|
||||
|
||||
engine, err := scriptorium.NewEngine(cfg, scriptorium.WithLLMClient(fakeLLM{}))
|
||||
```
|
||||
|
||||
Injected clients receive:
|
||||
|
||||
- rendered prompt;
|
||||
- effective execution target;
|
||||
- numeric target presence metadata;
|
||||
- structured-output spec when applicable;
|
||||
- direct request API key when provided.
|
||||
|
||||
Custom clients should not log raw prompts or API keys by default.
|
||||
|
||||
## Overrides And API Keys
|
||||
|
||||
`RunRequest` fields:
|
||||
|
||||
| Field | Description |
|
||||
| --- | --- |
|
||||
| `PromptID` | Prompt ID. |
|
||||
| `PromptVersion` | Optional prompt version filter. |
|
||||
| `ProfileID` | Optional profile override. |
|
||||
| `APIKey` | Direct per-request API key. |
|
||||
| `Inputs` | Input artifact references. |
|
||||
| `Vars` | Template variables. |
|
||||
| `Execution` | Per-request model overrides. |
|
||||
| `Validation` | Per-request output contract override. |
|
||||
| `Metadata` | Request metadata reserved for callers. |
|
||||
|
||||
`RunRequest.Execution` uses pointer fields for numeric values so explicit zero
|
||||
overrides are preserved:
|
||||
|
||||
```go
|
||||
zero := 0
|
||||
req.Execution = &scriptorium.ExecutionTargetOverride{
|
||||
MaxTokens: &zero,
|
||||
}
|
||||
```
|
||||
|
||||
Direct `RunRequest.APIKey` takes precedence over profile `api_key_env` for the
|
||||
default OpenAI-compatible client. It is request-scoped, uses `json:"-"`, and is
|
||||
not included in `PreparedRun` or `RunResult` JSON. Normal Go string formatting
|
||||
of `RunRequest` and `GenerateRequest` reports only whether a direct key is set.
|
||||
|
||||
Raw API keys do not belong in profile YAML, in-memory profiles, or app config.
|
||||
Avoid reflection-based debug dumps of request structs because exported fields
|
||||
remain visible to tools that bypass `String` and `GoString`.
|
||||
|
||||
## Errors
|
||||
|
||||
Public methods wrap context while preserving stable sentinel checks with
|
||||
`errors.Is`:
|
||||
|
||||
- `ErrInvalidConfig`
|
||||
- `ErrInvalidRequest`
|
||||
- `ErrPromptNotFound`
|
||||
- `ErrProfileNotFound`
|
||||
- `ErrPromptLoad`
|
||||
- `ErrProfileLoad`
|
||||
- `ErrArtifactLoad`
|
||||
- `ErrPromptRender`
|
||||
- `ErrLLMGenerate`
|
||||
- `ErrValidation`
|
||||
|
||||
Example:
|
||||
|
||||
```go
|
||||
if errors.Is(err, scriptorium.ErrPromptNotFound) {
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
Run the maintained prepare-only example from the repository root:
|
||||
|
||||
```bash
|
||||
go run ./examples/go-library/prepare
|
||||
```
|
||||
|
||||
See also:
|
||||
|
||||
- [Configuration reference](../config.md)
|
||||
- [Consumer integration overview](api.md)
|
||||
56
docs/development.md
Normal file
56
docs/development.md
Normal file
@@ -0,0 +1,56 @@
|
||||
# Development
|
||||
|
||||
This is the contributor entry point for Scriptorium. Scriptorium is an
|
||||
application that consumes the public
|
||||
[Promptkit v0.9.0 package](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md);
|
||||
framework implementation work belongs in Promptkit.
|
||||
|
||||
## Initial Orientation
|
||||
|
||||
Before starting work:
|
||||
|
||||
1. inspect the working tree and preserve unrelated changes;
|
||||
2. read the [architecture policy](policy/architecture.md);
|
||||
3. follow the task-specific contracts and internal documents below; and
|
||||
4. inspect the relevant implementation and tests before changing them.
|
||||
|
||||
Also read the [documentation policy](policy/documentation.md) before changing
|
||||
documentation and the [testing policy](policy/testing.md) before changing
|
||||
tests.
|
||||
|
||||
## Task-Specific Reading Guide
|
||||
|
||||
| Task | Read before changing |
|
||||
| --- | --- |
|
||||
| Repository orientation or component responsibility | [Internal component overview](internal/overview.md) and [architecture policy](policy/architecture.md) |
|
||||
| CLI commands, flags, output, or exit behavior | [CLI contract](cli.md) and [adapter internals](internal/adapters.md) |
|
||||
| HTTP routes, DTOs, limits, status mapping, or artifact policy | [HTTP API contract](api.md), [adapter internals](internal/adapters.md), and [source internals](internal/sources.md) |
|
||||
| Application configuration or precedence | [Configuration contract](config.md), [adapter internals](internal/adapters.md), and [source internals](internal/sources.md) |
|
||||
| Prepared-run presentation | [CLI contract](cli.md), [adapter internals](internal/adapters.md), and `internal/format` |
|
||||
| Prompt, profile, schema, generation, or validation semantics | [Promptkit framework formats](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md) and the [Promptkit consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md) |
|
||||
| OpenAI-compatible outbound behavior or timeout layering | [Promptkit integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/integrations/openai-compatible-chat.md) |
|
||||
| Subprocess behavior | [Subprocess integration](integrations/subprocess.md) and [CLI contract](cli.md) |
|
||||
| Runtime operation or recovery | [Operations](operations.md) |
|
||||
| Release packaging or publication | The [release procedure](release.md), [hosted release workflow](../.woodpecker/release.yml), and [architecture policy](policy/architecture.md) |
|
||||
| Examples or copyable assets | The owning Scriptorium contract, the relevant [Promptkit format contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md), and the related files under `examples/` |
|
||||
| Architecture decisions or future work | The [documentation policy](policy/documentation.md), relevant accepted ADRs, and relevant roadmap documents |
|
||||
|
||||
Cross-project changes land and release in Promptkit before Scriptorium adopts
|
||||
the tagged version. Do not commit a Go workspace, local replacement, vendored
|
||||
Promptkit source, or an import of a Promptkit `internal` package.
|
||||
|
||||
## Baseline Validation
|
||||
|
||||
For code changes, run:
|
||||
|
||||
```bash
|
||||
go test ./...
|
||||
go test -race ./...
|
||||
go vet ./...
|
||||
go build ./cmd/scriptorium
|
||||
```
|
||||
|
||||
Check formatting with `gofmt`, run `git diff --check`, and validate affected
|
||||
examples and documentation links. Documentation-only work does not require
|
||||
unrelated new tests, but commands and examples changed by documentation must be
|
||||
run.
|
||||
@@ -1,193 +0,0 @@
|
||||
# OpenAI-Compatible Chat Integration
|
||||
|
||||
## Scope
|
||||
|
||||
This document defines the outbound LLM contract implemented by `internal/llm/openai_compatible_client.go`.
|
||||
|
||||
It documents only fields and behaviors currently serialized by code.
|
||||
|
||||
## Endpoint Construction
|
||||
|
||||
Request endpoint is built as:
|
||||
|
||||
1. choose base URL:
|
||||
- `GenerateRequest.Target.Endpoint` if set
|
||||
- otherwise client config `BaseURL`
|
||||
2. trim trailing slash
|
||||
3. append `/chat/completions`
|
||||
|
||||
Example:
|
||||
|
||||
- base URL: `http://localhost:8000/v1`
|
||||
- final URL: `http://localhost:8000/v1/chat/completions`
|
||||
|
||||
## Request Fields Sent
|
||||
|
||||
Serialized JSON fields:
|
||||
|
||||
- `model` (required after fallback resolution)
|
||||
- `session_id` (only when the rendered prompt includes a non-empty session ID)
|
||||
- `messages` (rendered prompt messages)
|
||||
- `temperature` (when non-zero, or when explicitly overridden to zero)
|
||||
- `max_tokens` (when non-zero, or when explicitly overridden to zero)
|
||||
- `top_p` (when non-zero, or when explicitly overridden to zero)
|
||||
- `service_tier` (only when non-empty)
|
||||
- `reasoning_effort` (only when non-empty)
|
||||
- `response_format` (only when structured output is provided)
|
||||
- profile/request `extra_params` as additional provider-specific top-level fields
|
||||
|
||||
`service_tier` is provider-specific. OpenRouter currently documents request values such as `flex` and `priority`; Scriptorium forwards any non-empty configured value and lets the backend validate support.
|
||||
|
||||
`reasoning_effort` is provider-specific. Scriptorium forwards any non-empty configured value as top-level `reasoning_effort` and lets the backend validate support.
|
||||
|
||||
`extra_params` are flattened into the outbound JSON object. They are not wrapped in an `extra_params` object:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "gpt-4o-mini",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "rendered text"
|
||||
}
|
||||
],
|
||||
"provider_route": "primary",
|
||||
"provider_options": {
|
||||
"retry_budget": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`extra_params` values must be JSON-compatible. Supported value shapes include strings, numbers, booleans, objects, and arrays.
|
||||
|
||||
Reserved `extra_params` keys are rejected before the HTTP request is made:
|
||||
|
||||
- `model`
|
||||
- `session_id`
|
||||
- `messages`
|
||||
- `temperature`
|
||||
- `max_tokens`
|
||||
- `top_p`
|
||||
- `service_tier`
|
||||
- `reasoning_effort`
|
||||
- `response_format`
|
||||
|
||||
Empty `extra_params` keys and values that cannot be encoded as JSON are also rejected before the HTTP request is made.
|
||||
|
||||
`session_id` is rendered from prompt YAML using request variables and serialized as a top-level JSON request field. Scriptorium does not send an `x-session-id` header. Empty rendered session IDs are omitted, and values longer than 256 characters are rejected before the HTTP request.
|
||||
|
||||
Messages without prompt cache control serialize with string `content`:
|
||||
|
||||
```json
|
||||
{
|
||||
"role": "system",
|
||||
"content": "rendered text"
|
||||
}
|
||||
```
|
||||
|
||||
Messages with prompt cache control serialize as a single text content-block array:
|
||||
|
||||
```json
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "rendered text",
|
||||
"cache_control": {
|
||||
"type": "ephemeral",
|
||||
"ttl": "1h"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
When cache-control `ttl` is unset in the prompt definition, `ttl` is omitted from the outbound payload.
|
||||
|
||||
Structured output is currently `json_schema` only, serialized as:
|
||||
|
||||
```json
|
||||
{
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "...",
|
||||
"strict": true,
|
||||
"schema": {"type": "object"}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Authentication Header
|
||||
|
||||
If `Target.APIKey` is set:
|
||||
|
||||
- set `Authorization: Bearer <value>`
|
||||
- do not read `Target.APIKeyEnv`
|
||||
|
||||
If `Target.APIKey` is empty and `Target.APIKeyEnv` is set:
|
||||
|
||||
- resolve environment variable value at request time
|
||||
- set `Authorization: Bearer <value>`
|
||||
|
||||
If the environment variable is unset/empty:
|
||||
|
||||
- request fails before HTTP call (`ErrInvalidRequest`)
|
||||
|
||||
If both `Target.APIKey` and `Target.APIKeyEnv` are empty:
|
||||
|
||||
- no `Authorization` header is sent
|
||||
|
||||
## Timeout Behavior
|
||||
|
||||
Base timeout comes from client configuration.
|
||||
|
||||
Per-request override:
|
||||
|
||||
- if `Target.TimeoutSeconds > 0`, use that value for request timeout
|
||||
- if `Target.TimeoutSeconds == 0` and the value came from an explicit request override, disable the HTTP client timeout
|
||||
- if `Target.TimeoutSeconds < 0`, request is rejected (`ErrInvalidRequest`)
|
||||
|
||||
## Response Expectations
|
||||
|
||||
Expected successful response shape (subset used):
|
||||
|
||||
- `choices[0].message.content`
|
||||
- `usage.prompt_tokens`
|
||||
- `usage.completion_tokens`
|
||||
- `usage.total_tokens`
|
||||
- `usage.prompt_tokens_details.cached_tokens` (optional)
|
||||
- `usage.cache_write_tokens` (optional)
|
||||
|
||||
Absent cache usage fields are treated as zero. Parsed cache usage is exposed through run results and adapter response surfaces as:
|
||||
|
||||
- `cached_tokens`
|
||||
- `cache_write_tokens`
|
||||
|
||||
Malformed response conditions include:
|
||||
|
||||
- invalid JSON
|
||||
- empty `choices`
|
||||
- empty `choices[0].message.content`
|
||||
|
||||
Malformed responses return `ErrMalformedResponse`.
|
||||
|
||||
## Error Handling
|
||||
|
||||
- network/request-construction failures: `ErrRequestFailed`
|
||||
- non-2xx HTTP status: `ErrUnexpectedStatus` (includes status code; provider response bodies are not included)
|
||||
- malformed response shape/content: `ErrMalformedResponse`
|
||||
|
||||
## Unsupported Or Non-Serialized Fields
|
||||
|
||||
The client does not serialize top-level `cache_control`.
|
||||
|
||||
No built-in retries, tool-calls, or multi-request payload modes are implemented in this client.
|
||||
|
||||
## Relationship To Runner
|
||||
|
||||
When prompt validation mode is `json_schema`, runner prepares a structured-output schema spec and passes it to the client as `StructuredOutput`.
|
||||
|
||||
The client only serializes the provider request payload; it does not load schema files itself.
|
||||
@@ -1,130 +1,41 @@
|
||||
# Subprocess Integration
|
||||
|
||||
This document defines the supported subprocess contract for downstream
|
||||
applications invoking Scriptorium through the public CLI.
|
||||
This document covers process-boundary behavior for callers that invoke
|
||||
Scriptorium as a child process. Command syntax, flags, output, and exit codes
|
||||
are defined by the [CLI reference](../cli.md). Interface selection belongs in
|
||||
the [consumer integration overview](../consumers/api.md).
|
||||
|
||||
This is a CLI contract. Go callers that want an in-process typed API should use
|
||||
the [package guide](../consumers/pkg-scriptorium.md).
|
||||
## Process Contract
|
||||
|
||||
## Supported Commands
|
||||
Use `scriptorium render` when the caller needs prepared output without a model
|
||||
call, and `scriptorium run` for generation. Pass an explicit `--config` or
|
||||
make the configuration search paths available to the child process; configuration
|
||||
discovery, fields, profile selection, and credential mechanisms are defined in
|
||||
the [configuration reference](../config.md).
|
||||
|
||||
Downstream applications should invoke:
|
||||
Pass required API-key environment variables through the child environment. Do
|
||||
not place raw API keys in arguments. Keep the environment limited to the values
|
||||
needed for the selected profile.
|
||||
|
||||
- `scriptorium render` for preflight/debug output without LLM execution.
|
||||
- `scriptorium run` for generation.
|
||||
## Streams And Output Ownership
|
||||
|
||||
`scriptorium serve` is an HTTP service command, not the recommended subprocess
|
||||
contract for per-request execution.
|
||||
Capture stdout and stderr separately. Stdout contains the requested artifact or
|
||||
prepared output unless the caller selects an output file; stderr contains
|
||||
summaries, diagnostics, and server messages. The exact destinations and status
|
||||
meanings are part of the [CLI reference](../cli.md), not a stable stderr data
|
||||
protocol.
|
||||
|
||||
## Recommended Invocation Shapes
|
||||
When using `--out`, the caller owns the output path, its permissions, and
|
||||
cleanup. Treat rendered prompts, generated artifacts, stdout, and stderr as
|
||||
potentially sensitive.
|
||||
|
||||
Render:
|
||||
## Cancellation And Recovery
|
||||
|
||||
```bash
|
||||
scriptorium render \
|
||||
--config <config_path> \
|
||||
--prompt <prompt_id> \
|
||||
--input transcript=<path> \
|
||||
--format json
|
||||
```
|
||||
A CLI invocation performs one synchronous request and creates no durable run
|
||||
state. A supervising process that needs cancellation must terminate the child
|
||||
process according to its own process-management policy. A later invocation is a
|
||||
new request and can make another model call; there is no resume or checkpoint
|
||||
protocol.
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
scriptorium run \
|
||||
--config <config_path> \
|
||||
--prompt <prompt_id> \
|
||||
--input transcript=<path> \
|
||||
--out <artifact_path>
|
||||
```
|
||||
|
||||
Callers may add:
|
||||
|
||||
- `--profile <profile_id>`
|
||||
- repeatable `--input name=path`
|
||||
- repeatable `--var name=value`
|
||||
- runtime overrides when explicitly needed, such as `--model`, `--llm-base-url`, `--api-key-env`, and `--timeout`
|
||||
|
||||
Do not pass raw API keys as command arguments.
|
||||
|
||||
## Config And Directory Behavior
|
||||
|
||||
Callers can rely on resolved app config or pass explicit paths.
|
||||
|
||||
Default config search order:
|
||||
|
||||
1. `/usr/local/etc/scriptorium/config.yml`
|
||||
2. `/etc/scriptorium/config.yml`
|
||||
|
||||
Rules:
|
||||
|
||||
- Explicit `--config` requires file existence and valid syntax.
|
||||
- CLI flags override config values.
|
||||
- `run` and `render` require an effective `prompt_dir`.
|
||||
- `profile_dir` is optional because built-in profiles are available.
|
||||
|
||||
## Profile Selection
|
||||
|
||||
Profile selection follows runner behavior:
|
||||
|
||||
1. explicit `--profile`
|
||||
2. prompt `default_profile`
|
||||
3. error if neither is available
|
||||
|
||||
Treat prompt and profile IDs as deployment configuration, not hardcoded business
|
||||
logic.
|
||||
|
||||
## Input And Variable Contract
|
||||
|
||||
- Inputs use repeated `--input name=path`.
|
||||
- Input names must match prompt definition input names.
|
||||
- Variables use repeated `--var name=value`.
|
||||
- Both flags also accept comma-separated mappings.
|
||||
- Prefer file inputs for large content.
|
||||
|
||||
CLI inputs are file references. HTTP-only `inline` references are documented in
|
||||
the [HTTP API reference](../api.md).
|
||||
|
||||
## Environment Contract
|
||||
|
||||
- Pass through required API-key environment variables referenced by `api_key_env`.
|
||||
- Keep subprocess environments scoped to required variables.
|
||||
- Use `--api-key-env` only to name an environment variable.
|
||||
- Never pass raw API keys via argv.
|
||||
|
||||
## Stdout And Stderr
|
||||
|
||||
`run`:
|
||||
|
||||
- stdout: generated artifact body unless `--out` is used.
|
||||
- stderr: success summary and errors.
|
||||
|
||||
`render`:
|
||||
|
||||
- stdout: prepared-run output unless `--out` is used.
|
||||
- stderr: errors.
|
||||
|
||||
Capture stdout and stderr separately. Do not parse stderr as a stable data
|
||||
format beyond exit status handling.
|
||||
|
||||
## Exit Status Contract
|
||||
|
||||
- `0`: success.
|
||||
- `1`: parse, config, load, render, generation, IO, or runtime error.
|
||||
- `2`: `run` completed and output was written, but validation failed.
|
||||
|
||||
A `run` exit code `2` can still produce output on stdout or at `--out`.
|
||||
Consumers must decide whether to keep or discard that output.
|
||||
|
||||
## Security Notes
|
||||
|
||||
- Treat generated artifacts, rendered prompts, stdout, and stderr as potentially sensitive.
|
||||
- Use controlled output paths and access controls for persisted artifacts.
|
||||
- Avoid logging full rendered prompts or generated artifacts by default.
|
||||
|
||||
## Canonical References
|
||||
|
||||
- CLI behavior: [CLI reference](../cli.md)
|
||||
- Config and file formats: [Configuration reference](../config.md)
|
||||
- Operations: [Operations guide](../operations.md)
|
||||
- Troubleshooting: [Troubleshooting](../troubleshooting.md)
|
||||
For deployment, filesystem permissions, and sensitive-artifact handling, see
|
||||
the [operations guide](../operations.md).
|
||||
|
||||
@@ -2,154 +2,101 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
Adapters translate external interfaces into domain requests and translate domain results back out. They wire dependencies, apply app config, and own IO concerns, but they do not make runner decisions.
|
||||
Scriptorium adapters translate executable inputs into Promptkit public requests
|
||||
and translate Promptkit results or errors back to CLI or HTTP behavior. They
|
||||
own IO and presentation mechanics, not framework decisions.
|
||||
|
||||
Source-loading behavior belongs in `docs/internal/sources.md`. User-facing CLI, HTTP, and package contracts belong in `docs/cli.md`, `docs/api.md`, and `docs/consumers/pkg-scriptorium.md`.
|
||||
External contracts are canonical in the [CLI reference](../cli.md) and
|
||||
[HTTP API reference](../api.md). Promptkit's public engine contract is
|
||||
described by its tagged
|
||||
[Go consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md).
|
||||
|
||||
## Adapter Map
|
||||
## Components And Collaborators
|
||||
|
||||
- `cmd/scriptorium`: process entrypoint.
|
||||
- `internal/adapter/cli`: command parsing, config handoff, runner construction, stdout/stderr, exit codes.
|
||||
- `internal/adapter/http`: `POST /v1/runs` request/response mapping and HTTP error/status mapping.
|
||||
- root package `scriptorium`: public Go facade over internal runner types and dependencies.
|
||||
- `cmd/scriptorium` passes process arguments and streams to
|
||||
`internal/adapter/cli`.
|
||||
- `internal/adapter/cli` resolves settings through `internal/config`,
|
||||
constructs `promptkit.Engine`, maps CLI values to `promptkit.RunRequest`,
|
||||
and owns output files, summaries, and exit codes.
|
||||
- `internal/adapter/http` strictly decodes request DTOs, maps them to Promptkit
|
||||
public values, calls its adapter-owned `Runner` interface, and maps results
|
||||
and errors to HTTP DTOs.
|
||||
- `internal/format` renders `promptkit.PreparedRun` values as deterministic text
|
||||
or JSON.
|
||||
|
||||
Supporting implementation packages used during adapter wiring:
|
||||
## Wiring Flows
|
||||
|
||||
- `internal/config`
|
||||
- `internal/defaults`
|
||||
- `internal/format`
|
||||
- `internal/llm`
|
||||
- `internal/prompt`
|
||||
### CLI
|
||||
|
||||
## Inputs And Outputs
|
||||
`run` calls `promptkit.Engine.Run`; `render` calls
|
||||
`promptkit.Engine.Prepare`. Both share request mapping for prompt ID/version
|
||||
and profile selection, optional file inputs, variables, and presence-aware
|
||||
execution overrides. Omitted framework settings remain zero values so Promptkit
|
||||
resolves its own defaults and definition-required inputs.
|
||||
|
||||
CLI adapter:
|
||||
`inspect prompt` maps the selected ID and optional version to
|
||||
`promptkit.Engine.InspectPrompt`, then formats Scriptorium-owned inspection
|
||||
data. It constructs the same configuration-aware engine but does not perform
|
||||
preparation or generation.
|
||||
|
||||
- Input: process args, optional config file, filesystem sources, environment variables.
|
||||
- Output: process exit code, stdout artifact/prepared output, stderr summaries and errors.
|
||||
`inspect profile` maps an explicit profile ID to `Engine.InspectProfile` and
|
||||
formats a safe application-owned effective profile view. It intentionally does
|
||||
not require prompt, schema, or artifact sources.
|
||||
|
||||
HTTP adapter:
|
||||
`serve` constructs Scriptorium's restricted HTTP artifact reader, injects it
|
||||
with `promptkit.WithArtifactReader`, passes the engine through the HTTP
|
||||
adapter's consumer-owned `Runner` interface, and starts the server.
|
||||
|
||||
- Input: HTTP request method/path/headers/body for `POST /v1/runs`.
|
||||
- Output: JSON success or error body with mapped status code.
|
||||
All three CLI paths assemble the engine from the same resolved prompt, profile,
|
||||
and schema directories plus configured custom backends. Each backend is mapped
|
||||
to Promptkit's public `Backend` value and registered during engine construction,
|
||||
so one constructed server engine retains one immutable backend registry and its
|
||||
associated capacity state.
|
||||
|
||||
Public Go facade:
|
||||
### HTTP
|
||||
|
||||
- Input: typed `scriptorium.Config`, `Option`, and `RunRequest` values.
|
||||
- Output: typed `PreparedRun` and `RunResult` values plus public sentinel errors.
|
||||
The handler enforces transport limits and strict JSON decoding before mapping
|
||||
DTOs into `promptkit.RunRequest`, `promptkit.ArtifactRef`, and
|
||||
`promptkit.ExecutionTargetOverride`. On success it reads Promptkit artifact,
|
||||
validation, model, usage, and metadata values directly.
|
||||
|
||||
## Boundaries
|
||||
Failure mapping uses `errors.Is` against Promptkit's public sentinels and the
|
||||
HTTP reader's Scriptorium-owned containment and size errors. Wrapped reader
|
||||
errors preserve their identity through Promptkit's artifact-load boundary.
|
||||
|
||||
- Adapters convert external shapes to `domain.RunRequest` and back.
|
||||
- Runner orchestration remains in `internal/usecase`.
|
||||
- Prompt/profile/schema/artifact source rules remain in repository, validator, and artifact packages.
|
||||
- LLM provider request serialization remains in `internal/llm`.
|
||||
- Public package types are facade types; internal domain types do not leak across the package boundary.
|
||||
## Package-Local Guarantees
|
||||
|
||||
## Config Fields Used
|
||||
- Adapters contain no copied framework types or orchestration.
|
||||
- Configuration is resolved before Promptkit engine construction.
|
||||
- Explicit numeric overrides preserve presence, including zero.
|
||||
- HTTP DTO and error mapping remains stable and transport-owned.
|
||||
- Resolved secrets are not serialized or printed.
|
||||
- No adapter creates durable run state.
|
||||
|
||||
Adapter app settings:
|
||||
## Verification
|
||||
|
||||
- `prompt_dir`
|
||||
- `profile_dir`
|
||||
- `schema_dir`
|
||||
- `server.addr`
|
||||
- `server.artifact_root`
|
||||
- `server.max_request_bytes`
|
||||
- `server.max_artifact_bytes`
|
||||
- `server.max_response_bytes`
|
||||
- `defaults.render_format`
|
||||
|
||||
Execution request/profile settings passed through the runner:
|
||||
|
||||
- `endpoint`
|
||||
- `model`
|
||||
- `temperature`
|
||||
- `max_tokens`
|
||||
- `top_p`
|
||||
- `timeout_seconds`
|
||||
- `service_tier`
|
||||
- `api_key_env`
|
||||
- `reasoning_effort`
|
||||
- `extra_params`
|
||||
|
||||
CLI and HTTP preserve numeric override presence so omitted values and explicit zero values remain distinct.
|
||||
|
||||
## CLI Adapter
|
||||
|
||||
Implemented commands:
|
||||
|
||||
- `run`
|
||||
- `render`
|
||||
- `serve`
|
||||
|
||||
Behavior:
|
||||
|
||||
- `run` constructs a runner with direct filesystem artifact reading and calls `Runner.Run`.
|
||||
- `render` constructs a runner and calls `Runner.Prepare`; it does not call the LLM.
|
||||
- `serve` constructs a restricted artifact reader and HTTP handler, then starts an unauthenticated HTTP server.
|
||||
- `run` exits `2` when generation succeeds but validation fails.
|
||||
- parse, runtime, and output-write errors exit `1`.
|
||||
- deprecated `--prompt-id` and `--profile-id` aliases are accepted.
|
||||
|
||||
## HTTP Adapter
|
||||
|
||||
Behavior:
|
||||
|
||||
- Accepts only `POST /v1/runs`.
|
||||
- Decodes JSON strictly and rejects unknown fields and trailing JSON tokens.
|
||||
- Rejects empty `prompt_id` and empty `inputs` before calling the runner.
|
||||
- Does not accept raw API key values in the request body.
|
||||
- Returns validation failures as `200` responses with failed validation details.
|
||||
- Maps request-body, artifact, and encoded-response size failures to `413`.
|
||||
- Maps domain and repository errors to stable error codes without returning wrapped internal cause text.
|
||||
|
||||
The HTTP adapter has no built-in authentication or authorization. Deployment controls must be provided outside the process.
|
||||
|
||||
## Public Go Facade
|
||||
|
||||
Behavior:
|
||||
|
||||
- `NewEngine` wires the same default runner components as CLI/HTTP unless options override them.
|
||||
- Prompt, profile, and schema sources may come from directories, single files, or `fs.FS` roots.
|
||||
- `WithProfiles` adds in-memory profiles ahead of file-backed and built-in profiles.
|
||||
- `WithLLMClient` injects custom model behavior.
|
||||
- `RunRequest.APIKey` is request-scoped and direct; it is used only for generation and is stripped from public results.
|
||||
- internal errors are mapped to public sentinels in `errors.go`.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
Adapters should:
|
||||
|
||||
- keep external error payloads concise and stable.
|
||||
- avoid leaking raw secret values.
|
||||
- use sentinels and typed errors for mapping.
|
||||
- preserve strict external input decoding.
|
||||
- keep validation content failures distinct from runtime errors.
|
||||
|
||||
CLI writes human-readable summaries to stderr. HTTP writes JSON error envelopes. The public Go facade returns typed errors.
|
||||
|
||||
## State And Manifests
|
||||
|
||||
Adapters do not add durable run state.
|
||||
|
||||
- No adapter writes run manifests.
|
||||
- No adapter implements checkpoint, skip, or resume behavior.
|
||||
- CLI output files are caller-selected artifacts, not internal state.
|
||||
|
||||
## Tests To Inspect
|
||||
Inspect:
|
||||
|
||||
- `internal/adapter/cli/run_test.go`
|
||||
- `internal/adapter/http/handler_test.go`
|
||||
- `engine_test.go`
|
||||
- `internal/adapter/http/artifact_reader_test.go`
|
||||
- `internal/format/prepared_run_test.go`
|
||||
- `internal/llm/openai_compatible_client_test.go`
|
||||
- `internal/adapter/dependency_test.go`
|
||||
|
||||
## Architectural Invariants
|
||||
The adapter tests protect parsing, configuration mapping, output, status
|
||||
mapping, restricted artifacts, and representative real Promptkit-engine
|
||||
workflows. The dependency test protects the repository boundary.
|
||||
|
||||
- Adapter packages stay thin and translation-focused.
|
||||
- App config is resolved before dependency construction.
|
||||
- External input strictness is part of contract stability.
|
||||
- CLI and HTTP construct runners without a repairer.
|
||||
- HTTP endpoint details remain canonical in `docs/api.md`.
|
||||
- Public Go package details remain canonical in `docs/consumers/pkg-scriptorium.md`.
|
||||
## Change Recipes
|
||||
|
||||
For a CLI or HTTP change:
|
||||
|
||||
1. identify the Scriptorium-owned external contract;
|
||||
2. map through Promptkit public values without copying framework semantics;
|
||||
3. add or update the narrow application-owned test;
|
||||
4. update the canonical Scriptorium contract; and
|
||||
5. coordinate and tag Promptkit first if a required public capability is
|
||||
genuinely absent.
|
||||
|
||||
Update [source internals](sources.md) when application source locations or HTTP
|
||||
artifact containment changes.
|
||||
|
||||
18
docs/internal/overview.md
Normal file
18
docs/internal/overview.md
Normal file
@@ -0,0 +1,18 @@
|
||||
# Internal Component Overview
|
||||
|
||||
This is the complete inventory of Scriptorium's implemented Go components.
|
||||
The [architecture policy](../policy/architecture.md) owns normative boundaries;
|
||||
public behavior belongs in the linked contracts.
|
||||
|
||||
| Component | Implemented responsibility | References |
|
||||
| --- | --- | --- |
|
||||
| `cmd/scriptorium` | Process entrypoint that delegates arguments and streams to the CLI adapter. | [CLI contract](../cli.md), [adapter internals](adapters.md) |
|
||||
| `internal/adapter/cli` | Parses commands, resolves application settings, constructs Promptkit engines, maps requests, and owns process output and exit behavior. | [CLI contract](../cli.md), [adapter internals](adapters.md) |
|
||||
| `internal/adapter/http` | Owns routes, DTOs, strict decoding, limits, Promptkit request/result mapping, public error mapping, and restricted HTTP artifact reading. | [HTTP API](../api.md), [adapter internals](adapters.md), [source internals](sources.md) |
|
||||
| `internal/config` | Discovers and strictly decodes application configuration and applies built-in and CLI precedence. | [configuration contract](../config.md), [adapter internals](adapters.md) |
|
||||
| `internal/defaults` | Holds Scriptorium-owned application and HTTP defaults. | [configuration contract](../config.md) |
|
||||
| `internal/format` | Formats Promptkit prepared-run and prompt/profile inspection values for CLI text or JSON output. | [CLI contract](../cli.md), [adapter internals](adapters.md) |
|
||||
|
||||
Framework implementation packages are provided by
|
||||
[Promptkit v0.9.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md)
|
||||
and are not part of this repository.
|
||||
@@ -1,146 +0,0 @@
|
||||
# Runner Internals
|
||||
|
||||
## Purpose
|
||||
|
||||
`internal/usecase.Runner` is the core prompt-execution orchestrator. It prepares prompt requests, calls the configured LLM client for `Run`, validates generated output, and returns domain results.
|
||||
|
||||
Transport parsing, DTOs, CLI output, HTTP status mapping, and public package type conversion belong outside the runner.
|
||||
|
||||
## Inputs And Outputs
|
||||
|
||||
Primary inputs:
|
||||
|
||||
- `domain.RunRequest`
|
||||
- repositories/readers/renderers/validators injected at construction
|
||||
- `context.Context` for cancellation
|
||||
|
||||
Primary outputs:
|
||||
|
||||
- `domain.PreparedRun` from `Prepare`
|
||||
- `domain.RunResult` from `Run`
|
||||
- wrapped sentinel errors for adapter mapping
|
||||
|
||||
LLM boundary types:
|
||||
|
||||
- `domain.GenerateRequest`
|
||||
- `domain.GenerateResponse`
|
||||
|
||||
## Dependencies
|
||||
|
||||
`Runner` depends on package interfaces instead of concrete adapter types:
|
||||
|
||||
- `promptdef.Repository`
|
||||
- `profile.Repository`
|
||||
- `artifact.Reader`
|
||||
- `prompt.Renderer`
|
||||
- `llm.Client`
|
||||
- `validate.Validator`
|
||||
- optional `usecase.OutputRepairer`
|
||||
|
||||
The CLI, HTTP adapter, and public Go package construct these dependencies and pass them in.
|
||||
|
||||
## Config Fields
|
||||
|
||||
`Runner` does not read app config files. Effective behavior is determined by injected dependencies and the `domain.RunRequest`.
|
||||
|
||||
Adapter wiring commonly reflects these app config fields:
|
||||
|
||||
- `prompt_dir`
|
||||
- `profile_dir`
|
||||
- `schema_dir`
|
||||
- `server.artifact_root`
|
||||
- HTTP request/artifact/response size limits
|
||||
|
||||
Runtime model settings are resolved from the selected profile plus request overrides.
|
||||
|
||||
## Prepare Flow
|
||||
|
||||
`Prepare`:
|
||||
|
||||
1. requires a non-empty prompt ID.
|
||||
2. loads the prompt definition and computes its hash.
|
||||
3. selects the profile from request `profile_id`, then prompt `default_profile`.
|
||||
4. loads the selected execution profile.
|
||||
5. merges built-in execution defaults, profile values, and request overrides.
|
||||
6. applies request-scoped direct API key values for public Go callers.
|
||||
7. validates endpoint, model, and credential requirements.
|
||||
8. resolves the output contract and JSON Schema document when required.
|
||||
9. reads input artifacts.
|
||||
10. renders prompt messages and hashes the rendered prompt.
|
||||
11. returns a prepared run without calling the LLM.
|
||||
|
||||
Numeric request overrides are presence-aware: omitted values preserve the current effective value, while explicit zero values are real overrides.
|
||||
|
||||
## Run Flow
|
||||
|
||||
`Run`:
|
||||
|
||||
1. creates a run ID and start timestamp.
|
||||
2. calls `Prepare`.
|
||||
3. calls the injected LLM client with rendered messages, effective target, target presence, and structured-output settings.
|
||||
4. builds the output artifact.
|
||||
5. validates the output.
|
||||
6. optionally attempts bounded repair when a repairer is injected and the contract permits repair.
|
||||
7. returns the run result with artifact, raw output, validation, hashes, selected profile/model metadata, usage, and timing.
|
||||
|
||||
`Run` must reuse `Prepare`; prepare logic should not be duplicated elsewhere.
|
||||
|
||||
## Validation And Repair
|
||||
|
||||
Validation content failures are returned as successful run results with `Validation.Status == failed`. They are not runtime errors.
|
||||
|
||||
Validation runtime failures, such as schema load or compile errors, return `ErrValidation`.
|
||||
|
||||
Repair attempts occur only when all conditions are true:
|
||||
|
||||
- a repairer is injected
|
||||
- `repair_attempts` is greater than zero
|
||||
- validation status is `failed`
|
||||
- validation mode is `json` or `json_schema`
|
||||
|
||||
CLI and HTTP wiring call `usecase.NewRunner(...)`, which does not inject a repairer. Normal CLI and HTTP execution therefore does not repair invalid output.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
Stable runner sentinels include:
|
||||
|
||||
- `ErrInvalidRequest`
|
||||
- `ErrProfileRequired`
|
||||
- `ErrAPIKeyEnvMissing`
|
||||
- `ErrAPIKeyRequired`
|
||||
- `ErrPromptLoad`
|
||||
- `ErrProfileLoad`
|
||||
- `ErrArtifactLoad`
|
||||
- `ErrPromptRender`
|
||||
- `ErrLLMGenerate`
|
||||
- `ErrValidation`
|
||||
|
||||
Adapters should use `errors.Is` against sentinels and lower-level repository errors instead of matching message text.
|
||||
|
||||
Secret values must not appear in prepared output, run results, logs, HTTP responses, or serialized public package results. The effective API-key environment-variable name may appear.
|
||||
|
||||
## State And Manifests
|
||||
|
||||
The runner is stateless across requests.
|
||||
|
||||
- No durable run store.
|
||||
- No manifest files.
|
||||
- No checkpoint, skip, or resume behavior.
|
||||
- Recovery is a new request after correcting inputs, config, or environment.
|
||||
|
||||
## Tests To Inspect
|
||||
|
||||
- `internal/usecase/runner_test.go`
|
||||
- `internal/usecase/integration_test.go`
|
||||
- `engine_test.go`
|
||||
- `internal/adapter/cli/run_test.go`
|
||||
- `internal/adapter/http/handler_test.go`
|
||||
|
||||
## Architectural Invariants
|
||||
|
||||
- Use-case decisions stay in `internal/usecase`.
|
||||
- `Run` reuses `Prepare`.
|
||||
- Prompt/profile/artifact/schema loading remains behind injected boundaries.
|
||||
- Validation content failures are result state; validation runtime failures are errors.
|
||||
- Repair loops are bounded by `repair_attempts` and repairer presence.
|
||||
- Resolved secret values are never serialized or emitted.
|
||||
@@ -2,156 +2,69 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
This document covers implemented prompt, profile, schema, artifact, and catalog source behavior. It is for developers changing loaders or source wiring.
|
||||
This document covers Scriptorium-owned source locations and the restricted HTTP
|
||||
artifact reader. Prompt, profile, schema, and ordinary artifact semantics are
|
||||
owned by the tagged
|
||||
[Promptkit format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md).
|
||||
|
||||
Full user-facing YAML and config reference material belongs in `docs/config.md`.
|
||||
## Application Source Locations
|
||||
|
||||
## Prompt Definition Sources
|
||||
`internal/config` resolves `prompt_dir`, `profile_dir`, and `schema_dir` from
|
||||
Scriptorium defaults, configuration files, and CLI overrides. It also resolves
|
||||
the application-owned `backends` mapping into sorted engine settings.
|
||||
`internal/adapter/cli` passes the directories into `promptkit.Config` and maps
|
||||
each configured backend to Promptkit's public engine registration when
|
||||
constructing an engine shared by the command path.
|
||||
|
||||
`internal/promptdef` provides directory-backed and `fs.FS` repositories.
|
||||
Scriptorium does not search, parse, validate, or overlay framework source files
|
||||
itself. Promptkit owns prompt selection, profile built-ins and overlays, schema
|
||||
resolution, ordinary file artifacts, and the related error identities.
|
||||
|
||||
Behavior:
|
||||
The [configuration reference](../config.md) owns Scriptorium's source-location
|
||||
fields and precedence. Maintained files under `examples/` are application
|
||||
inputs that use Promptkit's tagged formats.
|
||||
|
||||
- recursively scans `.yaml` and `.yml` files.
|
||||
- decodes YAML with known-fields checking.
|
||||
- looks up prompts by YAML `id`, not by path.
|
||||
- optionally filters by prompt `version`.
|
||||
- rejects duplicate matching prompt IDs.
|
||||
- requires `id`, `version`, and at least one message.
|
||||
- requires each message to set exactly one of `content` or `content_file`.
|
||||
- resolves filesystem `content_file` values relative to the prompt YAML file.
|
||||
- resolves `fs.FS` `content_file` values inside the configured source root.
|
||||
- permits prompt subdirectories only as organization; they are not part of prompt identity.
|
||||
## Restricted HTTP Artifact Reader
|
||||
|
||||
For `fs.FS` roots, absolute paths and relative traversal outside the source root are rejected by catalog path helpers.
|
||||
`internal/adapter/http` implements `promptkit.ArtifactReader` for HTTP
|
||||
requests. The `serve` path injects it with
|
||||
`promptkit.WithArtifactReader`, replacing Promptkit's ordinary reader for
|
||||
inbound HTTP inputs.
|
||||
|
||||
## Profile Sources
|
||||
The reader:
|
||||
|
||||
`internal/profile` provides directory-backed, `fs.FS`, and overlay repositories. `internal/profile/builtin` embeds built-in profile YAML assets and exposes them through the same repository interface.
|
||||
- accepts inline references without an artifact root;
|
||||
- denies file references when no root is configured;
|
||||
- resolves relative paths below the configured root;
|
||||
- accepts absolute paths only when they are lexically within that root;
|
||||
- rejects lexical traversal outside the root;
|
||||
- applies the configured file byte limit, with zero meaning unlimited;
|
||||
- preserves content type, body, size, hash, name, and URI metadata; and
|
||||
- honors context cancellation.
|
||||
|
||||
Behavior:
|
||||
Containment is lexical and does not resolve symlinks. The operating system
|
||||
follows symlinks after the check. The [HTTP API](../api.md) owns observable
|
||||
request outcomes, and [operations](../operations.md) owns safe deployment
|
||||
permissions and root selection.
|
||||
|
||||
- recursively scans `.yaml` and `.yml` files.
|
||||
- decodes YAML with known-fields checking.
|
||||
- looks up profiles by YAML `id`, not by path.
|
||||
- rejects duplicate IDs inside the same source.
|
||||
- rejects raw `api_key` fields in YAML; file-backed profiles must use `api_key_env`.
|
||||
- validates required `endpoint` and `model` values.
|
||||
- validates numeric profile ranges.
|
||||
Reader errors remain identifiable after Promptkit wraps them as artifact-load
|
||||
failures, allowing the HTTP adapter to preserve Scriptorium status and error
|
||||
codes.
|
||||
|
||||
Overlay behavior:
|
||||
## Verification And Change Recipe
|
||||
|
||||
- custom profiles are primary.
|
||||
- built-in profiles are fallback.
|
||||
- fallback occurs only after a primary `ErrProfileNotFound`.
|
||||
- primary validation, YAML, duplicate, and raw-key errors are returned directly.
|
||||
- duplicate IDs across custom and built-in sources are allowed because the custom profile overrides the built-in one.
|
||||
Inspect:
|
||||
|
||||
The public Go facade can add in-memory profiles ahead of file-backed and built-in profiles.
|
||||
- `internal/config/config_test.go`
|
||||
- `internal/adapter/cli/run_test.go`
|
||||
- `internal/adapter/http/artifact_reader_test.go`
|
||||
- `internal/adapter/http/handler_test.go`
|
||||
|
||||
## Schema Sources
|
||||
When changing an application source location or HTTP artifact policy:
|
||||
|
||||
`internal/validate` provides:
|
||||
|
||||
- `StandardValidator` for filesystem paths.
|
||||
- `FSValidator` for `fs.FS` roots and single-file public schema sources.
|
||||
|
||||
Behavior:
|
||||
|
||||
- `json_schema` validation requires a non-empty `schema_path`.
|
||||
- filesystem schema paths resolve relative to `schema_dir` unless absolute.
|
||||
- directory-backed schema lookup uses the explicit `schema_path`; it does not search recursively by basename.
|
||||
- `fs.FS` schema paths must remain inside the configured source root.
|
||||
- single-file schema sources match by the configured file base name.
|
||||
- schema documents are loaded before the LLM call for structured output.
|
||||
- JSON parse failures are validation content failures.
|
||||
- schema access, decode, registration, and compile failures are runtime validation errors.
|
||||
|
||||
## Artifact Sources
|
||||
|
||||
`internal/artifact` supports two input artifact reference types:
|
||||
|
||||
- `inline`
|
||||
- `file`
|
||||
|
||||
Inline behavior:
|
||||
|
||||
- requires a non-empty body.
|
||||
- produces text/plain artifacts.
|
||||
- hashes the body bytes.
|
||||
|
||||
Direct file behavior:
|
||||
|
||||
- used by CLI `run`, CLI `render`, and the public Go facade.
|
||||
- requires a non-empty URI.
|
||||
- reads from the process filesystem without HTTP artifact-root restrictions.
|
||||
- infers content type from file extension, defaulting to text/plain.
|
||||
|
||||
Restricted file behavior:
|
||||
|
||||
- used by HTTP `serve`.
|
||||
- allows inline artifacts even when no artifact root is configured.
|
||||
- denies file artifacts when no artifact root is configured.
|
||||
- resolves relative file URIs against `server.artifact_root`.
|
||||
- accepts absolute file URIs only when they pass containment checks.
|
||||
- applies `server.max_artifact_bytes` when configured.
|
||||
|
||||
Restricted containment is lexical. It cleans paths and checks the relative path against the configured root; it does not resolve symlinks. Symlinks inside the root are followed by the operating system, including symlinks that target files outside the root.
|
||||
|
||||
## Catalog Helpers
|
||||
|
||||
`internal/filecatalog` centralizes shared source helpers:
|
||||
|
||||
- recursive YAML discovery for filesystem and `fs.FS` roots.
|
||||
- deterministic sorting.
|
||||
- `.yaml` and `.yml` filtering.
|
||||
- display paths for diagnostics.
|
||||
- YAML file stems.
|
||||
- `fs.FS` root cleaning and containment checks.
|
||||
|
||||
Repository code should use these helpers instead of reimplementing path traversal and containment rules.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
Common source failures:
|
||||
|
||||
- missing prompt/profile/schema/artifact files.
|
||||
- invalid YAML or JSON.
|
||||
- unknown YAML fields.
|
||||
- duplicate prompt or profile IDs.
|
||||
- prompt/profile validation errors.
|
||||
- raw API key fields in profile YAML.
|
||||
- unsupported artifact reference type.
|
||||
- missing inline body or file URI.
|
||||
- artifact outside HTTP root.
|
||||
- artifact exceeding HTTP size limit.
|
||||
- schema load or compile failure.
|
||||
|
||||
Prompt/profile repository lookup errors are mapped by adapters separately from runtime runner errors. Validation content failures remain result state; source and schema runtime failures return errors.
|
||||
|
||||
## State And Manifests
|
||||
|
||||
Source packages do not persist run state.
|
||||
|
||||
- No manifests are read or written.
|
||||
- No source package implements skip or resume behavior.
|
||||
- Source reads reflect the current filesystem or `fs.FS` state for each request.
|
||||
|
||||
## Tests To Inspect
|
||||
|
||||
- `internal/promptdef/repository_test.go`
|
||||
- `internal/profile/repository_test.go`
|
||||
- `internal/profile/builtin/repository_test.go`
|
||||
- `internal/artifact/reader_test.go`
|
||||
- `internal/validate/standard_validator_test.go`
|
||||
- `internal/usecase/integration_test.go`
|
||||
- `engine_test.go`
|
||||
|
||||
## Architectural Invariants
|
||||
|
||||
- Prompt/profile identity comes from YAML `id`.
|
||||
- External YAML decoding remains strict.
|
||||
- File-backed profile YAML never accepts raw API key values.
|
||||
- Built-in profiles are fallback, not a replacement for custom source validation.
|
||||
- HTTP file artifacts remain rooted by lexical containment.
|
||||
- Schema runtime failures remain errors, while JSON/schema content mismatches remain validation results.
|
||||
1. preserve strict configuration precedence and the Promptkit public boundary;
|
||||
2. keep containment and size policy in Scriptorium;
|
||||
3. update focused configuration, reader, and handler tests;
|
||||
4. update the [configuration](../config.md), [HTTP](../api.md), and
|
||||
[operations](../operations.md) contracts as applicable; and
|
||||
5. do not duplicate Promptkit loaders, formats, or ordinary artifact behavior.
|
||||
|
||||
@@ -1,164 +1,168 @@
|
||||
# Operations Guide
|
||||
|
||||
## Scope
|
||||
## Scope And References
|
||||
|
||||
This guide covers operating the implemented CLI commands and HTTP service. It
|
||||
does not replace the [CLI reference](cli.md), [Configuration reference](config.md),
|
||||
or [HTTP API reference](api.md).
|
||||
This runbook covers deployment, normal operation, capacity planning, and safe
|
||||
recovery for Scriptorium. It does not redefine invocation syntax, configuration
|
||||
fields, or HTTP wire behavior.
|
||||
|
||||
## Operational Model
|
||||
- [CLI reference](cli.md): commands, output destinations, and exit codes.
|
||||
- [Configuration reference](config.md): application settings, source
|
||||
locations, defaults, and credential mapping.
|
||||
- [Promptkit framework formats](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md):
|
||||
prompt, profile, schema, execution-setting, and framework credential
|
||||
contracts.
|
||||
- [HTTP API reference](api.md): route, request/response schema, status codes,
|
||||
limits, and HTTP artifact access.
|
||||
- [Consumer integration overview](consumers/api.md): caller responsibilities.
|
||||
|
||||
Scriptorium executes one prompt request per CLI invocation or HTTP request.
|
||||
## Operational Model And State
|
||||
|
||||
Important boundaries:
|
||||
Scriptorium handles one prompt request for each CLI invocation or HTTP request.
|
||||
It has no durable run store, archive, checkpoint, cache, or resume mechanism.
|
||||
A failed or interrupted request is recovered by correcting its inputs,
|
||||
configuration, or environment and submitting a new request.
|
||||
|
||||
- No durable run state is stored.
|
||||
- No manifest, archive, checkpoint, or built-in backup workflow is written.
|
||||
- No built-in resume behavior exists.
|
||||
- Recovery is rerun-based: correct inputs, config, or environment, then run again.
|
||||
Generated artifacts, rendered prompts, model output, and run metadata are
|
||||
caller-owned data. Retention, encryption, backup, and deletion are deployment
|
||||
responsibilities.
|
||||
|
||||
## Filesystem Layout
|
||||
## Deploy The Filesystem And Process
|
||||
|
||||
Operational deployments usually provide:
|
||||
Provide the process with readable configured Promptkit prompt, profile, and
|
||||
schema sources that follow the tagged framework formats. For an HTTP deployment
|
||||
that accepts file artifacts, use a dedicated, narrow artifact directory rather
|
||||
than a general-purpose or sensitive filesystem tree.
|
||||
|
||||
- `prompt_dir`: prompt definition YAML files and adjacent `content_file` templates.
|
||||
- `profile_dir`: optional custom profile YAML files.
|
||||
- `schema_dir`: optional JSON Schema files.
|
||||
- `server.artifact_root`: optional HTTP file-input root for `serve`.
|
||||
Run Scriptorium under an identity that can:
|
||||
|
||||
Keep these directories readable by the Scriptorium process. Keep
|
||||
`server.artifact_root` narrow and not writable by untrusted users.
|
||||
- read only the prompt, profile, schema, and allowed input-artifact paths it
|
||||
needs;
|
||||
- read the required credential environment variables without writing them to
|
||||
files or logs; and
|
||||
- write only caller-selected output locations when CLI output files are used.
|
||||
|
||||
## Normal CLI Workflow
|
||||
Do not make the HTTP artifact directory writable by untrusted users. The HTTP
|
||||
artifact containment behavior is lexical and the operating system follows
|
||||
symlinks; account for that when choosing ownership and mount boundaries. See
|
||||
the [HTTP API reference](api.md) for the externally observable behavior.
|
||||
|
||||
Use `render` before `run` when changing prompt/profile/input wiring:
|
||||
## Supply Credentials And Protect Runtime Data
|
||||
|
||||
```bash
|
||||
go run ./cmd/scriptorium render \
|
||||
--config ./examples/config.yml \
|
||||
--prompt generic.markdown_summary \
|
||||
--input transcript=./examples/fixtures/transcript.md \
|
||||
--input glossary=./examples/fixtures/glossary.yml \
|
||||
--format json
|
||||
```
|
||||
Set secret values in the process environment and configure only their
|
||||
environment-variable names. Do not put raw keys in configuration, prompt or
|
||||
profile files, process arguments, HTTP payloads, captured command lines, or
|
||||
debug dumps.
|
||||
|
||||
Use `run` for generation after preflight:
|
||||
Treat stdout, stderr, prepared-run output, generated artifacts, and HTTP
|
||||
responses as potentially sensitive. Send service logs to a controlled collector
|
||||
and apply the same retention and access rules as for model input and output.
|
||||
|
||||
```bash
|
||||
go run ./cmd/scriptorium run \
|
||||
--config ./examples/config.yml \
|
||||
--prompt generic.markdown_summary \
|
||||
--input transcript=./examples/fixtures/transcript.md \
|
||||
--input glossary=./examples/fixtures/glossary.yml \
|
||||
--out ./summary.md
|
||||
```
|
||||
## Run A Normal Workflow
|
||||
|
||||
Before production runs, confirm:
|
||||
Before changing production inputs, profiles, or schemas:
|
||||
|
||||
- the effective config path is the intended one;
|
||||
- prompt/profile/schema directories are readable;
|
||||
- input file paths exist and match prompt input names;
|
||||
- required API-key environment variables are set;
|
||||
- the selected model endpoint is reachable from the process environment.
|
||||
1. confirm the deployed configuration selects the intended sources and model
|
||||
credentials;
|
||||
2. use [`render`](cli.md) with the same request inputs and variables to confirm
|
||||
preparation without a model call;
|
||||
3. use [`run`](cli.md) for generation; and
|
||||
4. retain or discard validation-failed output according to the caller's
|
||||
policy.
|
||||
|
||||
## HTTP Service Operation
|
||||
The [maintained render script](../examples/render-markdown-summary.sh) is a
|
||||
copyable preflight example. The CLI reference owns its complete invocation and
|
||||
exit semantics.
|
||||
|
||||
Start the service with:
|
||||
## Expose The HTTP Service
|
||||
|
||||
```bash
|
||||
go run ./cmd/scriptorium serve --config ./examples/config.yml
|
||||
```
|
||||
The HTTP service has no built-in authentication or authorization. Place it on a
|
||||
trusted network or behind an authenticated reverse proxy, API gateway, or
|
||||
equivalent access control. Restrict who can reach it and who can read the
|
||||
artifact root.
|
||||
|
||||
The implemented HTTP route is `POST /v1/runs`; request and response fields are
|
||||
defined in the [HTTP API reference](api.md).
|
||||
Use a service manager or supervisor appropriate to the deployment to manage
|
||||
process lifetime, restart policy, log capture, and environment injection. The
|
||||
[HTTP API reference](api.md) owns client request shapes, status behavior, and
|
||||
artifact-access outcomes.
|
||||
|
||||
The maintained HTTP request-shape example is `examples/http-run.json`.
|
||||
## Plan Capacity And Limits
|
||||
|
||||
HTTP service notes:
|
||||
Capacity is primarily determined by concurrent model calls, input and output
|
||||
sizes, schema complexity, provider latency, and network behavior. Size limits
|
||||
protect request bodies, HTTP file artifacts, and encoded responses; configure
|
||||
them through the [configuration reference](config.md) and rely on the
|
||||
[HTTP API reference](api.md) for their response effects.
|
||||
|
||||
- Unknown JSON fields are rejected.
|
||||
- `inline` input references work without an artifact root.
|
||||
- `file` input references require `server.artifact_root` or `serve --artifact-root`.
|
||||
- Request bodies, HTTP file input artifacts, and encoded JSON responses are size-limited.
|
||||
- Validation content failures return `200 OK` with `validation.status: "failed"`.
|
||||
Configured backend concurrency and queue capacity are enforced per constructed
|
||||
Promptkit engine. A `serve` process constructs one engine for its handler, so
|
||||
concurrent HTTP requests share that transient admission state. Scriptorium does
|
||||
not retain workflow state: capacity is neither durable nor a queue of resumable
|
||||
runs. When admission is exhausted, HTTP returns `503 capacity_exceeded` without
|
||||
retry timing; callers choose any retry policy that is safe for another model
|
||||
call.
|
||||
|
||||
Security boundary:
|
||||
Before increasing a limit:
|
||||
|
||||
- `serve` has no built-in authentication or authorization.
|
||||
- Put it behind trusted controls such as a private network, authenticated reverse proxy, or API gateway.
|
||||
- Do not expose an artifact root containing unrelated sensitive files.
|
||||
- Symlinks inside the artifact root are followed by the operating system.
|
||||
1. measure representative input, generated-output, and optional raw-output
|
||||
sizes;
|
||||
2. confirm memory, network, and upstream-provider capacity;
|
||||
3. retain an upstream request-size and authentication boundary; and
|
||||
4. test the intended workload in a non-production environment.
|
||||
|
||||
## Secrets Handling
|
||||
For large local inputs, prefer a controlled file-artifact directory over
|
||||
placing arbitrary paths on the service host. Avoid disabling a limit unless an
|
||||
equivalent trusted control exists elsewhere.
|
||||
|
||||
Raw API keys are not accepted in app config, profiles, CLI flags, or HTTP
|
||||
request bodies.
|
||||
## Diagnose And Recover
|
||||
|
||||
Use this pattern:
|
||||
### Preparation Or Configuration Failure
|
||||
|
||||
1. Set an environment variable containing the secret value.
|
||||
2. Store only the variable name in profile `api_key_env` or request override `api_key_env`.
|
||||
3. Scope the process environment to the minimum required variables.
|
||||
Capture the CLI diagnostic or HTTP error response, then verify the selected
|
||||
configuration, prompt ID, profile selection, source readability, and input
|
||||
mapping. Use `render` with the same request when it is unclear whether failure
|
||||
occurs before model execution. Consult the [CLI reference](cli.md), the
|
||||
[configuration reference](config.md), and the [HTTP API reference](api.md) for
|
||||
the exact interface contract.
|
||||
|
||||
## Output, Logs, And Exit Codes
|
||||
### Credential Or Provider Failure
|
||||
|
||||
`run`:
|
||||
Confirm that the process environment contains the configured credential name
|
||||
without printing the secret. Check endpoint reachability and provider health
|
||||
from the process network. If preparation succeeds but generation fails, inspect
|
||||
the selected model settings in prepared output and the service's controlled
|
||||
logs. Correct the deployment or provider issue, then submit a new request.
|
||||
|
||||
- stdout: generated artifact body unless `--out` is used.
|
||||
- stderr: summary on success, errors on failure.
|
||||
- exit `2`: generation completed and output was written, but validation failed.
|
||||
### Artifact Or Permission Failure
|
||||
|
||||
`render`:
|
||||
Verify that the process can read the intended local input. For HTTP file
|
||||
artifacts, verify the deployment's artifact root, ownership, path layout, and
|
||||
file size. Do not widen filesystem permissions or the allowed root merely to
|
||||
make an arbitrary path work; move or copy the required artifact into the
|
||||
controlled location instead.
|
||||
|
||||
- stdout: prepared-run output unless `--out` is used.
|
||||
- stderr: errors.
|
||||
- exit `0` on success, `1` on failure.
|
||||
### Validation Failure
|
||||
|
||||
`serve`:
|
||||
A generated-content validation failure is distinct from a runtime failure.
|
||||
CLI `run` reports the validation result and error count in its success summary;
|
||||
it does not print the individual validation messages. For HTTP, inspect the
|
||||
validation object in the response according to the [HTTP API reference](api.md).
|
||||
|
||||
- stderr: startup and server errors.
|
||||
- HTTP response body: JSON success or error envelope.
|
||||
Use rendered input and generated output to determine whether prompt instructions,
|
||||
the selected model, or the schema needs correction. If schema loading or
|
||||
compilation itself fails, correct the source deployment or schema document
|
||||
before rerunning.
|
||||
|
||||
## Validation Behavior
|
||||
### HTTP Limit Or Request Failure
|
||||
|
||||
Prompt `output.validation_mode` controls validation:
|
||||
Compare the request, artifact, or expected response size with the deployed
|
||||
configuration, and validate the request against the [HTTP API reference](api.md).
|
||||
Reduce the payload, use an appropriate controlled artifact source, omit
|
||||
unneeded raw output, or adjust the deployment limit after capacity review.
|
||||
|
||||
- `none`: skipped.
|
||||
- `basic`: output body must not be empty.
|
||||
- `json`: output body must parse as JSON.
|
||||
- `json_schema`: output body must parse as JSON and satisfy the configured schema.
|
||||
## Cleanup And Reruns
|
||||
|
||||
Runtime/schema failures are hard failures (`run` exit `1`, HTTP error).
|
||||
Generated-content validation failures are soft failures (`run` exit `2`, HTTP
|
||||
`200 OK` with failed validation status).
|
||||
|
||||
## Size Limits
|
||||
|
||||
Defaults are documented in [Configuration reference](config.md). Operationally:
|
||||
|
||||
- Keep default HTTP limits unless larger payloads are measured and expected.
|
||||
- Prefer `inline` HTTP inputs for small payloads.
|
||||
- Prefer `file` HTTP inputs for larger local artifacts under a controlled artifact root.
|
||||
- Increase `server.max_response_bytes` when generated artifacts or requested raw output are expected to be large.
|
||||
- Use `0` only when another trusted layer enforces size limits.
|
||||
|
||||
## Maintained Examples
|
||||
|
||||
- `examples/config.yml`
|
||||
- `examples/config.full.yml`
|
||||
- `examples/render-markdown-summary.sh`
|
||||
- `examples/http-run.json`
|
||||
|
||||
## Safe Recovery
|
||||
|
||||
For failed CLI commands or HTTP requests:
|
||||
|
||||
1. Capture stderr or the HTTP error `code` and `message`.
|
||||
2. Confirm config path and effective directory settings.
|
||||
3. Verify prompt ID, profile ID, schema path, and input mappings.
|
||||
4. Verify required API-key environment variables.
|
||||
5. Reproduce with `render --format json` when pre-LLM resolution is uncertain.
|
||||
6. Rerun after correction.
|
||||
|
||||
Because Scriptorium does not persist run state, rerun is the supported recovery
|
||||
path.
|
||||
Because no run state is retained, cleanup concerns caller-owned output files,
|
||||
logs, and artifacts only. Remove or rotate them using the deployment's normal
|
||||
retention policy. After a correction, rerun the request from the beginning;
|
||||
there is no safe resume point.
|
||||
|
||||
@@ -1,118 +1,106 @@
|
||||
# Architecture
|
||||
|
||||
This document is the development architecture policy for Scriptorium.
|
||||
This document defines Scriptorium's current application architecture and
|
||||
durable development boundaries.
|
||||
|
||||
It is for developers and LLM coding agents. User-facing behavior belongs in `README.md` and the docs under `docs/` that target operators/users.
|
||||
## System Shape
|
||||
|
||||
## Project Shape
|
||||
Scriptorium is an executable application with four command entry paths: CLI
|
||||
`run`, CLI `render`, CLI `inspect`, and the HTTP service started by `serve`.
|
||||
The `inspect` command has prompt and profile modes. Scriptorium does not expose
|
||||
a reusable root Go package.
|
||||
|
||||
Scriptorium is a narrow prompt-execution application with three entry paths:
|
||||
The application consumes
|
||||
[Promptkit v0.9.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md)
|
||||
through its supported root package. Promptkit owns prompt execution,
|
||||
preparation, source formats, built-in profiles, model-client behavior, and
|
||||
validation. Scriptorium owns application configuration, executable adapters,
|
||||
prepared-run and definition-inspection presentation, process behavior, and HTTP
|
||||
deployment policy.
|
||||
|
||||
- CLI `run`
|
||||
- CLI `render`
|
||||
- HTTP `POST /v1/runs` through `serve`
|
||||
- public Go package `gitea.maximumdirect.net/eric/scriptorium`
|
||||
The concrete package inventory is maintained in the
|
||||
[internal overview](../internal/overview.md).
|
||||
|
||||
Domain behavior is centralized in `internal/usecase` and `internal/domain`.
|
||||
## Dependency Direction
|
||||
|
||||
## Core Principles
|
||||
```text
|
||||
cmd/scriptorium
|
||||
|
|
||||
v
|
||||
CLI and HTTP adapters, configuration, defaults, and formatting
|
||||
|
|
||||
v
|
||||
gitea.maximumdirect.net/eric/promptkit
|
||||
```
|
||||
|
||||
- Keep orchestration narrow: Scriptorium executes one prompt request; it is not a multi-step workflow engine.
|
||||
- Keep adapter logic thin: adapters map external shapes to domain requests/results and should not hold domain decisions.
|
||||
- Keep boundaries explicit: repositories/loaders/renderers/validators/LLM client stay behind package interfaces.
|
||||
- Keep config strict: YAML/JSON decoding for external inputs should reject unknown fields.
|
||||
- Keep secrets out of payloads: raw API key values must not be accepted or emitted.
|
||||
- Retained application packages may import Promptkit's root package.
|
||||
- They must not import Promptkit `internal` packages.
|
||||
- They must not import Promptkit catalog modules directly.
|
||||
- They must not import the removed Scriptorium root facade or recreate former
|
||||
framework package families.
|
||||
- Adapter-owned interfaces use Promptkit public values when a consumer-side
|
||||
substitution boundary is needed.
|
||||
- Scriptorium passes omitted framework settings as zero values so Promptkit
|
||||
applies its own defaults.
|
||||
|
||||
## Package Boundaries
|
||||
The repository architecture guard enforces these import and removal
|
||||
invariants.
|
||||
|
||||
Current package map:
|
||||
## Retained Boundaries
|
||||
|
||||
- root package `scriptorium`: public Go facade over engine construction, source options, request/result types, and error mapping.
|
||||
- `cmd/scriptorium`: process entrypoint.
|
||||
- `internal/adapter/cli`: command parsing, app wiring for CLI commands, output behavior.
|
||||
- `internal/adapter/http`: HTTP DTO mapping and error/status mapping.
|
||||
- `internal/config`: application settings loading and CLI override precedence.
|
||||
- `internal/defaults`: compile-time default constants.
|
||||
- `internal/domain`: core request/result and contract types.
|
||||
- `internal/usecase`: `Runner` prepare/run orchestration and repair-hook boundary.
|
||||
- `internal/promptdef`: filesystem prompt-definition repository.
|
||||
- `internal/profile`: filesystem, `fs.FS`, and overlay execution-profile repositories.
|
||||
- `internal/profile/builtin`: embedded built-in execution profiles.
|
||||
- `internal/filecatalog`: shared YAML discovery and `fs.FS` source helpers.
|
||||
- `internal/artifact`: artifact reference readers.
|
||||
- `internal/prompt`: template renderer.
|
||||
- `internal/llm`: provider-neutral LLM client interface and OpenAI-compatible implementation.
|
||||
- `internal/validate`: validator interfaces and standard implementation.
|
||||
- `internal/format`: prepared-run output formatting.
|
||||
- `internal/adapter/cli` owns commands, flags, configuration precedence,
|
||||
process streams, output files, summaries, and exit codes.
|
||||
- `internal/adapter/http` owns routes, strict JSON DTOs, size limits, response
|
||||
mapping, status mapping, and the restricted artifact reader.
|
||||
- `internal/config` owns discovery and strict decoding of Scriptorium
|
||||
application configuration.
|
||||
- `internal/defaults` owns Scriptorium application and HTTP defaults only.
|
||||
- `internal/format` owns deterministic prepared-run and definition-inspection
|
||||
text and JSON presentation.
|
||||
- Promptkit owns framework orchestration and contracts. Its
|
||||
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md)
|
||||
and
|
||||
[outbound integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/integrations/openai-compatible-chat.md)
|
||||
are canonical.
|
||||
|
||||
Detailed component behavior is documented in:
|
||||
## HTTP Artifact Security Boundary
|
||||
|
||||
- `docs/internal/runner.md`
|
||||
- `docs/internal/adapters.md`
|
||||
- `docs/internal/sources.md`
|
||||
Ordinary CLI file loading is provided by Promptkit. Scriptorium's HTTP adapter
|
||||
injects a restricted `promptkit.ArtifactReader` for inbound HTTP requests.
|
||||
That reader denies file references without an artifact root, enforces the
|
||||
configured byte limit, and applies Scriptorium's lexical root-containment rule.
|
||||
The operating system still follows symlinks after the lexical check.
|
||||
|
||||
## Configuration And Precedence
|
||||
The [HTTP API](../api.md) owns observable request outcomes, and
|
||||
[operations](../operations.md) owns deployment permissions and root selection.
|
||||
|
||||
Application settings are resolved as:
|
||||
## State, Errors, And Secrets
|
||||
|
||||
1. built-in defaults
|
||||
2. config file values
|
||||
3. CLI overrides
|
||||
Scriptorium has no durable run-state store, checkpoint, cache, or resume
|
||||
mechanism. Recovery is a new request after correcting inputs, configuration, or
|
||||
environment.
|
||||
|
||||
`config.yml` is for application wiring (directories, server address, render default format), not prompt/profile runtime execution settings.
|
||||
|
||||
Profile selection and runtime model resolution remain use-case concerns.
|
||||
|
||||
## State And Persistence Policy
|
||||
|
||||
Scriptorium has no durable run-state store.
|
||||
|
||||
- No built-in resume/checkpoint/archive behavior.
|
||||
- Recovery model is rerun after correcting inputs/config/environment.
|
||||
|
||||
## External Integration Policy
|
||||
|
||||
Current external contracts:
|
||||
|
||||
- inbound HTTP contract: `POST /v1/runs`, documented canonically in `docs/api.md`
|
||||
- outbound model contract: OpenAI-compatible chat completions subset
|
||||
- subprocess contract for integrators: CLI `run`/`render`
|
||||
- public Go package contract: `docs/consumers/pkg-scriptorium.md`
|
||||
|
||||
Integration docs belong under `docs/integrations/`.
|
||||
|
||||
## Error Handling And Logging
|
||||
|
||||
- Wrap errors with domain/operation context.
|
||||
- Map domain errors to adapter-appropriate statuses/codes without leaking sensitive internals.
|
||||
- Keep stderr summaries concise for CLI success/error paths.
|
||||
- Never emit raw secret values.
|
||||
|
||||
## Testing Expectations
|
||||
|
||||
- Core runner behavior should be covered with isolated unit tests and fixture-based integration tests.
|
||||
- Adapter behavior should be tested for parse/mapping/error semantics.
|
||||
- Config parsing, prompt/profile loading, validator behavior, and LLM client error handling should remain covered by package tests.
|
||||
- Repository-level docs/examples that claim runnable behavior should be validated by tests or smoke commands.
|
||||
|
||||
## Documentation Expectations
|
||||
|
||||
- Document implemented behavior only outside `docs/roadmap/`.
|
||||
- Keep canonical reference locations stable (`docs/cli.md`, `docs/config.md`, `docs/operations.md`, `docs/troubleshooting.md`, `docs/internal/`).
|
||||
- Update docs in the same change when architecture-relevant behavior changes.
|
||||
Adapters map Promptkit public error identities into CLI exits or HTTP statuses
|
||||
without classifying by message text. Raw API keys are not accepted in
|
||||
Scriptorium configuration, CLI arguments, or HTTP payloads, and resolved
|
||||
secrets must not be emitted.
|
||||
|
||||
## Architectural Invariants
|
||||
|
||||
- `Runner.Run` reuses `Runner.Prepare` flow.
|
||||
- CLI and HTTP currently instantiate `Runner` without a repairer.
|
||||
- Artifact reading supports `inline` and `file` references.
|
||||
- Unknown input fields in config/prompt/profile/http JSON should be rejected by strict decoding.
|
||||
- Raw API key values must not be accepted through config/HTTP payloads.
|
||||
- External YAML and JSON decoding remains strict.
|
||||
- CLI and HTTP behavior remains presentation and transport logic rather than
|
||||
framework orchestration.
|
||||
- Explicit numeric request overrides preserve presence, including zero.
|
||||
- HTTP artifact containment and byte limits remain Scriptorium policy.
|
||||
- No application package depends on Promptkit implementation packages.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Do not move orchestration responsibilities from external callers into Scriptorium.
|
||||
- Do not add adapter-specific business logic in `internal/adapter/*` packages.
|
||||
- Do not bypass repository/renderer/validator/LLM boundaries by introducing cross-package coupling.
|
||||
- Do not recreate an in-process Scriptorium framework API or compatibility
|
||||
facade.
|
||||
- Do not copy Promptkit types, defaults, built-in profiles, or implementation
|
||||
into Scriptorium.
|
||||
- Do not move CLI, inbound HTTP, process, or deployment policy into Promptkit.
|
||||
- Do not add durable workflow, archive, or resume behavior.
|
||||
|
||||
Work that is not implemented belongs in `docs/roadmap/`.
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
# Development Guide
|
||||
|
||||
This document defines contributor workflow for Scriptorium.
|
||||
|
||||
## Repository Layout
|
||||
|
||||
- root package `scriptorium`: public Go facade, options, types, and error mapping.
|
||||
- `cmd/scriptorium`: application entrypoint.
|
||||
- `internal/domain`: core contracts.
|
||||
- `internal/usecase`: runner orchestration.
|
||||
- `internal/adapter/cli`: CLI adapter.
|
||||
- `internal/adapter/http`: HTTP adapter.
|
||||
- `internal/config`: application settings loading and precedence.
|
||||
- `internal/defaults`: default constants.
|
||||
- `internal/promptdef`: prompt-definition repository.
|
||||
- `internal/profile`: execution-profile repository.
|
||||
- `internal/profile/builtin`: embedded built-in execution profiles.
|
||||
- `internal/filecatalog`: shared source discovery and path helpers.
|
||||
- `internal/artifact`: artifact readers.
|
||||
- `internal/prompt`: prompt rendering.
|
||||
- `internal/llm`: LLM client interface and OpenAI-compatible implementation.
|
||||
- `internal/validate`: validation interfaces and implementation.
|
||||
- `internal/format`: prepared-run formatting.
|
||||
- `docs/`: canonical documentation.
|
||||
- `examples/`: copyable maintained examples and fixtures.
|
||||
|
||||
## Common Commands
|
||||
|
||||
Build:
|
||||
|
||||
```bash
|
||||
go build ./cmd/scriptorium
|
||||
```
|
||||
|
||||
Test:
|
||||
|
||||
```bash
|
||||
go test ./...
|
||||
```
|
||||
|
||||
Targeted test runs commonly used during changes:
|
||||
|
||||
```bash
|
||||
go test .
|
||||
go test ./internal/adapter/cli ./internal/adapter/http ./internal/usecase
|
||||
go test ./internal/...
|
||||
```
|
||||
|
||||
## Coding Conventions
|
||||
|
||||
- Prefer small interfaces at package boundaries.
|
||||
- Keep adapter packages focused on translation and IO concerns.
|
||||
- Keep domain/use-case logic outside adapters.
|
||||
- Wrap errors with operation context.
|
||||
- Use strict decoding for user-provided YAML/JSON where applicable.
|
||||
- Avoid introducing dependencies unless they materially reduce risk/complexity.
|
||||
|
||||
## Dependency Policy
|
||||
|
||||
- Prefer standard library unless an external library is clearly justified.
|
||||
- Current non-stdlib dependencies are intentionally small:
|
||||
- `gopkg.in/yaml.v3` for YAML decoding.
|
||||
- `github.com/santhosh-tekuri/jsonschema/v6` for JSON Schema validation.
|
||||
- Do not leak dependency-specific types across unrelated package boundaries.
|
||||
|
||||
## How To Add App Config Fields
|
||||
|
||||
1. Add fields in `internal/config/config.go` (`Config`, `AppSettings`, and/or `CLIOverrides` as needed).
|
||||
2. Apply defaults in `BuiltInDefaults()` when required.
|
||||
3. Parse and validate in `applyConfig` / `ApplyCLIOverrides`.
|
||||
4. Wire the field through the consuming adapter(s).
|
||||
5. Add/update config tests in `internal/config/config_test.go`.
|
||||
6. Update canonical docs (`docs/config.md`, and other affected docs).
|
||||
|
||||
## How To Add CLI Flags
|
||||
|
||||
1. Add flags in `internal/adapter/cli/run.go` for the relevant command.
|
||||
2. Ensure precedence behavior remains consistent with app config rules.
|
||||
3. Keep `run`, `render`, and `serve` flag surfaces intentionally scoped.
|
||||
4. Add/update parser and command tests in `internal/adapter/cli/run_test.go`.
|
||||
5. Update `docs/cli.md` and any related docs/examples.
|
||||
|
||||
## How To Add Adapters Or Adapter Capabilities
|
||||
|
||||
1. Define or reuse the appropriate interface boundary in domain/use-case packages.
|
||||
2. Implement adapter code under `internal/adapter/<name>` (or relevant boundary package).
|
||||
3. Keep business decisions in `internal/usecase`.
|
||||
4. Add focused adapter tests for mapping, parse, and error behavior.
|
||||
5. Document the new/changed boundary in `docs/internal/adapters.md`.
|
||||
6. If source-loading behavior changes, update `docs/internal/sources.md`.
|
||||
7. If an external contract changes, update the canonical public or integration doc in the same change.
|
||||
|
||||
## How To Update Prompt/Profile/Schema Assets
|
||||
|
||||
1. Keep prompt/profile/schema files valid under strict loaders.
|
||||
2. Keep examples secret-free.
|
||||
3. Re-run tests that cover prompt/profile/validation behavior.
|
||||
4. Update `docs/config.md` and any docs that reference changed contracts.
|
||||
|
||||
## Documentation Update Expectations
|
||||
|
||||
When behavior changes:
|
||||
|
||||
1. Update canonical doc locations, not duplicate files.
|
||||
2. Keep non-roadmap docs limited to implemented behavior.
|
||||
3. Update links after file moves/renames.
|
||||
4. Re-run relevant tests and smoke commands.
|
||||
5. For internal boundary docs, check references with `rg "docs/internal|internal/sources" docs/policy docs/internal`.
|
||||
|
||||
Docs work is complete only when code/tests/examples/docs agree.
|
||||
@@ -1,446 +1,165 @@
|
||||
# Go Project Documentation Policy
|
||||
# Documentation Policy
|
||||
|
||||
## Purpose
|
||||
|
||||
Project documentation must help five audiences:
|
||||
|
||||
1. users who need to run the application;
|
||||
2. administrators/operators who need to configure and operate it;
|
||||
3. developers who need to understand and change it safely;
|
||||
4. LLM coding agents that need clear scope, boundaries, and invariants;
|
||||
5. developers and LLM coding agents integrating this project from another codebase.
|
||||
|
||||
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
||||
This policy assigns each documentation topic to one canonical owner. Its goal is
|
||||
to keep this repository's documentation accurate, concise, discoverable, and
|
||||
resistant to drift for users, operators, developers, integrators, and LLM
|
||||
coding agents.
|
||||
|
||||
## Core Rules
|
||||
|
||||
### 1. Keep docs concise
|
||||
### One Canonical Owner
|
||||
|
||||
Each authoritative fact belongs in one document. A non-owning document may give
|
||||
a short, stable summary for orientation, but it must link to the canonical owner
|
||||
instead of repeating volatile details.
|
||||
|
||||
Volatile details include commands, flags, configuration fields and defaults,
|
||||
module keys, schemas, file names, paths, status codes, retry behavior, and
|
||||
runtime guarantees. If readers could reasonably treat a statement as a
|
||||
contract, maintain it only in the owning document.
|
||||
|
||||
Each document should cover a defined scope and only the essentials for that scope.
|
||||
|
||||
Avoid:
|
||||
- long background explanations;
|
||||
- repeated reference material;
|
||||
- implementation detail in user-facing docs;
|
||||
- aspirational language outside roadmap docs;
|
||||
- verbose examples where one minimal example is clearer.
|
||||
|
||||
### 2. Document only implemented behavior outside roadmap files
|
||||
|
||||
Unimplemented, planned, aspirational, experimental, or future work may be described only under:
|
||||
|
||||
- `docs/roadmap/`
|
||||
|
||||
No other documentation file, including `README.md`, should describe code, features, modules, stages, commands, config fields, or behaviors that do not currently exist.
|
||||
|
||||
If a feature is partial, non-roadmap docs may describe only the implemented portion and its current boundary.
|
||||
|
||||
### 3. Use canonical homes
|
||||
|
||||
Each type of information should have one canonical location.
|
||||
|
||||
Canonical homes:
|
||||
|
||||
- project purpose and quickstart: `README.md`
|
||||
- development principles: `docs/policy/architecture.md`
|
||||
- public HTTP API reference: `docs/api.md`
|
||||
- configuration reference: `docs/config.md`
|
||||
- CLI reference: `docs/cli.md`
|
||||
- operations and recovery: `docs/operations.md`
|
||||
- troubleshooting: `docs/troubleshooting.md`
|
||||
- public API/package consumer guidance: `docs/consumers/`
|
||||
- implemented internals: `docs/internal/`
|
||||
- external protocol, service, and file-format contracts: `docs/integrations/`
|
||||
- future work: `docs/roadmap/`
|
||||
- contributor workflow: `docs/policy/development.md`
|
||||
- copyable examples: `examples/`
|
||||
|
||||
Other files should summarize briefly and link to the canonical source.
|
||||
|
||||
### 4. Keep examples real
|
||||
|
||||
Examples should be valid, maintained, and free of secrets.
|
||||
|
||||
Where practical:
|
||||
- example configs should load successfully;
|
||||
- example commands should match real CLI syntax;
|
||||
- important examples should be covered by tests.
|
||||
|
||||
## Documentation Profiles
|
||||
|
||||
All projects require:
|
||||
|
||||
- `README.md`
|
||||
- `docs/policy/architecture.md`
|
||||
|
||||
Additional docs depend on the project.
|
||||
|
||||
### Small library
|
||||
|
||||
Recommended:
|
||||
- `docs/policy/development.md`, if contributor conventions are non-obvious
|
||||
|
||||
### Simple CLI
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/policy/development.md`
|
||||
|
||||
### Config-driven CLI
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`
|
||||
- `docs/config.md`
|
||||
|
||||
Recommended:
|
||||
- `examples/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
### Stateful or operator-facing application
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- `examples/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
### Modular, service-oriented, or orchestration application
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
- `docs/internal/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- validated examples under `examples/`
|
||||
|
||||
### Public HTTP API service
|
||||
|
||||
Required:
|
||||
- `docs/api.md`
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
- `docs/internal/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- `docs/consumers/`, for task-oriented client integration guides
|
||||
- `docs/integrations/`, for upstream/downstream service contracts
|
||||
- validated examples under `examples/`
|
||||
|
||||
### Project with public packages or consumer APIs
|
||||
|
||||
Required:
|
||||
- `docs/consumers/api.md`
|
||||
- one `docs/consumers/pkg-<name>.md` file per public package, if public packages exist
|
||||
|
||||
Recommended:
|
||||
- copyable consumer examples under `examples/`, if practical
|
||||
|
||||
## Required Documents
|
||||
|
||||
### README.md
|
||||
|
||||
**Audience:** users, administrators, operators
|
||||
|
||||
The README is the outward-facing project orientation page.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. concise description;
|
||||
2. elevator pitch;
|
||||
3. shortest useful command or usage example;
|
||||
4. links to targeted docs.
|
||||
|
||||
The README should be short. It is not a manual.
|
||||
|
||||
The “shortest useful command” means the simplest command that performs the project’s core use case. (It does not mean `app --help`.)
|
||||
|
||||
### docs/policy/architecture.md
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
`docs/policy/architecture.md` is required for every project.
|
||||
|
||||
It is an inward-facing development policy document. It should describe how the project is intended to be built and changed.
|
||||
|
||||
It should include:
|
||||
|
||||
- project shape;
|
||||
- core design principles;
|
||||
- package and boundary philosophy;
|
||||
- state/persistence philosophy, if applicable;
|
||||
- external integration philosophy, if applicable;
|
||||
- error-handling and logging principles;
|
||||
- testing expectations;
|
||||
- documentation expectations;
|
||||
- architectural invariants;
|
||||
- explicit non-goals, if useful.
|
||||
|
||||
Notably, this file should prescribe a core development *policy* that should remain unchanged as the application evolves. It is not a place for details (e.g., CLI flags) that could change over time.
|
||||
|
||||
The contents of `architecture.md` should be trim and concise. LLMs may be directed to review it routinely via AGENTS.md, CLAUDE.md, or similar.
|
||||
|
||||
### docs/api.md
|
||||
|
||||
**Audience:** external HTTP API consumers, developers, LLM coding agents integrating by HTTP
|
||||
|
||||
Required for projects whose primary public interface is HTTP.
|
||||
|
||||
`docs/api.md` is the canonical public HTTP API contract. It should be normative for external consumers and should not be duplicated by README, operations docs, consumer guides, or integration docs.
|
||||
|
||||
It should include:
|
||||
|
||||
1. base URL conventions;
|
||||
2. authentication and authorization behavior, if implemented;
|
||||
3. response envelope;
|
||||
4. supported media types and content negotiation behavior;
|
||||
5. shared query parameters;
|
||||
6. endpoint reference grouped by route family;
|
||||
7. request parameters and validation rules;
|
||||
8. response fields, units, nullability, and optionality;
|
||||
9. error response shape and status codes;
|
||||
10. pagination, caching, rate-limit, idempotency, and retry behavior, if implemented;
|
||||
11. compact request and response examples.
|
||||
|
||||
It must document only implemented endpoints and behavior. Planned endpoints, proposed fields, future filters, and experimental response shapes belong only under `docs/roadmap/`.
|
||||
|
||||
For HTTP API projects, `docs/consumers/` may provide task-oriented client integration guides, but those guides should link to `docs/api.md` for the authoritative endpoint contract.
|
||||
|
||||
### docs/policy/development.md
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for projects maintained by humans and LLM coding agents.
|
||||
|
||||
It should include:
|
||||
|
||||
- repository layout;
|
||||
- build/test commands;
|
||||
- coding conventions;
|
||||
- dependency policy;
|
||||
- how to add config fields;
|
||||
- how to add CLI flags;
|
||||
- how to add modules or adapters, if applicable;
|
||||
- how to update examples;
|
||||
- documentation update expectations.
|
||||
|
||||
### docs/config.md
|
||||
|
||||
**Audience:** administrators, operators, advanced users
|
||||
|
||||
Required for applications with configuration files.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. config file locations and discovery precedence;
|
||||
2. minimal working config;
|
||||
3. production-oriented config;
|
||||
4. full configuration reference;
|
||||
5. secrets handling, if applicable;
|
||||
6. links to maintained examples.
|
||||
|
||||
The full configuration reference should be canonical.
|
||||
|
||||
### docs/cli.md
|
||||
|
||||
**Audience:** users, administrators, operators
|
||||
|
||||
Required for CLI applications.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. shortest useful command;
|
||||
2. command overview;
|
||||
3. complete flag reference;
|
||||
4. common workflows;
|
||||
5. diagnostic or recovery commands, if applicable.
|
||||
|
||||
Explain when commands are useful, not just their syntax.
|
||||
|
||||
### docs/operations.md
|
||||
|
||||
**Audience:** administrators, operators
|
||||
|
||||
Required for applications that maintain state, support resume behavior, run multi-step workflows, write durable artifacts, use remote storage, or require recovery procedures.
|
||||
|
||||
It should cover:
|
||||
|
||||
- normal workflow;
|
||||
- filesystem layout;
|
||||
- remote storage layout, if applicable;
|
||||
- logs and manifests;
|
||||
- resume/retry behavior;
|
||||
- cleanup behavior;
|
||||
- archive/backup behavior;
|
||||
- safe recovery procedures;
|
||||
- operational caveats.
|
||||
|
||||
### docs/troubleshooting.md
|
||||
|
||||
**Audience:** administrators, operators
|
||||
|
||||
Recommended once recurring failure modes exist.
|
||||
|
||||
Each entry should include:
|
||||
|
||||
- symptom;
|
||||
- likely cause;
|
||||
- diagnostic command or inspection step;
|
||||
- safe fix;
|
||||
- relevant links.
|
||||
|
||||
### docs/consumers/
|
||||
|
||||
**Audience:** developers and LLM coding agents integrating this project from another codebase
|
||||
|
||||
Required for projects with public packages, SDKs, client APIs, plugin APIs, or other application-facing integration surfaces.
|
||||
|
||||
This directory describes how an external codebase should consume the project's public API. It should be task-oriented and copyable where useful. It is not the place for internal implementation details or operator procedures.
|
||||
|
||||
For projects whose public API is HTTP, `docs/consumers/` is not required, and it should not duplicate the endpoint reference in `docs/api.md`. If present, it may provide practical integration workflows, client-specific examples, or migration notes that link back to `docs/api.md`.
|
||||
|
||||
`docs/consumers/api.md` should provide the consumer-facing overview and primary implementation workflow. It should include:
|
||||
|
||||
1. intended consumer audience and use cases;
|
||||
2. required inputs supplied by operators or deployment configuration;
|
||||
3. recommended public package or API workflow;
|
||||
4. minimal copyable example;
|
||||
5. consumer responsibilities and boundaries;
|
||||
6. retry, idempotency, or status behavior, if applicable;
|
||||
7. links to package-specific docs and canonical integration contracts.
|
||||
|
||||
Package-specific docs should be named `pkg-<name>.md` and should include:
|
||||
|
||||
1. import path;
|
||||
2. intended use cases;
|
||||
3. primary types and functions needed by consumers;
|
||||
4. minimal examples;
|
||||
5. validation, error, retry, and boundary behavior;
|
||||
6. links to canonical file-format or wire-protocol contracts.
|
||||
|
||||
### docs/internal/
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for modular, service-oriented, or orchestration projects.
|
||||
|
||||
This directory describes implemented internal components. It is not the roadmap.
|
||||
|
||||
Use one file per major component where useful.
|
||||
|
||||
Each component doc should include:
|
||||
|
||||
1. purpose;
|
||||
2. inputs and outputs;
|
||||
3. boundaries;
|
||||
4. config fields used;
|
||||
5. external adapters used;
|
||||
6. state or manifest behavior, if applicable;
|
||||
7. skip/resume behavior, if applicable;
|
||||
8. failure behavior;
|
||||
9. tests to inspect before changing;
|
||||
10. architectural invariants.
|
||||
|
||||
### docs/roadmap/
|
||||
|
||||
**Audience:** maintainers, developers, LLM coding agents
|
||||
|
||||
This is the only place for planned, future, aspirational, experimental, or unimplemented work.
|
||||
|
||||
Roadmap docs should clearly distinguish:
|
||||
|
||||
- proposed work;
|
||||
- accepted plans;
|
||||
- deferred ideas;
|
||||
- rejected ideas;
|
||||
- implementation prompts or task breakdowns, if useful.
|
||||
|
||||
Roadmap docs should not be confused with current behavior.
|
||||
|
||||
### docs/integrations/
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
||||
|
||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses or exposes.
|
||||
|
||||
For public HTTP API services, `docs/integrations/` should document upstream, downstream, storage, protocol, or runtime contracts that the service depends on or bridges. It should not become a second copy of the public HTTP endpoint reference; that belongs in `docs/api.md`.
|
||||
|
||||
Use one file per integration where useful.
|
||||
|
||||
## Examples Directory
|
||||
|
||||
Projects with non-trivial configuration or workflows should include `examples/`.
|
||||
|
||||
Useful examples include:
|
||||
|
||||
- minimal working config;
|
||||
- production-oriented config;
|
||||
- full annotated config;
|
||||
- local development config;
|
||||
- remote/object-storage config;
|
||||
- minimal session/input file.
|
||||
|
||||
Examples should be valid, maintained, tested when practical, and linked from relevant docs.
|
||||
|
||||
## Security and Privacy
|
||||
|
||||
Docs and examples must not include:
|
||||
|
||||
- real API keys;
|
||||
- tokens;
|
||||
- passwords;
|
||||
- private keys;
|
||||
- private environment dumps;
|
||||
- sensitive user data;
|
||||
- raw private transcripts;
|
||||
- private infrastructure details unless intentionally public.
|
||||
|
||||
Document secret-handling mechanisms, not actual secret values.
|
||||
|
||||
## Maintenance Rules
|
||||
|
||||
When docs change, verify the affected behavior.
|
||||
|
||||
Where practical:
|
||||
|
||||
- load example config files in tests;
|
||||
- test CLI examples or command parser behavior;
|
||||
- validate documented flags against real flags;
|
||||
- remove stale references;
|
||||
- update links after renames;
|
||||
- keep roadmap content out of non-roadmap docs.
|
||||
|
||||
If documentation and code disagree, fix the documentation and/or open a roadmap item; do not leave aspirational behavior in current-behavior docs.
|
||||
|
||||
Documentation is complete only when it matches the current code.
|
||||
|
||||
## Documentation Change Checklist
|
||||
|
||||
Before merging documentation changes, verify:
|
||||
|
||||
- README is concise and orientation-focused.
|
||||
- `docs/policy/architecture.md` describes development principles.
|
||||
- `docs/api.md` is the canonical HTTP contract for HTTP API services.
|
||||
- Future work appears only under `docs/roadmap/`.
|
||||
- User-facing docs avoid unnecessary internals.
|
||||
- Consumer-facing docs explain public APIs without duplicating HTTP endpoint or integration contracts.
|
||||
- Developer-facing docs preserve boundaries and invariants.
|
||||
- Config examples match the schema.
|
||||
- CLI examples match real commands and flags.
|
||||
- Defaults appear in the canonical config reference.
|
||||
- No secrets or private data are included.
|
||||
- Links are accurate.
|
||||
Minimal tested usage examples are allowed outside the owning contract when this
|
||||
policy assigns them an orientation or instructional purpose. They must link to
|
||||
the canonical contract and must not redefine complete syntax, defaults, or
|
||||
semantics.
|
||||
|
||||
### Current And Future Behavior
|
||||
|
||||
Outside `docs/roadmap/`, documentation describes implemented behavior only.
|
||||
Partial features may be described only to their implemented boundary.
|
||||
|
||||
ADRs are the narrow exception: an ADR may record an accepted architectural
|
||||
decision before implementation, but acceptance must not be presented as proof
|
||||
that the behavior exists. The roadmap owns implementation status and sequencing
|
||||
until the decision is implemented. Current architecture, user, operator,
|
||||
integration, and internal documentation are updated when the behavior lands.
|
||||
|
||||
### Audience And Detail
|
||||
|
||||
Write for the document's stated audience and include only the detail needed for
|
||||
its owned topic. User and operator docs should not expose implementation detail.
|
||||
Developer docs should link to user-facing and external contracts rather than
|
||||
restate them.
|
||||
|
||||
### Examples
|
||||
|
||||
Complete copyable files belong in `examples/`. Documentation may use the
|
||||
smallest illustrative snippet needed to explain its owned topic, but should link
|
||||
to maintained examples instead of embedding a second complete copy.
|
||||
|
||||
Examples must be valid, secret-free, and tested where practical. Commands and
|
||||
configuration used in documentation should match the application.
|
||||
|
||||
### Security And Privacy
|
||||
|
||||
Documentation and examples must not contain real credentials, private keys,
|
||||
private environment dumps, sensitive source material, or private infrastructure
|
||||
details unless intentionally public. Document secret-handling mechanisms, not
|
||||
secret values.
|
||||
|
||||
## Canonical Ownership
|
||||
|
||||
| Topic | Canonical owner | Owned content | Content owned elsewhere |
|
||||
| --- | --- | --- | --- |
|
||||
| Product orientation and minimal end-to-end quickstart | `README.md` | What this project is, why it is useful, one shortest successful invocation, and links onward. | Complete command reference, configuration reference, operational procedures, implementation detail. |
|
||||
| Contributor entry point | `docs/development.md` | Task-oriented reading guide, minimal contributor orientation, baseline validation commands, and links to canonical docs. | Package inventory, architecture rules, subsystem behavior, and detailed change recipes, which belong in the relevant internal component document. |
|
||||
| Current application architecture | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, architectural boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
|
||||
| Documentation organization | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
|
||||
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression-test policy, and criteria for adding, rewriting, or deleting tests. | Subsystem behavior, application contracts, subsystem-specific test inventories, and implementation plans. |
|
||||
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
|
||||
| Configuration contract | `docs/config.md` | Application discovery and precedence, source locations, server fields, render default, HTTP limits, and credential mapping. | Promptkit framework formats and defaults, complete example files, CLI syntax, runtime lifecycle, and implementation detail. |
|
||||
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
|
||||
| Release procedure | `docs/release.md` | Candidate validation, version and tag operations, hosted-workflow observation, and published-artifact verification. | Runtime operations, version-specific announcements, and complete application-interface contracts. |
|
||||
| Version-specific release notes | `docs/releases/` | Immutable release summaries, compatibility notices, and migration announcements for one published version. | Complete CLI, HTTP, configuration, operations, or dependency contracts. |
|
||||
| Public HTTP contract | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
|
||||
| Consumer guidance | `docs/consumers/` | Choosing between Scriptorium's executable interfaces and understanding consumer responsibilities. | HTTP wire semantics, CLI syntax, Promptkit's Go package, and internal implementation detail. |
|
||||
| External and durable integration contracts | `docs/integrations/` | Scriptorium-owned process and executable integration contracts. | Promptkit framework formats and outbound provider protocols, physical runtime placement, internal transformations, CLI syntax, and configuration defaults. |
|
||||
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
||||
| Internal component behavior | Other files under `docs/internal/` | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
||||
| Architectural decision history | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
|
||||
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status; sequencing; and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
||||
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other files intended to be copied or run. | Field-by-field reference, command reference, prose explanation. |
|
||||
|
||||
Documents that do not exist are required only when the corresponding interface
|
||||
or responsibility exists. Do not create placeholder API, consumer, integration,
|
||||
or operations documents for behavior the application does not have.
|
||||
|
||||
## Boundary Rules
|
||||
|
||||
### Orientation
|
||||
|
||||
The README owns product orientation. The developer guide routes contributors.
|
||||
Architecture owns normative structure. Internal overview owns the current
|
||||
concrete component map. These documents may link to one another but should not
|
||||
maintain parallel package or behavior descriptions.
|
||||
|
||||
### Commands, Configuration, And Operations
|
||||
|
||||
CLI documentation answers how to invoke the application. Configuration
|
||||
documentation answers what settings mean. Operations answers what happens to
|
||||
runtime state and how to operate or recover the application. When a workflow
|
||||
crosses these topics, choose the document that owns the task and link to the
|
||||
other contracts.
|
||||
|
||||
### Contracts And Implementation
|
||||
|
||||
Integration and API documents define externally observable shapes and
|
||||
semantics. Internal documents explain how this project implements or consumes
|
||||
those contracts. Internal docs may name a field, file, or protocol to identify
|
||||
a dependency, but must link to its canonical contract for the definition.
|
||||
|
||||
### Security Topics
|
||||
|
||||
This policy owns what documentation and examples may contain. Architecture owns
|
||||
application security invariants. Configuration owns credential-supply
|
||||
mechanisms. Operations owns permissions and handling of sensitive runtime
|
||||
artifacts. Internal docs own implementation mechanisms only.
|
||||
|
||||
## Architecture Decision Records
|
||||
|
||||
Use sequentially numbered ADR filenames such as
|
||||
`0001-record-architecture-decisions.md`. Follow the lightweight Nygard format:
|
||||
|
||||
1. title;
|
||||
2. status;
|
||||
3. date;
|
||||
4. context;
|
||||
5. decision;
|
||||
6. alternatives considered;
|
||||
7. consequences.
|
||||
|
||||
Use one of these statuses:
|
||||
|
||||
- **Proposed:** the decision is under consideration and may change;
|
||||
- **Accepted:** the decision is approved, whether or not implementation is
|
||||
complete;
|
||||
- **Rejected:** the proposed decision was considered and not adopted;
|
||||
- **Superseded:** a later ADR replaces the accepted decision.
|
||||
|
||||
A proposed ADR transitions to accepted or rejected. An accepted ADR transitions
|
||||
to superseded only when a later accepted ADR replaces it. An ADR may be created
|
||||
as accepted when the decision has already been made.
|
||||
|
||||
Treat the decision content of an accepted ADR as immutable. Its status and
|
||||
supersession metadata may be updated, but a changed decision requires a new ADR.
|
||||
A superseded ADR must link to its replacement, and the replacement must link
|
||||
back to the superseded ADR. Rejected architectural alternatives belong in the
|
||||
ADR; rejected product ideas belong in the roadmap.
|
||||
|
||||
## Maintenance
|
||||
|
||||
When behavior changes, update its canonical owner in the same change. If
|
||||
ownership moves, remove the old definition and replace it with a link where
|
||||
navigation remains useful.
|
||||
|
||||
Before completing documentation work:
|
||||
|
||||
- verify affected behavior and examples;
|
||||
- check commands, flags, fields, defaults, schemas, and paths against their
|
||||
implementation;
|
||||
- keep unimplemented behavior in the roadmap, subject to the ADR exception;
|
||||
- remove stale references and validate links;
|
||||
- confirm that non-owning documents summarize and link rather than redefine;
|
||||
- confirm that no secrets or sensitive private data were added.
|
||||
|
||||
301
docs/policy/testing.md
Normal file
301
docs/policy/testing.md
Normal file
@@ -0,0 +1,301 @@
|
||||
# Testing Policy
|
||||
|
||||
## Purpose
|
||||
|
||||
Our tests exist to make **incorrect changes expensive and correct changes cheap**.
|
||||
|
||||
We do not optimize for test count, line coverage, exhaustive isolation, or the fewest possible tests. We optimize for sufficient confidence in important behavior while imposing as little unnecessary friction as possible on future development.
|
||||
|
||||
## Every test has a cost
|
||||
|
||||
Testing is not an unqualified good. Every test imposes both an immediate cost and a continuing lifetime cost.
|
||||
|
||||
A test must be:
|
||||
|
||||
- written and reviewed;
|
||||
- understood by future maintainers and coding agents;
|
||||
- executed in local and CI workflows;
|
||||
- diagnosed when it fails;
|
||||
- updated when legitimate behavior changes;
|
||||
- maintained as fixtures, APIs, and dependencies evolve; and
|
||||
- removed or rewritten when it becomes redundant, brittle, misleading, or obsolete.
|
||||
|
||||
Tests also create cognitive and architectural friction. They can constrain refactoring, duplicate policy, slow feedback loops, add noise to failures, and cause harmless implementation changes to require unrelated edits across the suite.
|
||||
|
||||
A test is warranted only when the confidence it provides justifies these costs.
|
||||
|
||||
Apply this cost-benefit analysis at two levels:
|
||||
|
||||
1. **Per test:** What realistic defect does this test detect, how consequential would that defect be, and is that protection worth the test's lifetime cost?
|
||||
2. **Across the suite:** Does this collection provide materially more confidence than a smaller, simpler suite would?
|
||||
|
||||
The preferred test suite is a **lean suite that provides sufficient confidence in the risks that matter, without redundant or low-value tests**. We seek sufficient confidence with the least unnecessary testing friction, not the fewest possible tests.
|
||||
|
||||
Some friction is intentional. Tests should make dangerous changes—such as breaking compatibility, corrupting data, violating security boundaries, or reintroducing subtle bugs—require deliberate review. They should not make ordinary internal changes needlessly expensive.
|
||||
|
||||
The cost of a test is not a reason to omit testing by default. Do not cite maintenance cost abstractly. When omitting a plausible test, be able to state why the protected failure is low-risk, already covered, obvious, reversible, or cheaper to detect elsewhere. For consequential, subtle, or difficult-to-observe behavior, the presumption should favor testing.
|
||||
|
||||
## Default testing style
|
||||
|
||||
Use a **classical/Detroit-style** approach:
|
||||
|
||||
- Test observable behavior, resulting state, contracts, and invariants.
|
||||
- Use real internal collaborators when they are fast and deterministic.
|
||||
- Use fakes, stubs, or mocks primarily at expensive, nondeterministic, destructive, or external boundaries.
|
||||
- Prefer package-level behavioral tests over tests coupled to private helpers or internal call sequences.
|
||||
- Treat exact collaborator interactions as testable behavior only when the interaction itself is a requirement.
|
||||
|
||||
Examples of appropriate seams include clocks, randomness, subprocesses, remote APIs, object storage, email, and paid LLM calls.
|
||||
|
||||
## Test execution requirements
|
||||
|
||||
Tests in the default suite must be deterministic, offline, and independent of real credentials. They must not invoke paid APIs or depend on mutable external services. Tests that require live infrastructure must be explicitly opt-in and clearly separated from the default suite.
|
||||
|
||||
Control clocks, randomness, environment variables, and other process-global or machine-specific state when they affect behavior. Tests should be safe to run repeatedly and alongside other tests without depending on execution order or state left by an earlier test.
|
||||
|
||||
## What deserves tests
|
||||
|
||||
Prioritize tests for:
|
||||
|
||||
1. Public and package-level contracts.
|
||||
2. Domain rules and important invariants.
|
||||
3. Boundary conditions and malformed input.
|
||||
4. Failure handling, cancellation, retries, recovery, and partial success.
|
||||
5. Serialization, schemas, compatibility, and round trips.
|
||||
6. Previously observed or plausible regressions.
|
||||
7. Representative integration and end-to-end workflows.
|
||||
|
||||
A package-level contract is behavior relied upon by another package or major collaborator, not every observable detail of a package implementation.
|
||||
|
||||
For behavior involving **data integrity, destructive operations, compatibility, security, concurrency, idempotency, or recovery**, presume that durable tests are required unless the behavior is already credibly protected at another layer.
|
||||
|
||||
Do not add tests merely because a function, branch, or line exists. Do not add a test when the same meaningful risk is already adequately protected elsewhere.
|
||||
|
||||
## Choose the right test boundary
|
||||
|
||||
Test through the narrowest stable boundary that expresses the behavior clearly.
|
||||
|
||||
This is often the package API, but it may instead be:
|
||||
|
||||
- a smaller pure function when dense application logic is most clearly isolated there;
|
||||
- a package-level operation when several internal collaborators jointly produce the behavior; or
|
||||
- a larger integration boundary when correctness emerges from interaction with a real dependency.
|
||||
|
||||
Do not force all behavior through oversized end-to-end tests. Do not test every private helper merely because it exists. Choose the boundary that gives durable confidence with the least incidental coupling.
|
||||
|
||||
## Test behavior, not implementation
|
||||
|
||||
A test should protect a decision, contract, or invariant—not memorialize the current implementation.
|
||||
|
||||
Before adding or retaining a test, ask:
|
||||
|
||||
> What realistic defect would this test catch?
|
||||
|
||||
A test is suspect when its main purpose is to detect that someone:
|
||||
|
||||
- changed an internal constant;
|
||||
- renamed or split a private helper;
|
||||
- reordered equivalent internal operations;
|
||||
- changed incidental formatting;
|
||||
- replaced one correct algorithm with another; or
|
||||
- refactored internal object structure without changing behavior.
|
||||
|
||||
Refactoring should normally require no test edits unless the refactored structure is itself part of the contract.
|
||||
|
||||
A test can be factually correct and still have negative value. Accurately describing current behavior is not enough; the protected behavior must be important enough to justify the future friction.
|
||||
|
||||
## Expected effects of different changes
|
||||
|
||||
Use the following expectations when evaluating test failures and test maintenance:
|
||||
|
||||
| Change | Expected effect on tests |
|
||||
|---|---|
|
||||
| Internal refactor that preserves behavior | Existing tests should normally remain unchanged and continue to pass. |
|
||||
| Change to an internal default with no contractual significance | Behavioral tests should normally remain unchanged; tests should derive expectations from configuration or relationships rather than duplicate the old value. |
|
||||
| Intentional change to public behavior, policy, schema, or compatibility guarantees | The relevant tests should be reviewed and changed deliberately. |
|
||||
| Accidental violation of a contract or invariant | Tests should fail; fix the production code rather than rewriting the tests to accept the defect. |
|
||||
|
||||
A test failing is not the same as a test needing to be edited. Many tests may correctly fail because of one production defect. The maintenance smell is a correct internal change that requires unrelated expectation updates throughout the suite.
|
||||
|
||||
## Separate mechanism from policy
|
||||
|
||||
Configurable thresholds and defaults must not be duplicated throughout the test suite.
|
||||
|
||||
For example, do not encode an internal concurrency limit indirectly:
|
||||
|
||||
```go
|
||||
// Production policy:
|
||||
const maxConcurrency = 4
|
||||
|
||||
// Brittle test:
|
||||
err := startProcesses(5)
|
||||
require.Error(t, err)
|
||||
```
|
||||
|
||||
Instead, test the mechanism relationally:
|
||||
|
||||
```go
|
||||
const limit = 2
|
||||
runner := NewRunner(limit)
|
||||
|
||||
require.NoError(t, runner.Start(limit))
|
||||
require.ErrorIs(t, runner.Start(limit+1), ErrTooMuchConcurrency)
|
||||
```
|
||||
|
||||
The test should prove:
|
||||
|
||||
- the configured limit is accepted; and
|
||||
- one beyond the configured limit is rejected.
|
||||
|
||||
The production default should be tested exactly only when its literal value is itself a public, operational, safety, protocol, or compatibility requirement.
|
||||
|
||||
Apply the same rule to limits, timeouts, capacities, retry counts, and ranges: test relationships and behavior, not duplicated literals.
|
||||
|
||||
For concurrency limits, test both kinds of behavior when relevant:
|
||||
|
||||
1. **Configuration enforcement:** invalid or excessive requested values are handled correctly.
|
||||
2. **Runtime enforcement:** observed peak concurrency never exceeds the configured limit.
|
||||
|
||||
Use a test-controlled limit and measure the behavior relative to that limit. Do not merely assert today's default value.
|
||||
|
||||
## Avoid semantic duplication across layers
|
||||
|
||||
Each behavior should have a clear test owner.
|
||||
|
||||
- Configuration tests own application YAML, discovery, precedence, and
|
||||
application defaults.
|
||||
- CLI tests own argument mapping, streams, summaries, exit behavior, and
|
||||
representative command workflows.
|
||||
- HTTP tests own DTOs, strict decoding, limits, status mapping, and restricted
|
||||
artifact policy.
|
||||
- Formatter tests own prepared-run text and JSON presentation.
|
||||
- Architecture tests own dependency direction and removal invariants.
|
||||
- Promptkit owns framework parsing, orchestration, validation, profiles, and
|
||||
model-client behavior.
|
||||
|
||||
Higher-level tests should not repeat every lower-level case. A single intentional policy change should not require unrelated edits across many test files.
|
||||
|
||||
Tests that are individually reasonable may still be collectively redundant. Evaluate the marginal value of each additional test in light of the protection already provided by the rest of the suite.
|
||||
|
||||
## Use test doubles deliberately
|
||||
|
||||
Choose the least elaborate test double that provides the required control or observation.
|
||||
|
||||
As a default:
|
||||
|
||||
1. Prefer real collaborators when they are fast and deterministic.
|
||||
2. Use small in-memory fakes when realistic stateful behavior is helpful.
|
||||
3. Use stubs when a dependency only needs to provide controlled responses.
|
||||
4. Use mocks when the interaction itself is contractual.
|
||||
|
||||
Mocks are appropriate when the contract includes facts such as:
|
||||
|
||||
- a notification is sent exactly once;
|
||||
- a transaction is committed only after successful writes;
|
||||
- cancellation reaches a subprocess;
|
||||
- an expensive API is called no more than once; or
|
||||
- a security audit event is emitted.
|
||||
|
||||
Do not use mocks merely to isolate every object or reproduce the implementation's call graph.
|
||||
|
||||
## Go-specific guidance
|
||||
|
||||
Use:
|
||||
|
||||
- table-driven tests for meaningful behavioral categories and boundaries;
|
||||
- `t.TempDir()` for real filesystem behavior;
|
||||
- `httptest.Server` for realistic HTTP interactions;
|
||||
- fuzz tests for parsers, normalization, path handling, and broad input spaces;
|
||||
- golden files only when the complete output is intentionally stable;
|
||||
- integration tests where correctness depends on component interaction; and
|
||||
- a small number of representative end-to-end tests.
|
||||
|
||||
Avoid exact error-string assertions unless the wording is itself contractual. Prefer `errors.Is`, `errors.As`, typed errors, or structured error fields.
|
||||
|
||||
At CLI boundaries, prefer exit classifications, structured output, and the smallest stable semantic fragment needed to identify the error. Do not snapshot complete diagnostic wording unless it is contractual.
|
||||
|
||||
Golden-file updates must require an explicit local flag. CI must not update golden files automatically, and reviewers must inspect the semantic diff before accepting an update.
|
||||
|
||||
Keep tests readable and direct. Test helpers and fixture frameworks must earn their own maintenance cost; do not build elaborate test infrastructure for small or isolated needs.
|
||||
|
||||
## Coverage
|
||||
|
||||
Coverage is a diagnostic, not a target.
|
||||
|
||||
Use it to find untested critical branches and unexpectedly weak packages. Do not write low-value tests solely to increase a percentage, and do not infer test quality from coverage alone.
|
||||
|
||||
Security-sensitive HTTP containment and external mappings may warrant denser
|
||||
coverage than straightforward process wiring. Uneven coverage is acceptable
|
||||
when it reflects risk.
|
||||
|
||||
Increasing coverage is valuable only when the newly covered behavior protects a meaningful risk at an acceptable cost.
|
||||
|
||||
## Regression tests
|
||||
|
||||
A bug fix should normally include a regression test that fails before the fix and passes afterward.
|
||||
|
||||
Retain the test when the defect could realistically recur and its consequences justify the ongoing cost. Prefer the narrowest durable test of the violated contract or invariant; do not preserve accidental implementation details from the original bug.
|
||||
|
||||
Not every historical bug requires a permanent test. If the underlying design has made recurrence impossible, the test has become redundant, or a stronger invariant test now subsumes it, remove or consolidate it.
|
||||
|
||||
## Deleting or rewriting tests
|
||||
|
||||
Tests are maintained code, not permanent historical artifacts.
|
||||
|
||||
Delete or rewrite a test when its maintenance cost exceeds the confidence it provides.
|
||||
|
||||
Strong candidates include tests that:
|
||||
|
||||
- require updates after harmless internal changes;
|
||||
- directly assert private constants without protecting a real contract;
|
||||
- duplicate the same policy across several layers;
|
||||
- verify mock choreography rather than outcomes;
|
||||
- snapshot large amounts of incidental output;
|
||||
- test trivial private helpers already exercised through stable package behavior;
|
||||
- protect risks already covered more effectively elsewhere;
|
||||
- are flaky, misleading, obsolete, or disproportionately expensive to diagnose; or
|
||||
- no longer correspond to a plausible failure mode.
|
||||
|
||||
Several brittle tests may encode one genuine requirement. Replace them with one durable behavior-level or invariant test rather than preserving all of them.
|
||||
|
||||
Deleting a low-value test can improve the quality of the suite by reducing noise, maintenance burden, and friction around legitimate change.
|
||||
|
||||
## Reviewing a proposed test
|
||||
|
||||
Use the following questions when the value, boundary, or durability of a proposed test is not self-evident. Significant test additions should be reviewable against them, but written answers are not required for every routine test.
|
||||
|
||||
1. What realistic defect would it catch?
|
||||
2. How likely is that defect?
|
||||
3. How consequential would it be?
|
||||
4. Is the behavior already protected elsewhere?
|
||||
5. At which layer should this behavior be owned?
|
||||
6. Does the test assert a durable contract or an incidental implementation detail?
|
||||
7. Could the implementation be refactored without changing the behavior and without editing this test?
|
||||
8. What should cause this test to fail?
|
||||
9. What legitimate changes should not cause this test to fail?
|
||||
10. What ongoing maintenance, execution, and diagnostic cost will the test impose?
|
||||
11. Is there a smaller or more direct test that protects the same risk?
|
||||
|
||||
Do not add the test when its expected lifetime cost exceeds its expected protective value.
|
||||
|
||||
When deciding not to test plausible behavior, record or be able to explain why the risk is low, already protected, obvious, reversible, or cheaper to detect elsewhere.
|
||||
|
||||
## Definition of sufficient
|
||||
|
||||
A test suite is sufficient when:
|
||||
|
||||
- important contracts and invariants are protected;
|
||||
- meaningful boundaries and failure modes are exercised;
|
||||
- realistic and consequential regressions are credibly protected against silent recurrence;
|
||||
- behavior involving data integrity, destructive operations, compatibility, security, concurrency, idempotency, and recovery is credibly protected;
|
||||
- important external boundaries have realistic integration coverage;
|
||||
- representative complete workflows are tested;
|
||||
- failures provide useful signal rather than redundant noise;
|
||||
- legitimate internal changes usually do not require test edits; and
|
||||
- additional tests would mostly repeat existing protection or preserve inconsequential implementation details.
|
||||
|
||||
Sufficiency is a risk judgment, not a coverage percentage or test count. Reassess it as the application, its users, and the consequences of failure evolve.
|
||||
|
||||
The governing rule is:
|
||||
|
||||
> Test heavily where failure is consequential, subtle, or difficult to detect after the fact. Test lightly where failure is obvious, reversible, and inexpensive—and retain no test whose lifetime cost exceeds the confidence it provides.
|
||||
377
docs/release.md
Normal file
377
docs/release.md
Normal file
@@ -0,0 +1,377 @@
|
||||
# Release Procedure
|
||||
|
||||
## Release Model And Status
|
||||
|
||||
Scriptorium publishes annotated semantic tags and tag-triggered Linux binary
|
||||
releases. The hosted
|
||||
[release workflow](../.woodpecker/release.yml) builds `amd64` and `arm64`
|
||||
executables, publishes their SHA-256 checksums, and uses the matching file
|
||||
under `docs/releases/` as the hosted release body.
|
||||
|
||||
`v0.12.0` is the first published application-only release. For each later
|
||||
release, select a new `vMAJOR.MINOR.PATCH` version according to the intended
|
||||
compatibility change. A selected version remains an unreleased candidate until
|
||||
its annotated tag is published, the hosted workflow succeeds, and every
|
||||
published artifact is verified.
|
||||
|
||||
Run this procedure from the Scriptorium repository root. A release must not
|
||||
depend on a Go workspace, module replacement, vendor tree, sibling checkout,
|
||||
unpublished dependency, or unpushed source commit.
|
||||
|
||||
## Establish The Candidate
|
||||
|
||||
Start a POSIX shell, choose a semantic version that has not been published, and
|
||||
export it as `RELEASE_VERSION`. For example, if `v0.12.1` is the intended next
|
||||
version and remains unpublished, select:
|
||||
|
||||
```sh
|
||||
export RELEASE_VERSION=v0.12.1
|
||||
```
|
||||
|
||||
Use the version appropriate to the actual compatibility change rather than
|
||||
assuming that the example is the next release. Then run the following guard in
|
||||
that same shell:
|
||||
|
||||
```sh
|
||||
set -eu
|
||||
|
||||
: "${RELEASE_VERSION:?export an unpublished vMAJOR.MINOR.PATCH version}"
|
||||
if ! printf '%s\n' "$RELEASE_VERSION" |
|
||||
grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$'
|
||||
then
|
||||
printf '%s\n' "invalid release version: $RELEASE_VERSION" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
RELEASE_COMMIT=$(git rev-parse --verify 'HEAD^{commit}')
|
||||
export RELEASE_COMMIT
|
||||
|
||||
check_release_candidate() {
|
||||
test "$(git branch --show-current)" = main
|
||||
test -z "$(git status --porcelain)"
|
||||
|
||||
gowork_value=$(go env GOWORK)
|
||||
case "$gowork_value" in
|
||||
''|off) ;;
|
||||
*)
|
||||
printf '%s\n' "active Go workspace: $gowork_value" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
test -z "$(git ls-files go.work go.work.sum)"
|
||||
test ! -e vendor
|
||||
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||
then
|
||||
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
git fetch origin main --tags
|
||||
test "$RELEASE_COMMIT" = \
|
||||
"$(git rev-parse --verify 'refs/remotes/origin/main^{commit}')"
|
||||
|
||||
if git show-ref --verify --quiet "refs/tags/$RELEASE_VERSION"
|
||||
then
|
||||
printf '%s\n' "local tag already exists: $RELEASE_VERSION" >&2
|
||||
return 1
|
||||
fi
|
||||
if test -n "$(
|
||||
git ls-remote --tags origin \
|
||||
"refs/tags/$RELEASE_VERSION" \
|
||||
"refs/tags/$RELEASE_VERSION^{}"
|
||||
)"
|
||||
then
|
||||
printf '%s\n' "remote tag already exists: $RELEASE_VERSION" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
check_release_candidate
|
||||
```
|
||||
|
||||
Do not continue unless this guard succeeds. It deliberately requires the
|
||||
candidate to be the exact clean commit already published at `origin/main`.
|
||||
|
||||
## Verify Modules And Repository Boundaries
|
||||
|
||||
Confirm the module path and declared Go version:
|
||||
|
||||
```sh
|
||||
test "$(
|
||||
GOWORK=off go list -m -f '{{.Path}} {{.GoVersion}}'
|
||||
)" = 'gitea.maximumdirect.net/eric/scriptorium 1.25.5'
|
||||
```
|
||||
|
||||
Require Promptkit `v0.9.0` as both the direct module-graph edge and the selected
|
||||
module version:
|
||||
|
||||
```sh
|
||||
direct_promptkit=$(
|
||||
GOWORK=off go mod graph |
|
||||
awk '
|
||||
$1 == "gitea.maximumdirect.net/eric/scriptorium" &&
|
||||
$2 ~ /^gitea\.maximumdirect\.net\/eric\/promptkit@/ {
|
||||
print $2
|
||||
}
|
||||
'
|
||||
)
|
||||
test "$direct_promptkit" = \
|
||||
'gitea.maximumdirect.net/eric/promptkit@v0.9.0'
|
||||
test "$(
|
||||
GOWORK=off go list -m -f '{{.Path}}@{{.Version}}' \
|
||||
gitea.maximumdirect.net/eric/promptkit
|
||||
)" = 'gitea.maximumdirect.net/eric/promptkit@v0.9.0'
|
||||
GOWORK=off go list -m all
|
||||
```
|
||||
|
||||
Require tidy module metadata and recheck the repository exclusions:
|
||||
|
||||
```sh
|
||||
GOWORK=off go mod tidy -diff
|
||||
test -z "$(git ls-files go.work go.work.sum)"
|
||||
test ! -e vendor
|
||||
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||
then
|
||||
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||
exit 1
|
||||
fi
|
||||
test -z "$(git status --porcelain)"
|
||||
```
|
||||
|
||||
## Validate The Application
|
||||
|
||||
Run the complete application validation:
|
||||
|
||||
```sh
|
||||
GOWORK=off go test ./...
|
||||
GOWORK=off go test -race ./...
|
||||
GOWORK=off go vet ./...
|
||||
validation_build_dir=$(mktemp -d)
|
||||
GOWORK=off go build \
|
||||
-o "$validation_build_dir/scriptorium" \
|
||||
./cmd/scriptorium
|
||||
```
|
||||
|
||||
The ordinary test run includes the architecture guard that rejects a root Go
|
||||
package, former framework package families, and imports of Promptkit internal
|
||||
packages. Inspect the repository for generated binaries, credentials,
|
||||
temporary output, sibling paths, and other files that do not belong in the
|
||||
tracked release source.
|
||||
|
||||
Check every tracked Go file. This command must produce no output:
|
||||
|
||||
```sh
|
||||
unformatted=$(
|
||||
git ls-files '*.go' |
|
||||
while IFS= read -r go_file
|
||||
do
|
||||
gofmt -l "$go_file"
|
||||
done
|
||||
)
|
||||
test -z "$unformatted"
|
||||
```
|
||||
|
||||
Run the maintained render and Promptkit v0.9 feature scripts, then smoke-test
|
||||
both maintained configuration examples without a model call:
|
||||
|
||||
```sh
|
||||
GOWORK=off ./examples/render-markdown-summary.sh
|
||||
GOWORK=off ./examples/render-v0.9-features.sh
|
||||
for config_file in examples/config.yml examples/config.full.yml
|
||||
do
|
||||
GOWORK=off go run ./cmd/scriptorium render \
|
||||
--config "$config_file" \
|
||||
--prompt generic.markdown_summary \
|
||||
--input transcript=./examples/fixtures/transcript.md \
|
||||
--input glossary=./examples/fixtures/glossary.yml \
|
||||
--format json >/dev/null
|
||||
done
|
||||
```
|
||||
|
||||
Exercise usage output and offline rendering with the temporary native
|
||||
executable:
|
||||
|
||||
```sh
|
||||
usage_output="$validation_build_dir/usage.txt"
|
||||
if "$validation_build_dir/scriptorium" >"$usage_output" 2>&1
|
||||
then
|
||||
printf '%s\n' 'expected an invocation without a command to fail' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -F 'usage: scriptorium' "$usage_output"
|
||||
"$validation_build_dir/scriptorium" render \
|
||||
--config ./examples/config.yml \
|
||||
--prompt generic.markdown_summary \
|
||||
--input transcript=./examples/fixtures/transcript.md \
|
||||
--input glossary=./examples/fixtures/glossary.yml \
|
||||
--format text >/dev/null
|
||||
```
|
||||
|
||||
Follow every maintained local, Promptkit-tagged, and other external Markdown
|
||||
link. Confirm that all repository-relative link targets exist. Finish the
|
||||
application checks with:
|
||||
|
||||
```sh
|
||||
git diff --check
|
||||
test -z "$(git status --porcelain)"
|
||||
```
|
||||
|
||||
## Require Release Notes And Reproduce Packaging
|
||||
|
||||
The immutable release note must exist before tagging:
|
||||
|
||||
```sh
|
||||
release_notes="docs/releases/$RELEASE_VERSION.md"
|
||||
test -f "$release_notes"
|
||||
test -s "$release_notes"
|
||||
```
|
||||
|
||||
Validate every local and currently published link in the note. For links
|
||||
pinned to the candidate Scriptorium tag, confirm that the corresponding
|
||||
repository-relative path exists even though its tag URL is not live yet.
|
||||
|
||||
Reproduce the hosted build flags, targets, and filenames in a temporary
|
||||
directory:
|
||||
|
||||
```sh
|
||||
release_dist=$(mktemp -d)
|
||||
release_package='gitea.maximumdirect.net/eric/scriptorium/cmd/scriptorium'
|
||||
|
||||
build_release_binary() {
|
||||
target_os="$1"
|
||||
target_arch="$2"
|
||||
output="$release_dist/scriptorium-$RELEASE_VERSION-$target_os-$target_arch"
|
||||
|
||||
CGO_ENABLED=0 GOOS="$target_os" GOARCH="$target_arch" GOWORK=off \
|
||||
go build -trimpath -ldflags '-s -w' \
|
||||
-o "$output" "$release_package"
|
||||
}
|
||||
|
||||
build_release_binary linux amd64
|
||||
build_release_binary linux arm64
|
||||
|
||||
test -s "$release_dist/scriptorium-$RELEASE_VERSION-linux-amd64"
|
||||
test -s "$release_dist/scriptorium-$RELEASE_VERSION-linux-arm64"
|
||||
file "$release_dist/scriptorium-$RELEASE_VERSION-linux-amd64"
|
||||
file "$release_dist/scriptorium-$RELEASE_VERSION-linux-arm64"
|
||||
```
|
||||
|
||||
Require `file` to identify Linux executables for `x86-64` and `ARM aarch64`,
|
||||
respectively. Inspect the
|
||||
[hosted workflow](../.woodpecker/release.yml) and confirm that it uses the
|
||||
same build flags and names, copies the selected release note to
|
||||
`dist/RELEASE_NOTES.md`, publishes only `dist/scriptorium-*`, and keeps
|
||||
checksum generation enabled.
|
||||
|
||||
## Create And Publish The Tag
|
||||
|
||||
Run the candidate guard again immediately before creating the tag:
|
||||
|
||||
```sh
|
||||
check_release_candidate
|
||||
test -f "$release_notes"
|
||||
test -s "$release_notes"
|
||||
```
|
||||
|
||||
Create an annotated tag explicitly bound to the validated commit, using the
|
||||
version-specific release note as its message:
|
||||
|
||||
```sh
|
||||
git tag --annotate "$RELEASE_VERSION" \
|
||||
--file "$release_notes" \
|
||||
"$RELEASE_COMMIT"
|
||||
```
|
||||
|
||||
Inspect the tag and require it to resolve to the validated source:
|
||||
|
||||
```sh
|
||||
test "$(git cat-file -t "refs/tags/$RELEASE_VERSION")" = tag
|
||||
git show --no-patch --decorate "refs/tags/$RELEASE_VERSION"
|
||||
test "$(
|
||||
git rev-parse --verify "refs/tags/$RELEASE_VERSION^{commit}"
|
||||
)" = "$RELEASE_COMMIT"
|
||||
```
|
||||
|
||||
If inspection finds an error, delete only the unpublished local tag, correct
|
||||
the candidate, and repeat the complete validation. Never move or recreate a
|
||||
published tag.
|
||||
|
||||
Push only the selected tag ref:
|
||||
|
||||
```sh
|
||||
git push origin \
|
||||
"refs/tags/$RELEASE_VERSION:refs/tags/$RELEASE_VERSION"
|
||||
```
|
||||
|
||||
## Observe And Verify Publication
|
||||
|
||||
Open the hosted workflow run for the selected tag. Require
|
||||
`build-release-assets` to succeed before `publish-release`, then require the
|
||||
publication step and hosted release to succeed. A queued, running, failed, or
|
||||
partially published workflow is not a verified release.
|
||||
|
||||
Compare the local and remote annotated-tag objects and their source commits:
|
||||
|
||||
```sh
|
||||
remote_tag=$(
|
||||
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" |
|
||||
awk 'NR == 1 { print $1 }'
|
||||
)
|
||||
remote_commit=$(
|
||||
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION^{}" |
|
||||
awk 'NR == 1 { print $1 }'
|
||||
)
|
||||
test -n "$remote_tag"
|
||||
test "$remote_tag" = \
|
||||
"$(git rev-parse --verify "refs/tags/$RELEASE_VERSION")"
|
||||
test "$remote_commit" = "$RELEASE_COMMIT"
|
||||
```
|
||||
|
||||
Download the hosted binaries and checksum file into a temporary directory:
|
||||
|
||||
```sh
|
||||
release_base="https://gitea.maximumdirect.net/eric/scriptorium/releases/download/$RELEASE_VERSION"
|
||||
download_dir=$(mktemp -d)
|
||||
(
|
||||
cd "$download_dir"
|
||||
for asset in \
|
||||
"scriptorium-$RELEASE_VERSION-linux-amd64" \
|
||||
"scriptorium-$RELEASE_VERSION-linux-arm64" \
|
||||
SHA256SUMS
|
||||
do
|
||||
curl --fail --location --remote-name "$release_base/$asset"
|
||||
done
|
||||
|
||||
test -s "scriptorium-$RELEASE_VERSION-linux-amd64"
|
||||
test -s "scriptorium-$RELEASE_VERSION-linux-arm64"
|
||||
test -s SHA256SUMS
|
||||
sha256sum --check SHA256SUMS
|
||||
test "$(wc -l < SHA256SUMS | tr -d ' ')" = 2
|
||||
file "scriptorium-$RELEASE_VERSION-linux-amd64"
|
||||
file "scriptorium-$RELEASE_VERSION-linux-arm64"
|
||||
)
|
||||
```
|
||||
|
||||
Require the same Linux architectures observed in the local packaging check and
|
||||
confirm that the hosted release contains no unexpected asset. On a compatible
|
||||
Linux host, make the matching downloaded binary executable and repeat the
|
||||
usage-output and offline-render smoke checks against it.
|
||||
|
||||
Only after the tag, workflow, release body, binaries, architectures, and
|
||||
checksums all pass verification is the candidate a verified published release.
|
||||
|
||||
## Handle Failures
|
||||
|
||||
Before tag publication, correct the release commit or note and restart the
|
||||
complete procedure. After tag publication, never delete, move, overwrite, or
|
||||
recreate the tag. A transient hosted failure may be retried only against the
|
||||
same immutable tag and commit and only when doing so cannot overwrite or
|
||||
silently retain partial assets. A source, packaging, note, or artifact defect
|
||||
requires a new corrective semantic version from a new validated commit.
|
||||
|
||||
Record the selected version, validated commit, tag object, workflow result,
|
||||
artifact names, checksum result, and smoke-check outcome in the release
|
||||
checkpoint. Keep temporary builds and downloaded assets outside the repository,
|
||||
and require a clean `main` synchronized with `origin/main` when verification
|
||||
is complete.
|
||||
36
docs/releases/v0.12.0.md
Normal file
36
docs/releases/v0.12.0.md
Normal file
@@ -0,0 +1,36 @@
|
||||
# Scriptorium v0.12.0
|
||||
|
||||
## Breaking Project Boundary
|
||||
|
||||
Scriptorium is now an executable-only CLI and HTTP application. This is a
|
||||
breaking change for Go consumers: the former root Go package is not included,
|
||||
and no compatibility facade is provided.
|
||||
|
||||
Scriptorium `v0.11.1` was the final framework-bearing release. Former Go
|
||||
consumers should follow the
|
||||
[migration guide](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.12.0/docs/consumers/migrating-to-promptkit.md)
|
||||
and adopt
|
||||
[Promptkit `v0.1.0`](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/consumers/pkg-promptkit.md)
|
||||
for in-process prompt preparation and execution.
|
||||
|
||||
## Application Interfaces
|
||||
|
||||
The Scriptorium command-line and HTTP application interfaces remain. Their
|
||||
canonical documentation defines the supported commands, configuration,
|
||||
requests, responses, operational behavior, and deployment responsibilities:
|
||||
|
||||
- [CLI reference](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.12.0/docs/cli.md)
|
||||
- [HTTP API reference](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.12.0/docs/api.md)
|
||||
- [Configuration reference](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.12.0/docs/config.md)
|
||||
- [Operations guide](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.12.0/docs/operations.md)
|
||||
|
||||
## Framework Dependency And Consumers
|
||||
|
||||
The released Scriptorium binaries use Promptkit `v0.1.0` as their framework
|
||||
dependency. Promptkit owns the reusable engine, source formats, profiles,
|
||||
generation boundary, and validation contracts. See the
|
||||
[Promptkit Go consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/consumers/pkg-promptkit.md)
|
||||
for that supported API.
|
||||
|
||||
All known downstream Go consumers were migrated to Promptkit before this
|
||||
release.
|
||||
59
docs/releases/v0.13.0.md
Normal file
59
docs/releases/v0.13.0.md
Normal file
@@ -0,0 +1,59 @@
|
||||
# Scriptorium v0.13.0
|
||||
|
||||
## Promptkit v0.9 Adoption
|
||||
|
||||
Scriptorium now uses
|
||||
[Promptkit `v0.9.0`](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/)
|
||||
and supports the complete Promptkit v0.9 prompt and profile definition format
|
||||
through its configured directory sources. This includes prompt versions,
|
||||
optional inputs, all supported message roles, cache control, session templates,
|
||||
output schemas and repair budgets, profile inheritance, backend selection,
|
||||
provider controls, optional credential environment names, and JSON-compatible
|
||||
extra parameters.
|
||||
|
||||
Promptkit remains the canonical owner of definition parsing and framework
|
||||
validation. Scriptorium continues to own its CLI, HTTP, configuration,
|
||||
presentation, and deployment contracts.
|
||||
|
||||
## Application Features
|
||||
|
||||
- `run` and `render` can select an explicit prompt version and can execute
|
||||
prompts that do not declare or reference inputs.
|
||||
- CLI and HTTP requests can provide a direct session ID. Reasoning controls can
|
||||
inherit, replace, or explicitly clear the selected profile value.
|
||||
- Application configuration can register custom OpenAI-compatible backends
|
||||
with optional credential environment names, extra parameters, concurrency
|
||||
limits, and queue policy.
|
||||
- `inspect prompt` and `inspect profile` provide deterministic text or JSON
|
||||
views without invoking a model. Profile inspection does not read credential
|
||||
values or require a prompt definition.
|
||||
- Prepared output, CLI summaries, and HTTP metadata report effective backend
|
||||
identity when available.
|
||||
- HTTP requests rejected by backend capacity policy return `503` with the
|
||||
stable `capacity_exceeded` code. One server-scoped Promptkit engine enforces
|
||||
those limits across concurrent requests.
|
||||
|
||||
See the versioned application contracts for exact behavior:
|
||||
|
||||
- [CLI reference](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.13.0/docs/cli.md)
|
||||
- [HTTP API reference](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.13.0/docs/api.md)
|
||||
- [Configuration reference](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.13.0/docs/config.md)
|
||||
- [Operations guide](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.13.0/docs/operations.md)
|
||||
- [Promptkit v0.9 format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md)
|
||||
|
||||
## Compatibility And Operations
|
||||
|
||||
There are no intentional removals from the v0.12 application interfaces. The
|
||||
Promptkit upgrade does enforce its current definition-safety rules, including
|
||||
supported message roles, absolute HTTP or HTTPS endpoints, contained content
|
||||
and schema paths, bounded repair budgets, and JSON-compatible extra parameters.
|
||||
Definitions rejected by these rules must be corrected before use.
|
||||
|
||||
Credential values remain outside configuration, definition files, CLI flags,
|
||||
and HTTP payloads. Named environment sources are optional unless a selected
|
||||
Promptkit profile explicitly requires a key. A positive repair budget permits
|
||||
additional provider calls and can increase latency, token use, and cost.
|
||||
|
||||
The HTTP service retains only the shared operational state needed for backend
|
||||
admission and in-flight request handling. Scriptorium does not retain durable
|
||||
conversation, workflow, checkpoint, or resume state.
|
||||
601
docs/roadmap/implementation.md
Normal file
601
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,601 @@
|
||||
# Promptkit v0.9.0 Adoption Implementation Plan
|
||||
|
||||
## Status And Authority
|
||||
|
||||
Complete. Implement the stages below in numeric order. The accepted feature
|
||||
scope and target state are defined by the
|
||||
[Promptkit v0.9.0 adoption roadmap](promptkit-v0.9.0-adoption.md), and the
|
||||
durable product boundary is defined by
|
||||
[ADR 0004](../adr/0004-definition-boundary.md). If this plan and either source
|
||||
conflict, the ADR governs product boundaries and the feature roadmap governs
|
||||
scope.
|
||||
|
||||
Follow the architecture, documentation, and testing policies under
|
||||
`docs/policy/` throughout the work. In particular:
|
||||
|
||||
- use only Promptkit's public root package and its released v0.9.0 tag;
|
||||
- do not add a workspace, module replacement, vendored Promptkit source,
|
||||
direct catalog import, or Scriptorium reimplementation of Promptkit parsing
|
||||
and validation;
|
||||
- keep default tests deterministic, offline, and free of real credentials or
|
||||
provider calls;
|
||||
- update each current-state contract in the same stage that implements its
|
||||
behavior; and
|
||||
- prefer a small number of boundary and integration tests over duplicating
|
||||
Promptkit's own framework test matrix.
|
||||
|
||||
At the start of every stage, inspect the working tree and preserve unrelated
|
||||
changes. At the end of every stage, run the focused tests named in that stage,
|
||||
`gofmt` changed Go files, and run `git diff --check`. Do not proceed while a
|
||||
stage's completion conditions are unmet.
|
||||
|
||||
## Stage 1: Adopt The Promptkit v0.9.0 Compatibility Baseline
|
||||
|
||||
**Completion: Complete.**
|
||||
|
||||
Make the released dependency the build baseline before adding new application
|
||||
surface area.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Update `go.mod` to require
|
||||
`gitea.maximumdirect.net/eric/promptkit v0.9.0`, then run `go mod tidy` so
|
||||
`go.mod` and `go.sum` contain the module graph selected by that tag. Accept
|
||||
Promptkit's OpenRouter and Rakestrawhome catalog modules only as transitive
|
||||
dependencies; do not import them from Scriptorium.
|
||||
2. In the HTTP request DTO, change `model.reasoning_effort` from `string` to
|
||||
`*string`, and pass the pointer directly to
|
||||
`promptkit.ExecutionTargetOverride.ReasoningEffort`. This is required by the
|
||||
v0.9.0 API and preserves omitted versus explicit-empty input. Keep the JSON
|
||||
field itself a string when present.
|
||||
3. Compile every Promptkit public struct literal against v0.9.0. Keep literals
|
||||
keyed, do not restore the removed `RunRequest.Metadata` field, and do not
|
||||
add compatibility wrappers around Promptkit.
|
||||
4. Audit maintained prompt, profile, schema, configuration, and HTTP examples
|
||||
against v0.9.0. Correct only actual incompatibilities: endpoints must be
|
||||
absolute HTTP(S) URLs, roles must be `developer`, `system`, `user`, or
|
||||
`assistant`, positive repair budgets must be at most three and paired with
|
||||
`basic`, `json`, or `json_schema`, content and schema paths must remain
|
||||
within their Promptkit source rules, and extra parameters must be
|
||||
JSON-compatible and avoid reserved provider keys.
|
||||
5. Replace v0.1.0 links and current-version statements in current-state
|
||||
documentation with tagged v0.9.0 links. Do not rewrite historical version
|
||||
statements in accepted ADRs. Update the owning documentation to state the
|
||||
v0.9.0 credential and provider behavior: an optional missing environment
|
||||
credential may result in an unauthenticated provider request, while a
|
||||
profile that declares credentials required still fails as an invalid request
|
||||
when no source is supplied and with `ErrAPIKeyEnvMissing` when its selected
|
||||
environment source is empty; unset optional provider controls are omitted;
|
||||
and a positive repair budget can add provider calls, latency, token use, and
|
||||
cost.
|
||||
6. Preserve the architecture guard for forbidden replacements, workspaces,
|
||||
vendor trees, former facade packages, and Promptkit `internal` imports.
|
||||
Extend it only if the dependency graph introduces a new realistic bypass
|
||||
that the existing assertions do not cover.
|
||||
|
||||
### Verification
|
||||
|
||||
- Update HTTP mapping tests for the pointer-valued reasoning field, including
|
||||
separate omitted and explicit-empty cases, and update `docs/api.md` with the
|
||||
HTTP three-state behavior in this stage. Stage 4 adds the corresponding CLI
|
||||
control and session-ID support.
|
||||
- Run `go test ./...`, `go test -race ./...`, `go vet ./...`, and
|
||||
`go build ./cmd/scriptorium` against the tagged module with no local
|
||||
replacement.
|
||||
- Run the existing maintained render example and any example-validation tests.
|
||||
- Confirm that a search for `v0.1.0` finds only legitimate historical material,
|
||||
and that searches for `replace`, `go.work`, and Promptkit `/internal/`
|
||||
imports do not reveal a prohibited dependency path.
|
||||
|
||||
### Completion Conditions
|
||||
|
||||
The repository builds and all existing behavior tests pass on Promptkit v0.9.0;
|
||||
current documentation identifies v0.9.0 as the framework contract; no local or
|
||||
private Promptkit integration mechanism has been introduced.
|
||||
|
||||
## Stage 2: Remove Definition-Compatibility Restrictions In Run And Render
|
||||
|
||||
**Completion: Complete.**
|
||||
|
||||
Allow the executable adapters to address every valid directory-backed prompt
|
||||
definition without imposing input requirements of their own.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Add `promptVersion string` to the shared run/render request configuration.
|
||||
Register `--prompt-version <version>` for both `run` and `render`, include it
|
||||
in usage text, and map it without local normalization to
|
||||
`promptkit.RunRequest.PromptVersion`.
|
||||
2. Make `--input` optional for `run` and `render`. Do not weaken
|
||||
`parseMappings` for callers that actually supplied mappings; instead, skip
|
||||
that parser when no input flag was supplied and pass a nil or empty input
|
||||
map to Promptkit. Promptkit remains responsible for declared-required and
|
||||
template-referenced input failures.
|
||||
3. In `POST /v1/runs`, remove the adapter check that rejects a nil or empty
|
||||
`inputs` object. Preserve strict JSON decoding and all validation of input
|
||||
references that are present. Continue mapping the already-supported
|
||||
`prompt_version` field exactly once into `RunRequest.PromptVersion`.
|
||||
4. Keep extra supplied inputs legal and retain the CLI file-reader and HTTP
|
||||
artifact containment and size policies.
|
||||
5. Update `docs/cli.md` and `docs/api.md` in this stage: prompt ID remains
|
||||
required, prompt version is optional, inputs are optional at the adapter
|
||||
boundary, and Promptkit decides whether the selected definition requires
|
||||
them. Update the shortest request examples only where necessary to avoid
|
||||
implying that all prompts need inputs.
|
||||
|
||||
### Verification
|
||||
|
||||
- Add CLI behavior tests proving that an omitted version selects a sole prompt
|
||||
definition, an explicit version selects the requested definition, and an
|
||||
omitted version fails clearly when a prompt ID has multiple versions.
|
||||
- Add CLI and HTTP adapter coverage for: a prompt with no declared inputs,
|
||||
omitted optional inputs, a missing declared-required input, and a missing
|
||||
template-referenced input. Use a capturing runner for pure request mapping
|
||||
and one real Promptkit engine integration fixture for the definition-driven
|
||||
failures; do not reproduce Promptkit's parser tests.
|
||||
- Retain tests for malformed supplied mappings, invalid HTTP input references,
|
||||
artifact containment, and request-size limits.
|
||||
- Run `go test ./internal/adapter/cli ./internal/adapter/http` and the maintained
|
||||
render workflow.
|
||||
|
||||
### Completion Conditions
|
||||
|
||||
Both executable interfaces can select a specific prompt version and can submit
|
||||
no inputs; only Promptkit rejects missing definition-required data.
|
||||
|
||||
## Stage 3: Add Engine-Scoped Custom Backend Configuration
|
||||
|
||||
**Completion: Complete.**
|
||||
|
||||
Add the application-owned configuration needed for Promptkit profiles to select
|
||||
custom backend IDs and capacity policies.
|
||||
|
||||
### Configuration Contract
|
||||
|
||||
Use this exact strict-YAML shape:
|
||||
|
||||
```yaml
|
||||
backends:
|
||||
local-gpu:
|
||||
endpoint: http://localhost:11434/v1
|
||||
api_key_env: LOCAL_GPU_API_KEY
|
||||
extra_params:
|
||||
provider_option: enabled
|
||||
concurrency_limit: 2
|
||||
queue_capacity: 0
|
||||
```
|
||||
|
||||
The mapping key is the backend ID. `endpoint` is required by the effective
|
||||
contract. `api_key_env`, `extra_params`, `concurrency_limit`, and
|
||||
`queue_capacity` are optional. `concurrency_limit` uses zero as Promptkit's
|
||||
unlimited value. Represent `queue_capacity` as `*int` so omission uses
|
||||
Promptkit's default queue capacity and explicit zero disables queuing. There
|
||||
are no backend CLI overrides and no raw API-key field.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Add `Backends map[string]BackendConfig` to the on-disk `config.Config` and a
|
||||
`BackendConfig` with exactly the five YAML fields above. Add a resolved
|
||||
`BackendSettings` value containing `ID` plus the corresponding fields, and
|
||||
carry `[]BackendSettings` on `AppSettings`.
|
||||
2. During config application, sort the YAML map keys and build the resolved
|
||||
slice in that order. Preserve the `queue_capacity` pointer and the complete
|
||||
`extra_params` value. The deterministic order makes multi-error diagnostics
|
||||
and tests stable; it is not backend precedence.
|
||||
3. Do not duplicate Promptkit's backend validators in `internal/config`.
|
||||
Strict YAML decoding owns unknown-field and type errors. Promptkit
|
||||
`NewEngine` owns blank/reserved/duplicate IDs, endpoint and environment-name
|
||||
rules, reserved or invalid extra parameters, and capacity relationships.
|
||||
4. Refactor CLI engine construction around a small shared engine-settings value
|
||||
containing prompt, profile, and schema directories plus resolved backends.
|
||||
Convert every `BackendSettings` to `promptkit.Backend`, register it with
|
||||
`promptkit.WithBackend`, and use the same constructor for `run`, `render`,
|
||||
and `serve`. Wrap construction failures with application-configuration and
|
||||
engine-initialization context while preserving `errors.Is` identities.
|
||||
5. Retain one immutable backend registry per constructed engine. Do not create
|
||||
registries per HTTP request, and do not add direct imports of Promptkit's
|
||||
catalog modules; Promptkit's built-ins remain available automatically.
|
||||
6. Document `backends` in `docs/config.md`, including key identity,
|
||||
queue-capacity presence, validation ownership, optional environment
|
||||
credentials, and the raw-secret prohibition. Update
|
||||
`docs/internal/adapters.md` and `docs/internal/sources.md` to describe the
|
||||
implemented mapping and engine assembly. Add the exact shape to
|
||||
`examples/config.full.yml`; keep the minimal config minimal.
|
||||
|
||||
### Verification
|
||||
|
||||
- In `internal/config`, test strict decoding of all fields, unknown and raw
|
||||
secret-field rejection, deterministic resolved ordering, JSON-compatible
|
||||
nested `extra_params`, and omitted versus explicit-zero `queue_capacity`.
|
||||
- At the CLI/engine boundary, test a valid custom backend selected by a custom
|
||||
profile through `render`, and representative Promptkit validation failures:
|
||||
reserved backend ID, invalid endpoint, invalid capacity relationship, and a
|
||||
reserved `extra_params` key. Assert the stable error identity or a small
|
||||
diagnostic fragment, not Promptkit's full text.
|
||||
- Test that endpoint-only profiles and built-in profiles still work without a
|
||||
`backends` section.
|
||||
- Run `go test ./internal/config ./internal/adapter/cli` and render with the
|
||||
complete example without contacting a provider.
|
||||
|
||||
### Completion Conditions
|
||||
|
||||
Every command constructs its engine from the same resolved custom-backend
|
||||
configuration, Promptkit remains the validator, and profiles can select custom,
|
||||
built-in, or endpoint-only execution targets.
|
||||
|
||||
## Stage 4: Expose Session And Presence-Aware Reasoning Controls
|
||||
|
||||
**Completion: Complete.**
|
||||
|
||||
Complete the request mapping for direct session identifiers and the v0.9.0
|
||||
reasoning override semantics.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. Add `--session-id <id>` and `--reasoning-effort <value>` to both `run` and
|
||||
`render`, with no aliases. Map the session value to
|
||||
`RunRequest.SessionID`.
|
||||
2. Track flag presence for `--reasoning-effort`. Omission leaves
|
||||
`ExecutionTargetOverride.ReasoningEffort` nil, a nonblank value supplies a
|
||||
pointer to that value, and `--reasoning-effort=` supplies a pointer to the
|
||||
empty string. Include reasoning presence when deciding whether to allocate
|
||||
the enclosing execution override.
|
||||
3. Add optional `session_id` to the strict HTTP run-request DTO and map it to
|
||||
`RunRequest.SessionID` without application-level normalization. Keep
|
||||
`model.reasoning_effort` pointer-valued so omitted, nonblank, and empty JSON
|
||||
strings remain distinct.
|
||||
4. Add optional `session_id` to successful HTTP metadata and populate it from
|
||||
the effective `RunResult.SessionID`, not directly from the request. Use
|
||||
`omitempty`; absence means no effective session ID.
|
||||
5. Continue to reject unknown HTTP fields and do not add `api_key`, appended
|
||||
messages, or request-level output-contract fields.
|
||||
6. Update `docs/cli.md` and `docs/api.md` with the exact flag/field names,
|
||||
inheritance/replacement/clearing semantics, the non-secret nature and
|
||||
Promptkit length rules of session IDs, and effective-session response
|
||||
behavior.
|
||||
|
||||
### Verification
|
||||
|
||||
- CLI tests must distinguish an omitted reasoning flag, a nonblank replacement,
|
||||
and an explicit empty clear by observing the resulting prepared run through
|
||||
`render`. Add a mapping assertion that an explicit empty value still creates
|
||||
an execution override.
|
||||
- HTTP tests must capture and distinguish the same three reasoning states and
|
||||
continue to reject non-string values under the strict DTO contract. Treat
|
||||
JSON `null` like omission, matching ordinary pointer-field decoding; the
|
||||
documented explicit clear remains the empty JSON string.
|
||||
- CLI and HTTP tests must cover a direct session ID, a definition-rendered
|
||||
session ID when no direct value is supplied, no effective session ID, and an
|
||||
invalid overlong direct ID. Assert that successful HTTP metadata reports the
|
||||
engine's effective result.
|
||||
- Run `go test ./internal/adapter/cli ./internal/adapter/http ./internal/format`.
|
||||
|
||||
### Completion Conditions
|
||||
|
||||
CLI and HTTP callers can inherit, replace, or clear reasoning effort and can
|
||||
supply a direct session ID, with Promptkit retaining normalization and
|
||||
validation ownership.
|
||||
|
||||
## Stage 5: Present Backend Identity And Map Capacity Outcomes
|
||||
|
||||
**Completion: Complete.**
|
||||
|
||||
Expose Promptkit's selected routing identity and make overload behavior a
|
||||
stable application contract.
|
||||
|
||||
### Implementation
|
||||
|
||||
1. In prepared-run text output, add `selected_backend_id` immediately after
|
||||
`selected_profile_id` when it is nonempty. Preserve direct JSON marshaling of
|
||||
`promptkit.PreparedRun`; v0.9.0 already supplies `selected_backend_id` and
|
||||
`effective_model_params.backend_id` JSON fields.
|
||||
2. In the CLI run summary, append `backend=<id>` when
|
||||
`RunResult.SelectedBackendID` is nonempty. Omit the token for endpoint-only
|
||||
profiles rather than deriving an ID from the endpoint.
|
||||
3. Add `selected_backend_id,omitempty` to HTTP metadata and
|
||||
`backend_id,omitempty` to `metadata.model_params`, mapping them from
|
||||
`RunResult.SelectedBackendID` and `ExecutionTarget.BackendID` respectively.
|
||||
Keep the two values independently mapped so an upstream inconsistency is not
|
||||
hidden by Scriptorium.
|
||||
4. In HTTP error mapping, classify `promptkit.ErrCapacityExceeded` before the
|
||||
general generation failure and return HTTP 503 with code
|
||||
`capacity_exceeded` and message `model backend capacity is exhausted`. Do
|
||||
not emit `Retry-After` and do not expose `CapacityError` details.
|
||||
5. In CLI `run`, classify the same identity as the existing runtime-error exit
|
||||
status 1 and print the stable safe diagnostic
|
||||
`run error: model backend capacity is exhausted`. Keep all other runtime
|
||||
error handling unchanged.
|
||||
6. Preserve `serveCommand`'s one-engine/one-handler construction outside the
|
||||
request path. Do not add per-request engine construction; that would split
|
||||
capacity state and violate ADR 0004.
|
||||
7. Update `docs/cli.md`, `docs/api.md`, and `docs/operations.md` with backend
|
||||
identity, endpoint-only omission, capacity status, lack of retry timing,
|
||||
shared per-engine admission, and the distinction between transient
|
||||
in-flight state and durable workflow state.
|
||||
|
||||
### Verification
|
||||
|
||||
- Formatter, CLI, and HTTP tests must cover a registered backend and an
|
||||
endpoint-only profile. Assert supplied backend IDs and empty/omitted identity
|
||||
rather than endpoint-derived guesses.
|
||||
- Add HTTP error-table coverage proving capacity is 503
|
||||
`capacity_exceeded`, provider failure remains 502 `llm_failed`, and neither
|
||||
response leaks wrapped provider or capacity diagnostics.
|
||||
- Add one offline concurrent HTTP integration test using a single real
|
||||
Promptkit engine, a custom backend with `concurrency_limit: 1` and explicit
|
||||
`queue_capacity: 0`, and a blocking fake LLM client. Hold the first request
|
||||
in generation, issue a second request, assert the second receives the 503
|
||||
contract, release the first, assert it succeeds, and assert observed peak
|
||||
generation concurrency is one. This test owns the Scriptorium shared-engine
|
||||
interaction; do not replicate Promptkit's FIFO or broader scheduler suite.
|
||||
- Run `go test -race ./internal/adapter/http ./internal/adapter/cli ./internal/format`.
|
||||
|
||||
### Completion Conditions
|
||||
|
||||
All presentations report actual backend identity when present, endpoint-only
|
||||
profiles remain identity-free, and one server instance enforces one engine's
|
||||
backend capacity across concurrent requests.
|
||||
|
||||
## Stage 6: Add Stable Prompt Inspection Output And CLI
|
||||
|
||||
**Completion: Complete.**
|
||||
|
||||
Introduce reusable application-owned inspection presentation, then expose
|
||||
prompt inspection without model execution.
|
||||
|
||||
### Presentation Contract
|
||||
|
||||
1. Generalize the internal prepared-run format selector to an internal
|
||||
`OutputFormat` with `text` and `json` values, a text default, and one parser.
|
||||
Update config and prepared-run call sites to use it without changing
|
||||
existing prepared-run bytes or `defaults.render_format` behavior.
|
||||
2. Define a Scriptorium-owned prompt-inspection DTO; never marshal
|
||||
`promptkit.PromptInspection` directly. JSON contains these fields in this
|
||||
shape, including empty strings and an empty array where applicable:
|
||||
|
||||
```json
|
||||
{
|
||||
"prompt_id": "example",
|
||||
"prompt_version": "1",
|
||||
"prompt_hash": "opaque",
|
||||
"default_profile_id": "",
|
||||
"inputs": [
|
||||
{
|
||||
"name": "source",
|
||||
"required": true,
|
||||
"content_type": "text/plain",
|
||||
"description": "Input text"
|
||||
}
|
||||
],
|
||||
"output_contract": {
|
||||
"format": "text",
|
||||
"validation_mode": "none",
|
||||
"schema_path": "",
|
||||
"repair_attempts": 0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
3. Preserve Promptkit's declared input order. Emit JSON with deterministic
|
||||
indentation and one trailing newline. Text output uses the same field order,
|
||||
renders `inputs: []` when empty, lists input records in declaration order,
|
||||
quotes free-form descriptions safely, includes empty declared values, and
|
||||
ends in one newline. The prompt hash is opaque and must not be interpreted.
|
||||
|
||||
### CLI Contract And Implementation
|
||||
|
||||
1. Add `inspect` to top-level dispatch and usage, with implementation in a new
|
||||
focused CLI file rather than further expanding `run.go`.
|
||||
2. Implement:
|
||||
|
||||
```text
|
||||
scriptorium inspect prompt --prompt ID [--prompt-version VERSION]
|
||||
[--config PATH] [--prompt-dir DIR] [--format text|json] [--out PATH]
|
||||
```
|
||||
|
||||
`--prompt` is required. All other flags are optional. Positional arguments
|
||||
and unknown inspection modes are errors. There are no deprecated aliases.
|
||||
3. Apply ordinary config discovery and CLI-over-file precedence. Require an
|
||||
effective prompt directory, pass the optional version directly to
|
||||
`Engine.InspectPrompt`, and construct the engine through the shared settings
|
||||
path so configured backends are validated consistently. Do not require a
|
||||
profile or schema directory for this operation.
|
||||
4. Default inspection format to text independently of
|
||||
`defaults.render_format`; `--format` is the only inspection format override.
|
||||
Reuse ordinary output-file handling. Write output only after successful
|
||||
inspection and formatting. Use exit 0 for success and exit 1 for parsing,
|
||||
configuration, load, formatting, or write failure.
|
||||
5. Update `docs/cli.md`, `docs/internal/adapters.md`, and the internal format
|
||||
component description. Make clear that inspection loads and normalizes a
|
||||
definition but does not resolve a profile, load a schema, render templates,
|
||||
reserve backend capacity, or call a model.
|
||||
|
||||
### Verification
|
||||
|
||||
- Formatter tests cover deterministic text and JSON, declared order, empty
|
||||
inputs/default profile/schema path, all nonempty fields, and nil inspection
|
||||
handling. Prefer structural JSON assertions plus a few stable text fragments
|
||||
over large snapshots.
|
||||
- CLI tests cover required flags, rejected positional arguments, config source
|
||||
selection, `--prompt-dir` precedence, sole and explicit multi-version
|
||||
selection, default and explicit formats, stdout versus `--out`, no partial
|
||||
output on failure, and Promptkit prompt-not-found/load identities.
|
||||
- Use a fake or deliberately failing model client to prove inspection performs
|
||||
no generation; do not assert Promptkit's internal call sequence.
|
||||
- Run `go test ./internal/format ./internal/adapter/cli`.
|
||||
|
||||
### Completion Conditions
|
||||
|
||||
`inspect prompt` produces stable Scriptorium text or JSON for every valid
|
||||
directory-backed v0.9.0 prompt definition without inputs or model access, and
|
||||
prepared-run presentation remains backward compatible.
|
||||
|
||||
## Stage 7: Add Stable Profile Inspection Output And CLI
|
||||
|
||||
**Completion: Complete.**
|
||||
|
||||
Complete the inspection command family with inherited, built-in, custom, and
|
||||
endpoint-only profile support.
|
||||
|
||||
### Presentation Contract
|
||||
|
||||
1. Define a Scriptorium-owned profile-inspection DTO; never marshal
|
||||
`promptkit.ProfileInspection` directly. JSON has the following complete
|
||||
shape. Keep empty strings and zero numeric values because Promptkit reports
|
||||
unresolved optional controls as zero, and normalize absent `extra_params`
|
||||
to an empty object:
|
||||
|
||||
```json
|
||||
{
|
||||
"profile_id": "example",
|
||||
"effective_model_params": {
|
||||
"backend_id": "",
|
||||
"endpoint": "https://example.test/v1",
|
||||
"model": "model-name",
|
||||
"temperature": 0,
|
||||
"max_tokens": 0,
|
||||
"top_p": 0,
|
||||
"timeout_seconds": 0,
|
||||
"service_tier": "",
|
||||
"reasoning_effort": "",
|
||||
"api_key_env": "",
|
||||
"extra_params": {}
|
||||
},
|
||||
"api_key_required": false
|
||||
}
|
||||
```
|
||||
|
||||
2. Text output follows the same order, nests effective model parameters,
|
||||
encodes extra-parameter values as deterministic compact JSON with sorted
|
||||
keys, includes an empty `backend_id` for endpoint-only profiles, and ends in
|
||||
one newline. Never read or print an environment-variable value or direct
|
||||
API key.
|
||||
|
||||
### CLI Contract And Implementation
|
||||
|
||||
1. Implement the second mode:
|
||||
|
||||
```text
|
||||
scriptorium inspect profile --profile ID
|
||||
[--config PATH] [--profile-dir DIR] [--format text|json] [--out PATH]
|
||||
```
|
||||
|
||||
`--profile` is required. The common inspection parse, format, output, and
|
||||
exit rules from Stage 6 apply. No prompt, prompt directory, schema directory,
|
||||
or input is required, so built-in profiles are inspectable with only their
|
||||
ID.
|
||||
2. Resolve config and `--profile-dir` with normal precedence, construct the
|
||||
engine through the shared settings path so custom backends participate, and
|
||||
call `Engine.InspectProfile` exactly once.
|
||||
3. Update `docs/cli.md`, `docs/config.md`, and internal adapter/source docs.
|
||||
Explain that profile inspection resolves inheritance and backend defaults
|
||||
but does not read credentials, load a prompt, reserve capacity, or call a
|
||||
model. Document that zero provider-control values may mean unspecified,
|
||||
consistent with Promptkit v0.9.0.
|
||||
|
||||
### Verification
|
||||
|
||||
- Formatter tests cover deterministic text and JSON, registered and empty
|
||||
backend identity, optional environment-variable names, `api_key_required`,
|
||||
nested extra parameters, and the absence of credential values.
|
||||
- CLI tests cover built-in, inherited custom, custom-backend, and endpoint-only
|
||||
profiles; config and `--profile-dir` precedence; default and explicit format;
|
||||
stdout and `--out`; missing and invalid profiles; no partial output; and no
|
||||
requirement for `prompt_dir`.
|
||||
- Include a sentinel environment secret in a test and assert it appears in no
|
||||
inspection output or diagnostic while its variable name may appear.
|
||||
- Run `go test ./internal/format ./internal/adapter/cli ./internal/config`.
|
||||
|
||||
### Completion Conditions
|
||||
|
||||
`inspect profile` reports the complete safe effective profile view for every
|
||||
valid directory-backed v0.9.0 profile and configured backend combination
|
||||
without requiring a prompt or contacting a model.
|
||||
|
||||
## Stage 8: Complete Cross-Feature Conformance And Release Validation
|
||||
|
||||
**Completion: Complete.**
|
||||
|
||||
Close the roadmap with representative definition-level integration coverage,
|
||||
copyable examples, documentation alignment, and full validation. Do not defer
|
||||
contract documentation from earlier stages to this stage; this is a final
|
||||
cross-check.
|
||||
|
||||
### Implementation And Documentation
|
||||
|
||||
1. Maintain one compact shared fixture set under
|
||||
`testdata/promptkit-v0.9/` with `prompts`, `profiles`, `schemas`, and any
|
||||
message content files needed by CLI and HTTP integration tests. Through a
|
||||
small set of composable definitions, cover:
|
||||
|
||||
- inline and file-backed messages;
|
||||
- `developer`, `system`, `user`, and `assistant` roles and message cache
|
||||
control;
|
||||
- required and optional inputs plus template-referenced values;
|
||||
- prompt versions, default profiles, and session-ID templates;
|
||||
- text and JSON output plus `none`, `basic`, `json`, and `json_schema`
|
||||
validation, a schema path, and a valid positive repair budget;
|
||||
- standalone and inherited profiles;
|
||||
- built-in, configured custom, and endpoint-only backends;
|
||||
- model controls, service tier, reasoning effort, timeout, optional
|
||||
credential environment names, and nested JSON-compatible extra
|
||||
parameters.
|
||||
|
||||
Exercise these fixtures through Scriptorium's `render` and inspection
|
||||
boundaries. Assert representative effective outcomes; do not independently
|
||||
decode YAML or exhaustively retest every Promptkit field combination.
|
||||
2. Add a copyable `examples/render-v0.9-features.sh` workflow that uses only
|
||||
`render` and `inspect` operations to demonstrate prompt-version selection,
|
||||
a direct session ID, reasoning replacement or clearing, a configured custom
|
||||
backend, and both inspection modes without provider access. Keep
|
||||
`examples/config.full.yml` complete and keep raw credentials out of every
|
||||
example.
|
||||
3. Review every canonical current-state owner named by the feature roadmap.
|
||||
Remove stale claims that inputs are always required, that every command
|
||||
needs `prompt_dir`, or that v0.1.0 semantics still apply. Ensure CLI, API,
|
||||
configuration, operations, internal adapter/source, development, and
|
||||
overview documents link instead of duplicating one another's contracts.
|
||||
Change architecture policy or internal component boundaries only if the
|
||||
implementation actually changed those durable responsibilities.
|
||||
4. Verify that all explicit non-goals remain absent: appended messages,
|
||||
retained prepared handles, HTTP inspection routes, alternative Promptkit
|
||||
source kinds, request output-contract replacement, public provider details,
|
||||
direct catalog imports, raw API keys, and durable workflow state.
|
||||
5. After every completion criterion below passes, change the feature roadmap's
|
||||
Status from `Planned` to `Complete`. Do not mark it complete for a partial
|
||||
implementation.
|
||||
|
||||
### Final Validation
|
||||
|
||||
Run all of the following from the repository root:
|
||||
|
||||
```bash
|
||||
gofmt -w $(git ls-files '*.go')
|
||||
go test ./...
|
||||
go test -race ./...
|
||||
go vet ./...
|
||||
go build ./cmd/scriptorium
|
||||
go run ./cmd/scriptorium render --config ./examples/config.yml --prompt generic.markdown_summary --input transcript=./examples/fixtures/transcript.md --input glossary=./examples/fixtures/glossary.yml
|
||||
bash ./examples/render-v0.9-features.sh
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Also verify the architecture guard, strict example decoding, documentation
|
||||
links, executable permissions on shell examples, and absence of secrets. Review
|
||||
the suite for redundant upstream-semantic tests and retain only tests that
|
||||
protect Scriptorium mappings, presentation, configuration, concurrency
|
||||
integration, or complete workflows.
|
||||
|
||||
### Completion Conditions
|
||||
|
||||
Every roadmap validation criterion is satisfied on the tagged dependency; all
|
||||
current-state documentation and examples describe implemented behavior; the
|
||||
full application supports every Promptkit v0.9.0 feature expressible in its
|
||||
directory-backed prompt and profile definitions; and the ADR 0004 non-goals
|
||||
remain intact.
|
||||
|
||||
## Open Questions
|
||||
|
||||
None. The external syntax, application mappings, validation ownership,
|
||||
inspection wire shapes, concurrency behavior, testing boundaries, and stage
|
||||
order required for implementation are fixed above.
|
||||
351
docs/roadmap/promptkit-v0.9.0-adoption.md
Normal file
351
docs/roadmap/promptkit-v0.9.0-adoption.md
Normal file
@@ -0,0 +1,351 @@
|
||||
# Promptkit v0.9.0 Adoption Roadmap
|
||||
|
||||
## Status
|
||||
|
||||
Complete. This document records the delivered feature scope and targeted end
|
||||
state. The completed delivery plan is maintained in
|
||||
[implementation.md](implementation.md); current behavior remains defined by
|
||||
the canonical application contracts linked below.
|
||||
|
||||
The durable product and integration boundary for this work is established by
|
||||
[ADR 0004](../adr/0004-definition-boundary.md). This roadmap applies that
|
||||
decision to Promptkit v0.9.0 and owns implementation scope and completion
|
||||
status.
|
||||
|
||||
## Objective
|
||||
|
||||
Bring Scriptorium from Promptkit v0.1.0 to v0.9.0 and expose the newer
|
||||
framework capabilities that fit Scriptorium's existing CLI, HTTP,
|
||||
configuration, presentation, and process boundaries.
|
||||
|
||||
Scriptorium must accept the complete Promptkit v0.9.0 feature set expressed by
|
||||
directory-backed prompt, profile, and schema definitions. Application adapters
|
||||
must not narrow that format contract by requiring request values that the
|
||||
selected definition does not require or by omitting selection values needed to
|
||||
address a valid definition. Promptkit's tagged
|
||||
[framework format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md)
|
||||
remains canonical; Scriptorium must delegate parsing and framework validation
|
||||
rather than reproduce those rules.
|
||||
|
||||
The completed application will retain Promptkit as the owner of reusable
|
||||
prompt execution, source formats, profiles, backend definitions, capacity,
|
||||
inspection, generation, and validation. Scriptorium will continue to own
|
||||
application configuration, command and HTTP contracts, error and exit mapping,
|
||||
output presentation, and deployment policy.
|
||||
|
||||
## Targeted End State
|
||||
|
||||
When this roadmap is complete:
|
||||
|
||||
- Scriptorium pins the released Promptkit v0.9.0 module without a workspace,
|
||||
replacement, vendored source, direct catalog import, or Promptkit internal
|
||||
import.
|
||||
- Existing valid CLI, HTTP, configuration, prompt, profile, schema, and render
|
||||
workflows remain supported subject to Promptkit's documented v0.2.0 through
|
||||
v0.9.0 compatibility corrections.
|
||||
- Every Promptkit v0.9.0 prompt and profile feature expressible in definitions
|
||||
under Scriptorium's configured directory sources is usable through the
|
||||
executable application, including prompts with no inputs and explicitly
|
||||
selected prompt versions.
|
||||
- CLI and HTTP requests can supply a direct non-secret session ID.
|
||||
- HTTP requests and a presence-aware CLI flag can inherit, replace, or
|
||||
explicitly clear reasoning effort.
|
||||
- Prepared output, CLI run summaries, and HTTP result metadata expose effective
|
||||
backend identity when Promptkit supplies one.
|
||||
- HTTP capacity rejection has a stable Scriptorium-owned overload response.
|
||||
- Application configuration can register reusable engine-scoped custom
|
||||
backends, including bounded concurrency and queue policy, for selection by
|
||||
Promptkit profiles.
|
||||
- CLI users can inspect prompt and profile definitions without supplying
|
||||
placeholder inputs or invoking a model.
|
||||
- Scriptorium's contracts and examples point to Promptkit v0.9.0 and accurately
|
||||
describe its current credential, validation-repair, provider-control,
|
||||
message-role, backend, and source-validation semantics.
|
||||
|
||||
## Feature Scope
|
||||
|
||||
### Complete Prompt And Profile Definition Compatibility
|
||||
|
||||
Scriptorium will treat the tagged Promptkit v0.9.0 format contract as one
|
||||
indivisible downstream compatibility boundary. This includes prompt identity
|
||||
and version selection, optional and required input declarations, message and
|
||||
content-file templates, supported roles, cache control, session templates,
|
||||
default profiles, output contracts, schemas, and repair budgets. It also
|
||||
includes standalone and inherited profiles, built-in and custom backend
|
||||
selection, endpoint overrides, execution controls, optional credential
|
||||
environment sources, and JSON-compatible extra parameters.
|
||||
|
||||
Most of this support is provided by passing configured prompt, profile, and
|
||||
schema directories directly to Promptkit. Scriptorium will not independently
|
||||
decode these framework definitions or maintain a field-level subset. Two
|
||||
current adapter restrictions must be removed so the executable interfaces do
|
||||
not reject definitions that Promptkit accepts:
|
||||
|
||||
- `run` and `render` will accept an optional prompt-version flag and map it to
|
||||
`promptkit.RunRequest.PromptVersion`; and
|
||||
- `run`, `render`, and `POST /v1/runs` will permit omitted or empty input maps.
|
||||
Promptkit will decide whether an input is required by the selected
|
||||
definition or referenced template.
|
||||
|
||||
The HTTP API already carries `prompt_version`; its mapping must remain covered
|
||||
by adapter tests. CLI and HTTP callers may continue to supply extra inputs,
|
||||
subject to their existing file, containment, and size policies. Removing the
|
||||
application-level nonempty-input requirement does not weaken Promptkit's
|
||||
declared-required-input or template-reference validation.
|
||||
|
||||
Profiles that name built-in backend IDs require no application registration.
|
||||
Profiles that name other backend IDs become fully usable through the custom
|
||||
backend configuration in this roadmap. Profile inheritance and all other
|
||||
profile-field resolution remain Promptkit responsibilities.
|
||||
|
||||
This compatibility requirement concerns fields and behavior expressed in
|
||||
Promptkit YAML definitions. Go-consumer construction alternatives such as
|
||||
single-file sources, injected `fs.FS` sources, in-memory profiles, fallback
|
||||
profile filesystems, and prepared-execution handles are library-integration
|
||||
features and are not required merely to support the complete definition
|
||||
format.
|
||||
|
||||
### Promptkit v0.9.0 Compatibility Baseline
|
||||
|
||||
Required dependency and source changes:
|
||||
|
||||
- update `go.mod` and `go.sum` to select Promptkit v0.9.0 and its selected
|
||||
OpenRouter and Rakestrawhome catalog dependencies;
|
||||
- change Scriptorium's inbound HTTP reasoning-effort representation to preserve
|
||||
Promptkit's pointer semantics;
|
||||
- retain keyed Promptkit public struct literals and confirm no removed
|
||||
`RunRequest.Metadata` use exists;
|
||||
- update tagged Promptkit links and version references throughout Scriptorium;
|
||||
and
|
||||
- preserve Scriptorium's architecture guard against replacements, workspaces,
|
||||
vendored Promptkit code, former facade packages, and Promptkit internal
|
||||
imports.
|
||||
|
||||
Required compatibility review:
|
||||
|
||||
- verify application and maintained example endpoints satisfy Promptkit's
|
||||
absolute HTTP/HTTPS endpoint rules;
|
||||
- verify prompt content paths, file artifacts, identities, JSON documents, and
|
||||
JSON-compatible extra parameters satisfy the v0.6.0 safety boundaries;
|
||||
- verify maintained prompt roles are limited to `developer`, `system`, `user`,
|
||||
and `assistant`;
|
||||
- verify every positive repair budget is no greater than three and is paired
|
||||
with `basic`, `json`, or `json_schema` validation;
|
||||
- document that positive repair budgets authorize additional provider calls,
|
||||
latency, token use, and cost;
|
||||
- document that unset optional provider controls are omitted and that a zero
|
||||
effective value can represent an unspecified provider control unless an
|
||||
explicit request override supplied it; and
|
||||
- update credential guidance for Promptkit's optional environment lookup
|
||||
behavior, profile `api_key_env` semantics, and Scriptorium's prohibition on
|
||||
raw API-key inputs.
|
||||
|
||||
### Session And Reasoning Request Controls
|
||||
|
||||
The CLI `run` and `render` commands will accept a direct session-ID flag and map
|
||||
it to `promptkit.RunRequest.SessionID`. The value remains non-secret
|
||||
application correlation metadata and is subject to Promptkit's normalization
|
||||
and length rules.
|
||||
|
||||
The HTTP run request will accept `session_id` and map it through the existing
|
||||
strict DTO boundary. Successful HTTP metadata will report the effective
|
||||
session ID when present.
|
||||
|
||||
The CLI `run` and `render` commands will also accept a presence-aware reasoning
|
||||
effort flag. Omission inherits the selected profile, a nonblank value replaces
|
||||
it, and an explicitly supplied empty value clears it. The existing HTTP
|
||||
`model.reasoning_effort` field will gain the same three-state behavior while
|
||||
remaining a JSON string when present.
|
||||
|
||||
These additions must not introduce raw direct API-key flags or fields.
|
||||
|
||||
### Backend Identity And Capacity Outcomes
|
||||
|
||||
Scriptorium presentation will expose `SelectedBackendID` and the effective
|
||||
target's `BackendID` without deriving identity from endpoint text. Backend
|
||||
identity will be included where applicable in:
|
||||
|
||||
- prepared-run text and JSON output;
|
||||
- the CLI run summary; and
|
||||
- HTTP response metadata and effective model parameters.
|
||||
|
||||
Endpoint-only profiles continue to have no backend ID, and empty identities
|
||||
must remain distinguishable from registered built-in or custom backends.
|
||||
|
||||
The HTTP adapter will classify `promptkit.ErrCapacityExceeded` separately from
|
||||
provider generation failures. The target public outcome is HTTP `503` with a
|
||||
stable `capacity_exceeded` code and a generic message. Scriptorium will not
|
||||
invent retry timing or expose an untrusted diagnostic. CLI capacity rejection
|
||||
continues to be a runtime failure, with a clear safe diagnostic and the
|
||||
existing runtime-error exit status.
|
||||
|
||||
The HTTP server will use one appropriately scoped Promptkit engine across its
|
||||
requests so backend admission, active-generation limits, and queue capacity
|
||||
apply across concurrent in-flight work. The server may retain the operational
|
||||
state needed for those limits and for request cancellation and lifecycle
|
||||
management. This does not create durable per-request, conversation, or resume
|
||||
state.
|
||||
|
||||
Scriptorium will continue to return generic public model-generation failures.
|
||||
Promptkit `GenerationError` provider code, type, and message values will not be
|
||||
added to the public HTTP response because they are untrusted and potentially
|
||||
sensitive.
|
||||
|
||||
### Custom Backend Configuration
|
||||
|
||||
The strict Scriptorium application configuration will gain an optional custom
|
||||
backend collection. Each entry will support the application-owned mapping
|
||||
needed to construct a public `promptkit.Backend`:
|
||||
|
||||
- backend ID;
|
||||
- OpenAI-compatible endpoint;
|
||||
- optional API-key environment-variable name;
|
||||
- optional JSON-compatible request-wide extra parameters;
|
||||
- non-negative concurrency limit; and
|
||||
- queue capacity with presence preserved so omission and explicit zero remain
|
||||
different.
|
||||
|
||||
Raw API-key values will remain invalid. Unknown fields will remain errors.
|
||||
Configured backends will be registered through `promptkit.WithBackend` during
|
||||
engine construction and will be immutable and engine-scoped. Promptkit will
|
||||
remain the canonical validator for backend IDs, endpoints, reserved request
|
||||
parameters, capacity bounds, duplicate or reserved IDs, environment-variable
|
||||
names, and JSON-compatible values. Scriptorium will add context appropriate to
|
||||
its configuration error boundary without copying those framework rules.
|
||||
|
||||
The collection will be configuration-file-owned. This roadmap does not add
|
||||
per-backend CLI flags. Existing endpoint-only profiles remain valid, while
|
||||
profiles may select configured custom backend IDs through Promptkit's existing
|
||||
`backend` field. Promptkit's built-in OpenRouter and Rakestrawhome backends and
|
||||
profiles remain available without Scriptorium registration.
|
||||
|
||||
The application configuration shape is a `backends` mapping whose keys are the
|
||||
backend IDs registered with Promptkit:
|
||||
|
||||
```yaml
|
||||
backends:
|
||||
local-gpu:
|
||||
endpoint: http://localhost:11434/v1
|
||||
api_key_env: LOCAL_GPU_API_KEY
|
||||
extra_params:
|
||||
provider_option: enabled
|
||||
concurrency_limit: 2
|
||||
queue_capacity: 0
|
||||
```
|
||||
|
||||
`endpoint` is required for each entry. The other fields are optional. Omitted
|
||||
`queue_capacity` uses Promptkit's default queue policy when concurrency is
|
||||
bounded, while an explicit zero disables queuing. The configuration contract
|
||||
and complete maintained configuration example will document this shape when
|
||||
the feature is implemented.
|
||||
|
||||
### Prompt And Profile Inspection CLI
|
||||
|
||||
Scriptorium will add a CLI inspection command with prompt and profile modes.
|
||||
The intended command family is:
|
||||
|
||||
- `scriptorium inspect prompt`, backed by `Engine.InspectPrompt`; and
|
||||
- `scriptorium inspect profile`, backed by `Engine.InspectProfile`.
|
||||
|
||||
Prompt inspection will select an ID and optional version and report the
|
||||
normalized prompt identity, opaque prompt hash, declared default profile,
|
||||
declared inputs, and normalized output contract. It will not require input
|
||||
artifacts, resolve a profile, load a schema, render templates, reserve backend
|
||||
capacity, or contact a model.
|
||||
|
||||
Profile inspection will select an explicit profile ID and report its resolved
|
||||
effective target, backend identity, optional credential source name, and
|
||||
whether a later request must provide credential configuration. It will not
|
||||
read credential values, load a prompt, reserve capacity, or contact a model.
|
||||
|
||||
Inspection will support deterministic Scriptorium-owned text and JSON output
|
||||
and normal output-file handling. Scriptorium will define explicit output DTOs
|
||||
rather than treating Promptkit inspection structs as stable wire formats.
|
||||
Inspection failures will use existing public Promptkit error identities and
|
||||
Scriptorium-owned CLI diagnostics and exit behavior.
|
||||
|
||||
The prompt form accepts `--prompt`, optional `--prompt-version`, `--config`,
|
||||
`--prompt-dir`, `--format text|json`, and `--out`. The profile form accepts
|
||||
`--profile`, `--config`, `--profile-dir`, `--format text|json`, and `--out`.
|
||||
Both forms reject positional arguments and default to text output independently
|
||||
of `defaults.render_format`. Configuration discovery and CLI-over-file source
|
||||
precedence match the existing command family. Prompt inspection requires an
|
||||
effective prompt directory; profile inspection does not, because Promptkit's
|
||||
built-in profiles remain inspectable without one.
|
||||
|
||||
## Documentation And Example Changes
|
||||
|
||||
Implementation of this roadmap requires coordinated updates to the canonical
|
||||
owners of affected behavior:
|
||||
|
||||
- `docs/cli.md` for session and reasoning flags, inspection commands, output,
|
||||
prompt-version selection, optional inputs, and exit behavior;
|
||||
- `docs/api.md` for `session_id`, effective backend/session metadata, reasoning
|
||||
presence semantics, optional inputs, and the capacity response;
|
||||
- `docs/config.md` for custom backend configuration and current Promptkit
|
||||
credential and provider-default semantics;
|
||||
- `docs/operations.md` for capacity and custom-backend operational guidance
|
||||
where deployment handling is affected;
|
||||
- `docs/internal/adapters.md` and `docs/internal/sources.md` for implemented
|
||||
mapping and engine-assembly behavior;
|
||||
- `docs/policy/architecture.md` and `docs/internal/overview.md` only if concrete
|
||||
component responsibilities or durable boundaries change; and
|
||||
- maintained examples for valid v0.9.0 prompt/profile formats and custom
|
||||
backend configuration.
|
||||
|
||||
Current-state documents must not describe these features as implemented until
|
||||
the corresponding code lands.
|
||||
|
||||
## Validation And Completion Criteria
|
||||
|
||||
The roadmap is complete only when:
|
||||
|
||||
- ordinary and race-enabled tests pass against the tagged Promptkit v0.9.0
|
||||
dependency;
|
||||
- `go vet ./...` and `go build ./cmd/scriptorium` pass;
|
||||
- Go formatting and `git diff --check` pass;
|
||||
- the architecture guard passes with no workspace, replacement, vendor tree,
|
||||
former facade, or Promptkit internal import;
|
||||
- maintained render, custom-backend, session, reasoning, and inspection
|
||||
examples execute successfully without real credentials or provider calls;
|
||||
- CLI tests cover explicit and omitted prompt versions for prompt IDs with one
|
||||
or multiple definitions;
|
||||
- CLI and HTTP tests cover prompts with no declared inputs, omitted optional
|
||||
inputs, missing required inputs, and template-referenced inputs;
|
||||
- HTTP tests cover omitted, replacement, and clearing reasoning states;
|
||||
- CLI and HTTP tests cover session propagation and invalid session handling;
|
||||
- presentation tests cover registered and endpoint-only backend identity;
|
||||
- HTTP tests cover capacity rejection independently from provider failure;
|
||||
- concurrent HTTP tests demonstrate that one server's requests share backend
|
||||
admission and cannot bypass capacity through per-request engine creation;
|
||||
- configuration tests cover strict decoding, precedence, queue-capacity
|
||||
presence, invalid backends, and secret-field rejection;
|
||||
- inspection tests cover deterministic text and JSON, source selection,
|
||||
output-file handling, safe credential presentation, and public error
|
||||
mapping;
|
||||
- all maintained prompt roles and repair budgets are valid under Promptkit
|
||||
v0.9.0; and
|
||||
- representative definition fixtures cover inline and file-backed messages,
|
||||
cache control, session templates, every output validation mode, schemas,
|
||||
profile inheritance, built-in and custom backends, execution controls,
|
||||
credential environment names, and extra parameters without Scriptorium
|
||||
independently parsing those fields; and
|
||||
- affected documentation links and copyable examples validate successfully.
|
||||
|
||||
## Explicit Non-Goals
|
||||
|
||||
This roadmap does not include:
|
||||
|
||||
- appended request messages in either executable interface;
|
||||
- retained or durable prepared-execution handles;
|
||||
- HTTP prompt or profile inspection routes;
|
||||
- embedded application fallback profiles;
|
||||
- single-file, injected `fs.FS`, or in-memory prompt and profile source
|
||||
configuration;
|
||||
- request-level output-contract replacement through the executable interfaces;
|
||||
- public disclosure of structured provider error details;
|
||||
- direct imports or configuration of Promptkit's external catalog modules;
|
||||
- raw API-key configuration, CLI flags, or HTTP fields;
|
||||
- durable run state, conversations, retries, archives, checkpoints, or resume;
|
||||
or
|
||||
- changes to Promptkit itself.
|
||||
@@ -1,361 +0,0 @@
|
||||
# Troubleshooting
|
||||
|
||||
This guide lists common implemented failure modes and safe fixes.
|
||||
|
||||
Canonical references:
|
||||
|
||||
- [CLI reference](cli.md)
|
||||
- [Configuration reference](config.md)
|
||||
- [HTTP API reference](api.md)
|
||||
- [Operations guide](operations.md)
|
||||
|
||||
## Missing Or Invalid Config
|
||||
|
||||
Symptom:
|
||||
|
||||
- CLI error includes `application config error`, `config file not found`, `invalid config YAML`, or `invalid config`.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- `--config` points to a missing file.
|
||||
- YAML syntax is invalid.
|
||||
- Config contains unknown fields or negative HTTP size limits.
|
||||
|
||||
Diagnostic step:
|
||||
|
||||
```bash
|
||||
go run ./cmd/scriptorium render --config /path/to/config.yml --prompt generic.markdown_summary --input transcript=./examples/fixtures/transcript.md --input glossary=./examples/fixtures/glossary.yml
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- Correct the config path.
|
||||
- Fix YAML syntax.
|
||||
- Remove unknown fields.
|
||||
- Keep raw secrets out of config.
|
||||
|
||||
Relevant links: [Configuration reference](config.md), [CLI reference](cli.md)
|
||||
|
||||
## Missing Prompt Directory
|
||||
|
||||
Symptom:
|
||||
|
||||
- CLI parse error says the prompt directory is required.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- Neither config nor CLI flags provide an effective `prompt_dir`.
|
||||
|
||||
Diagnostic step:
|
||||
|
||||
- Re-run once with explicit `--prompt-dir`.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- Set `prompt_dir` in config or pass `--prompt-dir`.
|
||||
|
||||
Relevant links: [Configuration reference](config.md), [CLI reference](cli.md)
|
||||
|
||||
## Unknown Flags
|
||||
|
||||
Symptom:
|
||||
|
||||
- CLI parse error for an unknown flag.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- Typo.
|
||||
- Flag is valid for another command.
|
||||
- `serve` was given runtime model override flags.
|
||||
|
||||
Diagnostic step:
|
||||
|
||||
- Compare the command with the command-specific flag list.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- Remove unsupported flags.
|
||||
- Use `run` or `render` for runtime model overrides.
|
||||
|
||||
Relevant links: [CLI reference](cli.md)
|
||||
|
||||
## Prompt Load Failures
|
||||
|
||||
Symptom:
|
||||
|
||||
- CLI run/render fails during prompt loading.
|
||||
- HTTP returns `404 prompt_not_found` or `400 prompt_load_failed`.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- Prompt ID/version does not exist.
|
||||
- Prompt YAML is invalid or has unknown fields.
|
||||
- Prompt contract is invalid, such as missing messages, invalid output mode, bad `content_file`, or missing `schema_path` for `json_schema`.
|
||||
|
||||
Diagnostic step:
|
||||
|
||||
```bash
|
||||
go run ./cmd/scriptorium render --config ./examples/config.yml --prompt <prompt-id> --input transcript=./examples/fixtures/transcript.md --input glossary=./examples/fixtures/glossary.yml --format json
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- Correct prompt ID/version.
|
||||
- Fix prompt YAML and referenced `content_file` paths.
|
||||
- Fix output contract fields.
|
||||
|
||||
Relevant links: [Configuration reference](config.md), [CLI reference](cli.md)
|
||||
|
||||
## Profile Load Failures
|
||||
|
||||
Symptom:
|
||||
|
||||
- CLI run/render fails during profile loading.
|
||||
- HTTP returns `404 profile_not_found`, `400 profile_load_failed`, or `400 profile_required`.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- Profile ID does not exist.
|
||||
- Request omitted profile and prompt has no `default_profile`.
|
||||
- Profile YAML is invalid or has unknown fields.
|
||||
- Profile contains raw `api_key`.
|
||||
|
||||
Diagnostic step:
|
||||
|
||||
```bash
|
||||
go run ./cmd/scriptorium render --config ./examples/config.yml --prompt generic.markdown_summary --profile <profile-id> --input transcript=./examples/fixtures/transcript.md --input glossary=./examples/fixtures/glossary.yml
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- Correct profile ID or prompt `default_profile`.
|
||||
- Fix profile YAML and value ranges.
|
||||
- Replace raw `api_key` with `api_key_env`.
|
||||
|
||||
Relevant links: [Configuration reference](config.md), [CLI reference](cli.md)
|
||||
|
||||
## Input Artifact Failures
|
||||
|
||||
Symptom:
|
||||
|
||||
- CLI run/render fails while reading inputs.
|
||||
- HTTP returns `400 artifact_read_failed`, `400 artifact_not_allowed`, or `413 artifact_too_large`.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- Input file path is missing or unreadable.
|
||||
- HTTP input type is unsupported or missing required fields.
|
||||
- HTTP file refs are disabled because no artifact root is configured.
|
||||
- HTTP file path is lexically outside the artifact root.
|
||||
- HTTP file input exceeds `server.max_artifact_bytes`.
|
||||
|
||||
Diagnostic step:
|
||||
|
||||
- Verify each input path exists and is readable by the process.
|
||||
- For HTTP, verify input refs use `file` or `inline`.
|
||||
- For HTTP file refs, verify the artifact root and compare file size to `server.max_artifact_bytes`.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- Correct paths and permissions.
|
||||
- Configure a narrow artifact root for HTTP file refs.
|
||||
- Use relative paths under the artifact root or switch to `inline`.
|
||||
- Increase `server.max_artifact_bytes` only for expected larger inputs.
|
||||
|
||||
Relevant links: [HTTP API reference](api.md), [Configuration reference](config.md)
|
||||
|
||||
## Missing API-Key Environment Variable
|
||||
|
||||
Symptom:
|
||||
|
||||
- CLI render/run fails with an API-key environment error.
|
||||
- HTTP returns `400 api_key_env_missing`.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- Selected profile or runtime override sets `api_key_env`, but the environment variable is unset or empty.
|
||||
|
||||
Diagnostic step:
|
||||
|
||||
```bash
|
||||
printenv SCRIPTORIUM_API_KEY
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- Set the required environment variable before starting the CLI command or HTTP service.
|
||||
- Or use a profile that does not require provider API-key auth.
|
||||
|
||||
Relevant links: [Configuration reference](config.md), [Operations guide](operations.md)
|
||||
|
||||
## Prompt Template Render Failures
|
||||
|
||||
Symptom:
|
||||
|
||||
- CLI render/run fails during prompt rendering.
|
||||
- HTTP returns `400 prompt_render_failed`.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- Template references an input that was not supplied.
|
||||
- Template syntax or variable reference is invalid.
|
||||
|
||||
Diagnostic step:
|
||||
|
||||
- Run `render --format json` with the same prompt, inputs, vars, and profile.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- Align `{{input "name"}}` references with request input names.
|
||||
- Fix template syntax and variable names.
|
||||
|
||||
Relevant links: [Configuration reference](config.md), [CLI reference](cli.md)
|
||||
|
||||
## LLM Request Failures
|
||||
|
||||
Symptom:
|
||||
|
||||
- CLI `run` fails during generation.
|
||||
- HTTP returns `502 llm_failed`.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- Endpoint is unreachable.
|
||||
- Provider returns non-2xx.
|
||||
- Request times out.
|
||||
- Provider response is malformed.
|
||||
|
||||
Diagnostic step:
|
||||
|
||||
- Run `render` first to confirm pre-LLM preparation works.
|
||||
- Check selected endpoint/model in prepared output.
|
||||
- Check network/provider logs for timeout or non-2xx details.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- Correct endpoint/model/profile settings.
|
||||
- Adjust timeout when appropriate.
|
||||
- Resolve provider or network issue.
|
||||
|
||||
Relevant links: [Operations guide](operations.md), [Configuration reference](config.md)
|
||||
|
||||
## Validation Failed
|
||||
|
||||
Symptom:
|
||||
|
||||
- CLI `run` exits `2`.
|
||||
- HTTP returns `200 OK` with `validation.status` set to `failed`.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- Generated output failed `basic`, `json`, or `json_schema` content validation.
|
||||
|
||||
Diagnostic step:
|
||||
|
||||
- Inspect validation errors in CLI stderr or the HTTP response.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- Refine prompt instructions.
|
||||
- Adjust schema or model/profile settings.
|
||||
- Rerun after correction.
|
||||
|
||||
Relevant links: [Operations guide](operations.md), [HTTP API reference](api.md)
|
||||
|
||||
## Validation Runtime Failure
|
||||
|
||||
Symptom:
|
||||
|
||||
- CLI `run` fails with validation runtime error.
|
||||
- HTTP returns `500 validation_runtime_failed`.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- `json_schema` schema file is missing or unreadable.
|
||||
- Schema JSON is invalid.
|
||||
|
||||
Diagnostic step:
|
||||
|
||||
- Verify `schema_dir` and prompt `output.schema_path`.
|
||||
- Check schema file readability and JSON syntax.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- Correct schema path or permissions.
|
||||
- Fix schema JSON.
|
||||
- Rerun.
|
||||
|
||||
Relevant links: [Configuration reference](config.md), [Operations guide](operations.md)
|
||||
|
||||
## HTTP JSON Or Request Contract Errors
|
||||
|
||||
Symptom:
|
||||
|
||||
- HTTP returns `400 invalid_json` or `400 invalid_request`.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- JSON body is malformed.
|
||||
- Request has unknown fields or trailing JSON tokens.
|
||||
- Required `prompt_id` or `inputs` is missing.
|
||||
- Runtime override values are out of range.
|
||||
- `extra_params` collides with reserved outbound fields.
|
||||
|
||||
Diagnostic step:
|
||||
|
||||
- Revalidate request JSON and compare fields with the API reference.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- Send one JSON object with only supported fields.
|
||||
- Include `prompt_id` and at least one input.
|
||||
- Use valid model override ranges.
|
||||
- Remove reserved `extra_params` keys.
|
||||
|
||||
Relevant links: [HTTP API reference](api.md)
|
||||
|
||||
## HTTP Size Limit Errors
|
||||
|
||||
Symptom:
|
||||
|
||||
- HTTP returns `413 request_too_large`, `413 artifact_too_large`, or `413 response_too_large`.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- JSON request body exceeds `server.max_request_bytes`.
|
||||
- HTTP file input exceeds `server.max_artifact_bytes`.
|
||||
- Encoded JSON response exceeds `server.max_response_bytes`.
|
||||
|
||||
Diagnostic step:
|
||||
|
||||
- Compare request, file input, and expected response sizes with configured limits.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- Use smaller inline inputs or switch to file inputs under the artifact root.
|
||||
- Reduce generated output size.
|
||||
- Omit `include_raw_output`.
|
||||
- Increase limits only when the deployment expects larger payloads.
|
||||
|
||||
Relevant links: [HTTP API reference](api.md), [Operations guide](operations.md)
|
||||
|
||||
## HTTP Route Or Method Errors
|
||||
|
||||
Symptom:
|
||||
|
||||
- HTTP returns `404 not_found` or `405 method_not_allowed`.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- Path is not `/v1/runs`.
|
||||
- Method on `/v1/runs` is not `POST`.
|
||||
|
||||
Diagnostic step:
|
||||
|
||||
- Check the request URL and method.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- Send `POST /v1/runs`.
|
||||
|
||||
Relevant links: [HTTP API reference](api.md)
|
||||
318
engine.go
318
engine.go
@@ -1,318 +0,0 @@
|
||||
package scriptorium
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
artifactadapter "gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile/builtin"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
||||
)
|
||||
|
||||
// ErrInvalidConfig indicates invalid public engine configuration.
|
||||
var ErrInvalidConfig = errors.New("invalid engine configuration")
|
||||
|
||||
var (
|
||||
ErrInvalidRequest = errors.New("invalid run request")
|
||||
ErrPromptNotFound = errors.New("prompt not found")
|
||||
ErrProfileNotFound = errors.New("profile not found")
|
||||
ErrPromptLoad = errors.New("failed to load prompt definition")
|
||||
ErrProfileLoad = errors.New("failed to load execution profile")
|
||||
ErrArtifactLoad = errors.New("failed to load artifact")
|
||||
ErrPromptRender = errors.New("failed to render prompt")
|
||||
ErrLLMGenerate = errors.New("failed to generate output")
|
||||
ErrValidation = errors.New("failed to validate output")
|
||||
)
|
||||
|
||||
// Engine prepares and runs Scriptorium prompt requests.
|
||||
type Engine struct {
|
||||
runner *usecase.Runner
|
||||
}
|
||||
|
||||
// Config configures a public Scriptorium engine.
|
||||
type Config struct {
|
||||
PromptDir string
|
||||
ProfileDir string
|
||||
SchemaDir string
|
||||
Timeout time.Duration
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
// Option customizes engine construction.
|
||||
type Option interface {
|
||||
apply(*engineOptions) error
|
||||
}
|
||||
|
||||
type optionFunc func(*engineOptions) error
|
||||
|
||||
func (f optionFunc) apply(options *engineOptions) error {
|
||||
return f(options)
|
||||
}
|
||||
|
||||
type engineOptions struct {
|
||||
llmClient llm.Client
|
||||
promptDefs promptdef.Repository
|
||||
profiles profile.Repository
|
||||
memoryProfiles profile.Repository
|
||||
validator validate.Validator
|
||||
promptSource bool
|
||||
profileSource bool
|
||||
memorySource bool
|
||||
validatorSource bool
|
||||
}
|
||||
|
||||
// WithLLMClient injects a custom LLM client for execution.
|
||||
func WithLLMClient(client LLMClient) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if client == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.llmClient = publicLLMClientAdapter{client: client}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// WithPromptFS loads prompt definitions from fsys under root.
|
||||
//
|
||||
// The source uses the same strict prompt YAML rules as configured prompt
|
||||
// directories, and prompt content_file paths resolve within this source.
|
||||
func WithPromptFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.promptDefs = promptdef.NewFSRepository(fsys, root)
|
||||
options.promptSource = true
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// WithPromptFile loads prompt definitions from the single prompt file at path.
|
||||
//
|
||||
// Relative prompt content_file paths resolve from the file's directory.
|
||||
func WithPromptFile(path string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
fsys, root, err := fileSource(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
options.promptDefs = promptdef.NewFSRepository(fsys, root)
|
||||
options.promptSource = true
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// WithProfileFS loads execution profiles from fsys under root.
|
||||
//
|
||||
// Profiles from this source overlay built-in profiles. Profile YAML must use
|
||||
// api_key_env for environment-based credentials; raw API keys are rejected.
|
||||
func WithProfileFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.profiles = profile.NewFSRepository(fsys, root)
|
||||
options.profileSource = true
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// WithProfileFile loads execution profiles from the single profile file at path.
|
||||
//
|
||||
// The profile overlays built-in profiles. Profile YAML must use api_key_env for
|
||||
// environment-based credentials; raw API keys are rejected.
|
||||
func WithProfileFile(path string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
fsys, root, err := fileSource(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
options.profiles = profile.NewFSRepository(fsys, root)
|
||||
options.profileSource = true
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// WithProfiles configures in-memory profiles that take precedence over
|
||||
// configured profile files and built-in profiles.
|
||||
func WithProfiles(profiles ...Profile) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
repo, err := newMemoryProfileRepository(profiles)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
options.memoryProfiles = repo
|
||||
options.memorySource = true
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// WithSchemaFS loads JSON Schema documents from fsys under root.
|
||||
//
|
||||
// Prompt schema_path values resolve within this source when schema validation
|
||||
// or structured output is requested.
|
||||
func WithSchemaFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.validator = validate.NewFSValidator(fsys, root)
|
||||
options.validatorSource = true
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// WithSchemaFile loads JSON Schema documents from the single schema file at path.
|
||||
//
|
||||
// Prompt schema_path values refer to the file's base name.
|
||||
func WithSchemaFile(path string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
fsys, root, err := fileSource(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
options.validator = validate.NewFSValidator(fsys, root)
|
||||
options.validatorSource = true
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// NewEngine constructs an Engine using the same default internal components as
|
||||
// the CLI and HTTP adapters.
|
||||
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
var options engineOptions
|
||||
for _, opt := range opts {
|
||||
if opt == nil {
|
||||
continue
|
||||
}
|
||||
if err := opt.apply(&options); err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
}
|
||||
|
||||
promptDefs := options.promptDefs
|
||||
if !options.promptSource {
|
||||
if strings.TrimSpace(cfg.PromptDir) == "" {
|
||||
return nil, fmt.Errorf("%w: prompt directory is required", ErrInvalidConfig)
|
||||
}
|
||||
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
|
||||
}
|
||||
|
||||
profiles := builtin.NewRepositoryWithDirectory(cfg.ProfileDir)
|
||||
if options.profileSource {
|
||||
profiles = builtin.NewRepositoryWithPrimary(options.profiles)
|
||||
}
|
||||
if options.memorySource {
|
||||
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
|
||||
}
|
||||
|
||||
validator := options.validator
|
||||
if !options.validatorSource {
|
||||
schemaDir := cfg.SchemaDir
|
||||
if strings.TrimSpace(schemaDir) == "" {
|
||||
schemaDir = defaults.SchemaDirDefault
|
||||
}
|
||||
validator = validate.NewStandardValidator(schemaDir)
|
||||
}
|
||||
|
||||
llmClient := options.llmClient
|
||||
if llmClient == nil {
|
||||
var err error
|
||||
llmClient, err = llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
|
||||
Timeout: cfg.Timeout,
|
||||
HTTPClient: cfg.HTTPClient,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
}
|
||||
|
||||
return &Engine{
|
||||
runner: usecase.NewRunner(
|
||||
promptDefs,
|
||||
profiles,
|
||||
artifactadapter.NewCompositeReader(),
|
||||
prompt.NewGoRenderer(),
|
||||
llmClient,
|
||||
validator,
|
||||
),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func fileSource(name string) (fs.FS, string, error) {
|
||||
cleanName := strings.TrimSpace(name)
|
||||
if cleanName == "" {
|
||||
return nil, "", ErrInvalidConfig
|
||||
}
|
||||
dir := filepath.Dir(cleanName)
|
||||
base := filepath.Base(cleanName)
|
||||
if base == "." || base == string(filepath.Separator) || strings.TrimSpace(base) == "" {
|
||||
return nil, "", ErrInvalidConfig
|
||||
}
|
||||
info, err := os.Stat(cleanName)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, cleanName, err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, cleanName)
|
||||
}
|
||||
return os.DirFS(dir), filepath.ToSlash(base), nil
|
||||
}
|
||||
|
||||
// Prepare resolves a prompt request without calling an LLM.
|
||||
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
domainReq, err := toDomainRunRequest(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
prepared, err := e.runner.Prepare(ctx, domainReq)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return fromDomainPreparedRun(prepared), nil
|
||||
}
|
||||
|
||||
// Run executes a prompt request and returns the generated artifact and metadata.
|
||||
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
domainReq, err := toDomainRunRequest(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
result, err := e.runner.Run(ctx, domainReq)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return fromDomainRunResult(result), nil
|
||||
}
|
||||
1799
engine_test.go
1799
engine_test.go
File diff suppressed because it is too large
Load Diff
79
errors.go
79
errors.go
@@ -1,79 +0,0 @@
|
||||
package scriptorium
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||
)
|
||||
|
||||
func mapPublicError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if hasPublicError(err) {
|
||||
return err
|
||||
}
|
||||
publicErr := publicErrorFor(err)
|
||||
if publicErr == nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("%w: %w", publicErr, err)
|
||||
}
|
||||
|
||||
func hasPublicError(err error) bool {
|
||||
for _, publicErr := range []error{
|
||||
ErrInvalidConfig,
|
||||
ErrInvalidRequest,
|
||||
ErrPromptNotFound,
|
||||
ErrProfileNotFound,
|
||||
ErrPromptLoad,
|
||||
ErrProfileLoad,
|
||||
ErrArtifactLoad,
|
||||
ErrPromptRender,
|
||||
ErrLLMGenerate,
|
||||
ErrValidation,
|
||||
} {
|
||||
if errors.Is(err, publicErr) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func publicErrorFor(err error) error {
|
||||
switch {
|
||||
case errors.Is(err, promptdef.ErrPromptDefinitionNotFound):
|
||||
return ErrPromptNotFound
|
||||
case errors.Is(err, profile.ErrProfileNotFound):
|
||||
return ErrProfileNotFound
|
||||
case errors.Is(err, usecase.ErrPromptLoad):
|
||||
return ErrPromptLoad
|
||||
case errors.Is(err, usecase.ErrProfileLoad):
|
||||
return ErrProfileLoad
|
||||
case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition):
|
||||
return ErrPromptLoad
|
||||
case isProfileLoadCause(err):
|
||||
return ErrProfileLoad
|
||||
case errors.Is(err, usecase.ErrArtifactLoad):
|
||||
return ErrArtifactLoad
|
||||
case errors.Is(err, usecase.ErrPromptRender):
|
||||
return ErrPromptRender
|
||||
case errors.Is(err, usecase.ErrLLMGenerate):
|
||||
return ErrLLMGenerate
|
||||
case errors.Is(err, usecase.ErrValidation):
|
||||
return ErrValidation
|
||||
case errors.Is(err, usecase.ErrInvalidRequest):
|
||||
return ErrInvalidRequest
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func isProfileLoadCause(err error) bool {
|
||||
return errors.Is(err, profile.ErrInvalidYAML) ||
|
||||
errors.Is(err, profile.ErrInvalidProfile) ||
|
||||
errors.Is(err, profile.ErrRawAPIKeyNotAllowed)
|
||||
}
|
||||
@@ -2,6 +2,15 @@ prompt_dir: ./examples/prompts
|
||||
profile_dir: ./examples/profiles
|
||||
schema_dir: ./examples/schemas
|
||||
|
||||
backends:
|
||||
local-gpu:
|
||||
endpoint: http://localhost:11434/v1
|
||||
api_key_env: LOCAL_GPU_API_KEY
|
||||
extra_params:
|
||||
provider_option: enabled
|
||||
concurrency_limit: 2
|
||||
queue_capacity: 0
|
||||
|
||||
server:
|
||||
addr: 127.0.0.1:8080
|
||||
artifact_root: .
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
func main() {
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||
PromptDir: "./examples/prompts",
|
||||
ProfileDir: "./examples/profiles",
|
||||
SchemaDir: "./examples/schemas",
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
summary := struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
Model string `json:"model"`
|
||||
MessageCount int `json:"message_count"`
|
||||
InputHashes map[string]string `json:"input_hashes"`
|
||||
}{
|
||||
PromptID: prepared.PromptID,
|
||||
SelectedProfileID: prepared.SelectedProfileID,
|
||||
Model: prepared.EffectiveModelParams.Model,
|
||||
MessageCount: len(prepared.Messages),
|
||||
InputHashes: prepared.InputHashes,
|
||||
}
|
||||
|
||||
if err := json.NewEncoder(os.Stdout).Encode(summary); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
4
examples/profiles/local-gpu.yml
Normal file
4
examples/profiles/local-gpu.yml
Normal file
@@ -0,0 +1,4 @@
|
||||
id: local-gpu
|
||||
backend: local-gpu
|
||||
model: local-model
|
||||
reasoning_effort: low
|
||||
25
examples/render-v0.9-features.sh
Executable file
25
examples/render-v0.9-features.sh
Executable file
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$repo_root"
|
||||
|
||||
go run ./cmd/scriptorium render \
|
||||
--config ./examples/config.full.yml \
|
||||
--prompt generic.markdown_summary \
|
||||
--prompt-version 1.0.0 \
|
||||
--profile local-gpu \
|
||||
--session-id example-session \
|
||||
--reasoning-effort= \
|
||||
--input transcript=./examples/fixtures/transcript.md \
|
||||
--input glossary=./examples/fixtures/glossary.yml
|
||||
|
||||
go run ./cmd/scriptorium inspect prompt \
|
||||
--config ./examples/config.full.yml \
|
||||
--prompt generic.markdown_summary \
|
||||
--format json
|
||||
|
||||
go run ./cmd/scriptorium inspect profile \
|
||||
--config ./examples/config.full.yml \
|
||||
--profile local-gpu \
|
||||
--format json
|
||||
@@ -1,51 +0,0 @@
|
||||
package scriptorium
|
||||
|
||||
import "fmt"
|
||||
|
||||
// String returns a concise request summary without exposing direct API keys.
|
||||
func (r RunRequest) String() string {
|
||||
return r.redactedString()
|
||||
}
|
||||
|
||||
// GoString returns a concise request summary without exposing direct API keys.
|
||||
func (r RunRequest) GoString() string {
|
||||
return r.redactedString()
|
||||
}
|
||||
|
||||
func (r RunRequest) redactedString() string {
|
||||
return fmt.Sprintf(
|
||||
"scriptorium.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t Metadata:%d}",
|
||||
r.PromptID,
|
||||
r.PromptVersion,
|
||||
r.ProfileID,
|
||||
r.APIKey != "",
|
||||
len(r.Inputs),
|
||||
len(r.Vars),
|
||||
r.Execution != nil,
|
||||
r.Validation != nil,
|
||||
len(r.Metadata),
|
||||
)
|
||||
}
|
||||
|
||||
// String returns a concise request summary without exposing direct API keys or
|
||||
// rendered prompt content.
|
||||
func (r GenerateRequest) String() string {
|
||||
return r.redactedString()
|
||||
}
|
||||
|
||||
// GoString returns a concise request summary without exposing direct API keys or
|
||||
// rendered prompt content.
|
||||
func (r GenerateRequest) GoString() string {
|
||||
return r.redactedString()
|
||||
}
|
||||
|
||||
func (r GenerateRequest) redactedString() string {
|
||||
return fmt.Sprintf(
|
||||
"scriptorium.GenerateRequest{Messages:%d Model:%q APIKeySet:%t StructuredOutputSet:%t ExtraParams:%d}",
|
||||
len(r.Prompt.Messages),
|
||||
r.Target.Model,
|
||||
r.APIKey != "",
|
||||
r.StructuredOutput != nil,
|
||||
len(r.Target.ExtraParams),
|
||||
)
|
||||
}
|
||||
9
go.mod
9
go.mod
@@ -3,8 +3,13 @@ module gitea.maximumdirect.net/eric/scriptorium
|
||||
go 1.25.5
|
||||
|
||||
require (
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||
gitea.maximumdirect.net/eric/promptkit v0.9.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require golang.org/x/text v0.14.0 // indirect
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 // indirect
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 // indirect
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
)
|
||||
|
||||
6
go.sum
6
go.sum
@@ -1,3 +1,9 @@
|
||||
gitea.maximumdirect.net/eric/promptkit v0.9.0 h1:IpvDRC8L6xRxQ9hpuyKOmMc5b6MeLTKYyx+h1YAjy08=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.9.0/go.mod h1:oMJ/WUJImUtwJ5e+6MAGECPYAErAkOaKel0G+3T/b4E=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 h1:lc062euk2qseO//D762i3JaFyulDNML3eQQX7DkYTho=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0/go.mod h1:AIa7kAu2mfrRQgcspe4L+DW51WqgnALQT60lqkEywJI=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 h1:j9YY7wsTVjzke2kHH4YAzpU0oUpM+x+nXwl1IeS+2eg=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0/go.mod h1:4RNS+LILDg4JbS4Ts9Lwy1C92wauXJIbeQaalps4Koo=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
|
||||
200
internal/adapter/cli/inspect.go
Normal file
200
internal/adapter/cli/inspect.go
Normal file
@@ -0,0 +1,200 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||
appformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||
)
|
||||
|
||||
type promptInspectionConfig struct {
|
||||
configPath, promptDir, promptID, promptVersion, outputPath string
|
||||
configExplicit bool
|
||||
outputFormat appformat.OutputFormat
|
||||
}
|
||||
|
||||
type profileInspectionConfig struct {
|
||||
configPath, profileDir, profileID, outputPath string
|
||||
configExplicit bool
|
||||
outputFormat appformat.OutputFormat
|
||||
}
|
||||
|
||||
// emptyPromptDefinitionFS satisfies Promptkit's engine-level prompt-source
|
||||
// requirement without exposing the caller's working directory. Profile
|
||||
// inspection never reads this source.
|
||||
type emptyPromptDefinitionFS struct{}
|
||||
|
||||
func (emptyPromptDefinitionFS) Open(name string) (fs.File, error) {
|
||||
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
|
||||
}
|
||||
|
||||
func inspectCommand(args []string, stdout, stderr io.Writer) int {
|
||||
if len(args) == 0 {
|
||||
fmt.Fprintln(stderr, "inspect parse error: inspection mode is required")
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if args[0] == "profile" {
|
||||
return inspectProfileCommand(args[1:], stdout, stderr)
|
||||
}
|
||||
if args[0] != "prompt" {
|
||||
fmt.Fprintln(stderr, "inspect parse error: unknown inspection mode")
|
||||
return ExitRuntimeError
|
||||
}
|
||||
cfg, err := parsePromptInspectionArgs(args[1:])
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "inspect parse error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
settings, err := resolveAppSettingsForPromptInspection(cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
engine, err := newEngine(settings)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "engine error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
inspection, err := engine.InspectPrompt(context.Background(), cfg.promptID, cfg.promptVersion)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
data, err := appformat.FormatPromptInspection(inspection, cfg.outputFormat)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := writeOutput(stdout, cfg.outputPath, data); err != nil {
|
||||
fmt.Fprintf(stderr, "output write error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
return ExitOK
|
||||
}
|
||||
|
||||
func inspectProfileCommand(args []string, stdout, stderr io.Writer) int {
|
||||
cfg, err := parseProfileInspectionArgs(args)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "inspect parse error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
settings, err := resolveAppSettingsForProfileInspection(cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
engine, err := newEngine(settings, promptkit.WithPromptFS(emptyPromptDefinitionFS{}, "."))
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "engine error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
inspection, err := engine.InspectProfile(context.Background(), cfg.profileID)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
data, err := appformat.FormatProfileInspection(inspection, cfg.outputFormat)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := writeOutput(stdout, cfg.outputPath, data); err != nil {
|
||||
fmt.Fprintf(stderr, "output write error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
return ExitOK
|
||||
}
|
||||
|
||||
func parseProfileInspectionArgs(args []string) (*profileInspectionConfig, error) {
|
||||
cfg := &profileInspectionConfig{outputFormat: appformat.DefaultOutputFormat}
|
||||
fs := flag.NewFlagSet("inspect profile", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
registerConfigPathFlag(fs, &cfg.configPath)
|
||||
fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing execution profiles")
|
||||
fs.StringVar(&cfg.profileID, "profile", "", "profile ID to inspect")
|
||||
rawFormat := ""
|
||||
fs.StringVar(&rawFormat, "format", "", "output format: text or json")
|
||||
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.configExplicit = flagWasSet(fs, "config")
|
||||
if fs.NArg() > 0 {
|
||||
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
|
||||
}
|
||||
if strings.TrimSpace(cfg.profileID) == "" {
|
||||
return nil, errors.New("--profile is required")
|
||||
}
|
||||
format, err := appformat.ParseOutputFormat(rawFormat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.outputFormat = format
|
||||
if cfg.outputPath != "" {
|
||||
cfg.outputPath = filepath.Clean(cfg.outputPath)
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func resolveAppSettingsForProfileInspection(cfg *profileInspectionConfig) (engineSettings, error) {
|
||||
settings, err := resolveAppSettingsWithConfigPresence(cfg.configPath, cfg.configExplicit, appconfig.CLIOverrides{
|
||||
ProfileDir: cfg.profileDir,
|
||||
})
|
||||
if err != nil {
|
||||
return engineSettings{}, err
|
||||
}
|
||||
return engineSettings{profileDir: settings.ProfileDir, backends: settings.Backends}, nil
|
||||
}
|
||||
|
||||
func parsePromptInspectionArgs(args []string) (*promptInspectionConfig, error) {
|
||||
cfg := &promptInspectionConfig{outputFormat: appformat.DefaultOutputFormat}
|
||||
fs := flag.NewFlagSet("inspect prompt", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
registerConfigPathFlag(fs, &cfg.configPath)
|
||||
fs.StringVar(&cfg.promptDir, "prompt-dir", "", "directory containing prompt definitions")
|
||||
fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to inspect")
|
||||
fs.StringVar(&cfg.promptVersion, "prompt-version", "", "optional prompt version")
|
||||
rawFormat := ""
|
||||
fs.StringVar(&rawFormat, "format", "", "output format: text or json")
|
||||
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.configExplicit = flagWasSet(fs, "config")
|
||||
if fs.NArg() > 0 {
|
||||
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
|
||||
}
|
||||
if strings.TrimSpace(cfg.promptID) == "" {
|
||||
return nil, errors.New("--prompt is required")
|
||||
}
|
||||
format, err := appformat.ParseOutputFormat(rawFormat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.outputFormat = format
|
||||
if cfg.outputPath != "" {
|
||||
cfg.outputPath = filepath.Clean(cfg.outputPath)
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func resolveAppSettingsForPromptInspection(cfg *promptInspectionConfig) (engineSettings, error) {
|
||||
settings, err := resolveAppSettingsWithConfigPresence(cfg.configPath, cfg.configExplicit, appconfig.CLIOverrides{
|
||||
PromptDir: cfg.promptDir,
|
||||
})
|
||||
if err != nil {
|
||||
return engineSettings{}, err
|
||||
}
|
||||
if strings.TrimSpace(settings.PromptDir) == "" {
|
||||
return engineSettings{}, errors.New(errPromptDirRequired)
|
||||
}
|
||||
return engineSettings{promptDir: settings.PromptDir, backends: settings.Backends}, nil
|
||||
}
|
||||
@@ -12,18 +12,11 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
httpadapter "gitea.maximumdirect.net/eric/scriptorium/internal/adapter/http"
|
||||
artifactadapter "gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
||||
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile/builtin"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -42,7 +35,9 @@ type runConfig struct {
|
||||
promptDir string
|
||||
profileDir string
|
||||
promptID string
|
||||
promptVersion string
|
||||
profileID string
|
||||
sessionID string
|
||||
inputRaw listFlag
|
||||
varRaw listFlag
|
||||
outputPath string
|
||||
@@ -52,7 +47,9 @@ type runConfig struct {
|
||||
temperature float64
|
||||
maxTokens int
|
||||
topP float64
|
||||
reasoningEffort string
|
||||
schemaDir string
|
||||
backends []appconfig.BackendSettings
|
||||
timeout time.Duration
|
||||
|
||||
defaultRenderFormat renderformat.PreparedRunOutputFormat
|
||||
@@ -63,6 +60,7 @@ type runConfig struct {
|
||||
temperatureSet bool
|
||||
maxTokensSet bool
|
||||
topPSet bool
|
||||
reasoningEffortSet bool
|
||||
timeoutSet bool
|
||||
}
|
||||
|
||||
@@ -82,6 +80,7 @@ type serveConfig struct {
|
||||
maxRequestBytes int64
|
||||
maxArtifactBytes int64
|
||||
maxResponseBytes int64
|
||||
backends []appconfig.BackendSettings
|
||||
}
|
||||
|
||||
type commonCommandSettings struct {
|
||||
@@ -94,6 +93,7 @@ type commonCommandSettings struct {
|
||||
maxArtifactBytes int64
|
||||
maxResponseBytes int64
|
||||
defaultRenderFormat renderformat.PreparedRunOutputFormat
|
||||
backends []appconfig.BackendSettings
|
||||
}
|
||||
|
||||
type listFlag []string
|
||||
@@ -120,6 +120,8 @@ func Run(args []string, stdout, stderr io.Writer) int {
|
||||
return renderCommand(args[1:], stdout, stderr)
|
||||
case "serve":
|
||||
return serveCommand(args[1:], stderr)
|
||||
case "inspect":
|
||||
return inspectCommand(args[1:], stdout, stderr)
|
||||
default:
|
||||
fmt.Fprintf(stderr, "unknown command %q\n", args[0])
|
||||
printUsage(stderr)
|
||||
@@ -140,17 +142,15 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
llmClient, err := newOpenAIClient()
|
||||
engine, err := newEngine(cfg.engineSettings())
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "llm client error: %v\n", err)
|
||||
fmt.Fprintf(stderr, "engine error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
runner := newRunner(cfg.promptDir, cfg.profileDir, cfg.schemaDir, llmClient)
|
||||
|
||||
res, runErr := runner.Run(context.Background(), req)
|
||||
res, runErr := engine.Run(context.Background(), req)
|
||||
if runErr != nil {
|
||||
fmt.Fprintf(stderr, "run error: %v\n", runErr)
|
||||
fmt.Fprintln(stderr, runErrorMessage(runErr))
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
@@ -176,9 +176,13 @@ func renderCommand(args []string, stdout, stderr io.Writer) int {
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
runner := newRunner(cfg.promptDir, cfg.profileDir, cfg.schemaDir, nil)
|
||||
engine, err := newEngine(cfg.runConfig.engineSettings())
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "engine error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
prepared, prepErr := runner.Prepare(context.Background(), req)
|
||||
prepared, prepErr := engine.Prepare(context.Background(), req)
|
||||
if prepErr != nil {
|
||||
fmt.Fprintf(stderr, "render error: %v\n", prepErr)
|
||||
return ExitRuntimeError
|
||||
@@ -204,21 +208,19 @@ func serveCommand(args []string, stderr io.Writer) int {
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
llmClient, err := newOpenAIClient()
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "llm client error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
artifactReader, err := artifactadapter.NewRestrictedCompositeReaderWithLimit(cfg.artifactRoot, cfg.maxArtifactBytes)
|
||||
artifactReader, err := httpadapter.NewRestrictedArtifactReader(cfg.artifactRoot, cfg.maxArtifactBytes)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "artifact root error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
runner := newRunnerWithArtifactReader(cfg.promptDir, cfg.profileDir, cfg.schemaDir, llmClient, artifactReader)
|
||||
engine, err := newEngine(cfg.engineSettings(), promptkit.WithArtifactReader(artifactReader))
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "engine error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
h := httpadapter.NewHandlerWithOptions(runner, httpadapter.HandlerOptions{
|
||||
h := httpadapter.NewHandlerWithOptions(engine, httpadapter.HandlerOptions{
|
||||
MaxRequestBytes: cfg.maxRequestBytes,
|
||||
MaxResponseBytes: cfg.maxResponseBytes,
|
||||
})
|
||||
@@ -332,6 +334,7 @@ func parseServeArgs(args []string) (*serveConfig, error) {
|
||||
cfg.maxRequestBytes = settings.maxRequestBytes
|
||||
cfg.maxArtifactBytes = settings.maxArtifactBytes
|
||||
cfg.maxResponseBytes = settings.maxResponseBytes
|
||||
cfg.backends = settings.backends
|
||||
|
||||
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
|
||||
return nil, err
|
||||
@@ -352,7 +355,9 @@ func registerExecutionRequestFlags(fs *flag.FlagSet, cfg *runConfig) {
|
||||
fs.StringVar(&cfg.promptDir, "prompt-dir", "", "directory containing prompt definition YAML files")
|
||||
fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing execution profile YAML files")
|
||||
fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to run")
|
||||
fs.StringVar(&cfg.promptVersion, "prompt-version", "", "optional prompt definition version")
|
||||
fs.StringVar(&cfg.profileID, "profile", "", "optional execution profile ID; if omitted, prompt default_profile is used")
|
||||
fs.StringVar(&cfg.sessionID, "session-id", "", "optional session ID")
|
||||
fs.Var(&cfg.inputRaw, "input", "input mapping(s): name=path (repeatable, comma-separated)")
|
||||
fs.Var(&cfg.varRaw, "var", "variable mapping(s): name=value (repeatable, comma-separated)")
|
||||
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")
|
||||
@@ -362,7 +367,8 @@ func registerExecutionRequestFlags(fs *flag.FlagSet, cfg *runConfig) {
|
||||
fs.Float64Var(&cfg.temperature, "temperature", 0, "optional temperature override")
|
||||
fs.IntVar(&cfg.maxTokens, "max-tokens", 0, "optional max tokens override")
|
||||
fs.Float64Var(&cfg.topP, "top-p", 0, "optional top_p override")
|
||||
fs.DurationVar(&cfg.timeout, "timeout", defaults.LLMRequestTimeoutDefault, "LLM request timeout")
|
||||
fs.StringVar(&cfg.reasoningEffort, "reasoning-effort", "", "optional reasoning effort override")
|
||||
fs.DurationVar(&cfg.timeout, "timeout", 0, "LLM request timeout")
|
||||
fs.StringVar(&cfg.promptID, "prompt-id", "", "deprecated alias for --prompt")
|
||||
fs.StringVar(&cfg.profileID, "profile-id", "", "deprecated alias for --profile")
|
||||
}
|
||||
@@ -385,6 +391,7 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
|
||||
cfg.profileDir = settings.profileDir
|
||||
cfg.schemaDir = settings.schemaDir
|
||||
cfg.defaultRenderFormat = settings.defaultRenderFormat
|
||||
cfg.backends = settings.backends
|
||||
|
||||
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
|
||||
return err
|
||||
@@ -392,9 +399,6 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
|
||||
if strings.TrimSpace(cfg.promptID) == "" {
|
||||
return errors.New("--prompt is required")
|
||||
}
|
||||
if len(cfg.inputRaw) == 0 {
|
||||
return errors.New("at least one --input is required")
|
||||
}
|
||||
cfg.promptDir = filepath.Clean(cfg.promptDir)
|
||||
if strings.TrimSpace(cfg.profileDir) != "" {
|
||||
cfg.profileDir = filepath.Clean(cfg.profileDir)
|
||||
@@ -408,6 +412,7 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
|
||||
cfg.temperatureSet = flagWasSet(fs, "temperature")
|
||||
cfg.maxTokensSet = flagWasSet(fs, "max-tokens")
|
||||
cfg.topPSet = flagWasSet(fs, "top-p")
|
||||
cfg.reasoningEffortSet = flagWasSet(fs, "reasoning-effort")
|
||||
cfg.timeoutSet = flagWasSet(fs, "timeout")
|
||||
return nil
|
||||
}
|
||||
@@ -503,7 +508,11 @@ func registerConfigPathFlag(fs *flag.FlagSet, target *string) {
|
||||
}
|
||||
|
||||
func resolveAppSettings(fs *flag.FlagSet, configPath string, overrides appconfig.CLIOverrides) (appconfig.AppSettings, error) {
|
||||
settings, err := appconfig.LoadConfig(configPath, flagWasSet(fs, "config"))
|
||||
return resolveAppSettingsWithConfigPresence(configPath, flagWasSet(fs, "config"), overrides)
|
||||
}
|
||||
|
||||
func resolveAppSettingsWithConfigPresence(configPath string, configExplicit bool, overrides appconfig.CLIOverrides) (appconfig.AppSettings, error) {
|
||||
settings, err := appconfig.LoadConfig(configPath, configExplicit)
|
||||
if err != nil {
|
||||
return appconfig.AppSettings{}, fmt.Errorf("application config error: %w", err)
|
||||
}
|
||||
@@ -531,6 +540,7 @@ func resolveCommonSettings(fs *flag.FlagSet, configPath string, overrides appcon
|
||||
maxArtifactBytes: settings.MaxArtifactBytes,
|
||||
maxResponseBytes: settings.MaxResponseBytes,
|
||||
defaultRenderFormat: settings.DefaultRenderFormat,
|
||||
backends: settings.Backends,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -541,52 +551,74 @@ func validateRequiredLibraryDirs(promptDir string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func newRunner(promptDir, profileDir, schemaDir string, llmClient llm.Client) *usecase.Runner {
|
||||
return newRunnerWithArtifactReader(promptDir, profileDir, schemaDir, llmClient, artifactadapter.NewCompositeReader())
|
||||
type engineSettings struct {
|
||||
promptDir string
|
||||
profileDir string
|
||||
schemaDir string
|
||||
backends []appconfig.BackendSettings
|
||||
}
|
||||
|
||||
func newRunnerWithArtifactReader(promptDir, profileDir, schemaDir string, llmClient llm.Client, artifactReader artifactadapter.Reader) *usecase.Runner {
|
||||
if artifactReader == nil {
|
||||
artifactReader = artifactadapter.NewCompositeReader()
|
||||
func (c runConfig) engineSettings() engineSettings {
|
||||
return engineSettings{promptDir: c.promptDir, profileDir: c.profileDir, schemaDir: c.schemaDir, backends: c.backends}
|
||||
}
|
||||
|
||||
func (c serveConfig) engineSettings() engineSettings {
|
||||
return engineSettings{promptDir: c.promptDir, profileDir: c.profileDir, schemaDir: c.schemaDir, backends: c.backends}
|
||||
}
|
||||
|
||||
func newEngine(settings engineSettings, options ...promptkit.Option) (*promptkit.Engine, error) {
|
||||
engineOptions := make([]promptkit.Option, 0, len(settings.backends)+len(options))
|
||||
for _, configured := range settings.backends {
|
||||
engineOptions = append(engineOptions, promptkit.WithBackend(promptkit.Backend{
|
||||
ID: configured.ID,
|
||||
Endpoint: configured.Endpoint,
|
||||
APIKeyEnv: configured.APIKeyEnv,
|
||||
ExtraParams: configured.ExtraParams,
|
||||
ConcurrencyLimit: configured.ConcurrencyLimit,
|
||||
QueueCapacity: configured.QueueCapacity,
|
||||
}))
|
||||
}
|
||||
return usecase.NewRunner(
|
||||
promptdef.NewFilesystemRepository(promptDir),
|
||||
builtin.NewRepositoryWithDirectory(profileDir),
|
||||
artifactReader,
|
||||
prompt.NewGoRenderer(),
|
||||
llmClient,
|
||||
validate.NewStandardValidator(schemaDir),
|
||||
)
|
||||
}
|
||||
|
||||
func newOpenAIClient() (*llm.OpenAICompatibleClient, error) {
|
||||
return llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
|
||||
Timeout: defaults.LLMRequestTimeoutDefault,
|
||||
})
|
||||
}
|
||||
|
||||
func buildRunRequestFromConfig(cfg *runConfig) (domain.RunRequest, error) {
|
||||
inputMappings, err := parseMappings(cfg.inputRaw, false)
|
||||
engineOptions = append(engineOptions, options...)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: settings.promptDir,
|
||||
ProfileDir: settings.profileDir,
|
||||
SchemaDir: settings.schemaDir,
|
||||
}, engineOptions...)
|
||||
if err != nil {
|
||||
return domain.RunRequest{}, fmt.Errorf("input parse error: %w", err)
|
||||
return nil, fmt.Errorf("engine initialization from application configuration: %w", err)
|
||||
}
|
||||
return engine, nil
|
||||
}
|
||||
|
||||
func buildRunRequestFromConfig(cfg *runConfig) (promptkit.RunRequest, error) {
|
||||
var inputMappings map[string]string
|
||||
var err error
|
||||
if len(cfg.inputRaw) > 0 {
|
||||
inputMappings, err = parseMappings(cfg.inputRaw, false)
|
||||
if err != nil {
|
||||
return promptkit.RunRequest{}, fmt.Errorf("input parse error: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
varMappings := map[string]string{}
|
||||
if len(cfg.varRaw) > 0 {
|
||||
varMappings, err = parseMappings(cfg.varRaw, false)
|
||||
if err != nil {
|
||||
return domain.RunRequest{}, fmt.Errorf("var parse error: %w", err)
|
||||
return promptkit.RunRequest{}, fmt.Errorf("var parse error: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
inputs := make(map[string]domain.ArtifactRef, len(inputMappings))
|
||||
var inputs map[string]promptkit.ArtifactRef
|
||||
if len(inputMappings) > 0 {
|
||||
inputs = make(map[string]promptkit.ArtifactRef, len(inputMappings))
|
||||
for name, path := range inputMappings {
|
||||
inputs[name] = domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path}
|
||||
inputs[name] = promptkit.File(path)
|
||||
}
|
||||
}
|
||||
|
||||
var modelOverride *domain.ExecutionTargetOverride
|
||||
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.apiKeyEnvSet || cfg.timeoutSet {
|
||||
modelOverride = &domain.ExecutionTargetOverride{
|
||||
var modelOverride *promptkit.ExecutionTargetOverride
|
||||
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.reasoningEffortSet || cfg.apiKeyEnvSet || cfg.timeoutSet {
|
||||
modelOverride = &promptkit.ExecutionTargetOverride{
|
||||
Endpoint: cfg.llmBaseURL,
|
||||
Model: cfg.model,
|
||||
APIKeyEnv: cfg.apiKeyEnv,
|
||||
@@ -600,15 +632,20 @@ func buildRunRequestFromConfig(cfg *runConfig) (domain.RunRequest, error) {
|
||||
if cfg.topPSet {
|
||||
modelOverride.TopP = &cfg.topP
|
||||
}
|
||||
if cfg.reasoningEffortSet {
|
||||
modelOverride.ReasoningEffort = &cfg.reasoningEffort
|
||||
}
|
||||
if cfg.timeoutSet {
|
||||
timeoutSeconds := int(cfg.timeout.Seconds())
|
||||
modelOverride.TimeoutSeconds = &timeoutSeconds
|
||||
}
|
||||
}
|
||||
|
||||
return domain.RunRequest{
|
||||
return promptkit.RunRequest{
|
||||
PromptID: cfg.promptID,
|
||||
PromptVersion: cfg.promptVersion,
|
||||
ProfileID: cfg.profileID,
|
||||
SessionID: cfg.sessionID,
|
||||
Inputs: inputs,
|
||||
Vars: varMappings,
|
||||
Execution: modelOverride,
|
||||
@@ -670,17 +707,17 @@ func writeOutput(stdout io.Writer, outputPath string, body []byte) error {
|
||||
return os.WriteFile(outputPath, body, 0644)
|
||||
}
|
||||
|
||||
func determineExitCode(runErr error, result *domain.RunResult) int {
|
||||
func determineExitCode(runErr error, result *promptkit.RunResult) int {
|
||||
if runErr != nil {
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if result != nil && result.Validation.Status == domain.ValidationFailed {
|
||||
if result != nil && result.Validation.Status == promptkit.ValidationFailed {
|
||||
return ExitValidationFailed
|
||||
}
|
||||
return ExitOK
|
||||
}
|
||||
|
||||
func printSummary(stderr io.Writer, res *domain.RunResult) {
|
||||
func printSummary(stderr io.Writer, res *promptkit.RunResult) {
|
||||
if res == nil {
|
||||
return
|
||||
}
|
||||
@@ -701,12 +738,24 @@ func printSummary(stderr io.Writer, res *domain.RunResult) {
|
||||
if res.Usage.CachedTokens != 0 || res.Usage.CacheWriteTokens != 0 {
|
||||
fmt.Fprintf(stderr, " cached_tokens=%d cache_write_tokens=%d", res.Usage.CachedTokens, res.Usage.CacheWriteTokens)
|
||||
}
|
||||
if res.SelectedBackendID != "" {
|
||||
fmt.Fprintf(stderr, " backend=%s", res.SelectedBackendID)
|
||||
}
|
||||
fmt.Fprintln(stderr)
|
||||
}
|
||||
|
||||
func printUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, "usage: scriptorium <run|render|serve> ...")
|
||||
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID --input name=path [--input ...] [--profile ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--var k=v] [--out path] [--timeout 10m]")
|
||||
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID --input name=path [--input ...] [--profile ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--var k=v] [--format text|json] [--out path] [--timeout 10m]")
|
||||
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
|
||||
func runErrorMessage(err error) string {
|
||||
if errors.Is(err, promptkit.ErrCapacityExceeded) {
|
||||
return "run error: model backend capacity is exhausted"
|
||||
}
|
||||
return fmt.Sprintf("run error: %v", err)
|
||||
}
|
||||
|
||||
func printUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, "usage: scriptorium <run|render|serve|inspect> ...")
|
||||
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--timeout 10m]")
|
||||
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m]")
|
||||
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
|
||||
fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--prompt-dir DIR] [--format text|json] [--out path]")
|
||||
fmt.Fprintln(w, " inspect profile: scriptorium inspect profile --profile ID [--config PATH] [--profile-dir DIR] [--format text|json] [--out path]")
|
||||
}
|
||||
|
||||
@@ -17,9 +17,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||
)
|
||||
|
||||
@@ -87,9 +87,12 @@ func TestParseRunArgsRequiredFlags(t *testing.T) {
|
||||
t.Fatal("expected missing --prompt error")
|
||||
}
|
||||
|
||||
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p"})
|
||||
if err == nil {
|
||||
t.Fatal("expected missing --input error")
|
||||
cfg, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p"})
|
||||
if err != nil {
|
||||
t.Fatalf("expected omitted --input to be accepted, got %v", err)
|
||||
}
|
||||
if len(cfg.inputRaw) != 0 {
|
||||
t.Fatalf("expected no input mappings, got %#v", cfg.inputRaw)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,13 +101,16 @@ func TestParseRunArgsFlagMapping(t *testing.T) {
|
||||
"--prompt-dir", "./prompts",
|
||||
"--profile-dir", "./profiles",
|
||||
"--prompt", "prompt.a",
|
||||
"--prompt-version", "2",
|
||||
"--profile", "profile.a",
|
||||
"--session-id", "session-1",
|
||||
"--input", "a=b",
|
||||
"--llm-base-url", "http://x/v1",
|
||||
"--model", "m",
|
||||
"--temperature", "0.7",
|
||||
"--max-tokens", "111",
|
||||
"--top-p", "0.8",
|
||||
"--reasoning-effort", "medium",
|
||||
"--timeout", "30s",
|
||||
"--api-key-env", "SCRIPTORIUM_API_KEY",
|
||||
})
|
||||
@@ -114,8 +120,11 @@ func TestParseRunArgsFlagMapping(t *testing.T) {
|
||||
if cfg.promptDir != filepath.Clean("./prompts") || cfg.profileDir != filepath.Clean("./profiles") {
|
||||
t.Fatalf("unexpected dirs: prompt=%q profile=%q", cfg.promptDir, cfg.profileDir)
|
||||
}
|
||||
if cfg.promptID != "prompt.a" || cfg.profileID != "profile.a" {
|
||||
t.Fatalf("unexpected prompt/profile ids: %q %q", cfg.promptID, cfg.profileID)
|
||||
if cfg.promptID != "prompt.a" || cfg.promptVersion != "2" || cfg.profileID != "profile.a" {
|
||||
t.Fatalf("unexpected prompt/version/profile ids: %q %q %q", cfg.promptID, cfg.promptVersion, cfg.profileID)
|
||||
}
|
||||
if cfg.sessionID != "session-1" || cfg.reasoningEffort != "medium" || !cfg.reasoningEffortSet {
|
||||
t.Fatalf("unexpected session or reasoning configuration: %+v", cfg)
|
||||
}
|
||||
if !cfg.llmBaseURLSet || !cfg.modelSet || !cfg.temperatureSet || !cfg.maxTokensSet || !cfg.topPSet || !cfg.timeoutSet || !cfg.apiKeyEnvSet {
|
||||
t.Fatalf("expected override flags set, got %+v", cfg)
|
||||
@@ -192,7 +201,7 @@ func TestParseServeArgsRejectsRuntimeOverrideFlags(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUsageIncludesServeFileAndSizeLimitFlags(t *testing.T) {
|
||||
func TestUsageIncludesExecutionAndServeFlags(t *testing.T) {
|
||||
var stderr bytes.Buffer
|
||||
code := Run(nil, io.Discard, &stderr)
|
||||
if code != ExitRuntimeError {
|
||||
@@ -201,10 +210,15 @@ func TestUsageIncludesServeFileAndSizeLimitFlags(t *testing.T) {
|
||||
|
||||
usage := stderr.String()
|
||||
for _, want := range []string{
|
||||
"--prompt-version VERSION",
|
||||
"--session-id ID",
|
||||
"--reasoning-effort VALUE",
|
||||
"--artifact-root",
|
||||
"--max-request-bytes",
|
||||
"--max-artifact-bytes",
|
||||
"--max-response-bytes",
|
||||
"inspect prompt",
|
||||
"inspect profile",
|
||||
} {
|
||||
if !strings.Contains(usage, want) {
|
||||
t.Fatalf("expected usage to include %q, got %q", want, usage)
|
||||
@@ -222,8 +236,8 @@ func TestParseRunArgsTimeout(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("expected valid run args, got %v", err)
|
||||
}
|
||||
if cfg.timeout != defaults.LLMRequestTimeoutDefault {
|
||||
t.Fatalf("expected default timeout %s, got %s", defaults.LLMRequestTimeoutDefault, cfg.timeout)
|
||||
if cfg.timeout != 0 {
|
||||
t.Fatalf("expected omitted timeout to remain unset, got %s", cfg.timeout)
|
||||
}
|
||||
|
||||
cfg, err = parseRunArgs([]string{
|
||||
@@ -655,6 +669,89 @@ func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *tes
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRunRequestPreservesNumericOverridePresence(t *testing.T) {
|
||||
omitted, err := buildRunRequestFromConfig(&runConfig{
|
||||
promptID: "prompt-1",
|
||||
inputRaw: []string{"transcript=./transcript.md"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected omitted override request to build, got %v", err)
|
||||
}
|
||||
if omitted.Execution != nil {
|
||||
t.Fatalf("expected omitted numeric flags to leave execution override nil, got %#v", omitted.Execution)
|
||||
}
|
||||
|
||||
explicitZeros, err := buildRunRequestFromConfig(&runConfig{
|
||||
promptID: "prompt-1",
|
||||
inputRaw: []string{"transcript=./transcript.md"},
|
||||
temperatureSet: true,
|
||||
maxTokensSet: true,
|
||||
topPSet: true,
|
||||
timeoutSet: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected explicit zero override request to build, got %v", err)
|
||||
}
|
||||
if explicitZeros.Execution == nil {
|
||||
t.Fatal("expected explicit numeric flags to create execution override")
|
||||
}
|
||||
if explicitZeros.Execution.Temperature == nil || explicitZeros.Execution.MaxTokens == nil || explicitZeros.Execution.TopP == nil || explicitZeros.Execution.TimeoutSeconds == nil {
|
||||
t.Fatalf("expected explicit zero numeric overrides to remain non-nil, got %#v", explicitZeros.Execution)
|
||||
}
|
||||
if *explicitZeros.Execution.Temperature != 0 || *explicitZeros.Execution.MaxTokens != 0 || *explicitZeros.Execution.TopP != 0 || *explicitZeros.Execution.TimeoutSeconds != 0 {
|
||||
t.Fatalf("expected explicit numeric overrides to retain zero values, got %#v", explicitZeros.Execution)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRunRequestPreservesReasoningEffortPresenceAndSessionID(t *testing.T) {
|
||||
omitted, err := buildRunRequestFromConfig(&runConfig{promptID: "prompt-1"})
|
||||
if err != nil {
|
||||
t.Fatalf("expected omitted request to build, got %v", err)
|
||||
}
|
||||
if omitted.Execution != nil {
|
||||
t.Fatalf("expected omitted reasoning flag to leave execution nil, got %#v", omitted.Execution)
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
value string
|
||||
}{
|
||||
{name: "replacement", value: "high"},
|
||||
{name: "clear", value: ""},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
req, err := buildRunRequestFromConfig(&runConfig{
|
||||
promptID: "prompt-1",
|
||||
sessionID: "session-1",
|
||||
reasoningEffort: tc.value,
|
||||
reasoningEffortSet: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected request to build, got %v", err)
|
||||
}
|
||||
if req.SessionID != "session-1" || req.Execution == nil || req.Execution.ReasoningEffort == nil || *req.Execution.ReasoningEffort != tc.value {
|
||||
t.Fatalf("unexpected mapped request: %#v", req)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRunRequestAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) {
|
||||
req, err := buildRunRequestFromConfig(&runConfig{
|
||||
promptID: "prompt-1",
|
||||
promptVersion: "2",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected request without inputs to build, got %v", err)
|
||||
}
|
||||
if req.PromptVersion != "2" {
|
||||
t.Fatalf("expected prompt version to be mapped, got %q", req.PromptVersion)
|
||||
}
|
||||
if req.Inputs != nil {
|
||||
t.Fatalf("expected omitted inputs to remain nil, got %#v", req.Inputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRunArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) {
|
||||
configPath := writeAppConfigFile(t, `
|
||||
profile_dir: ./profiles
|
||||
@@ -731,13 +828,13 @@ func TestDetermineExitCode(t *testing.T) {
|
||||
if got := determineExitCode(errors.New("boom"), nil); got != ExitRuntimeError {
|
||||
t.Fatalf("expected runtime exit code, got %d", got)
|
||||
}
|
||||
if got := determineExitCode(nil, &domain.RunResult{Validation: domain.ValidationResult{Status: domain.ValidationFailed}}); got != ExitValidationFailed {
|
||||
if got := determineExitCode(nil, &promptkit.RunResult{Validation: promptkit.ValidationResult{Status: promptkit.ValidationFailed}}); got != ExitValidationFailed {
|
||||
t.Fatalf("expected validation exit code, got %d", got)
|
||||
}
|
||||
if got := determineExitCode(nil, &domain.RunResult{Validation: domain.ValidationResult{Status: domain.ValidationPassed}}); got != ExitOK {
|
||||
if got := determineExitCode(nil, &promptkit.RunResult{Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed}}); got != ExitOK {
|
||||
t.Fatalf("expected success exit code for passed validation, got %d", got)
|
||||
}
|
||||
if got := determineExitCode(nil, &domain.RunResult{Validation: domain.ValidationResult{Status: domain.ValidationSkipped}}); got != ExitOK {
|
||||
if got := determineExitCode(nil, &promptkit.RunResult{Validation: promptkit.ValidationResult{Status: promptkit.ValidationSkipped}}); got != ExitOK {
|
||||
t.Fatalf("expected success exit code for skipped validation, got %d", got)
|
||||
}
|
||||
}
|
||||
@@ -908,6 +1005,452 @@ profile_dir: %s
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderCommandUsesConfiguredCustomBackend(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom
|
||||
version: "1"
|
||||
default_profile: local-gpu
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`id: local-gpu
|
||||
backend: local-gpu
|
||||
model: local-model
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write profile fixture: %v", err)
|
||||
}
|
||||
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
||||
prompt_dir: %s
|
||||
profile_dir: %s
|
||||
backends:
|
||||
local-gpu:
|
||||
endpoint: http://localhost:11434/v1
|
||||
extra_params:
|
||||
provider_option: enabled
|
||||
concurrency_limit: 2
|
||||
queue_capacity: 0
|
||||
`, lib.promptDir, lib.profileDir))
|
||||
|
||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||
"--config", configPath,
|
||||
"--prompt", "custom",
|
||||
})
|
||||
if code != ExitOK {
|
||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||
}
|
||||
if !strings.Contains(stdout, "selected_backend_id: local-gpu") {
|
||||
t.Fatalf("expected configured backend in prepared output, got:\n%s", stdout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderCommandMapsReasoningEffortAndSessionID(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
writePromptDefinition(t, lib.promptDir, "session.yaml", `id: session
|
||||
version: "1"
|
||||
default_profile: local
|
||||
session_id: definition-session
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
if err := os.WriteFile(filepath.Join(lib.profileDir, "local.yaml"), []byte(`id: local
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
model: local-model
|
||||
reasoning_effort: low
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write profile fixture: %v", err)
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
args []string
|
||||
wantReasoning string
|
||||
wantSessionID string
|
||||
absentReasoning bool
|
||||
}{
|
||||
{name: "omitted reasoning inherits profile", wantReasoning: "low", wantSessionID: "definition-session"},
|
||||
{name: "nonblank reasoning replaces profile", args: []string{"--reasoning-effort", "high"}, wantReasoning: "high", wantSessionID: "definition-session"},
|
||||
{name: "empty reasoning clears profile", args: []string{"--reasoning-effort="}, wantSessionID: "definition-session", absentReasoning: true},
|
||||
{name: "direct session replaces definition", args: []string{"--session-id", "direct-session"}, wantReasoning: "low", wantSessionID: "direct-session"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
args := []string{"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "session"}
|
||||
args = append(args, tc.args...)
|
||||
code, stdout, stderr := runCLICommand(t, renderCommand, args)
|
||||
if code != ExitOK {
|
||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||
}
|
||||
if !strings.Contains(stdout, "session_id: "+tc.wantSessionID) {
|
||||
t.Fatalf("expected session ID %q, got:\n%s", tc.wantSessionID, stdout)
|
||||
}
|
||||
hasReasoning := strings.Contains(stdout, "reasoning_effort:")
|
||||
if tc.absentReasoning {
|
||||
if hasReasoning {
|
||||
t.Fatalf("expected cleared reasoning to be omitted, got:\n%s", stdout)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !strings.Contains(stdout, "reasoning_effort: "+tc.wantReasoning) {
|
||||
t.Fatalf("expected reasoning effort %q, got:\n%s", tc.wantReasoning, stdout)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderCommandOmitsEmptyEffectiveSessionID(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
writePromptDefinition(t, lib.promptDir, "plain.yaml", `id: plain
|
||||
version: "1"
|
||||
default_profile: local
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
|
||||
|
||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||
"--prompt-dir", lib.promptDir,
|
||||
"--profile-dir", lib.profileDir,
|
||||
"--prompt", "plain",
|
||||
})
|
||||
if code != ExitOK {
|
||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||
}
|
||||
if strings.Contains(stdout, "session_id:") {
|
||||
t.Fatalf("expected no effective session ID, got:\n%s", stdout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderCommandRejectsOverlongSessionID(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
writePromptDefinition(t, lib.promptDir, "session.yaml", `id: session
|
||||
version: "1"
|
||||
default_profile: local
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
|
||||
|
||||
code, _, stderr := runCLICommand(t, renderCommand, []string{
|
||||
"--prompt-dir", lib.promptDir,
|
||||
"--profile-dir", lib.profileDir,
|
||||
"--prompt", "session",
|
||||
"--session-id", strings.Repeat("x", 257),
|
||||
})
|
||||
if code != ExitRuntimeError {
|
||||
t.Fatalf("expected runtime error, got %d stderr=%q", code, stderr)
|
||||
}
|
||||
if !strings.Contains(stderr, "invalid run request") {
|
||||
t.Fatalf("expected invalid-request context, got %q", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectPromptCommandFormatsDefinitionWithoutProfileOrGeneration(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
writePromptDefinition(t, lib.promptDir, "inspect.yaml", `id: inspect
|
||||
version: "1"
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--prompt-dir", lib.promptDir, "--prompt", "inspect", "--format", "json"})
|
||||
if code != ExitOK {
|
||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||
}
|
||||
if !strings.Contains(stdout, `"prompt_id": "inspect"`) || !strings.Contains(stdout, `"inputs": []`) {
|
||||
t.Fatalf("unexpected inspection output: %s", stdout)
|
||||
}
|
||||
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--prompt-dir", lib.promptDir, "--prompt", "missing"})
|
||||
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
|
||||
t.Fatalf("expected failed inspection without output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectionParsersPreserveExplicitEmptyConfigPath(t *testing.T) {
|
||||
promptConfig, err := parsePromptInspectionArgs([]string{"--config=", "--prompt", "fixture"})
|
||||
if err != nil {
|
||||
t.Fatalf("parse prompt inspection: %v", err)
|
||||
}
|
||||
if promptConfig.configPath != "" || !promptConfig.configExplicit {
|
||||
t.Fatalf("expected explicit empty prompt config path, got %+v", promptConfig)
|
||||
}
|
||||
|
||||
profileConfig, err := parseProfileInspectionArgs([]string{"--config=", "--profile", "fixture"})
|
||||
if err != nil {
|
||||
t.Fatalf("parse profile inspection: %v", err)
|
||||
}
|
||||
if profileConfig.configPath != "" || !profileConfig.configExplicit {
|
||||
t.Fatalf("expected explicit empty profile config path, got %+v", profileConfig)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptkitV09DefinitionsRenderThroughCLI(t *testing.T) {
|
||||
fixtureRoot := promptkitV09FixtureRoot(t)
|
||||
configPath := writePromptkitV09Config(t, fixtureRoot, true)
|
||||
inputPath := filepath.Join(fixtureRoot, "inputs", "source.md")
|
||||
|
||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||
"--config", configPath,
|
||||
"--prompt", "compat.complete",
|
||||
"--prompt-version", "1.0.0",
|
||||
"--input", "source=" + inputPath,
|
||||
"--var", "topic=testing",
|
||||
"--format", "json",
|
||||
})
|
||||
if code != ExitOK {
|
||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||
}
|
||||
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal([]byte(stdout), &payload); err != nil {
|
||||
t.Fatalf("decode rendered fixture: %v\nbody=%s", err, stdout)
|
||||
}
|
||||
if payload["prompt_version"] != "1.0.0" || payload["selected_profile_id"] != "custom-derived" || payload["selected_backend_id"] != "fixture-custom" {
|
||||
t.Fatalf("unexpected selected definition and target: %#v", payload)
|
||||
}
|
||||
if payload["session_id"] != "fixture-testing" {
|
||||
t.Fatalf("expected rendered session template, got %#v", payload["session_id"])
|
||||
}
|
||||
|
||||
outputContract := payload["output_contract"].(map[string]any)
|
||||
if outputContract["format"] != "json" || outputContract["validation_mode"] != "json_schema" || outputContract["repair_attempts"] != float64(2) {
|
||||
t.Fatalf("unexpected output contract: %#v", outputContract)
|
||||
}
|
||||
if _, ok := payload["structured_output"].(map[string]any)["json_schema"]; !ok {
|
||||
t.Fatalf("expected loaded JSON schema metadata, got %#v", payload["structured_output"])
|
||||
}
|
||||
|
||||
messages := payload["messages"].([]any)
|
||||
wantRoles := []string{"developer", "system", "user", "assistant"}
|
||||
for i, wantRole := range wantRoles {
|
||||
message := messages[i].(map[string]any)
|
||||
if message["role"] != wantRole {
|
||||
t.Fatalf("message %d: expected role %q, got %#v", i, wantRole, message["role"])
|
||||
}
|
||||
}
|
||||
cacheControl := messages[0].(map[string]any)["cache_control"].(map[string]any)
|
||||
if cacheControl["type"] != "ephemeral" || cacheControl["ttl"] != "1h" {
|
||||
t.Fatalf("unexpected cache control: %#v", cacheControl)
|
||||
}
|
||||
if !strings.Contains(messages[2].(map[string]any)["content"].(string), "stable, synthetic material") {
|
||||
t.Fatalf("file-backed input template was not rendered: %#v", messages[2])
|
||||
}
|
||||
inputHashes := payload["input_hashes"].(map[string]any)
|
||||
if len(inputHashes) != 1 || inputHashes["source"] == "" {
|
||||
t.Fatalf("required and omitted optional inputs were not preserved: %#v", inputHashes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptkitV09PromptInspectionSelectsVersionsAndContracts(t *testing.T) {
|
||||
fixtureRoot := promptkitV09FixtureRoot(t)
|
||||
configPath := writePromptkitV09Config(t, fixtureRoot, true)
|
||||
tests := []struct {
|
||||
promptID string
|
||||
version string
|
||||
format string
|
||||
mode string
|
||||
}{
|
||||
{promptID: "compat.complete", version: "1.0.0", format: "json", mode: "json_schema"},
|
||||
{promptID: "compat.complete", version: "2.0.0", format: "text", mode: "basic"},
|
||||
{promptID: "compat.json", version: "1.0.0", format: "json", mode: "json"},
|
||||
{promptID: "compat.none", version: "1.0.0", format: "text", mode: "none"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.promptID+"@"+tc.version, func(t *testing.T) {
|
||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
||||
"prompt",
|
||||
"--config", configPath,
|
||||
"--prompt", tc.promptID,
|
||||
"--prompt-version", tc.version,
|
||||
"--format", "json",
|
||||
})
|
||||
if code != ExitOK {
|
||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||
}
|
||||
var inspection renderformat.PromptInspection
|
||||
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
|
||||
t.Fatalf("decode prompt inspection: %v\nbody=%s", err, stdout)
|
||||
}
|
||||
if inspection.PromptVersion != tc.version || inspection.OutputContract.Format != tc.format || inspection.OutputContract.ValidationMode != tc.mode {
|
||||
t.Fatalf("unexpected prompt inspection: %+v", inspection)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptkitV09ProfileInspectionResolvesSupportedTargets(t *testing.T) {
|
||||
const secret = "sentinel-profile-secret"
|
||||
t.Setenv("FIXTURE_PROFILE_API_KEY", secret)
|
||||
|
||||
fixtureRoot := promptkitV09FixtureRoot(t)
|
||||
configPath := writePromptkitV09Config(t, fixtureRoot, true)
|
||||
tests := []struct {
|
||||
name string
|
||||
profileID string
|
||||
wantBackend string
|
||||
wantModel string
|
||||
wantAPIKeyEnv string
|
||||
}{
|
||||
{name: "inherited custom backend", profileID: "custom-derived", wantBackend: "fixture-custom", wantModel: "fixture-derived-model", wantAPIKeyEnv: "FIXTURE_PROFILE_API_KEY"},
|
||||
{name: "endpoint only", profileID: "endpoint-only", wantModel: "fixture-endpoint-model"},
|
||||
{name: "built in", profileID: "deepseek-4-flash", wantBackend: "openrouter", wantModel: "deepseek/deepseek-v4-flash", wantAPIKeyEnv: "OPENROUTER_API_KEY"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
args := []string{"profile", "--config", configPath, "--profile", tc.profileID, "--format", "json"}
|
||||
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
|
||||
if code != ExitOK {
|
||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||
}
|
||||
if strings.Contains(stdout, secret) || strings.Contains(stderr, secret) {
|
||||
t.Fatalf("inspection exposed environment secret: stdout=%q stderr=%q", stdout, stderr)
|
||||
}
|
||||
var inspection renderformat.ProfileInspection
|
||||
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
|
||||
t.Fatalf("decode profile inspection: %v\nbody=%s", err, stdout)
|
||||
}
|
||||
params := inspection.EffectiveModelParams
|
||||
if inspection.ProfileID != tc.profileID || params.BackendID != tc.wantBackend || params.Model != tc.wantModel || params.APIKeyEnv != tc.wantAPIKeyEnv {
|
||||
t.Fatalf("unexpected effective profile: %+v", inspection)
|
||||
}
|
||||
if tc.profileID == "custom-derived" {
|
||||
if params.ServiceTier != "flex" || params.ReasoningEffort != "high" || params.TimeoutSeconds != 45 || len(params.ExtraParams) == 0 {
|
||||
t.Fatalf("inherited profile controls were not resolved: %+v", params)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
configWithoutPromptDir := writePromptkitV09Config(t, fixtureRoot, false)
|
||||
code, _, stderr := runCLICommand(t, inspectCommand, []string{
|
||||
"profile",
|
||||
"--config", configWithoutPromptDir,
|
||||
"--profile-dir", filepath.Join(fixtureRoot, "profiles"),
|
||||
"--profile", "custom-derived",
|
||||
})
|
||||
if code != ExitOK {
|
||||
t.Fatalf("profile inspection unexpectedly required a prompt directory: %q", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.T) {
|
||||
fixtureRoot := promptkitV09FixtureRoot(t)
|
||||
configPath := writePromptkitV09Config(t, fixtureRoot, true)
|
||||
overrideDir := t.TempDir()
|
||||
writeProfileFile(t, overrideDir, "custom-derived", "http://127.0.0.1:9000/v1", "override-model")
|
||||
outPath := filepath.Join(t.TempDir(), "inspection.json")
|
||||
|
||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
||||
"profile",
|
||||
"--config", configPath,
|
||||
"--profile-dir", overrideDir,
|
||||
"--profile", "custom-derived",
|
||||
"--format", "json",
|
||||
"--out", outPath,
|
||||
})
|
||||
if code != ExitOK || stdout != "" {
|
||||
t.Fatalf("expected file output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
output, err := os.ReadFile(outPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read profile inspection output: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(output), `"model": "override-model"`) || !strings.Contains(string(output), `"backend_id": ""`) {
|
||||
t.Fatalf("profile directory override was not used: %s", output)
|
||||
}
|
||||
|
||||
for _, profileID := range []string{"missing", "invalid"} {
|
||||
t.Run(profileID, func(t *testing.T) {
|
||||
profileDir := filepath.Join(fixtureRoot, "profiles")
|
||||
if profileID == "invalid" {
|
||||
profileDir = t.TempDir()
|
||||
writePromptDefinition(t, profileDir, "invalid.yaml", "id: invalid\nendpoint: not-a-url\nmodel: fixture\n")
|
||||
}
|
||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
||||
"profile",
|
||||
"--config", configPath,
|
||||
"--profile-dir", profileDir,
|
||||
"--profile", profileID,
|
||||
})
|
||||
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
|
||||
t.Fatalf("expected inspection failure without partial output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
backend string
|
||||
}{
|
||||
{
|
||||
name: "reserved ID",
|
||||
backend: `openrouter:
|
||||
endpoint: http://localhost:11434/v1`,
|
||||
},
|
||||
{
|
||||
name: "invalid endpoint",
|
||||
backend: `local-gpu:
|
||||
endpoint: not-a-url`,
|
||||
},
|
||||
{
|
||||
name: "invalid capacity relationship",
|
||||
backend: `local-gpu:
|
||||
endpoint: http://localhost:11434/v1
|
||||
queue_capacity: 0`,
|
||||
},
|
||||
{
|
||||
name: "reserved extra parameter",
|
||||
backend: `local-gpu:
|
||||
endpoint: http://localhost:11434/v1
|
||||
extra_params:
|
||||
model: forbidden`,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
||||
prompt_dir: %s
|
||||
profile_dir: %s
|
||||
backends:
|
||||
%s
|
||||
`, lib.promptDir, lib.profileDir, tc.backend))
|
||||
cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"})
|
||||
if err != nil {
|
||||
t.Fatalf("expected config decoding to succeed, got %v", err)
|
||||
}
|
||||
_, err = newEngine(cfg.runConfig.engineSettings())
|
||||
if !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("expected Promptkit ErrInvalidConfig, got %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "engine initialization from application configuration") {
|
||||
t.Fatalf("expected application context, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderCommandExplicitTextFormatWorks(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
||||
@@ -1088,6 +1631,104 @@ func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderCommandUsesDefinitionInputRulesAndPromptVersions(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "model")
|
||||
|
||||
writePromptDefinition(t, lib.promptDir, "sole.yaml", `id: sole
|
||||
version: "1"
|
||||
default_profile: local
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
writePromptDefinition(t, lib.promptDir, "versioned-one.yaml", `id: versioned
|
||||
version: "1"
|
||||
default_profile: local
|
||||
messages:
|
||||
- role: user
|
||||
content: "one"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
writePromptDefinition(t, lib.promptDir, "versioned-two.yaml", `id: versioned
|
||||
version: "2"
|
||||
default_profile: local
|
||||
messages:
|
||||
- role: user
|
||||
content: "two"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
writePromptDefinition(t, lib.promptDir, "optional.yaml", `id: optional
|
||||
version: "1"
|
||||
default_profile: local
|
||||
inputs:
|
||||
- name: note
|
||||
required: false
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
writePromptDefinition(t, lib.promptDir, "required.yaml", `id: required
|
||||
version: "1"
|
||||
default_profile: local
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
writePromptDefinition(t, lib.promptDir, "template.yaml", `id: template
|
||||
version: "1"
|
||||
default_profile: local
|
||||
messages:
|
||||
- role: user
|
||||
content: '{{input "note"}}'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
|
||||
baseArgs := []string{"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
args []string
|
||||
wantCode int
|
||||
wantText string
|
||||
}{
|
||||
{name: "sole version selected when omitted", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt_version: 1"},
|
||||
{name: "explicit version selected", args: []string{"--prompt", "versioned", "--prompt-version", "2"}, wantCode: ExitOK, wantText: "prompt_version: 2"},
|
||||
{name: "multiple versions require selection", args: []string{"--prompt", "versioned"}, wantCode: ExitRuntimeError, wantText: "duplicate prompt definition id"},
|
||||
{name: "no declared inputs", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt: sole"},
|
||||
{name: "optional input omitted", args: []string{"--prompt", "optional"}, wantCode: ExitOK, wantText: "prompt: optional"},
|
||||
{name: "required input omitted", args: []string{"--prompt", "required"}, wantCode: ExitRuntimeError, wantText: "required"},
|
||||
{name: "template input omitted", args: []string{"--prompt", "template"}, wantCode: ExitRuntimeError, wantText: "note"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
code, stdout, stderr := runCLICommand(t, renderCommand, append(append([]string{}, baseArgs...), tc.args...))
|
||||
if code != tc.wantCode {
|
||||
t.Fatalf("expected exit %d, got %d stderr=%q", tc.wantCode, code, stderr)
|
||||
}
|
||||
if !strings.Contains(stdout+stderr, tc.wantText) {
|
||||
t.Fatalf("expected output to contain %q, stdout=%q stderr=%q", tc.wantText, stdout, stderr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||
@@ -1208,12 +1849,12 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
|
||||
if err := writeOutput(&stdout, "", []byte("artifact-body")); err != nil {
|
||||
t.Fatalf("unexpected writeOutput error: %v", err)
|
||||
}
|
||||
printSummary(&stderr, &domain.RunResult{
|
||||
printSummary(&stderr, &promptkit.RunResult{
|
||||
PromptID: "p",
|
||||
PromptVersion: "1",
|
||||
SelectedProfileID: "exec",
|
||||
ModelName: "m",
|
||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic},
|
||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic},
|
||||
RenderedPromptHash: "h",
|
||||
InputHashes: map[string]string{"in": "x"},
|
||||
})
|
||||
@@ -1227,20 +1868,23 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
|
||||
if strings.Contains(stderr.String(), "cached_tokens=") || strings.Contains(stderr.String(), "cache_write_tokens=") {
|
||||
t.Fatalf("expected zero cache usage to be omitted from summary, got %q", stderr.String())
|
||||
}
|
||||
if strings.Contains(stderr.String(), "backend=") {
|
||||
t.Fatalf("expected endpoint-only backend to be omitted from summary, got %q", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) {
|
||||
var stderr bytes.Buffer
|
||||
|
||||
printSummary(&stderr, &domain.RunResult{
|
||||
printSummary(&stderr, &promptkit.RunResult{
|
||||
PromptID: "p",
|
||||
PromptVersion: "1",
|
||||
SelectedProfileID: "exec",
|
||||
ModelName: "m",
|
||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic},
|
||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic},
|
||||
RenderedPromptHash: "h",
|
||||
InputHashes: map[string]string{"in": "x"},
|
||||
Usage: domain.TokenUsage{
|
||||
Usage: promptkit.TokenUsage{
|
||||
PromptTokens: 10,
|
||||
CompletionTokens: 5,
|
||||
TotalTokens: 15,
|
||||
@@ -1256,6 +1900,32 @@ func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) {
|
||||
if !strings.Contains(summary, "cached_tokens=0 cache_write_tokens=3") {
|
||||
t.Fatalf("expected cache usage in summary, got %q", summary)
|
||||
}
|
||||
if strings.Contains(summary, "backend=") {
|
||||
t.Fatalf("expected endpoint-only backend to be omitted from summary, got %q", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrintSummaryIncludesBackendWhenPresent(t *testing.T) {
|
||||
var stderr bytes.Buffer
|
||||
printSummary(&stderr, &promptkit.RunResult{
|
||||
PromptID: "p",
|
||||
PromptVersion: "1",
|
||||
SelectedProfileID: "exec",
|
||||
SelectedBackendID: "local",
|
||||
ModelName: "m",
|
||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic},
|
||||
RenderedPromptHash: "h",
|
||||
})
|
||||
if !strings.Contains(stderr.String(), "backend=local") {
|
||||
t.Fatalf("expected backend in summary, got %q", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunErrorMessageDoesNotExposeCapacityDetails(t *testing.T) {
|
||||
got := runErrorMessage(&promptkit.CapacityError{BackendID: "private-backend"})
|
||||
if got != "run error: model backend capacity is exhausted" {
|
||||
t.Fatalf("unexpected capacity diagnostic: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
type cliTestLibrary struct {
|
||||
@@ -1301,11 +1971,48 @@ func runCLICommand(t *testing.T, command func([]string, io.Writer, io.Writer) in
|
||||
return code, stdout.String(), stderr.String()
|
||||
}
|
||||
|
||||
func promptkitV09FixtureRoot(t *testing.T) string {
|
||||
t.Helper()
|
||||
root, err := filepath.Abs(filepath.Join("..", "..", "..", "testdata", "promptkit-v0.9"))
|
||||
if err != nil {
|
||||
t.Fatalf("resolve Promptkit v0.9 fixture root: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(root); err != nil {
|
||||
t.Fatalf("stat Promptkit v0.9 fixture root: %v", err)
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func writePromptkitV09Config(t *testing.T, fixtureRoot string, includeSources bool) string {
|
||||
t.Helper()
|
||||
sources := ""
|
||||
if includeSources {
|
||||
sources = fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\nschema_dir: %q\n", filepath.Join(fixtureRoot, "prompts"), filepath.Join(fixtureRoot, "profiles"), filepath.Join(fixtureRoot, "schemas"))
|
||||
}
|
||||
return writeAppConfigFile(t, sources+`backends:
|
||||
fixture-custom:
|
||||
endpoint: http://127.0.0.1:11434/v1
|
||||
api_key_env: FIXTURE_BACKEND_API_KEY
|
||||
extra_params:
|
||||
backend_option:
|
||||
enabled: true
|
||||
concurrency_limit: 2
|
||||
queue_capacity: 0
|
||||
`)
|
||||
}
|
||||
|
||||
func writePromptFile(t *testing.T, dir, id, defaultProfile string) {
|
||||
t.Helper()
|
||||
writePromptFileWithTemplate(t, dir, id, defaultProfile, "Summarize: {{input \"transcript\"}}")
|
||||
}
|
||||
|
||||
func writePromptDefinition(t *testing.T, dir, name, definition string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(definition), 0o644); err != nil {
|
||||
t.Fatalf("write prompt definition: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writePromptFileWithTemplate(t *testing.T, dir, id, defaultProfile, templateContent string) {
|
||||
t.Helper()
|
||||
data := fmt.Sprintf(`id: %s
|
||||
|
||||
322
internal/adapter/dependency_test.go
Normal file
322
internal/adapter/dependency_test.go
Normal file
@@ -0,0 +1,322 @@
|
||||
package adapter_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const (
|
||||
scriptoriumModulePath = "gitea.maximumdirect.net/eric/scriptorium"
|
||||
promptkitInternalPath = "gitea.maximumdirect.net/eric/promptkit/internal"
|
||||
promptkitOpenRouterCatalogPath = "gitea.maximumdirect.net/eric/promptkit-backend-openrouter"
|
||||
promptkitRakestrawhomeCatalogPath = "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome"
|
||||
)
|
||||
|
||||
var (
|
||||
removedFrameworkPackageRoots = []string{
|
||||
scriptoriumModulePath + "/internal/artifact",
|
||||
scriptoriumModulePath + "/internal/domain",
|
||||
scriptoriumModulePath + "/internal/filecatalog",
|
||||
scriptoriumModulePath + "/internal/llm",
|
||||
scriptoriumModulePath + "/internal/profile",
|
||||
scriptoriumModulePath + "/internal/prompt",
|
||||
scriptoriumModulePath + "/internal/promptdef",
|
||||
scriptoriumModulePath + "/internal/usecase",
|
||||
scriptoriumModulePath + "/internal/validate",
|
||||
}
|
||||
removedFrameworkDirectories = []string{
|
||||
"internal/artifact",
|
||||
"internal/domain",
|
||||
"internal/filecatalog",
|
||||
"internal/llm",
|
||||
"internal/profile",
|
||||
"internal/prompt",
|
||||
"internal/promptdef",
|
||||
"internal/usecase",
|
||||
"internal/validate",
|
||||
}
|
||||
nonSourceDirectories = map[string]struct{}{
|
||||
".cache": {},
|
||||
".codebase-memory": {},
|
||||
".git": {},
|
||||
"build": {},
|
||||
"coverage": {},
|
||||
"dist": {},
|
||||
"node_modules": {},
|
||||
"out": {},
|
||||
"testdata": {},
|
||||
"vendor": {},
|
||||
}
|
||||
)
|
||||
|
||||
type forbiddenImport struct {
|
||||
filePath string
|
||||
importPath string
|
||||
}
|
||||
|
||||
func TestApplicationBoundary(t *testing.T) {
|
||||
moduleRoot := moduleRootFromTestFile(t)
|
||||
|
||||
violations, err := findForbiddenProductionImports(moduleRoot)
|
||||
if err != nil {
|
||||
t.Fatalf("scan production imports: %v", err)
|
||||
}
|
||||
for _, violation := range violations {
|
||||
t.Errorf("%s imports forbidden package %s", violation.filePath, violation.importPath)
|
||||
}
|
||||
|
||||
assertFrameworkImplementationAbsent(t, moduleRoot)
|
||||
}
|
||||
|
||||
func TestForbiddenImportScannerDetectsFormerRoot(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
sourcePath := writeGoSource(t, root, "nested/consumer/root.go", scriptoriumModulePath)
|
||||
|
||||
violations, err := findForbiddenProductionImports(root)
|
||||
if err != nil {
|
||||
t.Fatalf("scan source fixture: %v", err)
|
||||
}
|
||||
assertSingleViolation(t, violations, sourcePath, scriptoriumModulePath)
|
||||
}
|
||||
|
||||
func TestForbiddenImportScannerDetectsFormerFrameworkFamily(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
importPath := scriptoriumModulePath + "/internal/profile/builtin"
|
||||
sourcePath := writeGoSource(t, root, "nested/consumer/profile.go", importPath)
|
||||
|
||||
violations, err := findForbiddenProductionImports(root)
|
||||
if err != nil {
|
||||
t.Fatalf("scan source fixture: %v", err)
|
||||
}
|
||||
assertSingleViolation(t, violations, sourcePath, importPath)
|
||||
}
|
||||
|
||||
func TestForbiddenImportScannerDetectsPromptkitInternalPackages(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
importPath string
|
||||
}{
|
||||
{name: "exact internal root", importPath: promptkitInternalPath},
|
||||
{name: "internal descendant", importPath: promptkitInternalPath + "/domain"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
sourcePath := writeGoSource(t, root, "nested/consumer/promptkit.go", tc.importPath)
|
||||
|
||||
violations, err := findForbiddenProductionImports(root)
|
||||
if err != nil {
|
||||
t.Fatalf("scan source fixture: %v", err)
|
||||
}
|
||||
assertSingleViolation(t, violations, sourcePath, tc.importPath)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestForbiddenImportScannerDetectsPromptkitCatalogPackages(t *testing.T) {
|
||||
for _, importPath := range []string{
|
||||
promptkitOpenRouterCatalogPath,
|
||||
promptkitRakestrawhomeCatalogPath,
|
||||
} {
|
||||
t.Run(importPath, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
sourcePath := writeGoSource(t, root, "nested/consumer/catalog.go", importPath)
|
||||
|
||||
violations, err := findForbiddenProductionImports(root)
|
||||
if err != nil {
|
||||
t.Fatalf("scan source fixture: %v", err)
|
||||
}
|
||||
assertSingleViolation(t, violations, sourcePath, importPath)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestForbiddenImportScannerAllowsRetainedApplicationPackages(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
sourcePath := filepath.Join(root, "nested/consumer/application.go")
|
||||
if err := os.MkdirAll(filepath.Dir(sourcePath), 0o755); err != nil {
|
||||
t.Fatalf("create source fixture directory: %v", err)
|
||||
}
|
||||
source := `package consumer
|
||||
|
||||
import (
|
||||
_ "gitea.maximumdirect.net/eric/promptkit"
|
||||
_ "gitea.maximumdirect.net/eric/scriptorium/internal/adapter/http"
|
||||
_ "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||
_ "gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||
_ "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||
)
|
||||
`
|
||||
if err := os.WriteFile(sourcePath, []byte(source), 0o644); err != nil {
|
||||
t.Fatalf("write source fixture: %v", err)
|
||||
}
|
||||
|
||||
violations, err := findForbiddenProductionImports(root)
|
||||
if err != nil {
|
||||
t.Fatalf("scan source fixture: %v", err)
|
||||
}
|
||||
if len(violations) != 0 {
|
||||
t.Fatalf("expected retained application imports to be allowed, got %#v", violations)
|
||||
}
|
||||
}
|
||||
|
||||
func findForbiddenProductionImports(root string) ([]forbiddenImport, error) {
|
||||
var violations []forbiddenImport
|
||||
err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
if path != root && shouldSkipSourceDirectory(entry.Name()) {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !strings.HasSuffix(entry.Name(), ".go") || strings.HasSuffix(entry.Name(), "_test.go") {
|
||||
return nil
|
||||
}
|
||||
|
||||
file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.ImportsOnly)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse imports in %s: %w", path, err)
|
||||
}
|
||||
for _, imported := range file.Imports {
|
||||
importPath, err := strconv.Unquote(imported.Path.Value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse import path in %s: %w", path, err)
|
||||
}
|
||||
if isForbiddenProductionImport(importPath) {
|
||||
violations = append(violations, forbiddenImport{
|
||||
filePath: path,
|
||||
importPath: importPath,
|
||||
})
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("walk repository root %s: %w", root, err)
|
||||
}
|
||||
return violations, nil
|
||||
}
|
||||
|
||||
func shouldSkipSourceDirectory(name string) bool {
|
||||
_, skip := nonSourceDirectories[name]
|
||||
return skip
|
||||
}
|
||||
|
||||
func isForbiddenProductionImport(importPath string) bool {
|
||||
if importPath == scriptoriumModulePath {
|
||||
return true
|
||||
}
|
||||
if importPath == promptkitInternalPath || strings.HasPrefix(importPath, promptkitInternalPath+"/") {
|
||||
return true
|
||||
}
|
||||
if importPath == promptkitOpenRouterCatalogPath || strings.HasPrefix(importPath, promptkitOpenRouterCatalogPath+"/") {
|
||||
return true
|
||||
}
|
||||
if importPath == promptkitRakestrawhomeCatalogPath || strings.HasPrefix(importPath, promptkitRakestrawhomeCatalogPath+"/") {
|
||||
return true
|
||||
}
|
||||
for _, root := range removedFrameworkPackageRoots {
|
||||
if importPath == root || strings.HasPrefix(importPath, root+"/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func moduleRootFromTestFile(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
_, testFile, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
t.Fatal("locate dependency guard source")
|
||||
}
|
||||
root, err := findModuleRoot(filepath.Dir(testFile))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func findModuleRoot(start string) (string, error) {
|
||||
dir, err := filepath.Abs(start)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve module search path: %w", err)
|
||||
}
|
||||
for {
|
||||
goMod := filepath.Join(dir, "go.mod")
|
||||
if info, err := os.Stat(goMod); err == nil && !info.IsDir() {
|
||||
return dir, nil
|
||||
} else if err != nil && !os.IsNotExist(err) {
|
||||
return "", fmt.Errorf("inspect %s: %w", goMod, err)
|
||||
}
|
||||
|
||||
parent := filepath.Dir(dir)
|
||||
if parent == dir {
|
||||
return "", fmt.Errorf("locate go.mod from %s", start)
|
||||
}
|
||||
dir = parent
|
||||
}
|
||||
}
|
||||
|
||||
func assertFrameworkImplementationAbsent(t *testing.T, moduleRoot string) {
|
||||
t.Helper()
|
||||
|
||||
entries, err := os.ReadDir(moduleRoot)
|
||||
if err != nil {
|
||||
t.Fatalf("read module root: %v", err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".go") && !strings.HasSuffix(entry.Name(), "_test.go") {
|
||||
t.Errorf("module root contains production Go file %s", entry.Name())
|
||||
}
|
||||
}
|
||||
|
||||
for _, relativePath := range removedFrameworkDirectories {
|
||||
path := filepath.Join(moduleRoot, filepath.FromSlash(relativePath))
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
t.Errorf("removed framework directory still exists: %s", relativePath)
|
||||
} else if !os.IsNotExist(err) {
|
||||
t.Errorf("inspect removed framework directory %s: %v", relativePath, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func writeGoSource(t *testing.T, root, relativePath, importPath string) string {
|
||||
t.Helper()
|
||||
|
||||
sourcePath := filepath.Join(root, filepath.FromSlash(relativePath))
|
||||
if err := os.MkdirAll(filepath.Dir(sourcePath), 0o755); err != nil {
|
||||
t.Fatalf("create source fixture directory: %v", err)
|
||||
}
|
||||
source := fmt.Sprintf("package consumer\n\nimport _ %q\n", importPath)
|
||||
if err := os.WriteFile(sourcePath, []byte(source), 0o644); err != nil {
|
||||
t.Fatalf("write source fixture: %v", err)
|
||||
}
|
||||
return sourcePath
|
||||
}
|
||||
|
||||
func assertSingleViolation(t *testing.T, violations []forbiddenImport, sourcePath, importPath string) {
|
||||
t.Helper()
|
||||
|
||||
if len(violations) != 1 {
|
||||
t.Fatalf("expected one forbidden import, got %#v", violations)
|
||||
}
|
||||
if violations[0].filePath != sourcePath {
|
||||
t.Fatalf("unexpected importing file: %q", violations[0].filePath)
|
||||
}
|
||||
if violations[0].importPath != importPath {
|
||||
t.Fatalf("unexpected forbidden import: %q", violations[0].importPath)
|
||||
}
|
||||
}
|
||||
165
internal/adapter/http/artifact_reader.go
Normal file
165
internal/adapter/http/artifact_reader.go
Normal file
@@ -0,0 +1,165 @@
|
||||
package httpadapter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrFileNotAllowed = errors.New("file artifact references are not allowed")
|
||||
ErrFileOutsideRoot = errors.New("file artifact path is outside artifact root")
|
||||
ErrFileTooLarge = errors.New("file artifact exceeds size limit")
|
||||
)
|
||||
|
||||
const fallbackArtifactContentType = "text/plain"
|
||||
|
||||
// NewRestrictedArtifactReader creates the HTTP artifact reader for a rooted
|
||||
// filesystem and optional byte limit. An empty root permits inline artifacts
|
||||
// but denies file references; a zero limit permits artifacts of any size.
|
||||
func NewRestrictedArtifactReader(root string, maxBytes int64) (promptkit.ArtifactReader, error) {
|
||||
if maxBytes < 0 {
|
||||
return nil, fmt.Errorf("artifact size limit must be greater than or equal to 0")
|
||||
}
|
||||
|
||||
cleanRoot := strings.TrimSpace(root)
|
||||
if cleanRoot == "" {
|
||||
return &restrictedArtifactReader{maxBytes: maxBytes}, nil
|
||||
}
|
||||
absRoot, err := filepath.Abs(filepath.Clean(cleanRoot))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve artifact root: %w", err)
|
||||
}
|
||||
return &restrictedArtifactReader{root: absRoot, maxBytes: maxBytes}, nil
|
||||
}
|
||||
|
||||
type restrictedArtifactReader struct {
|
||||
root string
|
||||
maxBytes int64
|
||||
}
|
||||
|
||||
var _ promptkit.ArtifactReader = (*restrictedArtifactReader)(nil)
|
||||
|
||||
func (r *restrictedArtifactReader) Read(ctx context.Context, ref promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
switch ref.Type {
|
||||
case promptkit.ArtifactRefInline:
|
||||
return readInlineArtifact(ref)
|
||||
case promptkit.ArtifactRefFile:
|
||||
return r.readFileArtifact(ref)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported artifact reference type %q", ref.Type)
|
||||
}
|
||||
}
|
||||
|
||||
func readInlineArtifact(ref promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||
if ref.Body == "" {
|
||||
return nil, errors.New("inline artifact body is required")
|
||||
}
|
||||
|
||||
body := []byte(ref.Body)
|
||||
return &promptkit.Artifact{
|
||||
ContentType: fallbackArtifactContentType,
|
||||
Body: body,
|
||||
Size: int64(len(body)),
|
||||
Hash: artifactHash(body),
|
||||
URI: ref.URI,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *restrictedArtifactReader) readFileArtifact(ref promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||
if ref.URI == "" {
|
||||
return nil, errors.New("file artifact path is required")
|
||||
}
|
||||
if r.root == "" {
|
||||
return nil, ErrFileNotAllowed
|
||||
}
|
||||
|
||||
path, err := r.resolveLexicalPath(ref.URI)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return readArtifactFile(path, r.maxBytes)
|
||||
}
|
||||
|
||||
// resolveLexicalPath checks cleaned path containment without resolving symlinks.
|
||||
func (r *restrictedArtifactReader) resolveLexicalPath(rawPath string) (string, error) {
|
||||
cleanPath := filepath.Clean(strings.TrimSpace(rawPath))
|
||||
candidate := cleanPath
|
||||
if !filepath.IsAbs(cleanPath) {
|
||||
candidate = filepath.Join(r.root, cleanPath)
|
||||
}
|
||||
|
||||
absCandidate, err := filepath.Abs(candidate)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve artifact path: %w", err)
|
||||
}
|
||||
absCandidate = filepath.Clean(absCandidate)
|
||||
|
||||
rel, err := filepath.Rel(r.root, absCandidate)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("compare artifact path to root: %w", err)
|
||||
}
|
||||
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) {
|
||||
return "", ErrFileOutsideRoot
|
||||
}
|
||||
return absCandidate, nil
|
||||
}
|
||||
|
||||
func readArtifactFile(path string, maxBytes int64) (*promptkit.Artifact, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to stat file %s: %w", path, err)
|
||||
}
|
||||
if maxBytes > 0 && info.Size() > maxBytes {
|
||||
return nil, ErrFileTooLarge
|
||||
}
|
||||
|
||||
var reader io.Reader = file
|
||||
if maxBytes > 0 {
|
||||
reader = io.LimitReader(file, maxBytes+1)
|
||||
}
|
||||
body, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
||||
}
|
||||
if maxBytes > 0 && int64(len(body)) > maxBytes {
|
||||
return nil, ErrFileTooLarge
|
||||
}
|
||||
|
||||
contentType := mime.TypeByExtension(filepath.Ext(path))
|
||||
if contentType == "" {
|
||||
contentType = fallbackArtifactContentType
|
||||
}
|
||||
return &promptkit.Artifact{
|
||||
Name: filepath.Base(path),
|
||||
ContentType: contentType,
|
||||
Body: body,
|
||||
URI: path,
|
||||
Size: int64(len(body)),
|
||||
Hash: artifactHash(body),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func artifactHash(body []byte) string {
|
||||
return fmt.Sprintf("%x", sha256.Sum256(body))
|
||||
}
|
||||
185
internal/adapter/http/artifact_reader_test.go
Normal file
185
internal/adapter/http/artifact_reader_test.go
Normal file
@@ -0,0 +1,185 @@
|
||||
package httpadapter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"mime"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestRestrictedArtifactReaderReadsContainedFiles(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
inputPath := filepath.Join(root, "input.html")
|
||||
if err := os.WriteFile(inputPath, []byte("allowed"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "input.unknown"), []byte("unknown type"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Mkdir(filepath.Join(root, "nested"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(outside, "secret.txt"), []byte("denied"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expectedContentType := mime.TypeByExtension(filepath.Ext(inputPath))
|
||||
if expectedContentType == "" {
|
||||
t.Fatal("expected built-in HTML content type")
|
||||
}
|
||||
|
||||
reader, err := NewRestrictedArtifactReader(root, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("construct restricted reader: %v", err)
|
||||
}
|
||||
|
||||
for _, ref := range []promptkit.ArtifactRef{
|
||||
{Type: promptkit.ArtifactRefFile, URI: "nested/../input.html"},
|
||||
{Type: promptkit.ArtifactRefFile, URI: inputPath},
|
||||
} {
|
||||
artifact, err := reader.Read(context.Background(), ref)
|
||||
if err != nil {
|
||||
t.Fatalf("read contained path %q: %v", ref.URI, err)
|
||||
}
|
||||
if artifact.Name != "input.html" || artifact.URI != inputPath || artifact.Size != int64(len("allowed")) || string(artifact.Body) != "allowed" {
|
||||
t.Fatalf("unexpected artifact metadata: %#v", artifact)
|
||||
}
|
||||
if artifact.ContentType != expectedContentType {
|
||||
t.Fatalf("unexpected artifact content type: got %q, want %q", artifact.ContentType, expectedContentType)
|
||||
}
|
||||
if artifact.Hash != artifactHash([]byte("allowed")) {
|
||||
t.Fatalf("unexpected artifact hash: %q", artifact.Hash)
|
||||
}
|
||||
}
|
||||
|
||||
artifact, err := reader.Read(context.Background(), promptkit.File("input.unknown"))
|
||||
if err != nil {
|
||||
t.Fatalf("read unknown-extension path: %v", err)
|
||||
}
|
||||
if artifact.ContentType != fallbackArtifactContentType {
|
||||
t.Fatalf("unexpected fallback content type: %q", artifact.ContentType)
|
||||
}
|
||||
|
||||
for _, ref := range []promptkit.ArtifactRef{
|
||||
{Type: promptkit.ArtifactRefFile, URI: filepath.Join("..", filepath.Base(outside), "secret.txt")},
|
||||
{Type: promptkit.ArtifactRefFile, URI: filepath.Join(outside, "secret.txt")},
|
||||
} {
|
||||
_, err := reader.Read(context.Background(), ref)
|
||||
if !errors.Is(err, ErrFileOutsideRoot) {
|
||||
t.Fatalf("expected ErrFileOutsideRoot for %q, got %v", ref.URI, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestrictedArtifactReaderFollowsSymlinkAfterLexicalCheck(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
target := filepath.Join(outside, "linked.txt")
|
||||
if err := os.WriteFile(target, []byte("linked outside root"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(target, filepath.Join(root, "linked.txt")); err != nil {
|
||||
t.Skipf("symlink creation unavailable: %v", err)
|
||||
}
|
||||
|
||||
reader, err := NewRestrictedArtifactReader(root, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("construct restricted reader: %v", err)
|
||||
}
|
||||
artifact, err := reader.Read(context.Background(), promptkit.File("linked.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("read symlink inside root: %v", err)
|
||||
}
|
||||
if string(artifact.Body) != "linked outside root" {
|
||||
t.Fatalf("unexpected symlink artifact body: %q", artifact.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestrictedArtifactReaderWithoutRootDeniesFiles(t *testing.T) {
|
||||
reader, err := NewRestrictedArtifactReader("", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("construct rootless reader: %v", err)
|
||||
}
|
||||
|
||||
artifact, err := reader.Read(context.Background(), promptkit.Inline("inline"))
|
||||
if err != nil {
|
||||
t.Fatalf("read inline artifact: %v", err)
|
||||
}
|
||||
if artifact.ContentType != fallbackArtifactContentType || string(artifact.Body) != "inline" || artifact.Hash != artifactHash([]byte("inline")) {
|
||||
t.Fatalf("unexpected inline artifact: %#v", artifact)
|
||||
}
|
||||
|
||||
_, err = reader.Read(context.Background(), promptkit.File("input.txt"))
|
||||
if !errors.Is(err, ErrFileNotAllowed) {
|
||||
t.Fatalf("expected ErrFileNotAllowed, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestrictedArtifactReaderEnforcesLimits(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, "exact.txt"), []byte("12345"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "large.txt"), []byte("123456"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
reader, err := NewRestrictedArtifactReader(root, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("construct limited reader: %v", err)
|
||||
}
|
||||
artifact, err := reader.Read(context.Background(), promptkit.File("exact.txt"))
|
||||
if err != nil || string(artifact.Body) != "12345" {
|
||||
t.Fatalf("expected exact-limit artifact, got %#v and %v", artifact, err)
|
||||
}
|
||||
_, err = reader.Read(context.Background(), promptkit.File("large.txt"))
|
||||
if !errors.Is(err, ErrFileTooLarge) {
|
||||
t.Fatalf("expected ErrFileTooLarge, got %v", err)
|
||||
}
|
||||
|
||||
unlimited, err := NewRestrictedArtifactReader(root, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("construct unlimited reader: %v", err)
|
||||
}
|
||||
artifact, err = unlimited.Read(context.Background(), promptkit.File("large.txt"))
|
||||
if err != nil || string(artifact.Body) != "123456" {
|
||||
t.Fatalf("expected unlimited artifact, got %#v and %v", artifact, err)
|
||||
}
|
||||
|
||||
if _, err := NewRestrictedArtifactReader(root, -1); err == nil {
|
||||
t.Fatal("expected negative limit to fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestrictedArtifactReaderRejectsCanceledAndMalformedReferences(t *testing.T) {
|
||||
reader, err := NewRestrictedArtifactReader(t.TempDir(), 0)
|
||||
if err != nil {
|
||||
t.Fatalf("construct reader: %v", err)
|
||||
}
|
||||
|
||||
canceledCtx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
for _, ref := range []promptkit.ArtifactRef{
|
||||
promptkit.Inline("input"),
|
||||
promptkit.File("input.txt"),
|
||||
} {
|
||||
_, err := reader.Read(canceledCtx, ref)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected cancellation for %#v, got %v", ref, err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, ref := range []promptkit.ArtifactRef{
|
||||
{Type: promptkit.ArtifactRefType("unsupported")},
|
||||
{Type: promptkit.ArtifactRefInline},
|
||||
{Type: promptkit.ArtifactRefFile},
|
||||
} {
|
||||
if _, err := reader.Read(context.Background(), ref); err == nil {
|
||||
t.Fatalf("expected malformed reference %#v to fail", ref)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ type runRequestDTO struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Inputs map[string]inputRefDTO `json:"inputs"`
|
||||
Vars map[string]string `json:"vars,omitempty"`
|
||||
Model *modelOverrideRequestDTO `json:"model,omitempty"`
|
||||
@@ -28,7 +29,7 @@ type modelOverrideRequestDTO struct {
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||
ServiceTier string `json:"service_tier,omitempty"`
|
||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||
ReasoningEffort *string `json:"reasoning_effort,omitempty"`
|
||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||
}
|
||||
@@ -56,6 +57,8 @@ type metadataDTO struct {
|
||||
PromptHash string `json:"prompt_hash"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
ModelName string `json:"model_name"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
ModelParams modelParamsDTO `json:"model_params"`
|
||||
@@ -71,6 +74,7 @@ type metadataDTO struct {
|
||||
|
||||
type modelParamsDTO struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
BackendID string `json:"backend_id,omitempty"`
|
||||
Model string `json:"model"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
|
||||
@@ -8,16 +8,12 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||
)
|
||||
|
||||
type Runner interface {
|
||||
Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error)
|
||||
Run(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error)
|
||||
}
|
||||
|
||||
type Handler struct {
|
||||
@@ -80,29 +76,28 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request", "prompt_id is required")
|
||||
return
|
||||
}
|
||||
if len(req.Inputs) == 0 {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request", "inputs is required")
|
||||
return
|
||||
}
|
||||
|
||||
mappedInputs := make(map[string]domain.ArtifactRef, len(req.Inputs))
|
||||
var mappedInputs map[string]promptkit.ArtifactRef
|
||||
if len(req.Inputs) > 0 {
|
||||
mappedInputs = make(map[string]promptkit.ArtifactRef, len(req.Inputs))
|
||||
for name, in := range req.Inputs {
|
||||
mappedInputs[name] = domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefType(in.Type),
|
||||
mappedInputs[name] = promptkit.ArtifactRef{
|
||||
Type: promptkit.ArtifactRefType(in.Type),
|
||||
URI: in.URI,
|
||||
Body: in.Body,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var model *domain.ExecutionTargetOverride
|
||||
var model *promptkit.ExecutionTargetOverride
|
||||
if req.Model != nil {
|
||||
model = executionTargetOverrideFromModelOverrideDTO(req.Model)
|
||||
}
|
||||
|
||||
res, err := h.runner.Run(r.Context(), domain.RunRequest{
|
||||
res, err := h.runner.Run(r.Context(), promptkit.RunRequest{
|
||||
PromptID: req.PromptID,
|
||||
PromptVersion: req.PromptVersion,
|
||||
ProfileID: req.ProfileID,
|
||||
SessionID: req.SessionID,
|
||||
Inputs: mappedInputs,
|
||||
Vars: req.Vars,
|
||||
Execution: model,
|
||||
@@ -130,6 +125,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
PromptHash: res.PromptHash,
|
||||
RenderedPromptHash: res.RenderedPromptHash,
|
||||
SelectedProfileID: res.SelectedProfileID,
|
||||
SelectedBackendID: res.SelectedBackendID,
|
||||
SessionID: res.SessionID,
|
||||
ModelName: res.ModelName,
|
||||
Endpoint: res.Endpoint,
|
||||
ModelParams: modelParamsDTOFromExecutionTarget(res.EffectiveModelParams),
|
||||
@@ -156,11 +153,11 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
writeLimitedJSON(w, http.StatusOK, resp, h.options.MaxResponseBytes)
|
||||
}
|
||||
|
||||
func executionTargetOverrideFromModelOverrideDTO(dto *modelOverrideRequestDTO) *domain.ExecutionTargetOverride {
|
||||
func executionTargetOverrideFromModelOverrideDTO(dto *modelOverrideRequestDTO) *promptkit.ExecutionTargetOverride {
|
||||
if dto == nil {
|
||||
return nil
|
||||
}
|
||||
return &domain.ExecutionTargetOverride{
|
||||
return &promptkit.ExecutionTargetOverride{
|
||||
Endpoint: dto.Endpoint,
|
||||
Model: dto.Model,
|
||||
Temperature: dto.Temperature,
|
||||
@@ -174,9 +171,10 @@ func executionTargetOverrideFromModelOverrideDTO(dto *modelOverrideRequestDTO) *
|
||||
}
|
||||
}
|
||||
|
||||
func modelParamsDTOFromExecutionTarget(target domain.ExecutionTarget) modelParamsDTO {
|
||||
func modelParamsDTOFromExecutionTarget(target promptkit.ExecutionTarget) modelParamsDTO {
|
||||
return modelParamsDTO{
|
||||
Endpoint: target.Endpoint,
|
||||
BackendID: target.BackendID,
|
||||
Model: target.Model,
|
||||
Temperature: target.Temperature,
|
||||
MaxTokens: target.MaxTokens,
|
||||
@@ -189,7 +187,7 @@ func modelParamsDTOFromExecutionTarget(target domain.ExecutionTarget) modelParam
|
||||
}
|
||||
}
|
||||
|
||||
func mapValidation(v domain.ValidationResult) validationDTO {
|
||||
func mapValidation(v promptkit.ValidationResult) validationDTO {
|
||||
return validationDTO{
|
||||
Status: string(v.Status),
|
||||
Mode: string(v.Mode),
|
||||
@@ -202,35 +200,33 @@ func mapValidation(v domain.ValidationResult) validationDTO {
|
||||
|
||||
func mapRunError(err error) (int, string, string) {
|
||||
switch {
|
||||
case errors.Is(err, promptdef.ErrPromptDefinitionNotFound):
|
||||
case errors.Is(err, promptkit.ErrPromptNotFound):
|
||||
return http.StatusNotFound, "prompt_not_found", "prompt definition not found"
|
||||
case errors.Is(err, profile.ErrProfileNotFound):
|
||||
case errors.Is(err, promptkit.ErrProfileNotFound):
|
||||
return http.StatusNotFound, "profile_not_found", "execution profile not found"
|
||||
case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition):
|
||||
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
|
||||
case errors.Is(err, profile.ErrInvalidYAML), errors.Is(err, profile.ErrInvalidProfile), errors.Is(err, profile.ErrRawAPIKeyNotAllowed):
|
||||
return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile"
|
||||
case errors.Is(err, usecase.ErrProfileRequired):
|
||||
case errors.Is(err, promptkit.ErrProfileRequired):
|
||||
return http.StatusBadRequest, "profile_required", "profile_id is required when prompt default_profile is not set"
|
||||
case errors.Is(err, usecase.ErrAPIKeyEnvMissing):
|
||||
case errors.Is(err, promptkit.ErrAPIKeyEnvMissing):
|
||||
return http.StatusBadRequest, "api_key_env_missing", "api_key_env is set but the environment variable is missing"
|
||||
case errors.Is(err, usecase.ErrInvalidRequest):
|
||||
return http.StatusBadRequest, "invalid_request", "invalid run request"
|
||||
case errors.Is(err, usecase.ErrPromptLoad):
|
||||
case errors.Is(err, promptkit.ErrPromptLoad):
|
||||
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
|
||||
case errors.Is(err, usecase.ErrProfileLoad):
|
||||
case errors.Is(err, promptkit.ErrProfileLoad):
|
||||
return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile"
|
||||
case errors.Is(err, artifact.ErrFileNotAllowed), errors.Is(err, artifact.ErrFileOutsideRoot):
|
||||
case errors.Is(err, promptkit.ErrInvalidRequest):
|
||||
return http.StatusBadRequest, "invalid_request", "invalid run request"
|
||||
case errors.Is(err, ErrFileNotAllowed), errors.Is(err, ErrFileOutsideRoot):
|
||||
return http.StatusBadRequest, "artifact_not_allowed", "file input artifact is not allowed"
|
||||
case errors.Is(err, artifact.ErrFileTooLarge):
|
||||
case errors.Is(err, ErrFileTooLarge):
|
||||
return http.StatusRequestEntityTooLarge, "artifact_too_large", "file input artifact is too large"
|
||||
case errors.Is(err, usecase.ErrArtifactLoad):
|
||||
case errors.Is(err, promptkit.ErrArtifactLoad):
|
||||
return http.StatusBadRequest, "artifact_read_failed", "failed to read input artifact"
|
||||
case errors.Is(err, usecase.ErrPromptRender):
|
||||
case errors.Is(err, promptkit.ErrPromptRender):
|
||||
return http.StatusBadRequest, "prompt_render_failed", "failed to render prompt"
|
||||
case errors.Is(err, usecase.ErrLLMGenerate):
|
||||
case errors.Is(err, promptkit.ErrCapacityExceeded):
|
||||
return http.StatusServiceUnavailable, "capacity_exceeded", "model backend capacity is exhausted"
|
||||
case errors.Is(err, promptkit.ErrLLMGenerate):
|
||||
return http.StatusBadGateway, "llm_failed", "model generation request failed"
|
||||
case errors.Is(err, usecase.ErrValidation):
|
||||
case errors.Is(err, promptkit.ErrValidation):
|
||||
return http.StatusInternalServerError, "validation_runtime_failed", "validation runtime failed"
|
||||
default:
|
||||
return http.StatusInternalServerError, "internal_error", "internal server error"
|
||||
|
||||
@@ -11,24 +11,20 @@ import (
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
type fakeRunner struct {
|
||||
result *domain.RunResult
|
||||
result *promptkit.RunResult
|
||||
err error
|
||||
last domain.RunRequest
|
||||
last promptkit.RunRequest
|
||||
}
|
||||
|
||||
func (f *fakeRunner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) {
|
||||
func (f *fakeRunner) Run(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
|
||||
f.last = req
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
@@ -36,38 +32,70 @@ func (f *fakeRunner) Run(ctx context.Context, req domain.RunRequest) (*domain.Ru
|
||||
return f.result, nil
|
||||
}
|
||||
|
||||
type handlerPromptRepo struct {
|
||||
def *domain.PromptDefinition
|
||||
}
|
||||
func TestMaintainedHTTPRunExampleMatchesRequestContract(t *testing.T) {
|
||||
body, err := os.ReadFile(filepath.Join("..", "..", "..", "examples", "http-run.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read maintained HTTP request example: %v", err)
|
||||
}
|
||||
|
||||
func (r handlerPromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
||||
return r.def, nil
|
||||
}
|
||||
runner := &fakeRunner{result: &promptkit.RunResult{}}
|
||||
h := NewHandler(runner)
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewReader(body))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
type handlerProfileRepo struct {
|
||||
profile *domain.ExecutionProfile
|
||||
}
|
||||
h.ServeHTTP(w, req)
|
||||
|
||||
func (r handlerProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
return r.profile, nil
|
||||
}
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected maintained HTTP request example to be accepted, got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
type handlerArtifactReader struct{}
|
||||
|
||||
func (handlerArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
||||
return &domain.Artifact{Name: "input", Body: []byte("input"), Hash: "hash"}, nil
|
||||
}
|
||||
|
||||
type handlerRenderer struct{}
|
||||
|
||||
func (handlerRenderer) Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
|
||||
return &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, nil
|
||||
var invalidExample map[string]json.RawMessage
|
||||
if err := json.Unmarshal(body, &invalidExample); err != nil {
|
||||
t.Fatalf("decode maintained HTTP request example: %v", err)
|
||||
}
|
||||
invalidExample["unexpected"] = json.RawMessage(`true`)
|
||||
invalidBody, err := json.Marshal(invalidExample)
|
||||
if err != nil {
|
||||
t.Fatalf("encode structurally invalid request example: %v", err)
|
||||
}
|
||||
invalidReq := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewReader(invalidBody))
|
||||
invalidW := httptest.NewRecorder()
|
||||
h.ServeHTTP(invalidW, invalidReq)
|
||||
assertHTTPErrorCode(t, invalidW, http.StatusBadRequest, "invalid_json")
|
||||
}
|
||||
|
||||
type handlerLLMClient struct{}
|
||||
|
||||
func (handlerLLMClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
return &domain.GenerateResponse{Content: "ok"}, nil
|
||||
func (handlerLLMClient) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return &promptkit.GenerateResponse{Content: "ok"}, nil
|
||||
}
|
||||
|
||||
type blockingLLMClient struct {
|
||||
started chan struct{}
|
||||
release chan struct{}
|
||||
current int32
|
||||
peak int32
|
||||
}
|
||||
|
||||
func (c *blockingLLMClient) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
current := atomic.AddInt32(&c.current, 1)
|
||||
defer atomic.AddInt32(&c.current, -1)
|
||||
for {
|
||||
peak := atomic.LoadInt32(&c.peak)
|
||||
if current <= peak || atomic.CompareAndSwapInt32(&c.peak, peak, current) {
|
||||
break
|
||||
}
|
||||
}
|
||||
select {
|
||||
case c.started <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case <-c.release:
|
||||
return &promptkit.GenerateResponse{Content: "ok"}, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
||||
@@ -76,24 +104,26 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
||||
const envName = "SCRIPTORIUM_API_KEY"
|
||||
const secret = "never-include-me"
|
||||
|
||||
r := &fakeRunner{result: &domain.RunResult{
|
||||
r := &fakeRunner{result: &promptkit.RunResult{
|
||||
RunID: "11111111-1111-4111-8111-111111111111",
|
||||
Artifact: domain.Artifact{
|
||||
Artifact: promptkit.Artifact{
|
||||
Name: "output",
|
||||
ContentType: "text/plain",
|
||||
Body: []byte("hello"),
|
||||
Size: 5,
|
||||
Hash: "abc",
|
||||
},
|
||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||
PromptID: "prompt-1",
|
||||
PromptVersion: "1.0.0",
|
||||
PromptHash: "phash",
|
||||
RenderedPromptHash: "rhash",
|
||||
SelectedProfileID: "exec-default",
|
||||
SelectedBackendID: "local",
|
||||
ModelName: "m1",
|
||||
Endpoint: "http://llm/v1",
|
||||
EffectiveModelParams: domain.ExecutionTarget{
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{
|
||||
BackendID: "local",
|
||||
Endpoint: "http://llm/v1",
|
||||
Model: "m1",
|
||||
Temperature: 0.2,
|
||||
@@ -104,7 +134,7 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
||||
APIKeyEnv: envName,
|
||||
},
|
||||
InputHashes: map[string]string{"transcript": "h1"},
|
||||
Usage: domain.TokenUsage{
|
||||
Usage: promptkit.TokenUsage{
|
||||
PromptTokens: 1,
|
||||
CompletionTokens: 2,
|
||||
TotalTokens: 3,
|
||||
@@ -152,6 +182,9 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
||||
if metadata["selected_profile_id"] != "exec-default" {
|
||||
t.Fatalf("unexpected metadata.selected_profile_id: %#v", metadata["selected_profile_id"])
|
||||
}
|
||||
if metadata["selected_backend_id"] != "local" {
|
||||
t.Fatalf("unexpected metadata.selected_backend_id: %#v", metadata["selected_backend_id"])
|
||||
}
|
||||
if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" {
|
||||
t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"])
|
||||
}
|
||||
@@ -163,6 +196,9 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
||||
t.Fatalf("unexpected cache usage metadata: %#v", usage)
|
||||
}
|
||||
modelParams := metadata["model_params"].(map[string]any)
|
||||
if modelParams["backend_id"] != "local" {
|
||||
t.Fatalf("expected model_params.backend_id=local, got %#v", modelParams["backend_id"])
|
||||
}
|
||||
if modelParams["api_key_env"] != envName {
|
||||
t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"])
|
||||
}
|
||||
@@ -193,6 +229,124 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) {
|
||||
r := &fakeRunner{result: &promptkit.RunResult{
|
||||
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationNone, IsValid: true},
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||
}}
|
||||
h := NewHandler(r)
|
||||
|
||||
for _, body := range []string{
|
||||
`{"prompt_id":"prompt-1","prompt_version":"2"}`,
|
||||
`{"prompt_id":"prompt-1","prompt_version":"2","inputs":{}}`,
|
||||
} {
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(body))
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if r.last.PromptVersion != "2" {
|
||||
t.Fatalf("expected prompt version to be mapped, got %q", r.last.PromptVersion)
|
||||
}
|
||||
if r.last.Inputs != nil {
|
||||
t.Fatalf("expected omitted or empty inputs to remain nil, got %#v", r.last.Inputs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerDelegatesDefinitionInputRequirements(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
definition string
|
||||
wantStatus int
|
||||
wantCode string
|
||||
}{
|
||||
{
|
||||
name: "no declared inputs",
|
||||
definition: `id: p
|
||||
version: "1"
|
||||
default_profile: exec
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`,
|
||||
wantStatus: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "optional input omitted",
|
||||
definition: `id: p
|
||||
version: "1"
|
||||
default_profile: exec
|
||||
inputs:
|
||||
- name: note
|
||||
required: false
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`,
|
||||
wantStatus: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "required input omitted",
|
||||
definition: `id: p
|
||||
version: "1"
|
||||
default_profile: exec
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`,
|
||||
wantStatus: http.StatusBadRequest,
|
||||
wantCode: "prompt_render_failed",
|
||||
},
|
||||
{
|
||||
name: "template input omitted",
|
||||
definition: `id: p
|
||||
version: "1"
|
||||
default_profile: exec
|
||||
messages:
|
||||
- role: user
|
||||
content: '{{input "note"}}'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`,
|
||||
wantStatus: http.StatusBadRequest,
|
||||
wantCode: "prompt_render_failed",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
h := newDefinitionHandler(t, tc.definition)
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p"}`))
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != tc.wantStatus {
|
||||
t.Fatalf("expected %d, got %d body=%s", tc.wantStatus, w.Code, w.Body.String())
|
||||
}
|
||||
if tc.wantCode != "" {
|
||||
assertHTTPErrorCode(t, w, tc.wantStatus, tc.wantCode)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerInlineRefsWorkWithoutArtifactRoot(t *testing.T) {
|
||||
h := newArtifactRootHandler(t, "")
|
||||
|
||||
@@ -293,13 +447,13 @@ func TestHandlerFileRefsOutsideArtifactRootAreRejected(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
|
||||
r := &fakeRunner{result: &domain.RunResult{
|
||||
Artifact: domain.Artifact{Body: []byte("ok")},
|
||||
r := &fakeRunner{result: &promptkit.RunResult{
|
||||
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||
PromptID: "prompt-1",
|
||||
PromptVersion: "1.0.0",
|
||||
SelectedProfileID: "prompt-default",
|
||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||
}}
|
||||
h := NewHandler(r)
|
||||
|
||||
@@ -328,10 +482,10 @@ func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
|
||||
r := &fakeRunner{result: &domain.RunResult{
|
||||
Artifact: domain.Artifact{Body: []byte("ok")},
|
||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||
r := &fakeRunner{result: &promptkit.RunResult{
|
||||
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||
}}
|
||||
h := NewHandler(r)
|
||||
|
||||
@@ -366,10 +520,12 @@ func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
|
||||
if got.Endpoint != "http://override/v1" ||
|
||||
got.Model != "override-model" ||
|
||||
got.ServiceTier != "flex" ||
|
||||
got.ReasoningEffort != "medium" ||
|
||||
got.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
|
||||
t.Fatalf("unexpected mapped execution target: %+v", got)
|
||||
}
|
||||
if got.ReasoningEffort == nil || *got.ReasoningEffort != "medium" {
|
||||
t.Fatalf("unexpected mapped reasoning_effort: %#v", got.ReasoningEffort)
|
||||
}
|
||||
if got.Temperature == nil || *got.Temperature != 0.6 {
|
||||
t.Fatalf("unexpected mapped temperature: %#v", got.Temperature)
|
||||
}
|
||||
@@ -387,11 +543,256 @@ func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerModelOverridePreservesReasoningEffortPresence(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
model string
|
||||
wantPresent bool
|
||||
wantValue string
|
||||
}{
|
||||
{name: "omitted", model: `{}`},
|
||||
{name: "nonblank", model: `{"reasoning_effort":"high"}`, wantPresent: true, wantValue: "high"},
|
||||
{name: "explicit empty", model: `{"reasoning_effort":""}`, wantPresent: true},
|
||||
{name: "null", model: `{"reasoning_effort":null}`},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := &fakeRunner{result: &promptkit.RunResult{
|
||||
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||
}}
|
||||
h := NewHandler(r)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"prompt-1","inputs":{"transcript":{"type":"file","uri":"./t.md"}},"model":`+tc.model+`}`))
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if r.last.Execution == nil {
|
||||
t.Fatal("expected execution override")
|
||||
}
|
||||
if (r.last.Execution.ReasoningEffort != nil) != tc.wantPresent {
|
||||
t.Fatalf("unexpected reasoning_effort presence: %#v", r.last.Execution.ReasoningEffort)
|
||||
}
|
||||
if tc.wantPresent && *r.last.Execution.ReasoningEffort != tc.wantValue {
|
||||
t.Fatalf("unexpected reasoning_effort: got %q want %q", *r.last.Execution.ReasoningEffort, tc.wantValue)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerMapsSessionIDAndReportsEffectiveResultSessionID(t *testing.T) {
|
||||
r := &fakeRunner{result: &promptkit.RunResult{
|
||||
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||
SessionID: "effective-session",
|
||||
}}
|
||||
h := NewHandler(r)
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"prompt-1","session_id":"request-session"}`))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
h.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if r.last.SessionID != "request-session" {
|
||||
t.Fatalf("expected session ID in run request, got %q", r.last.SessionID)
|
||||
}
|
||||
var resp map[string]any
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("invalid JSON response: %v", err)
|
||||
}
|
||||
metadata := resp["metadata"].(map[string]any)
|
||||
if metadata["session_id"] != "effective-session" {
|
||||
t.Fatalf("expected effective session ID in response, got %#v", metadata["session_id"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerOmitsEmptyEffectiveSessionID(t *testing.T) {
|
||||
r := &fakeRunner{result: &promptkit.RunResult{
|
||||
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||
}}
|
||||
h := NewHandler(r)
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"prompt-1"}`))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
h.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
var resp map[string]any
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("invalid JSON response: %v", err)
|
||||
}
|
||||
metadata := resp["metadata"].(map[string]any)
|
||||
if _, ok := metadata["session_id"]; ok {
|
||||
t.Fatalf("expected empty effective session ID to be omitted, got %#v", metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerSessionIDUsesPromptkitResolution(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
definitionSession string
|
||||
requestSession string
|
||||
wantSession string
|
||||
}{
|
||||
{name: "definition session", definitionSession: "definition-session", wantSession: "definition-session"},
|
||||
{name: "direct session", definitionSession: "definition-session", requestSession: "direct-session", wantSession: "direct-session"},
|
||||
{name: "no effective session"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
promptDir := t.TempDir()
|
||||
profileDir := t.TempDir()
|
||||
sessionLine := ""
|
||||
if tc.definitionSession != "" {
|
||||
sessionLine = "session_id: " + tc.definitionSession + "\n"
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte("id: p\nversion: \"1\"\ndefault_profile: exec\n"+sessionLine+`messages:
|
||||
- role: user
|
||||
content: "hi"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write prompt fixture: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("id: exec\nendpoint: http://127.0.0.1:1/v1\nmodel: test\n"), 0o644); err != nil {
|
||||
t.Fatalf("write profile fixture: %v", err)
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{PromptDir: promptDir, ProfileDir: profileDir}, promptkit.WithLLMClient(handlerLLMClient{}))
|
||||
if err != nil {
|
||||
t.Fatalf("new engine: %v", err)
|
||||
}
|
||||
|
||||
body := `{"prompt_id":"p"}`
|
||||
if tc.requestSession != "" {
|
||||
body = `{"prompt_id":"p","session_id":"` + tc.requestSession + `"}`
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
NewHandler(engine).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(body)))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
var resp map[string]any
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("invalid JSON response: %v", err)
|
||||
}
|
||||
metadata := resp["metadata"].(map[string]any)
|
||||
if tc.wantSession == "" {
|
||||
if _, ok := metadata["session_id"]; ok {
|
||||
t.Fatalf("expected session_id to be omitted, got %#v", metadata)
|
||||
}
|
||||
return
|
||||
}
|
||||
if metadata["session_id"] != tc.wantSession {
|
||||
t.Fatalf("expected effective session ID %q, got %#v", tc.wantSession, metadata["session_id"])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerSharesBackendCapacityAcrossConcurrentRequests(t *testing.T) {
|
||||
promptDir := t.TempDir()
|
||||
profileDir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`id: p
|
||||
version: "1"
|
||||
default_profile: limited
|
||||
messages:
|
||||
- role: user
|
||||
content: "hi"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write prompt fixture: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`id: limited
|
||||
backend: limited
|
||||
model: test
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write profile fixture: %v", err)
|
||||
}
|
||||
queueCapacity := 0
|
||||
client := &blockingLLMClient{started: make(chan struct{}, 1), release: make(chan struct{})}
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{PromptDir: promptDir, ProfileDir: profileDir},
|
||||
promptkit.WithBackend(promptkit.Backend{
|
||||
ID: "limited",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
ConcurrencyLimit: 1,
|
||||
QueueCapacity: &queueCapacity,
|
||||
}),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("new engine: %v", err)
|
||||
}
|
||||
h := NewHandler(engine)
|
||||
first := httptest.NewRecorder()
|
||||
firstDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(firstDone)
|
||||
h.ServeHTTP(first, httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p"}`)))
|
||||
}()
|
||||
select {
|
||||
case <-client.started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("first request did not reach generation")
|
||||
}
|
||||
|
||||
second := httptest.NewRecorder()
|
||||
h.ServeHTTP(second, httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p"}`)))
|
||||
assertHTTPErrorCode(t, second, http.StatusServiceUnavailable, "capacity_exceeded")
|
||||
if second.Header().Get("Retry-After") != "" {
|
||||
t.Fatalf("expected no Retry-After header, got %q", second.Header().Get("Retry-After"))
|
||||
}
|
||||
if strings.Contains(second.Body.String(), "limited") {
|
||||
t.Fatalf("capacity response leaked backend details: %s", second.Body.String())
|
||||
}
|
||||
|
||||
close(client.release)
|
||||
select {
|
||||
case <-firstDone:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("first request did not complete")
|
||||
}
|
||||
if first.Code != http.StatusOK {
|
||||
t.Fatalf("expected first request to succeed, got %d body=%s", first.Code, first.Body.String())
|
||||
}
|
||||
if atomic.LoadInt32(&client.peak) != 1 {
|
||||
t.Fatalf("expected peak generation concurrency of one, got %d", atomic.LoadInt32(&client.peak))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerRejectsOverlongSessionIDAndNonStringReasoningEffort(t *testing.T) {
|
||||
engine := newHandlerEngine(t)
|
||||
for _, body := range []string{
|
||||
`{"prompt_id":"p","session_id":"` + strings.Repeat("x", 257) + `"}`,
|
||||
`{"prompt_id":"p","model":{"reasoning_effort":1}}`,
|
||||
} {
|
||||
w := httptest.NewRecorder()
|
||||
NewHandler(engine).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(body)))
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("expected 400, got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerModelOverrideAcceptsJSONCompatibleExtraParams(t *testing.T) {
|
||||
r := &fakeRunner{result: &domain.RunResult{
|
||||
Artifact: domain.Artifact{Body: []byte("ok")},
|
||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||
r := &fakeRunner{result: &promptkit.RunResult{
|
||||
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||
}}
|
||||
h := NewHandler(r)
|
||||
|
||||
@@ -432,10 +833,10 @@ func TestHandlerModelOverrideAcceptsJSONCompatibleExtraParams(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandlerModelOverrideExplicitZeroTemperatureMapsAsPresent(t *testing.T) {
|
||||
r := &fakeRunner{result: &domain.RunResult{
|
||||
Artifact: domain.Artifact{Body: []byte("ok")},
|
||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0},
|
||||
r := &fakeRunner{result: &promptkit.RunResult{
|
||||
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0},
|
||||
}}
|
||||
h := NewHandler(r)
|
||||
|
||||
@@ -459,10 +860,10 @@ func TestHandlerModelOverrideExplicitZeroTemperatureMapsAsPresent(t *testing.T)
|
||||
}
|
||||
|
||||
func TestHandlerModelOverrideOmittedTemperatureMapsAsAbsent(t *testing.T) {
|
||||
r := &fakeRunner{result: &domain.RunResult{
|
||||
Artifact: domain.Artifact{Body: []byte("ok")},
|
||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0.7},
|
||||
r := &fakeRunner{result: &promptkit.RunResult{
|
||||
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0.7},
|
||||
}}
|
||||
h := NewHandler(r)
|
||||
|
||||
@@ -496,16 +897,16 @@ func TestHandlerModelOverrideOmittedTemperatureMapsAsAbsent(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing.T) {
|
||||
r := &fakeRunner{result: &domain.RunResult{
|
||||
Artifact: domain.Artifact{
|
||||
r := &fakeRunner{result: &promptkit.RunResult{
|
||||
Artifact: promptkit.Artifact{
|
||||
Name: "output",
|
||||
ContentType: "text/plain",
|
||||
Body: []byte("ok"),
|
||||
Size: 2,
|
||||
Hash: "abc",
|
||||
},
|
||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: domain.ExecutionTarget{
|
||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{
|
||||
Endpoint: "http://llm/v1",
|
||||
Model: "gpt-test",
|
||||
Temperature: 0.4,
|
||||
@@ -625,10 +1026,10 @@ func TestHandlerMalformedJSONBelowLimitStillBadRequest(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandlerResponseTooLarge(t *testing.T) {
|
||||
h := NewHandlerWithOptions(&fakeRunner{result: &domain.RunResult{
|
||||
Artifact: domain.Artifact{Body: []byte(strings.Repeat("x", 128))},
|
||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||
h := NewHandlerWithOptions(&fakeRunner{result: &promptkit.RunResult{
|
||||
Artifact: promptkit.Artifact{Body: []byte(strings.Repeat("x", 128))},
|
||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||
}}, HandlerOptions{MaxRequestBytes: 1024, MaxResponseBytes: 64})
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
|
||||
w := httptest.NewRecorder()
|
||||
@@ -639,11 +1040,11 @@ func TestHandlerResponseTooLarge(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandlerRawOutputDoesNotBypassResponseLimit(t *testing.T) {
|
||||
h := NewHandlerWithOptions(&fakeRunner{result: &domain.RunResult{
|
||||
Artifact: domain.Artifact{Body: []byte("ok")},
|
||||
h := NewHandlerWithOptions(&fakeRunner{result: &promptkit.RunResult{
|
||||
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||
RawOutput: strings.Repeat("raw", 80),
|
||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||
}}, HandlerOptions{MaxRequestBytes: 1024, MaxResponseBytes: 128})
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||
"prompt_id":"p",
|
||||
@@ -677,35 +1078,12 @@ func TestHandlerMissingPromptID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerReservedExtraParamsThroughRunnerMapsToInvalidRequest(t *testing.T) {
|
||||
llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner := usecase.NewRunner(
|
||||
handlerPromptRepo{def: &domain.PromptDefinition{
|
||||
ID: "p",
|
||||
Version: "1",
|
||||
DefaultProfile: "exec",
|
||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "hi"}},
|
||||
OutputFormat: domain.FormatText,
|
||||
Validation: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone},
|
||||
}},
|
||||
handlerProfileRepo{profile: &domain.ExecutionProfile{
|
||||
ID: "exec",
|
||||
Endpoint: "http://example.invalid/v1",
|
||||
Model: "model",
|
||||
}},
|
||||
handlerArtifactReader{},
|
||||
handlerRenderer{},
|
||||
llmClient,
|
||||
nil,
|
||||
)
|
||||
h := NewHandler(runner)
|
||||
func TestHandlerReservedExtraParamsThroughEngineMapsToInvalidRequest(t *testing.T) {
|
||||
h := NewHandler(newHandlerEngineWithDefaultClient(t))
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||
"prompt_id":"p",
|
||||
"inputs":{"x":{"type":"file","uri":"a"}},
|
||||
"inputs":{"x":{"type":"inline","body":"input"}},
|
||||
"model":{"extra_params":{"model":"collision"}}
|
||||
}`))
|
||||
w := httptest.NewRecorder()
|
||||
@@ -725,7 +1103,7 @@ func TestHandlerReservedExtraParamsThroughRunnerMapsToInvalidRequest(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerUsecaseErrorMapping(t *testing.T) {
|
||||
func TestHandlerPublicErrorMapping(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
@@ -734,18 +1112,21 @@ func TestHandlerUsecaseErrorMapping(t *testing.T) {
|
||||
message string
|
||||
avoidCause string
|
||||
}{
|
||||
{name: "prompt not found", err: wrap(usecase.ErrPromptLoad, promptdef.ErrPromptDefinitionNotFound), status: http.StatusNotFound, code: "prompt_not_found", message: "prompt definition not found"},
|
||||
{name: "prompt load invalid", err: wrap(usecase.ErrPromptLoad, promptdef.ErrInvalidPromptDefinition), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition"},
|
||||
{name: "prompt load generic", err: wrap(usecase.ErrPromptLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition", avoidCause: "read failed"},
|
||||
{name: "missing profile/default", err: wrap(usecase.ErrInvalidRequest, usecase.ErrProfileRequired), status: http.StatusBadRequest, code: "profile_required", message: "profile_id is required when prompt default_profile is not set"},
|
||||
{name: "profile not found", err: wrap(usecase.ErrProfileLoad, profile.ErrProfileNotFound), status: http.StatusNotFound, code: "profile_not_found", message: "execution profile not found"},
|
||||
{name: "profile invalid", err: wrap(usecase.ErrProfileLoad, profile.ErrInvalidProfile), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile"},
|
||||
{name: "profile load generic", err: wrap(usecase.ErrProfileLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile", avoidCause: "read failed"},
|
||||
{name: "api key env missing", err: wrap(usecase.ErrInvalidRequest, usecase.ErrAPIKeyEnvMissing), status: http.StatusBadRequest, code: "api_key_env_missing", message: "api_key_env is set but the environment variable is missing"},
|
||||
{name: "artifact", err: wrap(usecase.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"},
|
||||
{name: "prompt render", err: wrap(usecase.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"},
|
||||
{name: "llm", err: wrap(usecase.ErrLLMGenerate, fmt.Errorf("llm failed")), status: http.StatusBadGateway, code: "llm_failed", message: "model generation request failed", avoidCause: "llm failed"},
|
||||
{name: "validation runtime", err: wrap(usecase.ErrValidation, fmt.Errorf("validator broke")), status: http.StatusInternalServerError, code: "validation_runtime_failed", message: "validation runtime failed", avoidCause: "validator broke"},
|
||||
{name: "prompt not found", err: promptkit.ErrPromptNotFound, status: http.StatusNotFound, code: "prompt_not_found", message: "prompt definition not found"},
|
||||
{name: "prompt load", err: wrap(promptkit.ErrPromptLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition", avoidCause: "read failed"},
|
||||
{name: "missing profile/default", err: wrap(promptkit.ErrProfileRequired, promptkit.ErrInvalidRequest), status: http.StatusBadRequest, code: "profile_required", message: "profile_id is required when prompt default_profile is not set"},
|
||||
{name: "profile not found", err: promptkit.ErrProfileNotFound, status: http.StatusNotFound, code: "profile_not_found", message: "execution profile not found"},
|
||||
{name: "profile load", err: wrap(promptkit.ErrProfileLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile", avoidCause: "read failed"},
|
||||
{name: "api key env missing", err: wrap(promptkit.ErrAPIKeyEnvMissing, promptkit.ErrInvalidRequest), status: http.StatusBadRequest, code: "api_key_env_missing", message: "api_key_env is set but the environment variable is missing"},
|
||||
{name: "invalid request", err: promptkit.ErrInvalidRequest, status: http.StatusBadRequest, code: "invalid_request", message: "invalid run request"},
|
||||
{name: "file denied", err: ErrFileNotAllowed, status: http.StatusBadRequest, code: "artifact_not_allowed", message: "file input artifact is not allowed"},
|
||||
{name: "file outside root", err: ErrFileOutsideRoot, status: http.StatusBadRequest, code: "artifact_not_allowed", message: "file input artifact is not allowed"},
|
||||
{name: "file too large", err: ErrFileTooLarge, status: http.StatusRequestEntityTooLarge, code: "artifact_too_large", message: "file input artifact is too large"},
|
||||
{name: "artifact", err: wrap(promptkit.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"},
|
||||
{name: "prompt render", err: wrap(promptkit.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"},
|
||||
{name: "capacity", err: &promptkit.CapacityError{BackendID: "private-backend"}, status: http.StatusServiceUnavailable, code: "capacity_exceeded", message: "model backend capacity is exhausted", avoidCause: "private-backend"},
|
||||
{name: "llm", err: wrap(promptkit.ErrLLMGenerate, fmt.Errorf("llm failed")), status: http.StatusBadGateway, code: "llm_failed", message: "model generation request failed", avoidCause: "llm failed"},
|
||||
{name: "validation runtime", err: wrap(promptkit.ErrValidation, fmt.Errorf("validator broke")), status: http.StatusInternalServerError, code: "validation_runtime_failed", message: "validation runtime failed", avoidCause: "validator broke"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
@@ -798,12 +1179,12 @@ func TestHandlerRawAPIKeyRejectedByStrictJSON(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandlerValidationFailureStillSuccessAndRawOutputOptIn(t *testing.T) {
|
||||
h := NewHandler(&fakeRunner{result: &domain.RunResult{
|
||||
Artifact: domain.Artifact{Body: []byte("bad json")},
|
||||
h := NewHandler(&fakeRunner{result: &promptkit.RunResult{
|
||||
Artifact: promptkit.Artifact{Body: []byte("bad json")},
|
||||
RawOutput: "bad json",
|
||||
Validation: domain.ValidationResult{
|
||||
Status: domain.ValidationFailed,
|
||||
Mode: domain.ValidationJSON,
|
||||
Validation: promptkit.ValidationResult{
|
||||
Status: promptkit.ValidationFailed,
|
||||
Mode: promptkit.ValidationJSON,
|
||||
Errors: []string{"invalid JSON"},
|
||||
},
|
||||
}})
|
||||
@@ -857,30 +1238,83 @@ func newArtifactRootHandler(t *testing.T, root string) *Handler {
|
||||
func newArtifactRootHandlerWithLimit(t *testing.T, root string, maxArtifactBytes int64) *Handler {
|
||||
t.Helper()
|
||||
|
||||
reader, err := artifact.NewRestrictedCompositeReaderWithLimit(root, maxArtifactBytes)
|
||||
reader, err := NewRestrictedArtifactReader(root, maxArtifactBytes)
|
||||
if err != nil {
|
||||
t.Fatalf("expected restricted artifact reader: %v", err)
|
||||
}
|
||||
runner := usecase.NewRunner(
|
||||
handlerPromptRepo{def: &domain.PromptDefinition{
|
||||
ID: "p",
|
||||
Version: "1",
|
||||
DefaultProfile: "exec",
|
||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "hi"}},
|
||||
OutputFormat: domain.FormatText,
|
||||
Validation: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone},
|
||||
}},
|
||||
handlerProfileRepo{profile: &domain.ExecutionProfile{
|
||||
ID: "exec",
|
||||
Endpoint: "http://example.invalid/v1",
|
||||
Model: "model",
|
||||
}},
|
||||
reader,
|
||||
handlerRenderer{},
|
||||
handlerLLMClient{},
|
||||
nil,
|
||||
)
|
||||
return NewHandler(runner)
|
||||
return NewHandler(newHandlerEngine(t, promptkit.WithArtifactReader(reader)))
|
||||
}
|
||||
|
||||
func newHandlerEngine(t *testing.T, options ...promptkit.Option) *promptkit.Engine {
|
||||
t.Helper()
|
||||
|
||||
return newHandlerEngineWithOptions(t, append(options, promptkit.WithLLMClient(handlerLLMClient{}))...)
|
||||
}
|
||||
|
||||
func newHandlerEngineWithDefaultClient(t *testing.T, options ...promptkit.Option) *promptkit.Engine {
|
||||
t.Helper()
|
||||
|
||||
return newHandlerEngineWithOptions(t, options...)
|
||||
}
|
||||
|
||||
func newHandlerEngineWithOptions(t *testing.T, options ...promptkit.Option) *promptkit.Engine {
|
||||
t.Helper()
|
||||
|
||||
promptDir := t.TempDir()
|
||||
profileDir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`id: p
|
||||
version: "1"
|
||||
default_profile: exec
|
||||
messages:
|
||||
- role: user
|
||||
content: "hi"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
repair_attempts: 0
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write prompt fixture: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`id: exec
|
||||
endpoint: http://example.invalid/v1
|
||||
model: model
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write profile fixture: %v", err)
|
||||
}
|
||||
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: promptDir,
|
||||
ProfileDir: profileDir,
|
||||
}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("construct public engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func newDefinitionHandler(t *testing.T, definition string) *Handler {
|
||||
t.Helper()
|
||||
|
||||
promptDir := t.TempDir()
|
||||
profileDir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(definition), 0o644); err != nil {
|
||||
t.Fatalf("write prompt fixture: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`id: exec
|
||||
endpoint: http://example.invalid/v1
|
||||
model: model
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write profile fixture: %v", err)
|
||||
}
|
||||
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: promptDir,
|
||||
ProfileDir: profileDir,
|
||||
}, promptkit.WithLLMClient(handlerLLMClient{}))
|
||||
if err != nil {
|
||||
t.Fatalf("construct public engine: %v", err)
|
||||
}
|
||||
return NewHandler(engine)
|
||||
}
|
||||
|
||||
func assertHTTPErrorCode(t *testing.T, w *httptest.ResponseRecorder, status int, code string) {
|
||||
|
||||
@@ -1,242 +0,0 @@
|
||||
package artifact
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"io"
|
||||
"mime"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrUnsupportedRefType = errors.New("unsupported artifact reference type")
|
||||
ErrMissingInlineBody = errors.New("missing body for inline artifact")
|
||||
ErrMissingFilePath = errors.New("missing file path for file artifact")
|
||||
ErrFileNotAllowed = errors.New("file artifact references are not allowed")
|
||||
ErrFileOutsideRoot = errors.New("file artifact path is outside artifact root")
|
||||
ErrFileTooLarge = errors.New("file artifact exceeds size limit")
|
||||
)
|
||||
|
||||
// Reader resolves artifact references into actual artifacts.
|
||||
type Reader interface {
|
||||
Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error)
|
||||
}
|
||||
|
||||
// CompositeReader routes artifact resolution based on the reference type.
|
||||
type CompositeReader struct {
|
||||
inlineReader *inlineReader
|
||||
fileReader Reader
|
||||
}
|
||||
|
||||
func NewCompositeReader() Reader {
|
||||
return &CompositeReader{
|
||||
inlineReader: &inlineReader{},
|
||||
fileReader: &fileReader{},
|
||||
}
|
||||
}
|
||||
|
||||
func NewRestrictedCompositeReader(root string) (Reader, error) {
|
||||
return NewRestrictedCompositeReaderWithLimit(root, 0)
|
||||
}
|
||||
|
||||
func NewRestrictedCompositeReaderWithLimit(root string, maxBytes int64) (Reader, error) {
|
||||
fileReader, err := newRestrictedFileReader(root, maxBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &CompositeReader{
|
||||
inlineReader: &inlineReader{},
|
||||
fileReader: fileReader,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *CompositeReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
switch ref.Type {
|
||||
case domain.ArtifactRefInline:
|
||||
return c.inlineReader.Read(ctx, ref)
|
||||
case domain.ArtifactRefFile:
|
||||
return c.fileReader.Read(ctx, ref)
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedRefType, ref.Type)
|
||||
}
|
||||
}
|
||||
|
||||
type inlineReader struct{}
|
||||
|
||||
func (r *inlineReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
if ref.Body == "" {
|
||||
return nil, ErrMissingInlineBody
|
||||
}
|
||||
|
||||
body := []byte(ref.Body)
|
||||
return &domain.Artifact{
|
||||
ContentType: defaults.ContentTypeTextPlain,
|
||||
Body: body,
|
||||
Size: int64(len(body)),
|
||||
Hash: fmt.Sprintf("%x", sha256.Sum256(body)),
|
||||
URI: ref.URI,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type fileReader struct{}
|
||||
|
||||
func (r *fileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
if ref.URI == "" {
|
||||
return nil, ErrMissingFilePath
|
||||
}
|
||||
|
||||
return readFileArtifact(ref.URI)
|
||||
}
|
||||
|
||||
type deniedFileReader struct{}
|
||||
|
||||
func (r deniedFileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
if ref.URI == "" {
|
||||
return nil, ErrMissingFilePath
|
||||
}
|
||||
return nil, ErrFileNotAllowed
|
||||
}
|
||||
|
||||
type restrictedFileReader struct {
|
||||
root string
|
||||
maxBytes int64
|
||||
}
|
||||
|
||||
func newRestrictedFileReader(root string, maxBytes int64) (Reader, error) {
|
||||
if maxBytes < 0 {
|
||||
return nil, fmt.Errorf("artifact size limit must be greater than or equal to 0")
|
||||
}
|
||||
cleanRoot := strings.TrimSpace(root)
|
||||
if cleanRoot == "" {
|
||||
return deniedFileReader{}, nil
|
||||
}
|
||||
absRoot, err := filepath.Abs(filepath.Clean(cleanRoot))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve artifact root: %w", err)
|
||||
}
|
||||
return &restrictedFileReader{root: absRoot, maxBytes: maxBytes}, nil
|
||||
}
|
||||
|
||||
func (r *restrictedFileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
if ref.URI == "" {
|
||||
return nil, ErrMissingFilePath
|
||||
}
|
||||
|
||||
path, err := r.resolveLexicalPath(ref.URI)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return readFileArtifactWithLimit(path, r.maxBytes)
|
||||
}
|
||||
|
||||
// resolveLexicalPath checks cleaned path containment without resolving symlinks.
|
||||
func (r *restrictedFileReader) resolveLexicalPath(rawPath string) (string, error) {
|
||||
cleanPath := filepath.Clean(strings.TrimSpace(rawPath))
|
||||
var candidate string
|
||||
if filepath.IsAbs(cleanPath) {
|
||||
candidate = cleanPath
|
||||
} else {
|
||||
candidate = filepath.Join(r.root, cleanPath)
|
||||
}
|
||||
|
||||
absCandidate, err := filepath.Abs(candidate)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve artifact path: %w", err)
|
||||
}
|
||||
absCandidate = filepath.Clean(absCandidate)
|
||||
|
||||
rel, err := filepath.Rel(r.root, absCandidate)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("compare artifact path to root: %w", err)
|
||||
}
|
||||
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) {
|
||||
return "", ErrFileOutsideRoot
|
||||
}
|
||||
return absCandidate, nil
|
||||
}
|
||||
|
||||
func readFileArtifact(path string) (*domain.Artifact, error) {
|
||||
return readFileArtifactWithLimit(path, 0)
|
||||
}
|
||||
|
||||
func readFileArtifactWithLimit(path string, maxBytes int64) (*domain.Artifact, error) {
|
||||
if maxBytes < 0 {
|
||||
return nil, fmt.Errorf("file size limit must be greater than or equal to 0")
|
||||
}
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to stat file %s: %w", path, err)
|
||||
}
|
||||
if maxBytes > 0 && info.Size() > maxBytes {
|
||||
return nil, ErrFileTooLarge
|
||||
}
|
||||
|
||||
var reader io.Reader = file
|
||||
if maxBytes > 0 {
|
||||
reader = io.LimitReader(file, maxBytes+1)
|
||||
}
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
||||
}
|
||||
if maxBytes > 0 && int64(len(data)) > maxBytes {
|
||||
return nil, ErrFileTooLarge
|
||||
}
|
||||
|
||||
contentType := mime.TypeByExtension(filepath.Ext(path))
|
||||
if contentType == "" {
|
||||
contentType = defaults.ContentTypeTextPlain
|
||||
}
|
||||
|
||||
return &domain.Artifact{
|
||||
Name: filepath.Base(path),
|
||||
ContentType: contentType,
|
||||
Body: data,
|
||||
URI: path,
|
||||
Size: int64(len(data)),
|
||||
Hash: fmt.Sprintf("%x", sha256.Sum256(data)),
|
||||
}, nil
|
||||
}
|
||||
@@ -1,257 +0,0 @@
|
||||
package artifact
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
)
|
||||
|
||||
func TestCompositeReader_Read(t *testing.T) {
|
||||
reader := NewCompositeReader()
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("inline artifact", func(t *testing.T) {
|
||||
ref := domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefInline,
|
||||
Body: "hello world",
|
||||
}
|
||||
art, err := reader.Read(ctx, ref)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if string(art.Body) != "hello world" {
|
||||
t.Errorf("expected 'hello world', got %s", string(art.Body))
|
||||
}
|
||||
if art.ContentType != "text/plain" {
|
||||
t.Errorf("expected text/plain content type, got %q", art.ContentType)
|
||||
}
|
||||
if art.Hash != "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" {
|
||||
t.Errorf("unexpected hash: %s", art.Hash)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("inline artifact missing body", func(t *testing.T) {
|
||||
ref := domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefInline,
|
||||
Body: "",
|
||||
}
|
||||
_, err := reader.Read(ctx, ref)
|
||||
if !errors.Is(err, ErrMissingInlineBody) {
|
||||
t.Errorf("expected ErrMissingInlineBody, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unsupported ref type", func(t *testing.T) {
|
||||
ref := domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefType("unsupported"),
|
||||
URI: "unsupported://bucket/key",
|
||||
}
|
||||
_, err := reader.Read(ctx, ref)
|
||||
if !errors.Is(err, ErrUnsupportedRefType) {
|
||||
t.Error("expected error for unsupported type")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRestrictedCompositeReader(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
|
||||
if err := os.WriteFile(filepath.Join(root, "input.txt"), []byte("allowed"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Mkdir(filepath.Join(root, "nested"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(outside, "secret.txt"), []byte("denied"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
reader, err := NewRestrictedCompositeReader(root)
|
||||
if err != nil {
|
||||
t.Fatalf("expected restricted reader construction, got %v", err)
|
||||
}
|
||||
|
||||
t.Run("accepts relative contained path", func(t *testing.T) {
|
||||
art, err := reader.Read(ctx, domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: "nested/../input.txt"})
|
||||
if err != nil {
|
||||
t.Fatalf("expected contained relative path to succeed, got %v", err)
|
||||
}
|
||||
if string(art.Body) != "allowed" {
|
||||
t.Fatalf("unexpected artifact body: %q", string(art.Body))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("accepts absolute contained path", func(t *testing.T) {
|
||||
art, err := reader.Read(ctx, domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: filepath.Join(root, "input.txt")})
|
||||
if err != nil {
|
||||
t.Fatalf("expected contained absolute path to succeed, got %v", err)
|
||||
}
|
||||
if art.Name != "input.txt" {
|
||||
t.Fatalf("unexpected artifact name: %q", art.Name)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects relative traversal outside root", func(t *testing.T) {
|
||||
_, err := reader.Read(ctx, domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: filepath.Join("..", filepath.Base(outside), "secret.txt")})
|
||||
if !errors.Is(err, ErrFileOutsideRoot) {
|
||||
t.Fatalf("expected ErrFileOutsideRoot, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects absolute path outside root", func(t *testing.T) {
|
||||
_, err := reader.Read(ctx, domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: filepath.Join(outside, "secret.txt")})
|
||||
if !errors.Is(err, ErrFileOutsideRoot) {
|
||||
t.Fatalf("expected ErrFileOutsideRoot, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRestrictedCompositeReaderFollowsSymlinkInsideRoot(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
|
||||
target := filepath.Join(outside, "linked.txt")
|
||||
if err := os.WriteFile(target, []byte("linked outside root"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
link := filepath.Join(root, "linked.txt")
|
||||
if err := os.Symlink(target, link); err != nil {
|
||||
t.Skipf("symlink creation unavailable: %v", err)
|
||||
}
|
||||
|
||||
reader, err := NewRestrictedCompositeReader(root)
|
||||
if err != nil {
|
||||
t.Fatalf("expected restricted reader construction, got %v", err)
|
||||
}
|
||||
|
||||
art, err := reader.Read(ctx, domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: "linked.txt"})
|
||||
if err != nil {
|
||||
t.Fatalf("expected symlink inside root to be followed, got %v", err)
|
||||
}
|
||||
if string(art.Body) != "linked outside root" {
|
||||
t.Fatalf("unexpected artifact body: %q", string(art.Body))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestrictedCompositeReaderWithoutRootDeniesFileRefs(t *testing.T) {
|
||||
reader, err := NewRestrictedCompositeReader("")
|
||||
if err != nil {
|
||||
t.Fatalf("expected restricted reader construction, got %v", err)
|
||||
}
|
||||
|
||||
art, err := reader.Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: "inline"})
|
||||
if err != nil {
|
||||
t.Fatalf("expected inline ref to work without artifact root, got %v", err)
|
||||
}
|
||||
if string(art.Body) != "inline" {
|
||||
t.Fatalf("unexpected inline body: %q", string(art.Body))
|
||||
}
|
||||
|
||||
_, err = reader.Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: "input.txt"})
|
||||
if !errors.Is(err, ErrFileNotAllowed) {
|
||||
t.Fatalf("expected ErrFileNotAllowed, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestrictedCompositeReaderFileSizeLimit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, "exact.txt"), []byte("12345"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "large.txt"), []byte("123456"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
reader, err := NewRestrictedCompositeReaderWithLimit(root, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("expected restricted reader construction, got %v", err)
|
||||
}
|
||||
|
||||
art, err := reader.Read(ctx, domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: "exact.txt"})
|
||||
if err != nil {
|
||||
t.Fatalf("expected file at limit to succeed, got %v", err)
|
||||
}
|
||||
if string(art.Body) != "12345" {
|
||||
t.Fatalf("unexpected artifact body: %q", string(art.Body))
|
||||
}
|
||||
|
||||
_, err = reader.Read(ctx, domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: "large.txt"})
|
||||
if !errors.Is(err, ErrFileTooLarge) {
|
||||
t.Fatalf("expected ErrFileTooLarge, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestrictedCompositeReaderFileSizeLimitZeroDisablesLimit(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, "large.txt"), []byte("123456"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
reader, err := NewRestrictedCompositeReaderWithLimit(root, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("expected restricted reader construction, got %v", err)
|
||||
}
|
||||
art, err := reader.Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: "large.txt"})
|
||||
if err != nil {
|
||||
t.Fatalf("expected unlimited reader to succeed, got %v", err)
|
||||
}
|
||||
if string(art.Body) != "123456" {
|
||||
t.Fatalf("unexpected artifact body: %q", string(art.Body))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileReader_Read(t *testing.T) {
|
||||
content := []byte("test file content")
|
||||
tmpFile, err := os.CreateTemp("", "artifact_test_*.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.Remove(tmpFile.Name())
|
||||
|
||||
if _, err := tmpFile.Write(content); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tmpFile.Close()
|
||||
|
||||
reader := NewCompositeReader()
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("file artifact loading", func(t *testing.T) {
|
||||
ref := domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefFile,
|
||||
URI: tmpFile.Name(),
|
||||
}
|
||||
art, err := reader.Read(ctx, ref)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if string(art.Body) != string(content) {
|
||||
t.Errorf("expected %s, got %s", string(content), string(art.Body))
|
||||
}
|
||||
if art.Name == "" {
|
||||
t.Error("expected name to be inferred from filename")
|
||||
}
|
||||
if art.Hash != "60f5237ed4049f0382661ef009d2bc42e48c3ceb3edb6600f7024e7ab3b838f3" {
|
||||
t.Errorf("unexpected hash: %s", art.Hash)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing file path", func(t *testing.T) {
|
||||
ref := domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefFile,
|
||||
URI: "",
|
||||
}
|
||||
_, err := reader.Read(ctx, ref)
|
||||
if !errors.Is(err, ErrMissingFilePath) {
|
||||
t.Errorf("expected ErrMissingFilePath, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||
@@ -37,6 +38,24 @@ type Config struct {
|
||||
SchemaDir string `yaml:"schema_dir"`
|
||||
Server ServerConfig `yaml:"server"`
|
||||
Defaults DefaultsConfig `yaml:"defaults"`
|
||||
Backends map[string]BackendConfig `yaml:"backends"`
|
||||
}
|
||||
|
||||
type BackendConfig struct {
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
APIKeyEnv string `yaml:"api_key_env"`
|
||||
ExtraParams map[string]any `yaml:"extra_params"`
|
||||
ConcurrencyLimit int `yaml:"concurrency_limit"`
|
||||
QueueCapacity *int `yaml:"queue_capacity"`
|
||||
}
|
||||
|
||||
type BackendSettings struct {
|
||||
ID string
|
||||
Endpoint string
|
||||
APIKeyEnv string
|
||||
ExtraParams map[string]any
|
||||
ConcurrencyLimit int
|
||||
QueueCapacity *int
|
||||
}
|
||||
|
||||
type ServerConfig struct {
|
||||
@@ -62,6 +81,7 @@ type AppSettings struct {
|
||||
MaxArtifactBytes int64
|
||||
MaxResponseBytes int64
|
||||
DefaultRenderFormat renderformat.PreparedRunOutputFormat
|
||||
Backends []BackendSettings
|
||||
}
|
||||
|
||||
// CLIOverrides can be applied after config load to enforce precedence.
|
||||
@@ -245,6 +265,25 @@ func applyConfig(base AppSettings, cfg Config) (AppSettings, error) {
|
||||
}
|
||||
out.DefaultRenderFormat = parsed
|
||||
}
|
||||
if len(cfg.Backends) > 0 {
|
||||
ids := make([]string, 0, len(cfg.Backends))
|
||||
for id := range cfg.Backends {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
out.Backends = make([]BackendSettings, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
backend := cfg.Backends[id]
|
||||
out.Backends = append(out.Backends, BackendSettings{
|
||||
ID: id,
|
||||
Endpoint: backend.Endpoint,
|
||||
APIKeyEnv: backend.APIKeyEnv,
|
||||
ExtraParams: backend.ExtraParams,
|
||||
ConcurrencyLimit: backend.ConcurrencyLimit,
|
||||
QueueCapacity: backend.QueueCapacity,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||
@@ -20,7 +21,7 @@ func TestLoadConfigMissingImplicitPathUsesBuiltInDefaults(t *testing.T) {
|
||||
}
|
||||
|
||||
want := BuiltInDefaults()
|
||||
if got != want {
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("unexpected settings: got=%+v want=%+v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -100,6 +101,78 @@ func TestLoadConfigAPIKeyFieldIsRejectedAsUnknown(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigBackendsRetainsResolvedSettingsInSortedOrder(t *testing.T) {
|
||||
path := writeConfigFile(t, "config.yml", `
|
||||
backends:
|
||||
zebra:
|
||||
endpoint: https://zebra.example/v1
|
||||
api_key_env: ZEBRA_API_KEY
|
||||
extra_params:
|
||||
provider_option: enabled
|
||||
nested:
|
||||
enabled: true
|
||||
attempts: 2
|
||||
concurrency_limit: 2
|
||||
alpha:
|
||||
endpoint: http://alpha.example/v1
|
||||
queue_capacity: 0
|
||||
`)
|
||||
|
||||
got, err := LoadConfig(path, true)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if len(got.Backends) != 2 {
|
||||
t.Fatalf("expected two backends, got %#v", got.Backends)
|
||||
}
|
||||
if got.Backends[0].ID != "alpha" || got.Backends[1].ID != "zebra" {
|
||||
t.Fatalf("expected sorted backend IDs, got %#v", got.Backends)
|
||||
}
|
||||
if got.Backends[0].QueueCapacity == nil || *got.Backends[0].QueueCapacity != 0 {
|
||||
t.Fatalf("expected explicit zero queue capacity, got %#v", got.Backends[0].QueueCapacity)
|
||||
}
|
||||
if got.Backends[1].QueueCapacity != nil {
|
||||
t.Fatalf("expected omitted queue capacity to remain nil, got %#v", got.Backends[1].QueueCapacity)
|
||||
}
|
||||
wantParams := map[string]any{
|
||||
"provider_option": "enabled",
|
||||
"nested": map[string]any{
|
||||
"enabled": true,
|
||||
"attempts": 2,
|
||||
},
|
||||
}
|
||||
if !reflect.DeepEqual(got.Backends[1].ExtraParams, wantParams) {
|
||||
t.Fatalf("unexpected extra params: got=%#v want=%#v", got.Backends[1].ExtraParams, wantParams)
|
||||
}
|
||||
if got.Backends[1].Endpoint != "https://zebra.example/v1" || got.Backends[1].APIKeyEnv != "ZEBRA_API_KEY" || got.Backends[1].ConcurrencyLimit != 2 {
|
||||
t.Fatalf("unexpected zebra backend: %#v", got.Backends[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigRejectsUnknownOrSecretBackendFields(t *testing.T) {
|
||||
for name, body := range map[string]string{
|
||||
"unknown": "backends:\n local:\n endpoint: http://localhost:11434/v1\n unexpected: value\n",
|
||||
"secret": "backends:\n local:\n endpoint: http://localhost:11434/v1\n api_key: secret\n",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
path := writeConfigFile(t, "config.yml", body)
|
||||
_, err := LoadConfig(path, true)
|
||||
if !errors.Is(err, ErrInvalidConfigYAML) {
|
||||
t.Fatalf("expected ErrInvalidConfigYAML, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigRejectsInvalidBackendFieldTypes(t *testing.T) {
|
||||
path := writeConfigFile(t, "config.yml", "backends:\n local:\n endpoint: http://localhost:11434/v1\n queue_capacity: not-a-number\n")
|
||||
|
||||
_, err := LoadConfig(path, true)
|
||||
if !errors.Is(err, ErrInvalidConfigYAML) {
|
||||
t.Fatalf("expected ErrInvalidConfigYAML, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigValidConfigSetsDirectoriesAndServerAddr(t *testing.T) {
|
||||
path := writeConfigFile(t, "config.yml", `
|
||||
prompt_dir: ./prompts
|
||||
@@ -196,7 +269,7 @@ func TestLoadConfigEmptyFileResolvesToBuiltInDefaults(t *testing.T) {
|
||||
}
|
||||
|
||||
want := BuiltInDefaults()
|
||||
if got != want {
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("unexpected settings: got=%+v want=%+v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,39 +1,13 @@
|
||||
package defaults
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
)
|
||||
import "time"
|
||||
|
||||
const (
|
||||
HTTPAddrDefault = ":8080"
|
||||
SchemaDirDefault = "."
|
||||
OutputArtifactName = "output"
|
||||
ContentTypeTextPlain = "text/plain"
|
||||
ContentTypeTextMarkdown = "text/markdown"
|
||||
ContentTypeApplicationJSON = "application/json"
|
||||
OpenAIChatCompletionsPath = "/chat/completions"
|
||||
HTTPMaxRequestBytesDefault = 16 * 1024 * 1024
|
||||
HTTPMaxArtifactBytesDefault = 16 * 1024 * 1024
|
||||
HTTPMaxResponseBytesDefault = 16 * 1024 * 1024
|
||||
|
||||
ExecutionDefaultTemperature = 0.0
|
||||
ExecutionDefaultMaxTokens = 0
|
||||
ExecutionDefaultTopP = 1.0
|
||||
ExecutionDefaultTimeoutSeconds = 600
|
||||
)
|
||||
|
||||
var (
|
||||
LLMRequestTimeoutDefault = 10 * time.Minute
|
||||
HTTPReadHeaderTimeoutDefault = 10 * time.Second
|
||||
)
|
||||
|
||||
func ExecutionTargetDefault() domain.ExecutionTarget {
|
||||
return domain.ExecutionTarget{
|
||||
Temperature: ExecutionDefaultTemperature,
|
||||
MaxTokens: ExecutionDefaultMaxTokens,
|
||||
TopP: ExecutionDefaultTopP,
|
||||
TimeoutSeconds: ExecutionDefaultTimeoutSeconds,
|
||||
}
|
||||
}
|
||||
var HTTPReadHeaderTimeoutDefault = 10 * time.Second
|
||||
|
||||
@@ -1,288 +0,0 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// ArtifactRefType defines how an artifact is referenced.
|
||||
type ArtifactRefType string
|
||||
|
||||
const (
|
||||
ArtifactRefInline ArtifactRefType = "inline"
|
||||
ArtifactRefFile ArtifactRefType = "file"
|
||||
)
|
||||
|
||||
// OutputFormat defines the desired format of the generated artifact.
|
||||
type OutputFormat string
|
||||
|
||||
const (
|
||||
FormatText OutputFormat = "text"
|
||||
FormatMarkdown OutputFormat = "markdown"
|
||||
FormatJSON OutputFormat = "json"
|
||||
)
|
||||
|
||||
// ValidationMode defines how the output should be validated.
|
||||
type ValidationMode string
|
||||
|
||||
const (
|
||||
ValidationNone ValidationMode = "none"
|
||||
ValidationBasic ValidationMode = "basic"
|
||||
ValidationJSON ValidationMode = "json"
|
||||
ValidationJSONSchema ValidationMode = "json_schema"
|
||||
)
|
||||
|
||||
// ValidationStatus defines the result of a validation check.
|
||||
type ValidationStatus string
|
||||
|
||||
const (
|
||||
ValidationPassed ValidationStatus = "passed"
|
||||
ValidationFailed ValidationStatus = "failed"
|
||||
ValidationSkipped ValidationStatus = "skipped"
|
||||
)
|
||||
|
||||
// CacheControlType defines provider cache behavior for prompt content.
|
||||
type CacheControlType string
|
||||
|
||||
const (
|
||||
CacheControlEphemeral CacheControlType = "ephemeral"
|
||||
)
|
||||
|
||||
const (
|
||||
// SessionIDMaxLength is OpenRouter's documented maximum session_id length.
|
||||
SessionIDMaxLength = 256
|
||||
)
|
||||
|
||||
// CacheControl describes provider cache metadata attached to prompt content.
|
||||
type CacheControl struct {
|
||||
Type CacheControlType `yaml:"type" json:"type"`
|
||||
TTL string `yaml:"ttl,omitempty" json:"ttl,omitempty"`
|
||||
}
|
||||
|
||||
// RunRequest represents a request to generate a single artifact.
|
||||
type RunRequest struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
APIKey string `json:"-" yaml:"-"`
|
||||
Inputs map[string]ArtifactRef
|
||||
Vars map[string]string
|
||||
Execution *ExecutionTargetOverride
|
||||
Validation *OutputContract
|
||||
Metadata map[string]string
|
||||
}
|
||||
|
||||
// RunResult represents the complete result of a prompt execution run.
|
||||
type RunResult struct {
|
||||
RunID string
|
||||
Artifact Artifact
|
||||
RawOutput string
|
||||
Validation ValidationResult
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
PromptHash string
|
||||
RenderedPromptHash string
|
||||
SelectedProfileID string
|
||||
ModelName string
|
||||
Endpoint string
|
||||
EffectiveModelParams ExecutionTarget
|
||||
InputHashes map[string]string
|
||||
Usage TokenUsage
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
Duration time.Duration
|
||||
}
|
||||
|
||||
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
|
||||
// It must never include resolved API key values, model output, or validation data.
|
||||
type PreparedRun struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
TargetPresence ExecutionTargetPresence `json:"-"`
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
EndTime time.Time `json:"end_time,omitempty"`
|
||||
DurationMS int64 `json:"duration_ms,omitempty"`
|
||||
}
|
||||
|
||||
// ArtifactRef represents a reference to an input artifact.
|
||||
type ArtifactRef struct {
|
||||
Type ArtifactRefType
|
||||
URI string
|
||||
Body string // Used for inline
|
||||
}
|
||||
|
||||
// Artifact represents the actual loaded content of a reference.
|
||||
type Artifact struct {
|
||||
Name string
|
||||
ContentType string
|
||||
Body []byte
|
||||
URI string
|
||||
Size int64
|
||||
Hash string
|
||||
}
|
||||
|
||||
// PromptDefinition represents a configured prompt execution definition.
|
||||
type PromptDefinition struct {
|
||||
ID string `yaml:"id"`
|
||||
Version string `yaml:"version"`
|
||||
DefaultProfile string `yaml:"default_profile"`
|
||||
Description string `yaml:"description"`
|
||||
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
|
||||
Inputs []PromptInput `yaml:"inputs"`
|
||||
Templates []PromptMessageTemplate `yaml:"templates"`
|
||||
OutputFormat OutputFormat `yaml:"output_format"`
|
||||
Validation OutputContract `yaml:"validation"`
|
||||
}
|
||||
|
||||
// PromptInput describes one named input expected by a prompt definition.
|
||||
type PromptInput struct {
|
||||
Name string `yaml:"name"`
|
||||
Required bool `yaml:"required"`
|
||||
ContentType string `yaml:"content_type"`
|
||||
Description string `yaml:"description"`
|
||||
}
|
||||
|
||||
// PromptMessageTemplate defines a template for a chat message.
|
||||
type PromptMessageTemplate struct {
|
||||
Role string `yaml:"role"`
|
||||
Content string `yaml:"content"`
|
||||
ContentFile string `yaml:"content_file"`
|
||||
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
||||
}
|
||||
|
||||
// ExecutionProfile describes how and where to execute a model.
|
||||
type ExecutionProfile struct {
|
||||
ID string `yaml:"id"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Model string `yaml:"model"`
|
||||
Temperature float64 `yaml:"temperature"`
|
||||
MaxTokens int `yaml:"max_tokens"`
|
||||
TopP float64 `yaml:"top_p"`
|
||||
TimeoutSeconds int `yaml:"timeout_seconds"`
|
||||
ServiceTier string `yaml:"service_tier"`
|
||||
ReasoningEffort string `yaml:"reasoning_effort"`
|
||||
APIKeyEnv string `yaml:"api_key_env"`
|
||||
APIKeyRequired bool `yaml:"-" json:"-"`
|
||||
ExtraParams map[string]any `yaml:"extra_params"`
|
||||
}
|
||||
|
||||
// ExecutionTargetOverride represents per-request runtime setting overrides.
|
||||
type ExecutionTargetOverride struct {
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||
ServiceTier string `json:"service_tier,omitempty"`
|
||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||
}
|
||||
|
||||
// ExecutionTargetPresence tracks which effective runtime fields came from an
|
||||
// explicit request override even when the resolved value is a zero value.
|
||||
type ExecutionTargetPresence struct {
|
||||
Temperature bool
|
||||
MaxTokens bool
|
||||
TopP bool
|
||||
TimeoutSeconds bool
|
||||
}
|
||||
|
||||
// ExecutionTarget represents effective model runtime settings for a run.
|
||||
type ExecutionTarget struct {
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||
Model string `yaml:"model" json:"model"`
|
||||
Temperature float64 `yaml:"temperature" json:"temperature"`
|
||||
MaxTokens int `yaml:"max_tokens" json:"max_tokens"`
|
||||
TopP float64 `yaml:"top_p" json:"top_p"`
|
||||
TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"`
|
||||
ServiceTier string `yaml:"service_tier" json:"service_tier"`
|
||||
ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"`
|
||||
APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"`
|
||||
APIKey string `yaml:"-" json:"-"`
|
||||
APIKeyRequired bool `yaml:"-" json:"-"`
|
||||
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
|
||||
}
|
||||
|
||||
// OutputContract defines the requirements for the output artifact.
|
||||
type OutputContract struct {
|
||||
Format OutputFormat `yaml:"format"`
|
||||
ValidationMode ValidationMode `yaml:"validation_mode"`
|
||||
SchemaPath string `yaml:"schema_path"`
|
||||
RepairAttempts int `yaml:"repair_attempts"`
|
||||
}
|
||||
|
||||
// RenderedPrompt represents the prompt after template application.
|
||||
type RenderedPrompt struct {
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
}
|
||||
|
||||
// RenderedMessage is a single message in a rendered prompt.
|
||||
type RenderedMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
CacheControl *CacheControl `json:"cache_control,omitempty"`
|
||||
}
|
||||
|
||||
// GenerateRequest is the internal request passed to the LLM client.
|
||||
type GenerateRequest struct {
|
||||
Prompt RenderedPrompt
|
||||
Target ExecutionTarget
|
||||
TargetPresence ExecutionTargetPresence
|
||||
StructuredOutput *StructuredOutputSpec
|
||||
}
|
||||
|
||||
// StructuredOutputType indicates which provider-level output mode is requested.
|
||||
type StructuredOutputType string
|
||||
|
||||
const (
|
||||
StructuredOutputJSONSchema StructuredOutputType = "json_schema"
|
||||
)
|
||||
|
||||
// StructuredOutputSpec describes provider-level structured output requirements.
|
||||
type StructuredOutputSpec struct {
|
||||
Type StructuredOutputType `json:"type"`
|
||||
JSONSchema *StructuredOutputJSONSpec `json:"json_schema,omitempty"`
|
||||
}
|
||||
|
||||
// StructuredOutputJSONSpec contains json_schema output constraints.
|
||||
type StructuredOutputJSONSpec struct {
|
||||
Name string `json:"name"`
|
||||
Strict bool `json:"strict"`
|
||||
Schema any `json:"schema"`
|
||||
}
|
||||
|
||||
// GenerateResponse is the response received from the LLM client.
|
||||
type GenerateResponse struct {
|
||||
Content string
|
||||
Usage TokenUsage
|
||||
}
|
||||
|
||||
// TokenUsage tracks token consumption.
|
||||
type TokenUsage struct {
|
||||
PromptTokens int
|
||||
CompletionTokens int
|
||||
TotalTokens int
|
||||
CachedTokens int
|
||||
CacheWriteTokens int
|
||||
}
|
||||
|
||||
// ValidationResult represents the outcome of an output validation.
|
||||
type ValidationResult struct {
|
||||
Status ValidationStatus
|
||||
Mode ValidationMode
|
||||
Errors []string
|
||||
SchemaPath string
|
||||
RepairAttempts int
|
||||
IsValid bool
|
||||
}
|
||||
@@ -1,141 +0,0 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
|
||||
const envName = "SCRIPTORIUM_TEST_API_KEY"
|
||||
const secret = "super-secret-value"
|
||||
t.Setenv(envName, secret)
|
||||
|
||||
prepared := PreparedRun{
|
||||
PromptID: "prompt.id",
|
||||
PromptVersion: "v1",
|
||||
PromptHash: "prompt-hash",
|
||||
SelectedProfileID: "local-fast",
|
||||
EffectiveModelParams: ExecutionTarget{
|
||||
Endpoint: "http://llm/v1",
|
||||
Model: "gpt-test",
|
||||
APIKeyEnv: envName,
|
||||
APIKey: secret,
|
||||
},
|
||||
InputHashes: map[string]string{"transcript": "hash-1"},
|
||||
RenderedPromptHash: "rendered-hash",
|
||||
Messages: []RenderedMessage{
|
||||
{Role: "system", Content: "You are helpful."},
|
||||
{Role: "user", Content: "Summarize this."},
|
||||
},
|
||||
}
|
||||
|
||||
b, err := json.Marshal(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal failed: %v", err)
|
||||
}
|
||||
|
||||
out := string(b)
|
||||
if strings.Contains(out, secret) {
|
||||
t.Fatalf("prepared run JSON unexpectedly contains secret value: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, `"api_key_env":"`+envName+`"`) {
|
||||
t.Fatalf("prepared run JSON should include api_key_env name: %s", out)
|
||||
}
|
||||
|
||||
var top map[string]any
|
||||
if err := json.Unmarshal(b, &top); err != nil {
|
||||
t.Fatalf("unmarshal failed: %v", err)
|
||||
}
|
||||
|
||||
for _, forbidden := range []string{"raw_output", "validation", "artifact"} {
|
||||
if _, ok := top[forbidden]; ok {
|
||||
t.Fatalf("prepared run JSON should not include %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedRunJSONIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
||||
prepared := PreparedRun{
|
||||
PromptID: "prompt.id",
|
||||
SelectedProfileID: "local-fast",
|
||||
EffectiveModelParams: ExecutionTarget{
|
||||
Endpoint: "http://llm/v1",
|
||||
Model: "gpt-test",
|
||||
},
|
||||
RenderedPromptHash: "rendered-hash",
|
||||
Messages: []RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "You are helpful.",
|
||||
CacheControl: &CacheControl{
|
||||
Type: CacheControlEphemeral,
|
||||
TTL: "1h",
|
||||
},
|
||||
},
|
||||
{Role: "user", Content: "Summarize this."},
|
||||
},
|
||||
}
|
||||
|
||||
b, err := json.Marshal(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal failed: %v", err)
|
||||
}
|
||||
|
||||
var decoded struct {
|
||||
Messages []map[string]any `json:"messages"`
|
||||
}
|
||||
if err := json.Unmarshal(b, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal failed: %v", err)
|
||||
}
|
||||
if len(decoded.Messages) != 2 {
|
||||
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
|
||||
}
|
||||
|
||||
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected cache_control on first message, got %#v", decoded.Messages[0])
|
||||
}
|
||||
if cacheControl["type"] != string(CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
||||
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
||||
}
|
||||
if _, ok := decoded.Messages[1]["cache_control"]; ok {
|
||||
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedRunJSONIncludesSessionIDOnlyWhenPresent(t *testing.T) {
|
||||
prepared := PreparedRun{
|
||||
PromptID: "prompt.id",
|
||||
SelectedProfileID: "local-fast",
|
||||
EffectiveModelParams: ExecutionTarget{
|
||||
Endpoint: "http://llm/v1",
|
||||
Model: "gpt-test",
|
||||
},
|
||||
SessionID: "session-123",
|
||||
RenderedPromptHash: "rendered-hash",
|
||||
Messages: []RenderedMessage{{Role: "user", Content: "Summarize this."}},
|
||||
}
|
||||
|
||||
b, err := json.Marshal(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal failed: %v", err)
|
||||
}
|
||||
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(b, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal failed: %v", err)
|
||||
}
|
||||
if decoded["session_id"] != "session-123" {
|
||||
t.Fatalf("expected session_id in prepared run JSON, got %#v", decoded["session_id"])
|
||||
}
|
||||
|
||||
prepared.SessionID = ""
|
||||
b, err = json.Marshal(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal failed: %v", err)
|
||||
}
|
||||
if strings.Contains(string(b), "session_id") {
|
||||
t.Fatalf("expected empty session_id to be omitted, got %s", b)
|
||||
}
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
package filecatalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// FindYAMLFiles returns sorted full paths for .yaml and .yml files under root.
|
||||
func FindYAMLFiles(ctx context.Context, root string) ([]string, error) {
|
||||
var files []string
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !IsYAMLFile(d.Name()) {
|
||||
return nil
|
||||
}
|
||||
files = append(files, path)
|
||||
return nil
|
||||
})
|
||||
sort.Strings(files)
|
||||
return files, err
|
||||
}
|
||||
|
||||
// FindFSYAMLFiles returns sorted paths for .yaml and .yml files under root in fsys.
|
||||
func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, error) {
|
||||
cleanRoot := CleanFSRoot(root)
|
||||
var files []string
|
||||
err := fs.WalkDir(fsys, cleanRoot, func(name string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !IsYAMLFile(d.Name()) {
|
||||
return nil
|
||||
}
|
||||
files = append(files, name)
|
||||
return nil
|
||||
})
|
||||
sort.Strings(files)
|
||||
return files, err
|
||||
}
|
||||
|
||||
// RelativePath computes a clean relative path from root to path.
|
||||
func RelativePath(root string, filePath string) string {
|
||||
rel, err := filepath.Rel(root, filePath)
|
||||
if err != nil {
|
||||
return filepath.Clean(filePath)
|
||||
}
|
||||
return filepath.Clean(rel)
|
||||
}
|
||||
|
||||
// CleanFSRoot normalizes a root path for use with fs.FS.
|
||||
func CleanFSRoot(root string) string {
|
||||
root = strings.TrimSpace(root)
|
||||
if root == "" || root == "." {
|
||||
return "."
|
||||
}
|
||||
return path.Clean(root)
|
||||
}
|
||||
|
||||
// DisplayPath returns name relative to root for messages about fs.FS paths.
|
||||
func DisplayPath(root string, name string) string {
|
||||
cleanRoot := CleanFSRoot(root)
|
||||
cleanName := path.Clean(name)
|
||||
if cleanRoot == "." {
|
||||
return cleanName
|
||||
}
|
||||
prefix := strings.TrimSuffix(cleanRoot, "/") + "/"
|
||||
if strings.HasPrefix(cleanName, prefix) {
|
||||
return strings.TrimPrefix(cleanName, prefix)
|
||||
}
|
||||
return cleanName
|
||||
}
|
||||
|
||||
// ResolveFSPath resolves userPath from baseDir and keeps it inside root.
|
||||
func ResolveFSPath(root string, baseDir string, userPath string) (string, string, error) {
|
||||
cleanRoot := CleanFSRoot(root)
|
||||
cleanBase := path.Clean(strings.TrimSpace(baseDir))
|
||||
if cleanBase == "" {
|
||||
cleanBase = cleanRoot
|
||||
}
|
||||
if !containsFSPath(cleanRoot, cleanBase) {
|
||||
return "", "", fmt.Errorf("base path %q is outside source root %q", cleanBase, cleanRoot)
|
||||
}
|
||||
|
||||
cleanUserPath := strings.TrimSpace(userPath)
|
||||
if cleanUserPath == "" {
|
||||
return "", "", fmt.Errorf("path is required")
|
||||
}
|
||||
cleanUserPath = path.Clean(cleanUserPath)
|
||||
if path.IsAbs(cleanUserPath) {
|
||||
return "", "", fmt.Errorf("path %q must be relative", userPath)
|
||||
}
|
||||
|
||||
resolved := path.Clean(path.Join(cleanBase, cleanUserPath))
|
||||
if !containsFSPath(cleanRoot, resolved) {
|
||||
return "", "", fmt.Errorf("path %q escapes source root %q", userPath, cleanRoot)
|
||||
}
|
||||
return resolved, DisplayPath(cleanRoot, resolved), nil
|
||||
}
|
||||
|
||||
func containsFSPath(root string, name string) bool {
|
||||
root = CleanFSRoot(root)
|
||||
name = path.Clean(name)
|
||||
if root == "." {
|
||||
return name == "." || (name != ".." && !strings.HasPrefix(name, "../"))
|
||||
}
|
||||
return name == root || strings.HasPrefix(name, strings.TrimSuffix(root, "/")+"/")
|
||||
}
|
||||
|
||||
// Stem strips .yaml or .yml from a file name.
|
||||
func Stem(name string) string {
|
||||
name = strings.TrimSuffix(name, ".yaml")
|
||||
name = strings.TrimSuffix(name, ".yml")
|
||||
return name
|
||||
}
|
||||
|
||||
func IsYAMLFile(name string) bool {
|
||||
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
|
||||
}
|
||||
@@ -1,270 +0,0 @@
|
||||
package filecatalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
)
|
||||
|
||||
func TestFindYAMLFilesNestedSortedAndFiltered(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mustWriteFile(t, filepath.Join(root, "z", "prompt.yml"), "id: z")
|
||||
mustWriteFile(t, filepath.Join(root, "a", "profile.yaml"), "id: a")
|
||||
mustWriteFile(t, filepath.Join(root, "a", "ignore.txt"), "not yaml")
|
||||
mustWriteFile(t, filepath.Join(root, "b", "ignore.yaml.bak"), "not yaml")
|
||||
|
||||
got, err := FindYAMLFiles(context.Background(), root)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
want := []string{
|
||||
filepath.Join(root, "a", "profile.yaml"),
|
||||
filepath.Join(root, "z", "prompt.yml"),
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("expected sorted YAML files %v, got %v", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindYAMLFilesHonorsContextCancellation(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mustWriteFile(t, filepath.Join(root, "one.yaml"), "id: one")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
_, err := FindYAMLFiles(ctx, root)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected context.Canceled, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindFSYAMLFilesNestedSortedAndFiltered(t *testing.T) {
|
||||
fsys := fstest.MapFS{
|
||||
"prompts/z/prompt.yml": &fstest.MapFile{Data: []byte("id: z")},
|
||||
"prompts/a/profile.yaml": &fstest.MapFile{Data: []byte("id: a")},
|
||||
"prompts/a/ignore.txt": &fstest.MapFile{Data: []byte("not yaml")},
|
||||
"prompts/b/ignore.yaml.bak": &fstest.MapFile{Data: []byte("not yaml")},
|
||||
"other/ignored.yaml": &fstest.MapFile{Data: []byte("id: ignored")},
|
||||
}
|
||||
|
||||
got, err := FindFSYAMLFiles(context.Background(), fsys, " prompts ")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
want := []string{
|
||||
"prompts/a/profile.yaml",
|
||||
"prompts/z/prompt.yml",
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("expected sorted YAML files %v, got %v", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindFSYAMLFilesHonorsContextCancellation(t *testing.T) {
|
||||
fsys := fstest.MapFS{
|
||||
"one.yaml": &fstest.MapFile{Data: []byte("id: one")},
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
_, err := FindFSYAMLFiles(ctx, fsys, ".")
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected context.Canceled, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRelativePathNested(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, "nested", "profiles", "local.yaml")
|
||||
got := RelativePath(root, path)
|
||||
want := filepath.Join("nested", "profiles", "local.yaml")
|
||||
if got != want {
|
||||
t.Fatalf("expected relative path %q, got %q", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanFSRoot(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
root string
|
||||
want string
|
||||
}{
|
||||
{name: "empty", root: "", want: "."},
|
||||
{name: "dot", root: ".", want: "."},
|
||||
{name: "trimmed", root: " prompts/../profiles ", want: "profiles"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := CleanFSRoot(tc.root); got != tc.want {
|
||||
t.Fatalf("expected %q, got %q", tc.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDisplayPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
root string
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{name: "root dot", root: ".", path: "profiles/local.yaml", want: "profiles/local.yaml"},
|
||||
{name: "nested root", root: "profiles", path: "profiles/local.yaml", want: "local.yaml"},
|
||||
{name: "outside root", root: "profiles", path: "other/local.yaml", want: "other/local.yaml"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := DisplayPath(tc.root, tc.path); got != tc.want {
|
||||
t.Fatalf("expected %q, got %q", tc.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveFSPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
root string
|
||||
baseDir string
|
||||
userPath string
|
||||
wantPath string
|
||||
wantDisplay string
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "sibling inside root",
|
||||
root: "prompts",
|
||||
baseDir: "prompts/nested",
|
||||
userPath: "./messages/user.tmpl",
|
||||
wantPath: "prompts/nested/messages/user.tmpl",
|
||||
wantDisplay: "nested/messages/user.tmpl",
|
||||
},
|
||||
{
|
||||
name: "parent inside root",
|
||||
root: "prompts",
|
||||
baseDir: "prompts/nested",
|
||||
userPath: "../shared/user.tmpl",
|
||||
wantPath: "prompts/shared/user.tmpl",
|
||||
wantDisplay: "shared/user.tmpl",
|
||||
},
|
||||
{
|
||||
name: "escape rejected",
|
||||
root: "prompts",
|
||||
baseDir: "prompts/nested",
|
||||
userPath: "../../outside.tmpl",
|
||||
wantErr: "escapes source root",
|
||||
},
|
||||
{
|
||||
name: "absolute path rejected",
|
||||
root: "prompts",
|
||||
baseDir: "prompts/nested",
|
||||
userPath: "/outside.tmpl",
|
||||
wantErr: "must be relative",
|
||||
},
|
||||
{
|
||||
name: "empty path rejected",
|
||||
root: "prompts",
|
||||
baseDir: "prompts/nested",
|
||||
userPath: " ",
|
||||
wantErr: "path is required",
|
||||
},
|
||||
{
|
||||
name: "dot root allows normal relative path",
|
||||
root: ".",
|
||||
baseDir: ".",
|
||||
userPath: "schemas/events.schema.json",
|
||||
wantPath: "schemas/events.schema.json",
|
||||
wantDisplay: "schemas/events.schema.json",
|
||||
},
|
||||
{
|
||||
name: "dot root rejects parent escape",
|
||||
root: ".",
|
||||
baseDir: ".",
|
||||
userPath: "../outside.tmpl",
|
||||
wantErr: "escapes source root",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
gotPath, gotDisplay, err := ResolveFSPath(tc.root, tc.baseDir, tc.userPath)
|
||||
if tc.wantErr != "" {
|
||||
if err == nil {
|
||||
t.Fatalf("expected error containing %q", tc.wantErr)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tc.wantErr) {
|
||||
t.Fatalf("expected error to contain %q, got %v", tc.wantErr, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if gotPath != tc.wantPath || gotDisplay != tc.wantDisplay {
|
||||
t.Fatalf("expected path/display %q/%q, got %q/%q", tc.wantPath, tc.wantDisplay, gotPath, gotDisplay)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStemStripsYAMLExtensions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{name: "yaml", in: "prompt.yaml", want: "prompt"},
|
||||
{name: "yml", in: "profile.yml", want: "profile"},
|
||||
{name: "other", in: "file.txt", want: "file.txt"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := Stem(tc.in); got != tc.want {
|
||||
t.Fatalf("expected %q, got %q", tc.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsYAMLFile(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want bool
|
||||
}{
|
||||
{name: "yaml", in: "prompt.yaml", want: true},
|
||||
{name: "yml", in: "profile.yml", want: true},
|
||||
{name: "backup", in: "profile.yaml.bak", want: false},
|
||||
{name: "uppercase", in: "profile.YAML", want: false},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := IsYAMLFile(tc.in); got != tc.want {
|
||||
t.Fatalf("expected %v, got %v", tc.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func mustWriteFile(t *testing.T, path string, content string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("failed to create directory: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("failed to write file %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
122
internal/format/inspection.go
Normal file
122
internal/format/inspection.go
Normal file
@@ -0,0 +1,122 @@
|
||||
package format
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
type PromptInspection struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version"`
|
||||
PromptHash string `json:"prompt_hash"`
|
||||
DefaultProfileID string `json:"default_profile_id"`
|
||||
Inputs []PromptInspectionInput `json:"inputs"`
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
}
|
||||
|
||||
type PromptInspectionInput struct {
|
||||
Name string `json:"name"`
|
||||
Required bool `json:"required"`
|
||||
ContentType string `json:"content_type"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
type OutputContract struct {
|
||||
Format string `json:"format"`
|
||||
ValidationMode string `json:"validation_mode"`
|
||||
SchemaPath string `json:"schema_path"`
|
||||
RepairAttempts int `json:"repair_attempts"`
|
||||
}
|
||||
|
||||
type ProfileInspection struct {
|
||||
ProfileID string `json:"profile_id"`
|
||||
EffectiveModelParams ProfileModelParams `json:"effective_model_params"`
|
||||
APIKeyRequired bool `json:"api_key_required"`
|
||||
}
|
||||
|
||||
type ProfileModelParams struct {
|
||||
BackendID string `json:"backend_id"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Model string `json:"model"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
TopP float64 `json:"top_p"`
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
ServiceTier string `json:"service_tier"`
|
||||
ReasoningEffort string `json:"reasoning_effort"`
|
||||
APIKeyEnv string `json:"api_key_env"`
|
||||
ExtraParams map[string]any `json:"extra_params"`
|
||||
}
|
||||
|
||||
func FormatPromptInspection(value *promptkit.PromptInspection, outputFormat OutputFormat) ([]byte, error) {
|
||||
if value == nil {
|
||||
return nil, errors.New("prompt inspection is nil")
|
||||
}
|
||||
dto := PromptInspection{
|
||||
PromptID: value.PromptID, PromptVersion: value.PromptVersion, PromptHash: value.PromptHash, DefaultProfileID: value.DefaultProfileID,
|
||||
Inputs: make([]PromptInspectionInput, len(value.Inputs)),
|
||||
OutputContract: OutputContract{Format: string(value.OutputContract.Format), ValidationMode: string(value.OutputContract.ValidationMode), SchemaPath: value.OutputContract.SchemaPath, RepairAttempts: value.OutputContract.RepairAttempts},
|
||||
}
|
||||
for i, input := range value.Inputs {
|
||||
dto.Inputs[i] = PromptInspectionInput{Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description}
|
||||
}
|
||||
switch outputFormat {
|
||||
case OutputFormatJSON:
|
||||
data, err := json.MarshalIndent(dto, "", " ")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(data, '\n'), nil
|
||||
case OutputFormatText:
|
||||
var b bytes.Buffer
|
||||
fmt.Fprintf(&b, "prompt_id: %s\nprompt_version: %s\nprompt_hash: %s\ndefault_profile_id: %s\ninputs:", dto.PromptID, dto.PromptVersion, dto.PromptHash, dto.DefaultProfileID)
|
||||
if len(dto.Inputs) == 0 {
|
||||
fmt.Fprintln(&b, " []")
|
||||
} else {
|
||||
fmt.Fprintln(&b)
|
||||
for _, input := range dto.Inputs {
|
||||
fmt.Fprintf(&b, " - name: %s\n required: %t\n content_type: %s\n description: %s\n", input.Name, input.Required, input.ContentType, strconv.Quote(input.Description))
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, "output_contract:\n format: %s\n validation_mode: %s\n schema_path: %s\n repair_attempts: %d\n", dto.OutputContract.Format, dto.OutputContract.ValidationMode, dto.OutputContract.SchemaPath, dto.OutputContract.RepairAttempts)
|
||||
return b.Bytes(), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
|
||||
}
|
||||
}
|
||||
|
||||
func FormatProfileInspection(value *promptkit.ProfileInspection, outputFormat OutputFormat) ([]byte, error) {
|
||||
if value == nil {
|
||||
return nil, errors.New("profile inspection is nil")
|
||||
}
|
||||
target := value.EffectiveModelParams
|
||||
params := map[string]any{}
|
||||
for key, item := range target.ExtraParams {
|
||||
params[key] = item
|
||||
}
|
||||
dto := ProfileInspection{ProfileID: value.ProfileID, APIKeyRequired: value.APIKeyRequired, EffectiveModelParams: ProfileModelParams{BackendID: target.BackendID, Endpoint: target.Endpoint, Model: target.Model, Temperature: target.Temperature, MaxTokens: target.MaxTokens, TopP: target.TopP, TimeoutSeconds: target.TimeoutSeconds, ServiceTier: target.ServiceTier, ReasoningEffort: target.ReasoningEffort, APIKeyEnv: target.APIKeyEnv, ExtraParams: params}}
|
||||
switch outputFormat {
|
||||
case OutputFormatJSON:
|
||||
data, err := json.MarshalIndent(dto, "", " ")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(data, '\n'), nil
|
||||
case OutputFormatText:
|
||||
var b bytes.Buffer
|
||||
fmt.Fprintf(&b, "profile_id: %s\neffective_model_params:\n backend_id: %s\n endpoint: %s\n model: %s\n temperature: %g\n max_tokens: %d\n top_p: %g\n timeout_seconds: %d\n service_tier: %s\n reasoning_effort: %s\n api_key_env: %s\n extra_params: ", dto.ProfileID, dto.EffectiveModelParams.BackendID, dto.EffectiveModelParams.Endpoint, dto.EffectiveModelParams.Model, dto.EffectiveModelParams.Temperature, dto.EffectiveModelParams.MaxTokens, dto.EffectiveModelParams.TopP, dto.EffectiveModelParams.TimeoutSeconds, dto.EffectiveModelParams.ServiceTier, dto.EffectiveModelParams.ReasoningEffort, dto.EffectiveModelParams.APIKeyEnv)
|
||||
paramsJSON, err := json.Marshal(dto.EffectiveModelParams.ExtraParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fmt.Fprintf(&b, "%s\napi_key_required: %t\n", paramsJSON, dto.APIKeyRequired)
|
||||
return b.Bytes(), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
|
||||
}
|
||||
}
|
||||
124
internal/format/inspection_test.go
Normal file
124
internal/format/inspection_test.go
Normal file
@@ -0,0 +1,124 @@
|
||||
package format
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestFormatPromptInspectionPreservesDeclaredOrderAndEmptyValues(t *testing.T) {
|
||||
value := &promptkit.PromptInspection{PromptID: "p", PromptVersion: "1", PromptHash: "hash", Inputs: []promptkit.PromptInputDefinition{{Name: "first", Required: true, ContentType: "text/plain", Description: "first input"}, {Name: "second"}}, OutputContract: promptkit.OutputContract{Format: "text", ValidationMode: promptkit.ValidationNone}}
|
||||
text, err := FormatPromptInspection(value, OutputFormatText)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(text), "default_profile_id: \ninputs:\n - name: first") || strings.Index(string(text), "name: first") > strings.Index(string(text), "name: second") {
|
||||
t.Fatalf("unexpected text inspection: %s", text)
|
||||
}
|
||||
jsonOutput, err := FormatPromptInspection(value, OutputFormatJSON)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasSuffix(string(jsonOutput), "\n") || !strings.Contains(string(jsonOutput), `"default_profile_id": ""`) {
|
||||
t.Fatalf("unexpected json inspection: %s", jsonOutput)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatPromptInspectionRejectsNil(t *testing.T) {
|
||||
if _, err := FormatPromptInspection(nil, OutputFormatText); err == nil {
|
||||
t.Fatal("expected nil inspection error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatProfileInspectionPreservesSafeEffectiveValues(t *testing.T) {
|
||||
const secret = "sentinel-secret-must-not-appear"
|
||||
t.Setenv("FIXTURE_PROFILE_API_KEY", secret)
|
||||
|
||||
value := &promptkit.ProfileInspection{
|
||||
ProfileID: "custom-derived",
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{
|
||||
BackendID: "fixture-custom",
|
||||
Endpoint: "http://127.0.0.1:11434/v1",
|
||||
Model: "fixture-model",
|
||||
Temperature: 0.25,
|
||||
MaxTokens: 640,
|
||||
TopP: 0.9,
|
||||
TimeoutSeconds: 45,
|
||||
ServiceTier: "flex",
|
||||
ReasoningEffort: "high",
|
||||
APIKeyEnv: "FIXTURE_PROFILE_API_KEY",
|
||||
ExtraParams: map[string]any{
|
||||
"zeta": true,
|
||||
"alpha": map[string]any{"nested": []any{"first", 2.0}},
|
||||
},
|
||||
},
|
||||
APIKeyRequired: true,
|
||||
}
|
||||
|
||||
jsonOutput, err := FormatProfileInspection(value, OutputFormatJSON)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondJSONOutput, err := FormatProfileInspection(value, OutputFormatJSON)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(jsonOutput) != string(secondJSONOutput) || !strings.HasSuffix(string(jsonOutput), "\n") {
|
||||
t.Fatalf("expected deterministic newline-terminated JSON, got %q", jsonOutput)
|
||||
}
|
||||
if strings.Contains(string(jsonOutput), secret) {
|
||||
t.Fatalf("inspection exposed an environment secret: %s", jsonOutput)
|
||||
}
|
||||
|
||||
var decoded ProfileInspection
|
||||
if err := json.Unmarshal(jsonOutput, &decoded); err != nil {
|
||||
t.Fatalf("decode profile inspection: %v", err)
|
||||
}
|
||||
if decoded.ProfileID != "custom-derived" || decoded.EffectiveModelParams.BackendID != "fixture-custom" {
|
||||
t.Fatalf("unexpected profile identity: %+v", decoded)
|
||||
}
|
||||
if decoded.EffectiveModelParams.APIKeyEnv != "FIXTURE_PROFILE_API_KEY" || !decoded.APIKeyRequired {
|
||||
t.Fatalf("unexpected credential metadata: %+v", decoded)
|
||||
}
|
||||
alpha, ok := decoded.EffectiveModelParams.ExtraParams["alpha"].(map[string]any)
|
||||
if !ok || len(alpha["nested"].([]any)) != 2 {
|
||||
t.Fatalf("nested extra parameters were not preserved: %#v", decoded.EffectiveModelParams.ExtraParams)
|
||||
}
|
||||
|
||||
textOutput, err := FormatProfileInspection(value, OutputFormatText)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
textValue := string(textOutput)
|
||||
if !strings.Contains(textValue, "backend_id: fixture-custom") ||
|
||||
!strings.Contains(textValue, "api_key_env: FIXTURE_PROFILE_API_KEY") ||
|
||||
!strings.Contains(textValue, `extra_params: {"alpha":{"nested":["first",2]},"zeta":true}`) ||
|
||||
!strings.Contains(textValue, "api_key_required: true") {
|
||||
t.Fatalf("unexpected text inspection: %s", textValue)
|
||||
}
|
||||
if strings.Contains(textValue, secret) {
|
||||
t.Fatalf("text inspection exposed an environment secret: %s", textValue)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatProfileInspectionPreservesEmptyBackendAndRejectsNil(t *testing.T) {
|
||||
output, err := FormatProfileInspection(&promptkit.ProfileInspection{
|
||||
ProfileID: "endpoint-only",
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{
|
||||
Endpoint: "http://127.0.0.1:8000/v1",
|
||||
Model: "fixture-model",
|
||||
},
|
||||
}, OutputFormatJSON)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(output), `"backend_id": ""`) ||
|
||||
!strings.Contains(string(output), `"extra_params": {}`) {
|
||||
t.Fatalf("expected explicit empty backend and object extra params, got %s", output)
|
||||
}
|
||||
if _, err := FormatProfileInspection(nil, OutputFormatText); err == nil {
|
||||
t.Fatal("expected nil profile inspection error")
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
// Package format formats already-prepared domain data for adapters.
|
||||
// Package format formats already-prepared public data for adapters.
|
||||
package format
|
||||
|
||||
import (
|
||||
@@ -9,37 +9,52 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
var ErrUnknownPreparedRunFormat = errors.New("unknown prepared run format")
|
||||
|
||||
// PreparedRunOutputFormat is the output format for prepared render data.
|
||||
type PreparedRunOutputFormat string
|
||||
// OutputFormat selects an application-owned textual or JSON representation.
|
||||
type OutputFormat string
|
||||
|
||||
const (
|
||||
PreparedRunFormatText PreparedRunOutputFormat = "text"
|
||||
PreparedRunFormatJSON PreparedRunOutputFormat = "json"
|
||||
OutputFormatText OutputFormat = "text"
|
||||
OutputFormatJSON OutputFormat = "json"
|
||||
|
||||
DefaultPreparedRunOutputFormat PreparedRunOutputFormat = PreparedRunFormatText
|
||||
DefaultOutputFormat OutputFormat = OutputFormatText
|
||||
)
|
||||
|
||||
// PreparedRunOutputFormat is the output format for prepared render data.
|
||||
type PreparedRunOutputFormat = OutputFormat
|
||||
|
||||
const (
|
||||
PreparedRunFormatText = OutputFormatText
|
||||
PreparedRunFormatJSON = OutputFormatJSON
|
||||
|
||||
DefaultPreparedRunOutputFormat = DefaultOutputFormat
|
||||
)
|
||||
|
||||
// PreparedRunFormatter serializes a prepared run without performing use case work.
|
||||
type PreparedRunFormatter interface {
|
||||
Format(prepared *domain.PreparedRun) ([]byte, error)
|
||||
Format(prepared *promptkit.PreparedRun) ([]byte, error)
|
||||
}
|
||||
|
||||
// ParsePreparedRunOutputFormat parses a format name.
|
||||
func ParsePreparedRunOutputFormat(raw string) (PreparedRunOutputFormat, error) {
|
||||
switch PreparedRunOutputFormat(strings.ToLower(strings.TrimSpace(raw))) {
|
||||
return ParseOutputFormat(raw)
|
||||
}
|
||||
|
||||
// ParseOutputFormat parses the shared inspection and prepared-run format names.
|
||||
func ParseOutputFormat(raw string) (OutputFormat, error) {
|
||||
switch OutputFormat(strings.ToLower(strings.TrimSpace(raw))) {
|
||||
case "":
|
||||
return DefaultPreparedRunOutputFormat, nil
|
||||
case PreparedRunFormatText:
|
||||
return PreparedRunFormatText, nil
|
||||
case PreparedRunFormatJSON:
|
||||
return PreparedRunFormatJSON, nil
|
||||
return DefaultOutputFormat, nil
|
||||
case OutputFormatText:
|
||||
return OutputFormatText, nil
|
||||
case OutputFormatJSON:
|
||||
return OutputFormatJSON, nil
|
||||
default:
|
||||
return "", fmt.Errorf("%w: %q (supported: %s, %s)", ErrUnknownPreparedRunFormat, raw, PreparedRunFormatText, PreparedRunFormatJSON)
|
||||
return "", fmt.Errorf("%w: %q (supported: %s, %s)", ErrUnknownPreparedRunFormat, raw, OutputFormatText, OutputFormatJSON)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,7 +71,7 @@ func FormatterForPreparedRun(outputFormat PreparedRunOutputFormat) (PreparedRunF
|
||||
}
|
||||
|
||||
// FormatPreparedRun formats a prepared run using the selected format.
|
||||
func FormatPreparedRun(prepared *domain.PreparedRun, outputFormat PreparedRunOutputFormat) ([]byte, error) {
|
||||
func FormatPreparedRun(prepared *promptkit.PreparedRun, outputFormat PreparedRunOutputFormat) ([]byte, error) {
|
||||
formatter, err := FormatterForPreparedRun(outputFormat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -65,7 +80,7 @@ func FormatPreparedRun(prepared *domain.PreparedRun, outputFormat PreparedRunOut
|
||||
}
|
||||
|
||||
// FormatPreparedRunByName parses a format name and formats a prepared run.
|
||||
func FormatPreparedRunByName(prepared *domain.PreparedRun, rawFormat string) ([]byte, error) {
|
||||
func FormatPreparedRunByName(prepared *promptkit.PreparedRun, rawFormat string) ([]byte, error) {
|
||||
outputFormat, err := ParsePreparedRunOutputFormat(rawFormat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -75,7 +90,7 @@ func FormatPreparedRunByName(prepared *domain.PreparedRun, rawFormat string) ([]
|
||||
|
||||
type jsonPreparedRunFormatter struct{}
|
||||
|
||||
func (jsonPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, error) {
|
||||
func (jsonPreparedRunFormatter) Format(prepared *promptkit.PreparedRun) ([]byte, error) {
|
||||
if prepared == nil {
|
||||
return nil, errors.New("prepared run is nil")
|
||||
}
|
||||
@@ -84,7 +99,7 @@ func (jsonPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
||||
|
||||
type textPreparedRunFormatter struct{}
|
||||
|
||||
func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, error) {
|
||||
func (textPreparedRunFormatter) Format(prepared *promptkit.PreparedRun) ([]byte, error) {
|
||||
if prepared == nil {
|
||||
return nil, errors.New("prepared run is nil")
|
||||
}
|
||||
@@ -93,6 +108,9 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
||||
fmt.Fprintf(&b, "prompt: %s\n", prepared.PromptID)
|
||||
fmt.Fprintf(&b, "prompt_version: %s\n", prepared.PromptVersion)
|
||||
fmt.Fprintf(&b, "selected_profile_id: %s\n", prepared.SelectedProfileID)
|
||||
if prepared.SelectedBackendID != "" {
|
||||
fmt.Fprintf(&b, "selected_backend_id: %s\n", prepared.SelectedBackendID)
|
||||
}
|
||||
if prepared.PromptHash != "" {
|
||||
fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash)
|
||||
}
|
||||
@@ -146,7 +164,7 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
||||
|
||||
fmt.Fprintln(&b, "messages:")
|
||||
roleOrder := make([]string, 0)
|
||||
byRole := make(map[string][]domain.RenderedMessage)
|
||||
byRole := make(map[string][]promptkit.RenderedMessage)
|
||||
for _, msg := range prepared.Messages {
|
||||
if _, exists := byRole[msg.Role]; !exists {
|
||||
roleOrder = append(roleOrder, msg.Role)
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) {
|
||||
@@ -22,6 +22,7 @@ func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) {
|
||||
"prompt: prompt.id",
|
||||
"prompt_version: v1",
|
||||
"selected_profile_id: local-fast",
|
||||
"selected_backend_id: local",
|
||||
"endpoint: http://llm/v1",
|
||||
"model: gpt-test",
|
||||
"temperature: 0.4",
|
||||
@@ -94,9 +95,8 @@ func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
||||
|
||||
func TestTextFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) {
|
||||
const directKey = "direct-format-key"
|
||||
// PreparedRun intentionally has no field for direct API keys.
|
||||
prepared := samplePreparedRun()
|
||||
prepared.EffectiveModelParams.APIKey = directKey
|
||||
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
@@ -108,12 +108,12 @@ func TestTextFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) {
|
||||
|
||||
func TestTextFormatterIncludesMessageCacheControlBeforeContent(t *testing.T) {
|
||||
prepared := samplePreparedRun()
|
||||
prepared.Messages = []domain.RenderedMessage{
|
||||
prepared.Messages = []promptkit.RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "System guidance.",
|
||||
CacheControl: &domain.CacheControl{
|
||||
Type: domain.CacheControlEphemeral,
|
||||
CacheControl: &promptkit.CacheControl{
|
||||
Type: promptkit.CacheControlEphemeral,
|
||||
TTL: "1h",
|
||||
},
|
||||
},
|
||||
@@ -148,12 +148,12 @@ func TestTextFormatterIncludesSessionIDWhenPresent(t *testing.T) {
|
||||
|
||||
func TestTextFormatterOmitsEmptyCacheControlTTL(t *testing.T) {
|
||||
prepared := samplePreparedRun()
|
||||
prepared.Messages = []domain.RenderedMessage{
|
||||
prepared.Messages = []promptkit.RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "System guidance.",
|
||||
CacheControl: &domain.CacheControl{
|
||||
Type: domain.CacheControlEphemeral,
|
||||
CacheControl: &promptkit.CacheControl{
|
||||
Type: promptkit.CacheControlEphemeral,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -231,12 +231,12 @@ func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
||||
|
||||
func TestJSONFormatterIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
||||
prepared := samplePreparedRun()
|
||||
prepared.Messages = []domain.RenderedMessage{
|
||||
prepared.Messages = []promptkit.RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "System guidance.",
|
||||
CacheControl: &domain.CacheControl{
|
||||
Type: domain.CacheControlEphemeral,
|
||||
CacheControl: &promptkit.CacheControl{
|
||||
Type: promptkit.CacheControlEphemeral,
|
||||
TTL: "1h",
|
||||
},
|
||||
},
|
||||
@@ -262,7 +262,7 @@ func TestJSONFormatterIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
||||
if !ok {
|
||||
t.Fatalf("expected first message cache_control, got %#v", decoded.Messages[0])
|
||||
}
|
||||
if cacheControl["type"] != string(domain.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
||||
if cacheControl["type"] != string(promptkit.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
||||
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
||||
}
|
||||
if _, ok := decoded.Messages[1]["cache_control"]; ok {
|
||||
@@ -285,9 +285,8 @@ func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
||||
|
||||
func TestJSONFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) {
|
||||
const directKey = "direct-format-key"
|
||||
// PreparedRun intentionally has no field for direct API keys.
|
||||
prepared := samplePreparedRun()
|
||||
prepared.EffectiveModelParams.APIKey = directKey
|
||||
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
@@ -342,13 +341,15 @@ func TestFormatPreparedRunByNameUnknownFailsClearly(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func samplePreparedRun() *domain.PreparedRun {
|
||||
return &domain.PreparedRun{
|
||||
func samplePreparedRun() *promptkit.PreparedRun {
|
||||
return &promptkit.PreparedRun{
|
||||
PromptID: "prompt.id",
|
||||
PromptVersion: "v1",
|
||||
PromptHash: "prompt-hash",
|
||||
SelectedProfileID: "local-fast",
|
||||
EffectiveModelParams: domain.ExecutionTarget{
|
||||
SelectedBackendID: "local",
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{
|
||||
BackendID: "local",
|
||||
Endpoint: "http://llm/v1",
|
||||
Model: "gpt-test",
|
||||
Temperature: 0.4,
|
||||
@@ -364,7 +365,7 @@ func samplePreparedRun() *domain.PreparedRun {
|
||||
"glossary": "hash-glossary",
|
||||
},
|
||||
RenderedPromptHash: "rendered-hash",
|
||||
Messages: []domain.RenderedMessage{
|
||||
Messages: []promptkit.RenderedMessage{
|
||||
{Role: "system", Content: "System guidance."},
|
||||
{Role: "user", Content: "Summarize the transcript.\nInclude key entities."},
|
||||
{Role: "user", Content: "Second user message."},
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
)
|
||||
|
||||
// Client executes a rendered prompt against an LLM endpoint.
|
||||
type Client interface {
|
||||
Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error)
|
||||
}
|
||||
@@ -1,388 +0,0 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidConfig = errors.New("invalid llm client configuration")
|
||||
ErrInvalidRequest = errors.New("invalid generate request")
|
||||
ErrRequestFailed = errors.New("llm request failed")
|
||||
ErrUnexpectedStatus = errors.New("llm returned non-success status")
|
||||
ErrMalformedResponse = errors.New("malformed llm response")
|
||||
)
|
||||
|
||||
type OpenAICompatibleConfig struct {
|
||||
BaseURL string
|
||||
Model string
|
||||
Timeout time.Duration
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
type OpenAICompatibleClient struct {
|
||||
baseURL string
|
||||
defaultModel string
|
||||
timeout time.Duration
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) {
|
||||
baseURL := strings.TrimSpace(cfg.BaseURL)
|
||||
if baseURL != "" {
|
||||
if _, err := url.ParseRequestURI(baseURL); err != nil {
|
||||
return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
}
|
||||
|
||||
timeout := cfg.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = defaults.LLMRequestTimeoutDefault
|
||||
}
|
||||
|
||||
var client *http.Client
|
||||
if cfg.HTTPClient != nil {
|
||||
cloned := *cfg.HTTPClient
|
||||
if cloned.Timeout == 0 {
|
||||
cloned.Timeout = timeout
|
||||
}
|
||||
client = &cloned
|
||||
} else {
|
||||
client = &http.Client{Timeout: timeout}
|
||||
}
|
||||
|
||||
return &OpenAICompatibleClient{
|
||||
baseURL: strings.TrimRight(baseURL, "/"),
|
||||
defaultModel: cfg.Model,
|
||||
timeout: timeout,
|
||||
httpClient: client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
if req.Target.TimeoutSeconds < 0 {
|
||||
return nil, fmt.Errorf("%w: timeout_seconds must be greater than or equal to 0", ErrInvalidRequest)
|
||||
}
|
||||
|
||||
endpoint := strings.TrimSpace(req.Target.Endpoint)
|
||||
if endpoint == "" {
|
||||
endpoint = c.baseURL
|
||||
}
|
||||
if endpoint == "" {
|
||||
return nil, fmt.Errorf("%w: endpoint is required", ErrInvalidRequest)
|
||||
}
|
||||
endpoint = strings.TrimRight(endpoint, "/") + defaults.OpenAIChatCompletionsPath
|
||||
|
||||
wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
wirePayload, err := openAIChatRequestPayload(wireReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(wirePayload)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
if apiKey := strings.TrimSpace(req.Target.APIKey); apiKey != "" {
|
||||
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
} else if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" {
|
||||
apiKey := strings.TrimSpace(os.Getenv(envName))
|
||||
if apiKey == "" {
|
||||
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
|
||||
}
|
||||
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
}
|
||||
|
||||
effectiveTimeout := c.timeout
|
||||
if req.Target.TimeoutSeconds > 0 {
|
||||
effectiveTimeout = time.Duration(req.Target.TimeoutSeconds) * time.Second
|
||||
} else if req.TargetPresence.TimeoutSeconds {
|
||||
effectiveTimeout = 0
|
||||
}
|
||||
|
||||
httpClient := c.httpClient
|
||||
if httpClient == nil {
|
||||
httpClient = &http.Client{Timeout: effectiveTimeout}
|
||||
} else if httpClient.Timeout != effectiveTimeout {
|
||||
cloned := *httpClient
|
||||
cloned.Timeout = effectiveTimeout
|
||||
httpClient = &cloned
|
||||
}
|
||||
|
||||
httpResp, err := httpClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrRequestFailed, err)
|
||||
}
|
||||
defer httpResp.Body.Close()
|
||||
|
||||
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(httpResp.Body, 4096))
|
||||
return nil, fmt.Errorf("%w: status=%d", ErrUnexpectedStatus, httpResp.StatusCode)
|
||||
}
|
||||
|
||||
var wireResp openAIChatResponse
|
||||
if err := json.NewDecoder(httpResp.Body).Decode(&wireResp); err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to decode response: %v", ErrMalformedResponse, err)
|
||||
}
|
||||
|
||||
if len(wireResp.Choices) == 0 {
|
||||
return nil, fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
|
||||
}
|
||||
content := wireResp.Choices[0].Message.Content
|
||||
if content == "" {
|
||||
return nil, fmt.Errorf("%w: first choice has empty message content", ErrMalformedResponse)
|
||||
}
|
||||
|
||||
return &domain.GenerateResponse{
|
||||
Content: content,
|
||||
Usage: domain.TokenUsage{
|
||||
PromptTokens: wireResp.Usage.PromptTokens,
|
||||
CompletionTokens: wireResp.Usage.CompletionTokens,
|
||||
TotalTokens: wireResp.Usage.TotalTokens,
|
||||
CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens,
|
||||
CacheWriteTokens: wireResp.Usage.CacheWriteTokens,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) {
|
||||
model := strings.TrimSpace(req.Target.Model)
|
||||
if model == "" {
|
||||
model = strings.TrimSpace(defaultModel)
|
||||
}
|
||||
if model == "" {
|
||||
return openAIChatRequest{}, errors.New("model is required")
|
||||
}
|
||||
|
||||
wireReq := openAIChatRequest{
|
||||
Model: model,
|
||||
}
|
||||
if sessionID := strings.TrimSpace(req.Prompt.SessionID); sessionID != "" {
|
||||
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
|
||||
return openAIChatRequest{}, fmt.Errorf("session_id length %d exceeds maximum %d", n, domain.SessionIDMaxLength)
|
||||
}
|
||||
wireReq.SessionID = sessionID
|
||||
}
|
||||
|
||||
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
|
||||
for _, msg := range req.Prompt.Messages {
|
||||
wireReq.Messages = append(wireReq.Messages, openAIChatRequestMessageFromRenderedMessage(msg))
|
||||
}
|
||||
|
||||
if req.Target.Temperature != 0 || req.TargetPresence.Temperature {
|
||||
wireReq.Temperature = &req.Target.Temperature
|
||||
}
|
||||
if req.Target.MaxTokens != 0 || req.TargetPresence.MaxTokens {
|
||||
wireReq.MaxTokens = &req.Target.MaxTokens
|
||||
}
|
||||
if req.Target.TopP != 0 || req.TargetPresence.TopP {
|
||||
wireReq.TopP = &req.Target.TopP
|
||||
}
|
||||
if strings.TrimSpace(req.Target.ServiceTier) != "" {
|
||||
wireReq.ServiceTier = req.Target.ServiceTier
|
||||
}
|
||||
if strings.TrimSpace(req.Target.ReasoningEffort) != "" {
|
||||
wireReq.ReasoningEffort = req.Target.ReasoningEffort
|
||||
}
|
||||
if len(req.Target.ExtraParams) > 0 {
|
||||
wireReq.ExtraParams = req.Target.ExtraParams
|
||||
}
|
||||
if req.StructuredOutput != nil {
|
||||
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
|
||||
if err != nil {
|
||||
return openAIChatRequest{}, err
|
||||
}
|
||||
wireReq.ResponseFormat = responseFormat
|
||||
}
|
||||
|
||||
return wireReq, nil
|
||||
}
|
||||
|
||||
type openAIChatRequest struct {
|
||||
Model string `json:"model"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Messages []openAIChatRequestMessage `json:"messages"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
ServiceTier string `json:"service_tier,omitempty"`
|
||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"`
|
||||
ExtraParams map[string]any `json:"-"`
|
||||
}
|
||||
|
||||
func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
||||
out := map[string]any{
|
||||
"model": req.Model,
|
||||
"messages": req.Messages,
|
||||
}
|
||||
if req.SessionID != "" {
|
||||
out["session_id"] = req.SessionID
|
||||
}
|
||||
if req.Temperature != nil {
|
||||
out["temperature"] = *req.Temperature
|
||||
}
|
||||
if req.MaxTokens != nil {
|
||||
out["max_tokens"] = *req.MaxTokens
|
||||
}
|
||||
if req.TopP != nil {
|
||||
out["top_p"] = *req.TopP
|
||||
}
|
||||
if req.ServiceTier != "" {
|
||||
out["service_tier"] = req.ServiceTier
|
||||
}
|
||||
if req.ReasoningEffort != "" {
|
||||
out["reasoning_effort"] = req.ReasoningEffort
|
||||
}
|
||||
if req.ResponseFormat != nil {
|
||||
out["response_format"] = req.ResponseFormat
|
||||
}
|
||||
|
||||
for key, value := range req.ExtraParams {
|
||||
if key == "" {
|
||||
return nil, errors.New("extra_params key must not be empty")
|
||||
}
|
||||
if _, reserved := reservedOpenAIChatRequestFields[key]; reserved {
|
||||
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
|
||||
}
|
||||
if _, err := json.Marshal(value); err != nil {
|
||||
return nil, fmt.Errorf("extra_params.%s must be JSON-serializable: %w", key, err)
|
||||
}
|
||||
out[key] = value
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
var reservedOpenAIChatRequestFields = map[string]struct{}{
|
||||
"model": {},
|
||||
"session_id": {},
|
||||
"messages": {},
|
||||
"temperature": {},
|
||||
"max_tokens": {},
|
||||
"top_p": {},
|
||||
"service_tier": {},
|
||||
"reasoning_effort": {},
|
||||
"response_format": {},
|
||||
}
|
||||
|
||||
type openAIChatRequestMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content any `json:"content"`
|
||||
}
|
||||
|
||||
type openAIChatTextContentBlock struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
CacheControl *openAICacheControl `json:"cache_control,omitempty"`
|
||||
}
|
||||
|
||||
type openAICacheControl struct {
|
||||
Type string `json:"type"`
|
||||
TTL string `json:"ttl,omitempty"`
|
||||
}
|
||||
|
||||
type openAIChatResponseMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
type openAIChatResponse struct {
|
||||
Choices []struct {
|
||||
Message openAIChatResponseMessage `json:"message"`
|
||||
} `json:"choices"`
|
||||
Usage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
PromptTokensDetails struct {
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
} `json:"prompt_tokens_details"`
|
||||
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||
} `json:"usage"`
|
||||
}
|
||||
|
||||
type openAIResponseFormat struct {
|
||||
Type string `json:"type"`
|
||||
JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"`
|
||||
}
|
||||
|
||||
type openAIJSONSchemaEnvelope struct {
|
||||
Name string `json:"name"`
|
||||
Strict bool `json:"strict"`
|
||||
Schema any `json:"schema"`
|
||||
}
|
||||
|
||||
func openAIChatRequestMessageFromRenderedMessage(msg domain.RenderedMessage) openAIChatRequestMessage {
|
||||
wireMsg := openAIChatRequestMessage{
|
||||
Role: msg.Role,
|
||||
Content: msg.Content,
|
||||
}
|
||||
if msg.CacheControl == nil {
|
||||
return wireMsg
|
||||
}
|
||||
|
||||
wireMsg.Content = []openAIChatTextContentBlock{
|
||||
{
|
||||
Type: "text",
|
||||
Text: msg.Content,
|
||||
CacheControl: &openAICacheControl{
|
||||
Type: string(msg.CacheControl.Type),
|
||||
TTL: msg.CacheControl.TTL,
|
||||
},
|
||||
},
|
||||
}
|
||||
return wireMsg
|
||||
}
|
||||
|
||||
func toOpenAIResponseFormat(spec *domain.StructuredOutputSpec) (*openAIResponseFormat, error) {
|
||||
if spec == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
switch spec.Type {
|
||||
case domain.StructuredOutputJSONSchema:
|
||||
if spec.JSONSchema == nil {
|
||||
return nil, errors.New("json_schema structured output requires schema payload")
|
||||
}
|
||||
if strings.TrimSpace(spec.JSONSchema.Name) == "" {
|
||||
return nil, errors.New("json_schema structured output requires non-empty schema name")
|
||||
}
|
||||
if spec.JSONSchema.Schema == nil {
|
||||
return nil, errors.New("json_schema structured output requires schema document")
|
||||
}
|
||||
return &openAIResponseFormat{
|
||||
Type: "json_schema",
|
||||
JSONSchema: &openAIJSONSchemaEnvelope{
|
||||
Name: spec.JSONSchema.Name,
|
||||
Strict: spec.JSONSchema.Strict,
|
||||
Schema: spec.JSONSchema.Schema,
|
||||
},
|
||||
}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported structured output type %q", spec.Type)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,9 +0,0 @@
|
||||
id: aion-2
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: aion-labs/aion-2.0
|
||||
temperature: 0.72
|
||||
reasoning_effort: high
|
||||
top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,7 +0,0 @@
|
||||
id: claude-fable-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "~anthropic/claude-fable-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 600
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,7 +0,0 @@
|
||||
id: claude-haiku-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "~anthropic/claude-haiku-latest"
|
||||
reasoning_effort: medium
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,7 +0,0 @@
|
||||
id: claude-opus-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "~anthropic/claude-opus-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,7 +0,0 @@
|
||||
id: claude-sonnet-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "~anthropic/claude-sonnet-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,7 +0,0 @@
|
||||
id: deepseek-3-2
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: deepseek/deepseek-v3.2
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,7 +0,0 @@
|
||||
id: deepseek-4-flash
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: deepseek/deepseek-v4-flash
|
||||
#reasoning_effort: medium
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,7 +0,0 @@
|
||||
id: deepseek-4-pro
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: deepseek/deepseek-v4-pro
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,9 +0,0 @@
|
||||
id: gemini-2-flash-lite
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "google/gemini-2.5-flash-lite"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,9 +0,0 @@
|
||||
id: gemini-2-flash
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "google/gemini-2.5-flash"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,9 +0,0 @@
|
||||
id: gemini-2-pro
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "google/gemini-2.5-pro"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,9 +0,0 @@
|
||||
id: gemini-3-flash-lite
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "google/gemini-3.1-flash-lite"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,9 +0,0 @@
|
||||
id: gemini-flash-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "~google/gemini-flash-latest"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,9 +0,0 @@
|
||||
id: gemini-pro-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "~google/gemini-pro-latest"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,9 +0,0 @@
|
||||
id: gemma-4-31b
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: google/gemma-4-31b-it:exacto
|
||||
temperature: 0.15
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,9 +0,0 @@
|
||||
id: minimax-m2
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: minimax/minimax-m2.5
|
||||
temperature: 0.5
|
||||
reasoning_effort: high
|
||||
top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,9 +0,0 @@
|
||||
id: minimax-m3
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: minimax/minimax-m3
|
||||
#temperature: 0.5
|
||||
reasoning_effort: high
|
||||
#top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,7 +0,0 @@
|
||||
id: mistral-large-2512
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: mistralai/mistral-large-2512
|
||||
temperature: 0.15
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
@@ -1,8 +0,0 @@
|
||||
id: mistral-medium-3-5
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: mistralai/mistral-medium-3-5
|
||||
temperature: 0.15
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
@@ -1,7 +0,0 @@
|
||||
id: mistral-small-3
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: mistralai/mistral-small-3.2-24b-instruct
|
||||
temperature: 0.05
|
||||
top_p: 1.0
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
@@ -1,8 +0,0 @@
|
||||
id: mistral-small-4
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: mistralai/mistral-small-2603
|
||||
temperature: 0.1
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
@@ -1,7 +0,0 @@
|
||||
id: nemotron-3-ultra
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: nvidia/nemotron-3-ultra-550b-a55b
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,7 +0,0 @@
|
||||
id: gpt-5-mini
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "openai/gpt-5.4-mini"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,7 +0,0 @@
|
||||
id: gpt-5-nano
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
model: "openai/gpt-5.4-nano"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
@@ -1,31 +0,0 @@
|
||||
package builtin
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
)
|
||||
|
||||
const assetRoot = "assets"
|
||||
|
||||
//go:embed assets/**/*.yml
|
||||
var assets embed.FS
|
||||
|
||||
func NewRepository() profile.Repository {
|
||||
return profile.NewFSRepository(assets, assetRoot)
|
||||
}
|
||||
|
||||
func NewRepositoryWithPrimary(primary profile.Repository) profile.Repository {
|
||||
if primary == nil {
|
||||
return NewRepository()
|
||||
}
|
||||
return profile.NewOverlayRepository(primary, NewRepository())
|
||||
}
|
||||
|
||||
func NewRepositoryWithDirectory(dir string) profile.Repository {
|
||||
if strings.TrimSpace(dir) == "" {
|
||||
return NewRepository()
|
||||
}
|
||||
return NewRepositoryWithPrimary(profile.NewFilesystemRepository(dir))
|
||||
}
|
||||
@@ -1,127 +0,0 @@
|
||||
package builtin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
|
||||
repo := NewRepository()
|
||||
ids := loadBuiltInProfileIDs(t)
|
||||
if len(ids) == 0 {
|
||||
t.Fatal("expected built-in profiles")
|
||||
}
|
||||
|
||||
for id := range ids {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
p, err := repo.GetProfile(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("expected built-in profile %q to load, got %v", id, err)
|
||||
}
|
||||
if p.ID != id {
|
||||
t.Fatalf("expected profile id %q, got %q", id, p.ID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltInProfilesDoNotContainDuplicateIDsOrRawAPIKeys(t *testing.T) {
|
||||
loadBuiltInProfileIDs(t)
|
||||
}
|
||||
|
||||
func loadBuiltInProfileIDs(t *testing.T) map[string]string {
|
||||
t.Helper()
|
||||
|
||||
ids := map[string]string{}
|
||||
err := fs.WalkDir(assets, assetRoot, func(name string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() || !strings.HasSuffix(name, ".yml") {
|
||||
return nil
|
||||
}
|
||||
|
||||
data, err := assets.ReadFile(name)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read built-in profile %s: %v", name, err)
|
||||
}
|
||||
|
||||
var raw map[string]any
|
||||
if err := yaml.Unmarshal(data, &raw); err != nil {
|
||||
t.Fatalf("failed to decode built-in profile %s: %v", name, err)
|
||||
}
|
||||
if _, ok := raw["api_key"]; ok {
|
||||
t.Fatalf("built-in profile %s contains raw api_key", name)
|
||||
}
|
||||
id, ok := raw["id"].(string)
|
||||
if !ok || strings.TrimSpace(id) == "" {
|
||||
t.Fatalf("built-in profile %s has missing id", name)
|
||||
}
|
||||
if previous, ok := ids[id]; ok {
|
||||
t.Fatalf("duplicate built-in profile id %q in %s and %s", id, previous, name)
|
||||
}
|
||||
ids[id] = name
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to walk built-in profiles: %v", err)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func TestRepositoryWithPrimaryUsesPrimaryBeforeBuiltIns(t *testing.T) {
|
||||
repo := NewRepositoryWithPrimary(staticProfileRepo{
|
||||
profiles: map[string]string{"mistral-small-3": "custom-model"},
|
||||
})
|
||||
|
||||
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
||||
if err != nil {
|
||||
t.Fatalf("expected profile to load, got %v", err)
|
||||
}
|
||||
if p.Model != "custom-model" {
|
||||
t.Fatalf("expected primary profile to override built-in, got %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryWithPrimaryFallsBackToBuiltIns(t *testing.T) {
|
||||
repo := NewRepositoryWithPrimary(staticProfileRepo{})
|
||||
|
||||
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
||||
if err != nil {
|
||||
t.Fatalf("expected built-in profile to load, got %v", err)
|
||||
}
|
||||
if p.ID != "mistral-small-3" {
|
||||
t.Fatalf("unexpected profile: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryWithPrimaryDoesNotFallBackAfterPrimaryError(t *testing.T) {
|
||||
repo := NewRepositoryWithPrimary(staticProfileRepo{err: profile.ErrInvalidProfile})
|
||||
|
||||
_, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
||||
if !errors.Is(err, profile.ErrInvalidProfile) {
|
||||
t.Fatalf("expected primary error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type staticProfileRepo struct {
|
||||
profiles map[string]string
|
||||
err error
|
||||
}
|
||||
|
||||
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
if r.err != nil {
|
||||
return nil, r.err
|
||||
}
|
||||
if model, ok := r.profiles[id]; ok {
|
||||
return &domain.ExecutionProfile{ID: id, Endpoint: "http://primary/v1", Model: model}, nil
|
||||
}
|
||||
return nil, profile.ErrProfileNotFound
|
||||
}
|
||||
@@ -1,213 +0,0 @@
|
||||
package profile
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/filecatalog"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrProfileNotFound = errors.New("execution profile not found")
|
||||
ErrInvalidYAML = errors.New("invalid YAML format")
|
||||
ErrInvalidProfile = errors.New("invalid execution profile configuration")
|
||||
ErrRawAPIKeyNotAllowed = errors.New("raw api_key is not allowed; use api_key_env")
|
||||
)
|
||||
|
||||
type filesystemRepository struct {
|
||||
dir string
|
||||
}
|
||||
|
||||
func NewFilesystemRepository(dir string) Repository {
|
||||
return &filesystemRepository{dir: dir}
|
||||
}
|
||||
|
||||
func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
return loadProfile(ctx, os.DirFS(r.dir), ".", id)
|
||||
}
|
||||
|
||||
type fsRepository struct {
|
||||
fsys fs.FS
|
||||
root string
|
||||
}
|
||||
|
||||
func NewFSRepository(fsys fs.FS, root string) Repository {
|
||||
return &fsRepository{fsys: fsys, root: root}
|
||||
}
|
||||
|
||||
func (r *fsRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
return loadProfile(ctx, r.fsys, r.root, id)
|
||||
}
|
||||
|
||||
type overlayRepository struct {
|
||||
primary Repository
|
||||
fallback Repository
|
||||
}
|
||||
|
||||
func NewOverlayRepository(primary, fallback Repository) Repository {
|
||||
return &overlayRepository{primary: primary, fallback: fallback}
|
||||
}
|
||||
|
||||
func (r *overlayRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
if r.primary != nil {
|
||||
prof, err := r.primary.GetProfile(ctx, id)
|
||||
if err == nil {
|
||||
return prof, nil
|
||||
}
|
||||
if !errors.Is(err, ErrProfileNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if r.fallback == nil {
|
||||
return nil, ErrProfileNotFound
|
||||
}
|
||||
return r.fallback.GetProfile(ctx, id)
|
||||
}
|
||||
|
||||
func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*domain.ExecutionProfile, error) {
|
||||
if strings.TrimSpace(id) == "" {
|
||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
|
||||
}
|
||||
if fsys == nil {
|
||||
return nil, fmt.Errorf("failed to read profile directory: filesystem is nil")
|
||||
}
|
||||
|
||||
files, err := filecatalog.FindFSYAMLFiles(ctx, fsys, root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read profile directory: %w", err)
|
||||
}
|
||||
|
||||
var matches []profileMatch
|
||||
for _, fullPath := range files {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
relPath := filecatalog.DisplayPath(root, fullPath)
|
||||
fileMatch := filecatalog.Stem(path.Base(fullPath)) == id
|
||||
data, err := fs.ReadFile(fsys, fullPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
|
||||
}
|
||||
metadata := readProfileFileMetadata(data)
|
||||
idMatch := fileMatch || metadata.id == id
|
||||
if metadata.hasRawAPIKey {
|
||||
if idMatch {
|
||||
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
var prof domain.ExecutionProfile
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||
decoder.KnownFields(true)
|
||||
if err := decoder.Decode(&prof); err != nil {
|
||||
if idMatch {
|
||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if prof.ID != id {
|
||||
continue
|
||||
}
|
||||
if err := validateProfile(&prof); err != nil {
|
||||
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||
return nil, fmt.Errorf("%w: %s", err, relPath)
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
|
||||
}
|
||||
matches = append(matches, profileMatch{
|
||||
profile: &prof,
|
||||
path: relPath,
|
||||
})
|
||||
}
|
||||
|
||||
if len(matches) > 1 {
|
||||
paths := make([]string, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
paths = append(paths, match.path)
|
||||
}
|
||||
return nil, fmt.Errorf("%w: duplicate execution profile id %q found in: %s", ErrInvalidProfile, id, strings.Join(paths, ", "))
|
||||
}
|
||||
|
||||
if len(matches) == 1 {
|
||||
return matches[0].profile, nil
|
||||
}
|
||||
|
||||
return nil, ErrProfileNotFound
|
||||
}
|
||||
|
||||
type profileMatch struct {
|
||||
profile *domain.ExecutionProfile
|
||||
path string
|
||||
}
|
||||
|
||||
type profileFileMetadata struct {
|
||||
id string
|
||||
hasRawAPIKey bool
|
||||
}
|
||||
|
||||
func readProfileFileMetadata(data []byte) profileFileMetadata {
|
||||
var node yaml.Node
|
||||
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&node); err != nil {
|
||||
return profileFileMetadata{}
|
||||
}
|
||||
if node.Kind != yaml.DocumentNode || len(node.Content) == 0 {
|
||||
return profileFileMetadata{}
|
||||
}
|
||||
mapping := node.Content[0]
|
||||
if mapping.Kind != yaml.MappingNode {
|
||||
return profileFileMetadata{}
|
||||
}
|
||||
|
||||
var metadata profileFileMetadata
|
||||
for i := 0; i+1 < len(mapping.Content); i += 2 {
|
||||
key := mapping.Content[i]
|
||||
value := mapping.Content[i+1]
|
||||
switch key.Value {
|
||||
case "id":
|
||||
metadata.id = strings.TrimSpace(value.Value)
|
||||
case "api_key":
|
||||
metadata.hasRawAPIKey = true
|
||||
}
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
|
||||
func validateProfile(p *domain.ExecutionProfile) error {
|
||||
if strings.TrimSpace(p.ID) == "" {
|
||||
return errors.New("id is required")
|
||||
}
|
||||
if strings.TrimSpace(p.Endpoint) == "" {
|
||||
return errors.New("endpoint is required")
|
||||
}
|
||||
if strings.TrimSpace(p.Model) == "" {
|
||||
return errors.New("model is required")
|
||||
}
|
||||
|
||||
if p.Temperature < 0 || p.Temperature > 2 {
|
||||
return errors.New("temperature must be between 0 and 2")
|
||||
}
|
||||
if p.MaxTokens < 0 {
|
||||
return errors.New("max_tokens must be greater than or equal to 0")
|
||||
}
|
||||
if p.TopP < 0 || p.TopP > 1 {
|
||||
return errors.New("top_p must be between 0 and 1")
|
||||
}
|
||||
if p.TimeoutSeconds < 0 {
|
||||
return errors.New("timeout_seconds must be greater than or equal to 0")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
package profile
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
)
|
||||
|
||||
// Repository loads execution profiles.
|
||||
type Repository interface {
|
||||
GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error)
|
||||
}
|
||||
@@ -1,479 +0,0 @@
|
||||
package profile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
)
|
||||
|
||||
func TestFilesystemRepository_GetProfile(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "execution_profile_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
files, err := os.ReadDir("testdata")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read testdata: %v", err)
|
||||
}
|
||||
for _, f := range files {
|
||||
src := filepath.Join("testdata", f.Name())
|
||||
dst := filepath.Join(tmpDir, f.Name())
|
||||
data, err := os.ReadFile(src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(dst, data, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
repo := NewFilesystemRepository(tmpDir)
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("valid local profile", func(t *testing.T) {
|
||||
p, err := repo.GetProfile(ctx, "local-default")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if p.ID != "local-default" {
|
||||
t.Fatalf("unexpected id: %q", p.ID)
|
||||
}
|
||||
if p.Endpoint == "" || p.Model == "" {
|
||||
t.Fatalf("expected endpoint/model to be set: %+v", p)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid profile with api_key_env", func(t *testing.T) {
|
||||
p, err := repo.GetProfile(ctx, "local-secure")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if p.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
|
||||
t.Fatalf("unexpected api_key_env: %q", p.APIKeyEnv)
|
||||
}
|
||||
if p.ReasoningEffort != "medium" {
|
||||
t.Fatalf("unexpected reasoning_effort: %q", p.ReasoningEffort)
|
||||
}
|
||||
if p.ServiceTier != "priority" {
|
||||
t.Fatalf("unexpected service_tier: %q", p.ServiceTier)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid nested profile", func(t *testing.T) {
|
||||
nestedDir := filepath.Join(tmpDir, "local")
|
||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeProfileTestFile(t, filepath.Join(nestedDir, "nested-local.yaml"), `
|
||||
id: nested-local
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: nested-model
|
||||
temperature: 0.1
|
||||
`)
|
||||
|
||||
p, err := repo.GetProfile(ctx, "nested-local")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if p.Model != "nested-model" {
|
||||
t.Fatalf("unexpected model: %q", p.Model)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid profile with JSON-compatible extra params", func(t *testing.T) {
|
||||
writeProfileTestFile(t, filepath.Join(tmpDir, "json-extra-params.yaml"), `
|
||||
id: json-extra-params
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: nested-model
|
||||
extra_params:
|
||||
string_value: enabled
|
||||
number_value: 42
|
||||
boolean_value: true
|
||||
object_value:
|
||||
nested: value
|
||||
count: 2
|
||||
array_value:
|
||||
- first
|
||||
- 3
|
||||
- false
|
||||
`)
|
||||
|
||||
p, err := repo.GetProfile(ctx, "json-extra-params")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
var got map[string]any
|
||||
encoded, err := json.Marshal(p.ExtraParams)
|
||||
if err != nil {
|
||||
t.Fatalf("expected extra_params to marshal as JSON, got %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(encoded, &got); err != nil {
|
||||
t.Fatalf("expected extra_params JSON to decode, got %v", err)
|
||||
}
|
||||
|
||||
if got["string_value"] != "enabled" {
|
||||
t.Fatalf("unexpected string extra param: %#v", got["string_value"])
|
||||
}
|
||||
if got["number_value"] != float64(42) {
|
||||
t.Fatalf("unexpected number extra param: %#v", got["number_value"])
|
||||
}
|
||||
if got["boolean_value"] != true {
|
||||
t.Fatalf("unexpected boolean extra param: %#v", got["boolean_value"])
|
||||
}
|
||||
objectValue, ok := got["object_value"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected object extra param, got %#v", got["object_value"])
|
||||
}
|
||||
if objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
|
||||
t.Fatalf("unexpected object extra param: %#v", objectValue)
|
||||
}
|
||||
arrayValue, ok := got["array_value"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected array extra param, got %#v", got["array_value"])
|
||||
}
|
||||
if len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
|
||||
t.Fatalf("unexpected array extra param: %#v", arrayValue)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
|
||||
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
|
||||
id: duplicate-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: first-model
|
||||
`)
|
||||
nestedDir := filepath.Join(tmpDir, "duplicates")
|
||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeProfileTestFile(t, filepath.Join(nestedDir, "duplicate-profile-b.yaml"), `
|
||||
id: duplicate-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: second-model
|
||||
`)
|
||||
|
||||
_, err := repo.GetProfile(ctx, "duplicate-profile")
|
||||
if !errors.Is(err, ErrInvalidProfile) {
|
||||
t.Fatalf("expected duplicate profile to return ErrInvalidProfile, got %v", err)
|
||||
}
|
||||
for _, want := range []string{"duplicate execution profile id", "duplicate-profile-a.yaml", filepath.Join("duplicates", "duplicate-profile-b.yaml")} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("expected error to contain %q, got %v", want, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nested raw api_key rejected for likely target file", func(t *testing.T) {
|
||||
nestedDir := filepath.Join(tmpDir, "secure")
|
||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeProfileTestFile(t, filepath.Join(nestedDir, "not_named_like_id.yaml"), `
|
||||
id: nested_raw_api_key
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: m
|
||||
api_key: secret
|
||||
`)
|
||||
|
||||
_, err := repo.GetProfile(ctx, "nested_raw_api_key")
|
||||
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), filepath.Join("secure", "not_named_like_id.yaml")) {
|
||||
t.Fatalf("expected nested path in error, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("raw api_key in non-target profile is ignored", func(t *testing.T) {
|
||||
writeProfileTestFile(t, filepath.Join(tmpDir, "raw-api-key-non-target.yaml"), `
|
||||
id: raw-api-key-non-target
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: m
|
||||
api_key: secret
|
||||
`)
|
||||
|
||||
_, err := repo.GetProfile(ctx, "does-not-exist-with-raw-key-nearby")
|
||||
if !errors.Is(err, ErrProfileNotFound) {
|
||||
t.Fatalf("expected ErrProfileNotFound for non-target raw api_key file, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid yaml", func(t *testing.T) {
|
||||
_, err := repo.GetProfile(ctx, "invalid_yaml")
|
||||
if !errors.Is(err, ErrInvalidYAML) {
|
||||
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing id", func(t *testing.T) {
|
||||
_, err := repo.GetProfile(ctx, "missing_id")
|
||||
if !errors.Is(err, ErrProfileNotFound) {
|
||||
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing endpoint", func(t *testing.T) {
|
||||
_, err := repo.GetProfile(ctx, "missing-endpoint")
|
||||
if !errors.Is(err, ErrInvalidProfile) {
|
||||
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing model", func(t *testing.T) {
|
||||
_, err := repo.GetProfile(ctx, "missing-model")
|
||||
if !errors.Is(err, ErrInvalidProfile) {
|
||||
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unknown field", func(t *testing.T) {
|
||||
_, err := repo.GetProfile(ctx, "unknown_field")
|
||||
if !errors.Is(err, ErrInvalidYAML) {
|
||||
t.Fatalf("expected ErrInvalidYAML for strict decode unknown field, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("raw api_key rejected", func(t *testing.T) {
|
||||
_, err := repo.GetProfile(ctx, "raw_api_key")
|
||||
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("profile not found", func(t *testing.T) {
|
||||
_, err := repo.GetProfile(ctx, "does-not-exist")
|
||||
if !errors.Is(err, ErrProfileNotFound) {
|
||||
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func writeProfileTestFile(t *testing.T, path string, content string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil {
|
||||
t.Fatalf("failed to write profile test file %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSRepository(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("loads valid profiles from nested directories", func(t *testing.T) {
|
||||
repo := NewFSRepository(fstest.MapFS{
|
||||
"profiles/provider/nested.yaml": profileMapFile(`
|
||||
id: nested-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: nested-model
|
||||
temperature: 0.1
|
||||
`),
|
||||
}, "profiles")
|
||||
|
||||
p, err := repo.GetProfile(ctx, "nested-profile")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if p.ID != "nested-profile" || p.Model != "nested-model" {
|
||||
t.Fatalf("unexpected profile: %+v", p)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects unknown YAML fields", func(t *testing.T) {
|
||||
repo := NewFSRepository(fstest.MapFS{
|
||||
"profiles/unknown.yaml": profileMapFile(`
|
||||
id: unknown-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: model
|
||||
unknown: value
|
||||
`),
|
||||
}, "profiles")
|
||||
|
||||
_, err := repo.GetProfile(ctx, "unknown-profile")
|
||||
if !errors.Is(err, ErrInvalidYAML) {
|
||||
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects raw api_key in selected profile", func(t *testing.T) {
|
||||
repo := NewFSRepository(fstest.MapFS{
|
||||
"profiles/raw.yaml": profileMapFile(`
|
||||
id: raw-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: model
|
||||
api_key: secret
|
||||
`),
|
||||
}, "profiles")
|
||||
|
||||
_, err := repo.GetProfile(ctx, "raw-profile")
|
||||
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ignores raw api_key in non-selected profiles", func(t *testing.T) {
|
||||
repo := NewFSRepository(fstest.MapFS{
|
||||
"profiles/raw.yaml": profileMapFile(`
|
||||
id: raw-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: model
|
||||
api_key: secret
|
||||
`),
|
||||
"profiles/valid.yaml": profileMapFile(`
|
||||
id: valid-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: model
|
||||
`),
|
||||
}, "profiles")
|
||||
|
||||
p, err := repo.GetProfile(ctx, "valid-profile")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if p.ID != "valid-profile" {
|
||||
t.Fatalf("unexpected profile: %+v", p)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects duplicate IDs within one source", func(t *testing.T) {
|
||||
repo := NewFSRepository(fstest.MapFS{
|
||||
"profiles/a.yaml": profileMapFile(`
|
||||
id: duplicate-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: first
|
||||
`),
|
||||
"profiles/nested/b.yaml": profileMapFile(`
|
||||
id: duplicate-profile
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: second
|
||||
`),
|
||||
}, "profiles")
|
||||
|
||||
_, err := repo.GetProfile(ctx, "duplicate-profile")
|
||||
if !errors.Is(err, ErrInvalidProfile) {
|
||||
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||
}
|
||||
for _, want := range []string{"duplicate execution profile id", "a.yaml", "nested/b.yaml"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("expected error to contain %q, got %v", want, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestOverlayRepository(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"}
|
||||
fallbackProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://fallback", Model: "fallback"}
|
||||
|
||||
t.Run("returns primary matches before fallback matches", func(t *testing.T) {
|
||||
repo := NewOverlayRepository(
|
||||
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": primaryProfile}},
|
||||
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
|
||||
)
|
||||
|
||||
p, err := repo.GetProfile(ctx, "shared")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if p.Model != "primary" {
|
||||
t.Fatalf("expected primary profile, got %+v", p)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("falls back on primary not found", func(t *testing.T) {
|
||||
repo := NewOverlayRepository(
|
||||
staticProfileRepo{},
|
||||
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
|
||||
)
|
||||
|
||||
p, err := repo.GetProfile(ctx, "shared")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if p.Model != "fallback" {
|
||||
t.Fatalf("expected fallback profile, got %+v", p)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("does not fall back after primary load errors", func(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
err error
|
||||
}{
|
||||
{name: "invalid yaml", err: ErrInvalidYAML},
|
||||
{name: "invalid profile", err: ErrInvalidProfile},
|
||||
{name: "raw api key", err: ErrRawAPIKeyNotAllowed},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
repo := NewOverlayRepository(
|
||||
staticProfileRepo{err: tc.err},
|
||||
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
|
||||
)
|
||||
|
||||
_, err := repo.GetProfile(ctx, "shared")
|
||||
if !errors.Is(err, tc.err) {
|
||||
t.Fatalf("expected %v, got %v", tc.err, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("returns not found when both sources miss", func(t *testing.T) {
|
||||
repo := NewOverlayRepository(staticProfileRepo{}, staticProfileRepo{})
|
||||
|
||||
_, err := repo.GetProfile(ctx, "missing")
|
||||
if !errors.Is(err, ErrProfileNotFound) {
|
||||
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nil primary uses fallback", func(t *testing.T) {
|
||||
repo := NewOverlayRepository(nil, staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}})
|
||||
|
||||
p, err := repo.GetProfile(ctx, "shared")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if p.Model != "fallback" {
|
||||
t.Fatalf("expected fallback profile, got %+v", p)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nil fallback returns not found after primary miss", func(t *testing.T) {
|
||||
repo := NewOverlayRepository(staticProfileRepo{}, nil)
|
||||
|
||||
_, err := repo.GetProfile(ctx, "missing")
|
||||
if !errors.Is(err, ErrProfileNotFound) {
|
||||
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func profileMapFile(content string) *fstest.MapFile {
|
||||
return &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))}
|
||||
}
|
||||
|
||||
type staticProfileRepo struct {
|
||||
profiles map[string]*domain.ExecutionProfile
|
||||
err error
|
||||
}
|
||||
|
||||
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
if r.err != nil {
|
||||
return nil, r.err
|
||||
}
|
||||
if p, ok := r.profiles[id]; ok {
|
||||
cp := *p
|
||||
return &cp, nil
|
||||
}
|
||||
return nil, ErrProfileNotFound
|
||||
}
|
||||
3
internal/profile/testdata/invalid_yaml.yaml
vendored
3
internal/profile/testdata/invalid_yaml.yaml
vendored
@@ -1,3 +0,0 @@
|
||||
id: invalid_yaml
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: [broken
|
||||
@@ -1,2 +0,0 @@
|
||||
id: missing-endpoint
|
||||
model: gpt-4o-mini
|
||||
2
internal/profile/testdata/missing_id.yaml
vendored
2
internal/profile/testdata/missing_id.yaml
vendored
@@ -1,2 +0,0 @@
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: gpt-4o-mini
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user