Compare commits
6 Commits
v0.10.0
...
89cafcefec
| Author | SHA1 | Date | |
|---|---|---|---|
| 89cafcefec | |||
| 1d7fac0a47 | |||
| 03d4f27d2b | |||
| 4ac2038331 | |||
| 14a7e7e04c | |||
| 5e522bad8b |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1,6 +1,5 @@
|
|||||||
# ---> Codex
|
# ---> Codex
|
||||||
.codex
|
.codex
|
||||||
AGENTS.md
|
|
||||||
|
|
||||||
# ---> Go
|
# ---> Go
|
||||||
# If you prefer the allow list template instead of the deny list, see community template:
|
# If you prefer the allow list template instead of the deny list, see community template:
|
||||||
|
|||||||
4
AGENTS.md
Normal file
4
AGENTS.md
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
Please carefully review the relevant documents in `docs/policy` before making any changes to this repository.
|
||||||
|
- `development.md` defines the contributor workflow for this application.
|
||||||
|
- `architecture.md` provides the canonical high-level architecture policy for this repository, and should be reviewed before writing or changing any code.
|
||||||
|
- `documentation.md` provides the canonical documentation policy for this repository, and should be reviewed before writing or changing any documentation.
|
||||||
@@ -25,6 +25,7 @@ This command renders the prepared prompt and effective runtime settings without
|
|||||||
- [Configuration reference](docs/config.md)
|
- [Configuration reference](docs/config.md)
|
||||||
- [Operations guide](docs/operations.md)
|
- [Operations guide](docs/operations.md)
|
||||||
- [Troubleshooting](docs/troubleshooting.md)
|
- [Troubleshooting](docs/troubleshooting.md)
|
||||||
|
- [Go library package](docs/consumers/pkg-scriptorium.md)
|
||||||
- [HTTP API integration](docs/integrations/http-api.md)
|
- [HTTP API integration](docs/integrations/http-api.md)
|
||||||
- [OpenAI-compatible chat integration](docs/integrations/openai-compatible-chat.md)
|
- [OpenAI-compatible chat integration](docs/integrations/openai-compatible-chat.md)
|
||||||
- [Narratio subprocess integration](docs/integrations/narratio.md)
|
- [Narratio subprocess integration](docs/integrations/narratio.md)
|
||||||
@@ -34,3 +35,4 @@ This command renders the prepared prompt and effective runtime settings without
|
|||||||
|
|
||||||
- `examples/render-markdown-summary.sh`
|
- `examples/render-markdown-summary.sh`
|
||||||
- `examples/http-run.json`
|
- `examples/http-run.json`
|
||||||
|
- `examples/go-library/prepare`
|
||||||
|
|||||||
396
convert.go
Normal file
396
convert.go
Normal file
@@ -0,0 +1,396 @@
|
|||||||
|
package scriptorium
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func toDomainRunRequest(req RunRequest) domain.RunRequest {
|
||||||
|
return domain.RunRequest{
|
||||||
|
PromptID: req.PromptID,
|
||||||
|
PromptVersion: req.PromptVersion,
|
||||||
|
ProfileID: req.ProfileID,
|
||||||
|
Inputs: toDomainArtifactRefMap(req.Inputs),
|
||||||
|
Vars: copyStringMap(req.Vars),
|
||||||
|
Execution: toDomainExecutionTargetOverride(req.Execution),
|
||||||
|
Validation: toDomainOutputContractPtr(req.Validation),
|
||||||
|
Metadata: copyStringMap(req.Metadata),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
||||||
|
if prepared == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &PreparedRun{
|
||||||
|
PromptID: prepared.PromptID,
|
||||||
|
PromptVersion: prepared.PromptVersion,
|
||||||
|
PromptHash: prepared.PromptHash,
|
||||||
|
SelectedProfileID: prepared.SelectedProfileID,
|
||||||
|
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
|
||||||
|
OutputContract: fromDomainOutputContract(prepared.OutputContract),
|
||||||
|
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
|
||||||
|
InputHashes: copyStringMap(prepared.InputHashes),
|
||||||
|
SessionID: prepared.SessionID,
|
||||||
|
RenderedPromptHash: prepared.RenderedPromptHash,
|
||||||
|
Messages: fromDomainRenderedMessages(prepared.Messages),
|
||||||
|
StartTime: prepared.StartTime,
|
||||||
|
EndTime: prepared.EndTime,
|
||||||
|
DurationMS: prepared.DurationMS,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
||||||
|
if result == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &RunResult{
|
||||||
|
RunID: result.RunID,
|
||||||
|
Artifact: fromDomainArtifact(result.Artifact),
|
||||||
|
RawOutput: result.RawOutput,
|
||||||
|
Validation: fromDomainValidationResult(result.Validation),
|
||||||
|
PromptID: result.PromptID,
|
||||||
|
PromptVersion: result.PromptVersion,
|
||||||
|
PromptHash: result.PromptHash,
|
||||||
|
RenderedPromptHash: result.RenderedPromptHash,
|
||||||
|
SelectedProfileID: result.SelectedProfileID,
|
||||||
|
ModelName: result.ModelName,
|
||||||
|
Endpoint: result.Endpoint,
|
||||||
|
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
|
||||||
|
InputHashes: copyStringMap(result.InputHashes),
|
||||||
|
Usage: fromDomainTokenUsage(result.Usage),
|
||||||
|
StartTime: result.StartTime,
|
||||||
|
EndTime: result.EndTime,
|
||||||
|
Duration: result.Duration,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainGenerateRequest(req domain.GenerateRequest) GenerateRequest {
|
||||||
|
return GenerateRequest{
|
||||||
|
Prompt: fromDomainRenderedPrompt(req.Prompt),
|
||||||
|
Target: fromDomainExecutionTarget(req.Target),
|
||||||
|
TargetPresence: fromDomainExecutionTargetPresence(req.TargetPresence),
|
||||||
|
StructuredOutput: fromDomainStructuredOutputSpec(req.StructuredOutput),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse {
|
||||||
|
if resp == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &domain.GenerateResponse{
|
||||||
|
Content: resp.Content,
|
||||||
|
Usage: toDomainTokenUsage(resp.Usage),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainRenderedPrompt(prompt domain.RenderedPrompt) RenderedPrompt {
|
||||||
|
return RenderedPrompt{
|
||||||
|
SessionID: prompt.SessionID,
|
||||||
|
Messages: fromDomainRenderedMessages(prompt.Messages),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func toDomainArtifactRefMap(src map[string]ArtifactRef) map[string]domain.ArtifactRef {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]domain.ArtifactRef, len(src))
|
||||||
|
for k, v := range src {
|
||||||
|
out[k] = toDomainArtifactRef(v)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func toDomainArtifactRef(ref ArtifactRef) domain.ArtifactRef {
|
||||||
|
return domain.ArtifactRef{
|
||||||
|
Type: domain.ArtifactRefType(ref.Type),
|
||||||
|
URI: ref.URI,
|
||||||
|
Body: ref.Body,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainArtifact(artifact domain.Artifact) Artifact {
|
||||||
|
return Artifact{
|
||||||
|
Name: artifact.Name,
|
||||||
|
ContentType: artifact.ContentType,
|
||||||
|
Body: copyBytes(artifact.Body),
|
||||||
|
URI: artifact.URI,
|
||||||
|
Size: artifact.Size,
|
||||||
|
Hash: artifact.Hash,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) *domain.ExecutionTargetOverride {
|
||||||
|
if override == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &domain.ExecutionTargetOverride{
|
||||||
|
Endpoint: override.Endpoint,
|
||||||
|
Model: override.Model,
|
||||||
|
Temperature: copyFloat64Ptr(override.Temperature),
|
||||||
|
MaxTokens: copyIntPtr(override.MaxTokens),
|
||||||
|
TopP: copyFloat64Ptr(override.TopP),
|
||||||
|
TimeoutSeconds: copyIntPtr(override.TimeoutSeconds),
|
||||||
|
ServiceTier: override.ServiceTier,
|
||||||
|
ReasoningEffort: override.ReasoningEffort,
|
||||||
|
APIKeyEnv: override.APIKeyEnv,
|
||||||
|
ExtraParams: copyAnyMap(override.ExtraParams),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
||||||
|
return ExecutionTarget{
|
||||||
|
Endpoint: target.Endpoint,
|
||||||
|
Model: target.Model,
|
||||||
|
Temperature: target.Temperature,
|
||||||
|
MaxTokens: target.MaxTokens,
|
||||||
|
TopP: target.TopP,
|
||||||
|
TimeoutSeconds: target.TimeoutSeconds,
|
||||||
|
ServiceTier: target.ServiceTier,
|
||||||
|
ReasoningEffort: target.ReasoningEffort,
|
||||||
|
APIKeyEnv: target.APIKeyEnv,
|
||||||
|
ExtraParams: copyAnyMap(target.ExtraParams),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
||||||
|
return ExecutionTargetPresence{
|
||||||
|
Temperature: presence.Temperature,
|
||||||
|
MaxTokens: presence.MaxTokens,
|
||||||
|
TopP: presence.TopP,
|
||||||
|
TimeoutSeconds: presence.TimeoutSeconds,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func toDomainOutputContractPtr(contract *OutputContract) *domain.OutputContract {
|
||||||
|
if contract == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := toDomainOutputContract(*contract)
|
||||||
|
return &out
|
||||||
|
}
|
||||||
|
|
||||||
|
func toDomainOutputContract(contract OutputContract) domain.OutputContract {
|
||||||
|
return domain.OutputContract{
|
||||||
|
Format: domain.OutputFormat(contract.Format),
|
||||||
|
ValidationMode: domain.ValidationMode(contract.ValidationMode),
|
||||||
|
SchemaPath: contract.SchemaPath,
|
||||||
|
RepairAttempts: contract.RepairAttempts,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainOutputContract(contract domain.OutputContract) OutputContract {
|
||||||
|
return OutputContract{
|
||||||
|
Format: OutputFormat(contract.Format),
|
||||||
|
ValidationMode: ValidationMode(contract.ValidationMode),
|
||||||
|
SchemaPath: contract.SchemaPath,
|
||||||
|
RepairAttempts: contract.RepairAttempts,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainValidationResult(result domain.ValidationResult) ValidationResult {
|
||||||
|
return ValidationResult{
|
||||||
|
Status: ValidationStatus(result.Status),
|
||||||
|
Mode: ValidationMode(result.Mode),
|
||||||
|
Errors: copyStringSlice(result.Errors),
|
||||||
|
SchemaPath: result.SchemaPath,
|
||||||
|
RepairAttempts: result.RepairAttempts,
|
||||||
|
IsValid: result.IsValid,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainTokenUsage(usage domain.TokenUsage) TokenUsage {
|
||||||
|
return TokenUsage{
|
||||||
|
PromptTokens: usage.PromptTokens,
|
||||||
|
CompletionTokens: usage.CompletionTokens,
|
||||||
|
TotalTokens: usage.TotalTokens,
|
||||||
|
CachedTokens: usage.CachedTokens,
|
||||||
|
CacheWriteTokens: usage.CacheWriteTokens,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func toDomainTokenUsage(usage TokenUsage) domain.TokenUsage {
|
||||||
|
return domain.TokenUsage{
|
||||||
|
PromptTokens: usage.PromptTokens,
|
||||||
|
CompletionTokens: usage.CompletionTokens,
|
||||||
|
TotalTokens: usage.TotalTokens,
|
||||||
|
CachedTokens: usage.CachedTokens,
|
||||||
|
CacheWriteTokens: usage.CacheWriteTokens,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainRenderedMessages(messages []domain.RenderedMessage) []RenderedMessage {
|
||||||
|
if messages == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]RenderedMessage, len(messages))
|
||||||
|
for i, msg := range messages {
|
||||||
|
out[i] = RenderedMessage{
|
||||||
|
Role: msg.Role,
|
||||||
|
Content: msg.Content,
|
||||||
|
CacheControl: fromDomainCacheControl(msg.CacheControl),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl {
|
||||||
|
if cacheControl == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &CacheControl{
|
||||||
|
Type: CacheControlType(cacheControl.Type),
|
||||||
|
TTL: cacheControl.TTL,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainStructuredOutputSpec(spec *domain.StructuredOutputSpec) *StructuredOutputSpec {
|
||||||
|
if spec == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := &StructuredOutputSpec{
|
||||||
|
Type: StructuredOutputType(spec.Type),
|
||||||
|
}
|
||||||
|
if spec.JSONSchema != nil {
|
||||||
|
out.JSONSchema = &StructuredOutputJSONSpec{
|
||||||
|
Name: spec.JSONSchema.Name,
|
||||||
|
Strict: spec.JSONSchema.Strict,
|
||||||
|
Schema: copyAny(spec.JSONSchema.Schema),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyStringMap(src map[string]string) map[string]string {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]string, len(src))
|
||||||
|
for k, v := range src {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyAnyMap(src map[string]any) map[string]any {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]any, len(src))
|
||||||
|
for k, v := range src {
|
||||||
|
out[k] = copyAny(v)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyAny(value any) any {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
switch v := value.(type) {
|
||||||
|
case map[string]any:
|
||||||
|
return copyAnyMap(v)
|
||||||
|
case []any:
|
||||||
|
out := make([]any, len(v))
|
||||||
|
for i, item := range v {
|
||||||
|
out[i] = copyAny(item)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case []string:
|
||||||
|
return copyStringSlice(v)
|
||||||
|
case []byte:
|
||||||
|
return copyBytes(v)
|
||||||
|
default:
|
||||||
|
return copyReflectValue(reflect.ValueOf(value)).Interface()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyReflectValue(value reflect.Value) reflect.Value {
|
||||||
|
if !value.IsValid() {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
switch value.Kind() {
|
||||||
|
case reflect.Interface:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
copied := copyReflectValue(value.Elem())
|
||||||
|
if copied.IsValid() && copied.Type().AssignableTo(value.Type()) {
|
||||||
|
return copied
|
||||||
|
}
|
||||||
|
out := reflect.New(value.Type()).Elem()
|
||||||
|
out.Set(copied)
|
||||||
|
return out
|
||||||
|
case reflect.Pointer:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
out := reflect.New(value.Type().Elem())
|
||||||
|
out.Elem().Set(copyReflectValue(value.Elem()))
|
||||||
|
return out
|
||||||
|
case reflect.Map:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
out := reflect.MakeMapWithSize(value.Type(), value.Len())
|
||||||
|
iter := value.MapRange()
|
||||||
|
for iter.Next() {
|
||||||
|
out.SetMapIndex(copyReflectValue(iter.Key()), copyReflectValue(iter.Value()))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case reflect.Slice:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap())
|
||||||
|
for i := 0; i < value.Len(); i++ {
|
||||||
|
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case reflect.Array:
|
||||||
|
out := reflect.New(value.Type()).Elem()
|
||||||
|
for i := 0; i < value.Len(); i++ {
|
||||||
|
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
default:
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyStringSlice(src []string) []string {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]string, len(src))
|
||||||
|
copy(out, src)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyBytes(src []byte) []byte {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]byte, len(src))
|
||||||
|
copy(out, src)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyFloat64Ptr(src *float64) *float64 {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
v := *src
|
||||||
|
return &v
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyIntPtr(src *int) *int {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
v := *src
|
||||||
|
return &v
|
||||||
|
}
|
||||||
13
docs/consumers/api.md
Normal file
13
docs/consumers/api.md
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
# Consumer API Overview
|
||||||
|
|
||||||
|
Scriptorium can be used by consumers through three implemented surfaces:
|
||||||
|
|
||||||
|
- CLI commands, documented in [CLI reference](../cli.md).
|
||||||
|
- HTTP `POST /v1/runs`, documented in [HTTP API integration](../integrations/http-api.md).
|
||||||
|
- Go package `gitea.maximumdirect.net/eric/scriptorium`, documented in [pkg-scriptorium](pkg-scriptorium.md).
|
||||||
|
|
||||||
|
The Go package is the typed in-process API. It prepares prompts, runs prompts, accepts file or inline artifacts, supports per-request execution overrides, and exposes stable public errors for `errors.Is`.
|
||||||
|
|
||||||
|
Use the Go package when the caller is a Go program that wants typed requests/results, context cancellation, repeated calls without subprocess overhead, or fake LLM injection for tests. Use the CLI or HTTP surfaces when process isolation, language neutrality, or an HTTP boundary is preferred.
|
||||||
|
|
||||||
|
Raw API key values are not accepted in public payloads and are not returned in prepared or run results. Execution profiles may reference an environment variable name through `api_key_env`.
|
||||||
129
docs/consumers/pkg-scriptorium.md
Normal file
129
docs/consumers/pkg-scriptorium.md
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
# Package scriptorium
|
||||||
|
|
||||||
|
Import path:
|
||||||
|
|
||||||
|
```go
|
||||||
|
import "gitea.maximumdirect.net/eric/scriptorium"
|
||||||
|
```
|
||||||
|
|
||||||
|
The root package is a public facade over Scriptorium's prompt execution use case. It keeps `internal/*` packages private while exposing typed construction, preparation, execution, inputs, results, and errors.
|
||||||
|
|
||||||
|
## Construct An Engine
|
||||||
|
|
||||||
|
```go
|
||||||
|
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||||
|
PromptDir: "./examples/prompts",
|
||||||
|
ProfileDir: "./examples/profiles",
|
||||||
|
SchemaDir: "./examples/schemas",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`PromptDir` and `ProfileDir` are required. `SchemaDir` defaults to the built-in schema directory. `Timeout` and `HTTPClient` configure the default OpenAI-compatible client used by `Run` when no custom LLM client is supplied.
|
||||||
|
|
||||||
|
## Prepare A Prompt
|
||||||
|
|
||||||
|
`Prepare` resolves the prompt definition, profile, inputs, variables, output contract, structured-output metadata, and rendered messages without calling an LLM.
|
||||||
|
|
||||||
|
```go
|
||||||
|
prepared, err := engine.Prepare(ctx, scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||||
|
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_ = prepared.Messages
|
||||||
|
```
|
||||||
|
|
||||||
|
Input helpers:
|
||||||
|
|
||||||
|
- `scriptorium.File(path)` loads an input artifact from a file.
|
||||||
|
- `scriptorium.Inline(body)` passes inline input content.
|
||||||
|
- `scriptorium.InlineWithURI(uri, body)` passes inline content with URI metadata.
|
||||||
|
|
||||||
|
## Run A Prompt
|
||||||
|
|
||||||
|
`Run` prepares the prompt, calls the configured LLM client, builds the output artifact, and validates the output.
|
||||||
|
|
||||||
|
```go
|
||||||
|
result, err := engine.Run(ctx, scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||||
|
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_ = result.Artifact
|
||||||
|
```
|
||||||
|
|
||||||
|
`RunResult` includes the run ID, output artifact, raw output, validation result, prompt/profile/model metadata, effective model parameters, input hashes, token/cache usage, and timing fields. Validation content failures return a successful `RunResult` with failed validation status. Runtime validation errors return `ErrValidation`.
|
||||||
|
|
||||||
|
## Inject An LLM Client
|
||||||
|
|
||||||
|
Use `WithLLMClient` for tests or custom model integrations:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type fakeLLM struct{}
|
||||||
|
|
||||||
|
func (fakeLLM) Generate(ctx context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
|
||||||
|
return &scriptorium.GenerateResponse{
|
||||||
|
Content: "generated text",
|
||||||
|
Usage: scriptorium.TokenUsage{TotalTokens: 12},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
engine, err := scriptorium.NewEngine(cfg, scriptorium.WithLLMClient(fakeLLM{}))
|
||||||
|
```
|
||||||
|
|
||||||
|
The injected client receives the rendered prompt, effective execution target, target presence metadata for explicit numeric overrides, and structured-output spec. `WithLLMClient(nil)` returns `ErrInvalidConfig`.
|
||||||
|
|
||||||
|
## Request Overrides
|
||||||
|
|
||||||
|
`RunRequest.Execution` accepts per-request overrides. Numeric override fields are pointers so explicit zero values are preserved:
|
||||||
|
|
||||||
|
```go
|
||||||
|
zero := 0
|
||||||
|
req.Execution = &scriptorium.ExecutionTargetOverride{
|
||||||
|
MaxTokens: &zero,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Errors
|
||||||
|
|
||||||
|
Public methods wrap context while preserving stable sentinel checks with `errors.Is`:
|
||||||
|
|
||||||
|
- `ErrInvalidConfig`
|
||||||
|
- `ErrInvalidRequest`
|
||||||
|
- `ErrPromptNotFound`
|
||||||
|
- `ErrProfileNotFound`
|
||||||
|
- `ErrPromptLoad`
|
||||||
|
- `ErrProfileLoad`
|
||||||
|
- `ErrArtifactLoad`
|
||||||
|
- `ErrPromptRender`
|
||||||
|
- `ErrLLMGenerate`
|
||||||
|
- `ErrValidation`
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```go
|
||||||
|
if errors.Is(err, scriptorium.ErrPromptNotFound) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Examples
|
||||||
|
|
||||||
|
Run the prepare-only example from the repository root:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go run ./examples/go-library/prepare
|
||||||
|
```
|
||||||
@@ -8,6 +8,7 @@ This document describes implemented adapter/repository boundaries and their curr
|
|||||||
|
|
||||||
- `internal/adapter/cli`: CLI command parsing, app wiring, stdout/stderr handling, exit codes.
|
- `internal/adapter/cli`: CLI command parsing, app wiring, stdout/stderr handling, exit codes.
|
||||||
- `internal/adapter/http`: HTTP request/response mapping for `POST /v1/runs`.
|
- `internal/adapter/http`: HTTP request/response mapping for `POST /v1/runs`.
|
||||||
|
- root package `scriptorium`: public Go library facade for preparing and running prompt requests.
|
||||||
- `internal/promptdef`: filesystem prompt-definition repository.
|
- `internal/promptdef`: filesystem prompt-definition repository.
|
||||||
- `internal/profile`: filesystem execution-profile repository.
|
- `internal/profile`: filesystem execution-profile repository.
|
||||||
- `internal/artifact`: input artifact reader.
|
- `internal/artifact`: input artifact reader.
|
||||||
@@ -30,6 +31,13 @@ HTTP adapter:
|
|||||||
- Output: JSON success/error body with mapped status codes.
|
- Output: JSON success/error body with mapped status codes.
|
||||||
- Success metadata includes token usage plus cache usage counters.
|
- Success metadata includes token usage plus cache usage counters.
|
||||||
|
|
||||||
|
Public library facade:
|
||||||
|
|
||||||
|
- Input: typed `scriptorium.RunRequest` values.
|
||||||
|
- Output: typed `PreparedRun` and `RunResult` values plus public sentinel errors.
|
||||||
|
- Custom LLM behavior is injected with `WithLLMClient`; otherwise the default OpenAI-compatible client is used.
|
||||||
|
- Public types are facade types converted at the package boundary; internal domain types remain internal.
|
||||||
|
|
||||||
Filesystem repositories:
|
Filesystem repositories:
|
||||||
|
|
||||||
- Input: prompt/profile YAML files under configured directories.
|
- Input: prompt/profile YAML files under configured directories.
|
||||||
|
|||||||
@@ -2,12 +2,13 @@
|
|||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
Project documentation must help four audiences:
|
Project documentation must help five audiences:
|
||||||
|
|
||||||
1. users who need to run the application;
|
1. users who need to run the application;
|
||||||
2. administrators/operators who need to configure and operate it;
|
2. administrators/operators who need to configure and operate it;
|
||||||
3. developers who need to understand and change it safely;
|
3. developers who need to understand and change it safely;
|
||||||
4. LLM coding agents that need clear scope, boundaries, and invariants.
|
4. LLM coding agents that need clear scope, boundaries, and invariants;
|
||||||
|
5. developers and LLM coding agents integrating this project from another codebase.
|
||||||
|
|
||||||
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
||||||
|
|
||||||
@@ -42,11 +43,14 @@ Canonical homes:
|
|||||||
|
|
||||||
- project purpose and quickstart: `README.md`
|
- project purpose and quickstart: `README.md`
|
||||||
- development principles: `docs/policy/architecture.md`
|
- development principles: `docs/policy/architecture.md`
|
||||||
|
- public HTTP API reference: `docs/api.md`
|
||||||
- configuration reference: `docs/config.md`
|
- configuration reference: `docs/config.md`
|
||||||
- CLI reference: `docs/cli.md`
|
- CLI reference: `docs/cli.md`
|
||||||
- operations and recovery: `docs/operations.md`
|
- operations and recovery: `docs/operations.md`
|
||||||
- troubleshooting: `docs/troubleshooting.md`
|
- troubleshooting: `docs/troubleshooting.md`
|
||||||
|
- public API/package consumer guidance: `docs/consumers/`
|
||||||
- implemented internals: `docs/internal/`
|
- implemented internals: `docs/internal/`
|
||||||
|
- external protocol, service, and file-format contracts: `docs/integrations/`
|
||||||
- future work: `docs/roadmap/`
|
- future work: `docs/roadmap/`
|
||||||
- contributor workflow: `docs/policy/development.md`
|
- contributor workflow: `docs/policy/development.md`
|
||||||
- copyable examples: `examples/`
|
- copyable examples: `examples/`
|
||||||
@@ -106,7 +110,7 @@ Recommended:
|
|||||||
- `examples/`
|
- `examples/`
|
||||||
- `docs/policy/development.md`
|
- `docs/policy/development.md`
|
||||||
|
|
||||||
### Modular, staged, service-oriented, or orchestration application
|
### Modular, service-oriented, or orchestration application
|
||||||
|
|
||||||
Required:
|
Required:
|
||||||
- `docs/cli.md`, if CLI-based
|
- `docs/cli.md`, if CLI-based
|
||||||
@@ -119,6 +123,31 @@ Recommended:
|
|||||||
- `docs/troubleshooting.md`
|
- `docs/troubleshooting.md`
|
||||||
- validated examples under `examples/`
|
- validated examples under `examples/`
|
||||||
|
|
||||||
|
### Public HTTP API service
|
||||||
|
|
||||||
|
Required:
|
||||||
|
- `docs/api.md`
|
||||||
|
- `docs/cli.md`, if CLI-based
|
||||||
|
- `docs/config.md`, if config-driven
|
||||||
|
- `docs/operations.md`
|
||||||
|
- `docs/internal/`
|
||||||
|
- `docs/policy/development.md`
|
||||||
|
|
||||||
|
Recommended:
|
||||||
|
- `docs/troubleshooting.md`
|
||||||
|
- `docs/consumers/`, for task-oriented client integration guides
|
||||||
|
- `docs/integrations/`, for upstream/downstream service contracts
|
||||||
|
- validated examples under `examples/`
|
||||||
|
|
||||||
|
### Project with public packages or consumer APIs
|
||||||
|
|
||||||
|
Required:
|
||||||
|
- `docs/consumers/api.md`
|
||||||
|
- one `docs/consumers/pkg-<name>.md` file per public package, if public packages exist
|
||||||
|
|
||||||
|
Recommended:
|
||||||
|
- copyable consumer examples under `examples/`, if practical
|
||||||
|
|
||||||
## Required Documents
|
## Required Documents
|
||||||
|
|
||||||
### README.md
|
### README.md
|
||||||
@@ -159,7 +188,35 @@ It should include:
|
|||||||
- architectural invariants;
|
- architectural invariants;
|
||||||
- explicit non-goals, if useful.
|
- explicit non-goals, if useful.
|
||||||
|
|
||||||
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
|
Notably, this file should prescribe a core development *policy* that should remain unchanged as the application evolves. It is not a place for details (e.g., CLI flags) that could change over time.
|
||||||
|
|
||||||
|
The contents of `architecture.md` should be trim and concise. LLMs may be directed to review it routinely via AGENTS.md, CLAUDE.md, or similar.
|
||||||
|
|
||||||
|
### docs/api.md
|
||||||
|
|
||||||
|
**Audience:** external HTTP API consumers, developers, LLM coding agents integrating by HTTP
|
||||||
|
|
||||||
|
Required for projects whose primary public interface is HTTP.
|
||||||
|
|
||||||
|
`docs/api.md` is the canonical public HTTP API contract. It should be normative for external consumers and should not be duplicated by README, operations docs, consumer guides, or integration docs.
|
||||||
|
|
||||||
|
It should include:
|
||||||
|
|
||||||
|
1. base URL conventions;
|
||||||
|
2. authentication and authorization behavior, if implemented;
|
||||||
|
3. response envelope;
|
||||||
|
4. supported media types and content negotiation behavior;
|
||||||
|
5. shared query parameters;
|
||||||
|
6. endpoint reference grouped by route family;
|
||||||
|
7. request parameters and validation rules;
|
||||||
|
8. response fields, units, nullability, and optionality;
|
||||||
|
9. error response shape and status codes;
|
||||||
|
10. pagination, caching, rate-limit, idempotency, and retry behavior, if implemented;
|
||||||
|
11. compact request and response examples.
|
||||||
|
|
||||||
|
It must document only implemented endpoints and behavior. Planned endpoints, proposed fields, future filters, and experimental response shapes belong only under `docs/roadmap/`.
|
||||||
|
|
||||||
|
For HTTP API projects, `docs/consumers/` may provide task-oriented client integration guides, but those guides should link to `docs/api.md` for the authoritative endpoint contract.
|
||||||
|
|
||||||
### docs/policy/development.md
|
### docs/policy/development.md
|
||||||
|
|
||||||
@@ -175,7 +232,7 @@ It should include:
|
|||||||
- dependency policy;
|
- dependency policy;
|
||||||
- how to add config fields;
|
- how to add config fields;
|
||||||
- how to add CLI flags;
|
- how to add CLI flags;
|
||||||
- how to add stages/modules/adapters, if applicable;
|
- how to add modules or adapters, if applicable;
|
||||||
- how to update examples;
|
- how to update examples;
|
||||||
- documentation update expectations.
|
- documentation update expectations.
|
||||||
|
|
||||||
@@ -216,7 +273,7 @@ Explain when commands are useful, not just their syntax.
|
|||||||
|
|
||||||
**Audience:** administrators, operators
|
**Audience:** administrators, operators
|
||||||
|
|
||||||
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
|
Required for applications that maintain state, support resume behavior, run multi-step workflows, write durable artifacts, use remote storage, or require recovery procedures.
|
||||||
|
|
||||||
It should cover:
|
It should cover:
|
||||||
|
|
||||||
@@ -244,11 +301,40 @@ Each entry should include:
|
|||||||
- safe fix;
|
- safe fix;
|
||||||
- relevant links.
|
- relevant links.
|
||||||
|
|
||||||
|
### docs/consumers/
|
||||||
|
|
||||||
|
**Audience:** developers and LLM coding agents integrating this project from another codebase
|
||||||
|
|
||||||
|
Required for projects with public packages, SDKs, client APIs, plugin APIs, or other application-facing integration surfaces.
|
||||||
|
|
||||||
|
This directory describes how an external codebase should consume the project's public API. It should be task-oriented and copyable where useful. It is not the place for internal implementation details or operator procedures.
|
||||||
|
|
||||||
|
For projects whose public API is HTTP, `docs/consumers/` is not required, and it should not duplicate the endpoint reference in `docs/api.md`. If present, it may provide practical integration workflows, client-specific examples, or migration notes that link back to `docs/api.md`.
|
||||||
|
|
||||||
|
`docs/consumers/api.md` should provide the consumer-facing overview and primary implementation workflow. It should include:
|
||||||
|
|
||||||
|
1. intended consumer audience and use cases;
|
||||||
|
2. required inputs supplied by operators or deployment configuration;
|
||||||
|
3. recommended public package or API workflow;
|
||||||
|
4. minimal copyable example;
|
||||||
|
5. consumer responsibilities and boundaries;
|
||||||
|
6. retry, idempotency, or status behavior, if applicable;
|
||||||
|
7. links to package-specific docs and canonical integration contracts.
|
||||||
|
|
||||||
|
Package-specific docs should be named `pkg-<name>.md` and should include:
|
||||||
|
|
||||||
|
1. import path;
|
||||||
|
2. intended use cases;
|
||||||
|
3. primary types and functions needed by consumers;
|
||||||
|
4. minimal examples;
|
||||||
|
5. validation, error, retry, and boundary behavior;
|
||||||
|
6. links to canonical file-format or wire-protocol contracts.
|
||||||
|
|
||||||
### docs/internal/
|
### docs/internal/
|
||||||
|
|
||||||
**Audience:** developers, LLM coding agents
|
**Audience:** developers, LLM coding agents
|
||||||
|
|
||||||
Required for modular, staged, service-oriented, or orchestration projects.
|
Required for modular, service-oriented, or orchestration projects.
|
||||||
|
|
||||||
This directory describes implemented internal components. It is not the roadmap.
|
This directory describes implemented internal components. It is not the roadmap.
|
||||||
|
|
||||||
@@ -289,7 +375,9 @@ Roadmap docs should not be confused with current behavior.
|
|||||||
|
|
||||||
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
||||||
|
|
||||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
|
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses or exposes.
|
||||||
|
|
||||||
|
For public HTTP API services, `docs/integrations/` should document upstream, downstream, storage, protocol, or runtime contracts that the service depends on or bridges. It should not become a second copy of the public HTTP endpoint reference; that belongs in `docs/api.md`.
|
||||||
|
|
||||||
Use one file per integration where useful.
|
Use one file per integration where useful.
|
||||||
|
|
||||||
@@ -346,8 +434,10 @@ Before merging documentation changes, verify:
|
|||||||
|
|
||||||
- README is concise and orientation-focused.
|
- README is concise and orientation-focused.
|
||||||
- `docs/policy/architecture.md` describes development principles.
|
- `docs/policy/architecture.md` describes development principles.
|
||||||
|
- `docs/api.md` is the canonical HTTP contract for HTTP API services.
|
||||||
- Future work appears only under `docs/roadmap/`.
|
- Future work appears only under `docs/roadmap/`.
|
||||||
- User-facing docs avoid unnecessary internals.
|
- User-facing docs avoid unnecessary internals.
|
||||||
|
- Consumer-facing docs explain public APIs without duplicating HTTP endpoint or integration contracts.
|
||||||
- Developer-facing docs preserve boundaries and invariants.
|
- Developer-facing docs preserve boundaries and invariants.
|
||||||
- Config examples match the schema.
|
- Config examples match the schema.
|
||||||
- CLI examples match real commands and flags.
|
- CLI examples match real commands and flags.
|
||||||
|
|||||||
142
docs/roadmap/builtins.md
Normal file
142
docs/roadmap/builtins.md
Normal file
@@ -0,0 +1,142 @@
|
|||||||
|
# Built-In Profiles Roadmap
|
||||||
|
|
||||||
|
This roadmap defines the target behavior for adding built-in execution profiles to Scriptorium.
|
||||||
|
|
||||||
|
Built-in profiles are useful for CLI, HTTP, subprocess, and library consumers, and they provide a clean foundation for public-package ergonomics.
|
||||||
|
|
||||||
|
## Motivation
|
||||||
|
|
||||||
|
Scriptorium currently requires a profile source for every run path. That is appropriate for fully custom deployments, but it creates unnecessary setup for common model targets where stable profile definitions can be shipped with the application.
|
||||||
|
|
||||||
|
Built-in profiles should let callers select standard profile IDs without creating local profile files. Users and downstream applications should still be able to override any built-in profile by providing a custom profile with the same ID.
|
||||||
|
|
||||||
|
## Target Behavior
|
||||||
|
|
||||||
|
Scriptorium should include a built-in set of execution profiles compiled into the binary/package.
|
||||||
|
|
||||||
|
Profile lookup should use this precedence:
|
||||||
|
|
||||||
|
1. user-provided or downstream-provided profiles;
|
||||||
|
2. built-in profiles;
|
||||||
|
3. profile-not-found error.
|
||||||
|
|
||||||
|
If a user profile and a built-in profile share the same ID, the user profile wins. This is intentional override behavior and should not be treated as a duplicate-profile error.
|
||||||
|
|
||||||
|
Duplicate profile IDs within the user profile source should remain invalid. Duplicate profile IDs within the built-in profile set should be prevented by tests. Duplicate IDs across the user source and built-in source are valid because they express override intent.
|
||||||
|
|
||||||
|
Once built-ins exist, `profile_dir` should no longer be required for CLI, HTTP, or public library engine construction. When no custom profile source is configured, Scriptorium should use the built-in profile repository alone. When a custom profile source is configured, Scriptorium should overlay it on top of the built-in repository.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
Built-in profiles should be modeled as another implementation of the existing `profile.Repository` boundary.
|
||||||
|
|
||||||
|
Recommended repository structure:
|
||||||
|
|
||||||
|
- filesystem or custom profile repository for user-provided profiles;
|
||||||
|
- built-in profile repository backed by embedded profile YAML;
|
||||||
|
- overlay repository that checks the primary repository first and falls back to built-ins only when the primary returns `profile.ErrProfileNotFound`.
|
||||||
|
|
||||||
|
The runner should continue to depend only on `profile.Repository`. It should not know whether a selected profile came from a file, a built-in definition, or a future public-package source.
|
||||||
|
|
||||||
|
### Built-In Repository
|
||||||
|
|
||||||
|
Built-in definitions should be stored as normal profile YAML and embedded into the binary with Go `embed`.
|
||||||
|
|
||||||
|
Recommended package shape:
|
||||||
|
|
||||||
|
- `internal/profile/builtin` owns embedded built-in profile assets and exposes a repository constructor.
|
||||||
|
- built-in profile files live under that package in a stable asset directory.
|
||||||
|
- the built-in repository reuses the same strict decoding and validation rules as normal profiles.
|
||||||
|
|
||||||
|
Using YAML for built-ins keeps the built-in profile format aligned with the documented profile format and lets maintainers add stable definitions without duplicating profile construction logic in Go.
|
||||||
|
|
||||||
|
### FS Repository
|
||||||
|
|
||||||
|
The implementation should introduce or reuse an `fs.FS`-based profile repository rather than making the built-in loader special-purpose.
|
||||||
|
|
||||||
|
That repository supports:
|
||||||
|
|
||||||
|
- embedded built-in profile assets;
|
||||||
|
- embedded or virtual profile sources in public library work;
|
||||||
|
- fixture-based tests without temporary directory setup where useful.
|
||||||
|
|
||||||
|
The existing filesystem repository can remain as a thin path-based adapter, or it can delegate internally to the `fs.FS` repository where that is clean and maintainable.
|
||||||
|
|
||||||
|
### Overlay Repository
|
||||||
|
|
||||||
|
An overlay repository should compose two repositories:
|
||||||
|
|
||||||
|
- primary: user-provided, custom, or downstream profile source;
|
||||||
|
- fallback: built-in profile source.
|
||||||
|
|
||||||
|
Lookup behavior:
|
||||||
|
|
||||||
|
- return the primary result if primary lookup succeeds;
|
||||||
|
- if primary returns `profile.ErrProfileNotFound`, try fallback;
|
||||||
|
- if primary returns any other error, return that error and do not try fallback;
|
||||||
|
- return fallback result or fallback error.
|
||||||
|
|
||||||
|
This preserves strict validation of user profile sources. A malformed selected user profile should not silently fall through to a built-in with the same ID.
|
||||||
|
|
||||||
|
## CLI And HTTP Behavior
|
||||||
|
|
||||||
|
The CLI and HTTP server should no longer require `profile_dir` once built-in profiles are available.
|
||||||
|
|
||||||
|
Expected behavior:
|
||||||
|
|
||||||
|
- `profile_dir` omitted: built-ins are available.
|
||||||
|
- `profile_dir` provided: profiles from that directory override built-ins with the same ID.
|
||||||
|
- selected profile ID present only in built-ins: run succeeds.
|
||||||
|
- selected profile ID present in both custom profiles and built-ins: custom profile is used.
|
||||||
|
- selected profile ID missing from both sources: existing profile-not-found behavior is preserved.
|
||||||
|
- selected profile ID matches a malformed custom profile: profile-load failure is returned, not fallback to built-in.
|
||||||
|
|
||||||
|
Configuration and CLI documentation should describe `profile_dir` as optional once built-in profiles are available.
|
||||||
|
|
||||||
|
## Public Library Interaction
|
||||||
|
|
||||||
|
This feature should support the current public package behavior and the production library roadmap.
|
||||||
|
|
||||||
|
For the current public engine, `ProfileDir` should become optional once built-ins exist. A caller that does not configure a custom profile directory should still be able to use built-in profile IDs.
|
||||||
|
|
||||||
|
The production library roadmap may add `fs.FS`, single-file, and in-memory profile sources. Those sources should become overlay primaries above the same built-in repository.
|
||||||
|
|
||||||
|
Credential behavior for built-ins should follow the active execution path:
|
||||||
|
|
||||||
|
- current CLI/HTTP behavior may continue to use `api_key_env` in profile definitions;
|
||||||
|
- the public library API may supply direct API-key values without changing built-in profile IDs;
|
||||||
|
- built-in profile files must never contain raw API keys.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
In scope:
|
||||||
|
|
||||||
|
- built-in execution profile assets;
|
||||||
|
- strict validation of all built-in profiles;
|
||||||
|
- `fs.FS` profile repository support where needed for embedded assets;
|
||||||
|
- overlay profile repository with user-over-built-in precedence;
|
||||||
|
- optional `profile_dir` for CLI, HTTP, and public engine construction;
|
||||||
|
- tests for lookup precedence, override behavior, duplicate handling, and error behavior;
|
||||||
|
- documentation of CLI/config/profile behavior.
|
||||||
|
|
||||||
|
Out of scope:
|
||||||
|
|
||||||
|
- changing the profile YAML format;
|
||||||
|
- accepting raw API keys in profile YAML;
|
||||||
|
- adding a mutable runtime profile registry;
|
||||||
|
- adding a provider/model catalog that must track rapidly changing model availability;
|
||||||
|
- changing prompt `default_profile` semantics beyond allowing built-in IDs;
|
||||||
|
- implementing the broader `fs.FS` prompt/schema/library source work from `docs/roadmap/library.md`.
|
||||||
|
|
||||||
|
## Acceptance Criteria
|
||||||
|
|
||||||
|
- Scriptorium can run or render using a built-in profile ID with no configured `profile_dir`.
|
||||||
|
- CLI `run`, CLI `render`, and HTTP `serve` no longer fail solely because `profile_dir` is omitted.
|
||||||
|
- A profile in `profile_dir` overrides a built-in profile with the same ID.
|
||||||
|
- Duplicate profile IDs inside `profile_dir` remain invalid.
|
||||||
|
- Duplicate profile IDs inside the built-in profile set are caught by tests.
|
||||||
|
- A malformed selected custom profile does not fall back to a built-in profile with the same ID.
|
||||||
|
- Profile-not-found behavior remains clear when an ID exists in neither custom profiles nor built-ins.
|
||||||
|
- Built-in profiles are loaded through the same validation rules as file-based profiles.
|
||||||
|
- Existing runtime override behavior continues to apply to built-in profiles.
|
||||||
|
- Existing CLI, HTTP, and public error mapping remains consistent with current profile-load and profile-not-found semantics.
|
||||||
@@ -1,245 +1,420 @@
|
|||||||
# Runtime Parameter Implementation Plan
|
# Built-In Profiles And Library API Implementation Plan
|
||||||
|
|
||||||
This plan implements the target state in `docs/roadmap/params.md`.
|
This plan implements the target states in:
|
||||||
|
|
||||||
Audience: LLM coding agents implementing the feature in order. Follow `docs/policy/architecture.md`, `docs/policy/development.md`, and `docs/policy/documentation.md` before changing code.
|
- `docs/roadmap/builtins.md`
|
||||||
|
- `docs/roadmap/library.md`
|
||||||
|
|
||||||
## Constraints
|
Audience: LLM coding agents implementing the work in order. Review and follow `docs/policy/architecture.md`, `docs/policy/development.md`, and `docs/policy/documentation.md` before changing code.
|
||||||
|
|
||||||
- Keep adapters thin. CLI and HTTP should capture caller intent and map it into domain request types; merge decisions belong in `internal/usecase`.
|
## Global Constraints
|
||||||
- Keep external decoding strict. Unknown YAML/JSON fields must continue to fail.
|
|
||||||
- Do not accept or emit raw API key values.
|
- Implement built-in profiles before the public library production upgrades.
|
||||||
- Do not add dependencies unless there is a clear need. This feature should use the standard library plus existing dependencies.
|
- Keep orchestration in `internal/usecase`; adapters and the public package should translate inputs and wire components.
|
||||||
- Do not expand the HTTP API surface beyond `POST /v1/runs`.
|
- Keep `internal/*` packages internal. Public package types must remain facade types.
|
||||||
- Do not add provider-specific adapter packages.
|
- Do not add a mutable global profile registry.
|
||||||
|
- Do not add a credential resolver or secret-manager abstraction.
|
||||||
|
- Do not accept raw API keys in YAML/JSON config, profile files, prompt files, CLI flags, or HTTP request bodies.
|
||||||
|
- Do not emit raw API keys in prepared output, run results, logs, or examples.
|
||||||
|
- Prefer standard library APIs. Do not add dependencies unless a later implementation prompt explicitly approves one.
|
||||||
- Keep each stage passing `go test ./...` before moving to the next stage.
|
- Keep each stage passing `go test ./...` before moving to the next stage.
|
||||||
|
|
||||||
## Stage 1: Presence-Aware Request Overrides
|
## Stage 1: Profile Repository Foundations
|
||||||
|
|
||||||
Goal: make per-request numeric execution overrides presence-aware while keeping resolved execution settings concrete.
|
Goal: add reusable profile repository primitives that support built-ins without changing runner behavior.
|
||||||
|
|
||||||
### Domain Changes
|
### Implementation Steps
|
||||||
|
|
||||||
1. In `internal/domain/domain.go`, add a request-only type:
|
1. Add an `fs.FS`-backed profile repository in `internal/profile`.
|
||||||
|
- Constructor shape should be similar to `NewFSRepository(fsys fs.FS, root string) Repository`.
|
||||||
|
- It must scan YAML files recursively below `root`.
|
||||||
|
- It must use the same strict YAML decoding, profile validation, raw `api_key` rejection, and duplicate-ID behavior as the existing filesystem repository.
|
||||||
|
- It must return the existing profile package sentinel errors where applicable.
|
||||||
|
|
||||||
|
2. Refactor shared profile-loading behavior.
|
||||||
|
- Avoid duplicating validation and metadata logic between filesystem and `fs.FS` repositories.
|
||||||
|
- The existing `NewFilesystemRepository(dir)` API should remain available.
|
||||||
|
- It may either delegate to the `fs.FS` repository through `os.DirFS` or share unexported loader helpers.
|
||||||
|
|
||||||
|
3. Add an overlay profile repository in `internal/profile`.
|
||||||
|
- Constructor shape should be similar to `NewOverlayRepository(primary, fallback Repository) Repository`.
|
||||||
|
- Lookup must return the primary result when primary succeeds.
|
||||||
|
- Lookup must fall back only when `errors.Is(err, profile.ErrProfileNotFound)` for the primary.
|
||||||
|
- Lookup must return primary load/validation errors directly and must not fall back after those errors.
|
||||||
|
- Nil repository inputs should be handled deliberately. Prefer treating nil primary as "no primary" and requiring a non-nil fallback for built-in-only operation.
|
||||||
|
|
||||||
|
### Tests
|
||||||
|
|
||||||
|
Add focused tests under `internal/profile`.
|
||||||
|
|
||||||
|
Required coverage:
|
||||||
|
|
||||||
|
- `NewFSRepository` loads valid profiles from nested directories.
|
||||||
|
- `NewFSRepository` rejects unknown YAML fields.
|
||||||
|
- `NewFSRepository` rejects raw `api_key` in the selected profile.
|
||||||
|
- `NewFSRepository` ignores raw `api_key` in non-selected profiles, matching current filesystem behavior.
|
||||||
|
- `NewFSRepository` rejects duplicate IDs within one source.
|
||||||
|
- `NewFilesystemRepository` still satisfies all existing repository tests.
|
||||||
|
- `NewOverlayRepository` returns primary matches before fallback matches.
|
||||||
|
- `NewOverlayRepository` falls back on primary not found.
|
||||||
|
- `NewOverlayRepository` does not fall back after a primary invalid YAML/profile/raw-key error.
|
||||||
|
- `NewOverlayRepository` returns not found when both sources miss.
|
||||||
|
|
||||||
|
### Verification
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go test ./internal/profile
|
||||||
|
go test ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
## Stage 2: Built-In Profile Assets And Wiring
|
||||||
|
|
||||||
|
Goal: compile built-in profiles into Scriptorium and make profile lookup use user-over-built-in precedence.
|
||||||
|
|
||||||
|
### Implementation Steps
|
||||||
|
|
||||||
|
1. Add an `internal/profile/builtin` package.
|
||||||
|
- Store built-in profile YAML files in a stable asset directory under that package.
|
||||||
|
- Use Go `embed` to compile those files into the binary/package.
|
||||||
|
- Expose a constructor such as `builtin.NewRepository() profile.Repository`.
|
||||||
|
- The repository should use the `fs.FS` profile repository from Stage 1.
|
||||||
|
|
||||||
|
2. Add built-in profile validation tests.
|
||||||
|
- Tests should load every built-in profile through the real profile loader.
|
||||||
|
- Tests should fail if the built-in profile set contains duplicate IDs.
|
||||||
|
- Tests should fail if any built-in profile contains raw `api_key`.
|
||||||
|
|
||||||
|
3. Add built-in profiles.
|
||||||
|
- Use `docs/roadmap/profiles/` as the source catalog for the initial built-in profile set.
|
||||||
|
- Copy those YAML files into the built-in profile asset directory, preserving provider subdirectories unless the implementation has a clear reason to flatten them.
|
||||||
|
- Do not invent a broad provider/model catalog.
|
||||||
|
- Do not add raw API keys.
|
||||||
|
- Built-in profiles may use `api_key_env` for CLI/HTTP compatibility when the provider requires authentication.
|
||||||
|
|
||||||
|
4. Wire repositories through a small helper.
|
||||||
|
- Add an internal helper near adapter wiring, or in `internal/profile`, that returns:
|
||||||
|
- built-in repository only when no custom profile source is configured;
|
||||||
|
- overlay repository when a custom profile source is configured.
|
||||||
|
- The runner should still receive only a `profile.Repository`.
|
||||||
|
|
||||||
|
5. Make `profile_dir` optional.
|
||||||
|
- CLI `run`, CLI `render`, and HTTP `serve` argument/config validation should require `prompt_dir` but no longer require `profile_dir`.
|
||||||
|
- Public `NewEngine` should no longer reject an empty `Config.ProfileDir`.
|
||||||
|
- When `profile_dir` is empty, wire only built-ins.
|
||||||
|
- When `profile_dir` is non-empty, wire filesystem profiles over built-ins.
|
||||||
|
|
||||||
|
### Tests
|
||||||
|
|
||||||
|
Add or update tests under `internal/adapter/cli`, `internal/adapter/http`, root public package tests, and profile/builtin tests.
|
||||||
|
|
||||||
|
Required coverage:
|
||||||
|
|
||||||
|
- CLI parse/config tests accept missing `profile_dir` when `prompt_dir` is present.
|
||||||
|
- HTTP serve parse/config tests accept missing `profile_dir` when `prompt_dir` is present.
|
||||||
|
- Public `NewEngine` accepts missing `ProfileDir`.
|
||||||
|
- A built-in profile ID can be selected with no custom `profile_dir`.
|
||||||
|
- A prompt `default_profile` can refer to a built-in profile ID.
|
||||||
|
- A custom profile in `profile_dir` overrides a built-in with the same ID.
|
||||||
|
- A malformed selected custom profile does not fall back to a built-in with the same ID.
|
||||||
|
- A missing profile ID still maps to the existing profile-not-found behavior.
|
||||||
|
- Existing duplicate-ID tests for filesystem profiles continue to fail within the custom source.
|
||||||
|
|
||||||
|
### Documentation
|
||||||
|
|
||||||
|
After code behavior exists, update non-roadmap docs:
|
||||||
|
|
||||||
|
- `docs/config.md`: document `profile_dir` as optional and describe built-in fallback/override behavior.
|
||||||
|
- `docs/cli.md`: remove claims that `--profile-dir` is required.
|
||||||
|
- `docs/internal/adapters.md`: document built-in profile repository composition.
|
||||||
|
- Any affected examples or README snippets that imply `profile_dir` is mandatory.
|
||||||
|
|
||||||
|
Do not document built-in profile IDs outside implemented assets.
|
||||||
|
|
||||||
|
### Verification
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go test ./internal/profile ./internal/adapter/cli ./internal/adapter/http .
|
||||||
|
go test ./...
|
||||||
|
go run ./cmd/scriptorium render \
|
||||||
|
--config ./examples/config.yml \
|
||||||
|
--prompt generic.markdown_summary \
|
||||||
|
--input transcript=./examples/fixtures/transcript.md \
|
||||||
|
--input glossary=./examples/fixtures/glossary.yml \
|
||||||
|
--format json
|
||||||
|
```
|
||||||
|
|
||||||
|
Also run a new smoke command that uses a built-in profile without `--profile-dir` once a concrete built-in profile ID is available.
|
||||||
|
|
||||||
|
## Stage 3: Public API Credential Value
|
||||||
|
|
||||||
|
Goal: support the production library credential model: a direct API-key Go value, without adding a resolver or accepting raw keys in serialized config.
|
||||||
|
|
||||||
|
### Public API Decision
|
||||||
|
|
||||||
|
Use a single public credential-supply method:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
type ExecutionTargetOverride struct {
|
type RunRequest struct {
|
||||||
Endpoint string `json:"endpoint,omitempty"`
|
// existing fields...
|
||||||
Model string `json:"model,omitempty"`
|
APIKey string `json:"-"`
|
||||||
Temperature *float64 `json:"temperature,omitempty"`
|
|
||||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
|
||||||
TopP *float64 `json:"top_p,omitempty"`
|
|
||||||
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
|
||||||
ServiceTier string `json:"service_tier,omitempty"`
|
|
||||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
|
||||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
|
||||||
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
2. Change `domain.RunRequest.Execution` from `*ExecutionTarget` to `*ExecutionTargetOverride`.
|
Do not add `Config.APIKey`, `WithAPIKey`, or a credential resolver in this stage. A request-level value avoids storing secrets on long-lived engines and supports per-tenant callers.
|
||||||
3. Change `ExecutionProfile.ExtraParams` and `ExecutionTarget.ExtraParams` from `map[string]string` to `map[string]any`.
|
|
||||||
4. Keep `ExecutionTarget` concrete. It represents the resolved effective runtime target after defaults, profile, and request overrides are merged.
|
|
||||||
|
|
||||||
### Runner Changes
|
### Implementation Steps
|
||||||
|
|
||||||
1. Update `internal/usecase/runner.go` so profile values still merge over built-in defaults and request overrides merge over that result.
|
1. Add direct API-key plumbing through internal request/target types.
|
||||||
2. Keep the existing concrete profile merge semantics for profile numeric fields.
|
- Add an internal direct API-key field where needed, with `json:"-"` and `yaml:"-"` tags.
|
||||||
3. Add a separate request override merge path that uses pointer presence:
|
- Convert public `RunRequest.APIKey` into the internal request.
|
||||||
- `nil` numeric pointer means omitted; preserve the current value.
|
- Carry the value to the effective execution target used for LLM generation.
|
||||||
- non-nil numeric pointer means explicit override, even when the value is `0`.
|
- Do not include the value in prepared/run public results, formatted output, logs, hashes, or docs examples.
|
||||||
4. Validate request override numeric values before or during merge:
|
|
||||||
- `temperature`: `0 <= value <= 2`
|
|
||||||
- `max_tokens`: `value >= 0`
|
|
||||||
- `top_p`: `0 <= value <= 1`
|
|
||||||
- `timeout_seconds`: `value >= 0`
|
|
||||||
5. Preserve existing validation after merge:
|
|
||||||
- effective endpoint required
|
|
||||||
- effective model required
|
|
||||||
- `api_key_env`, when set, must name a non-empty environment variable
|
|
||||||
6. Preserve secret handling. The resolved API key value must never be stored in `PreparedRun`, `RunResult`, logs, or HTTP responses.
|
|
||||||
|
|
||||||
### CLI Changes
|
2. Update credential validation.
|
||||||
|
- If a selected/effective profile requires authentication and a direct API key is provided, do not require the environment variable to be set for the public path.
|
||||||
|
- Preserve existing CLI/HTTP behavior that uses `api_key_env`.
|
||||||
|
- Preserve existing errors for missing environment variables in CLI/HTTP paths.
|
||||||
|
|
||||||
1. Update `internal/adapter/cli/run.go` request construction to build `domain.ExecutionTargetOverride`.
|
3. Update the OpenAI-compatible LLM client.
|
||||||
2. Use the existing `flagWasSet` booleans to populate numeric pointers only when the user provided the flag.
|
- Prefer the direct API-key value when present.
|
||||||
3. Required behavior:
|
- Fall back to existing `api_key_env` behavior for CLI/HTTP compatibility.
|
||||||
- omitted `--temperature` preserves profile/default temperature;
|
- Never serialize or log the direct API-key value.
|
||||||
- `--temperature 0` explicitly sets temperature to zero;
|
|
||||||
- omitted `--top-p` preserves profile/default top-p;
|
|
||||||
- `--top-p 0` explicitly sets top-p to zero;
|
|
||||||
- omitted `--max-tokens` preserves profile/default max tokens;
|
|
||||||
- `--max-tokens 0` explicitly sets max tokens to zero;
|
|
||||||
- omitted `--timeout` preserves profile/default timeout;
|
|
||||||
- `--timeout 0s` explicitly sets timeout seconds to zero.
|
|
||||||
4. Do not add new CLI flags in this stage.
|
|
||||||
|
|
||||||
### HTTP Changes
|
4. Update public LLM injection conversion.
|
||||||
|
- Do not expose the raw API key to injected public `LLMClient` implementations unless that is strictly necessary for custom LLM execution.
|
||||||
1. Update `internal/adapter/http/dto.go` so numeric model override fields are pointers:
|
- If custom LLM clients need the key, expose it only on the public `GenerateRequest` with `json:"-"` and document that fake/test clients should avoid logging it.
|
||||||
- `Temperature *float64`
|
|
||||||
- `MaxTokens *int`
|
|
||||||
- `TopP *float64`
|
|
||||||
- `TimeoutSeconds *int`
|
|
||||||
2. Update DTO mapping in `internal/adapter/http/handler.go` to build `domain.ExecutionTargetOverride`.
|
|
||||||
3. Preserve strict JSON decoding and existing error mapping.
|
|
||||||
4. Required behavior:
|
|
||||||
- omitted numeric JSON fields preserve profile/default values;
|
|
||||||
- explicit numeric zero JSON fields override profile/default values.
|
|
||||||
|
|
||||||
### Tests
|
### Tests
|
||||||
|
|
||||||
Add or update tests in:
|
Required coverage:
|
||||||
|
|
||||||
- `internal/usecase/runner_test.go`
|
- Public `Run` can call the default OpenAI-compatible client path with a direct API key without requiring the configured `api_key_env` environment variable.
|
||||||
- `internal/adapter/cli/run_test.go`
|
- CLI/HTTP behavior using `api_key_env` still works.
|
||||||
- `internal/adapter/http/handler_test.go`
|
- Missing credentials still fail clearly when a selected profile requires authentication and neither direct key nor usable env value is available.
|
||||||
|
- Public `PreparedRun` and `RunResult` JSON do not include the direct API key.
|
||||||
Required test coverage:
|
- Formatted prepared output does not include the direct API key.
|
||||||
|
- Direct API-key values are not included in hashes.
|
||||||
- Runner preserves profile value when request numeric override is omitted.
|
- Injected fake LLM tests either receive no key or receive it only through a `json:"-"` field, depending on the implementation choice above.
|
||||||
- Runner applies explicit zero request override for `temperature`.
|
|
||||||
- Runner applies explicit zero request override for `top_p`.
|
|
||||||
- Runner applies explicit zero request override for `max_tokens`.
|
|
||||||
- Runner applies explicit zero request override for `timeout_seconds`.
|
|
||||||
- Invalid request override ranges fail as invalid request errors.
|
|
||||||
- CLI `--temperature 0` reaches effective settings as zero.
|
|
||||||
- HTTP `"temperature": 0` reaches effective settings as zero.
|
|
||||||
- HTTP omitted `temperature` preserves profile/default value.
|
|
||||||
|
|
||||||
### Verification
|
### Verification
|
||||||
|
|
||||||
Run:
|
Run:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
go test ./internal/llm ./internal/usecase ./internal/adapter/cli ./internal/adapter/http .
|
||||||
go test ./...
|
go test ./...
|
||||||
```
|
```
|
||||||
|
|
||||||
## Stage 2: JSON-Compatible `extra_params`
|
## Stage 4: Public Asset Source Options
|
||||||
|
|
||||||
Goal: allow provider-specific parameters to carry JSON-compatible values throughout profile, HTTP, prepared output, metadata, and LLM request construction.
|
Goal: let library consumers load standard Scriptorium prompt/profile/schema assets from directories, single files, and `fs.FS` sources.
|
||||||
|
|
||||||
### Domain And Loader Changes
|
### Public API
|
||||||
|
|
||||||
1. Complete all compile fixes from changing `ExtraParams` to `map[string]any`.
|
Keep existing `Config` directory fields working. Add options:
|
||||||
2. Ensure `internal/profile/filesystem_repository.go` continues to decode profiles strictly while allowing nested JSON-compatible values under `extra_params`.
|
|
||||||
3. Add profile repository tests for `extra_params` containing:
|
|
||||||
- string
|
|
||||||
- number
|
|
||||||
- boolean
|
|
||||||
- nested object or array
|
|
||||||
4. Ensure formatter output remains deterministic:
|
|
||||||
- keep sorting `extra_params` keys in `internal/format/prepared_run.go`;
|
|
||||||
- render non-string values with stable JSON encoding in text output.
|
|
||||||
5. Preserve JSON formatter behavior through normal `encoding/json` output.
|
|
||||||
|
|
||||||
### HTTP Changes
|
```go
|
||||||
|
func WithPromptFS(fsys fs.FS, root string) Option
|
||||||
1. Change HTTP model override `ExtraParams` to `map[string]any`.
|
func WithPromptFile(path string) Option
|
||||||
2. Add handler tests proving HTTP accepts JSON-compatible `extra_params` values.
|
func WithProfileFS(fsys fs.FS, root string) Option
|
||||||
3. Preserve strict rejection of unknown fields and raw API-key payload fields.
|
func WithProfileFile(path string) Option
|
||||||
|
func WithSchemaFS(fsys fs.FS, root string) Option
|
||||||
### Verification
|
func WithSchemaFile(path string) Option
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go test ./...
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Stage 3: Outbound Serialization
|
Rules:
|
||||||
|
|
||||||
Goal: serialize `reasoning_effort` and `extra_params` to the OpenAI-compatible chat-completions request.
|
- Directory `Config` fields remain the compatibility path.
|
||||||
|
- Explicit options override the corresponding `Config` directory field.
|
||||||
|
- Prompt source is required.
|
||||||
|
- Profile source is optional because built-in profiles exist.
|
||||||
|
- Schema source is optional and should default to the current schema default behavior when not configured.
|
||||||
|
- Nil `fs.FS` values, empty required roots, and invalid option combinations return `ErrInvalidConfig`.
|
||||||
|
|
||||||
### LLM Adapter Changes
|
### Implementation Steps
|
||||||
|
|
||||||
1. In `internal/llm/openai_compatible_client.go`, add first-class outbound support for `reasoning_effort`.
|
1. Add `fs.FS` prompt-definition support.
|
||||||
2. Add `extra_params` support by flattening `domain.ExecutionTarget.ExtraParams` into additional top-level JSON request fields.
|
- Implement an `fs.FS` prompt repository that preserves existing strict prompt YAML behavior.
|
||||||
3. Implement reserved-field collision checks before the HTTP request is made.
|
- Preserve prompt lookup by YAML `id`, not path.
|
||||||
4. Reserved keys must include:
|
- Preserve duplicate prompt ID errors within one source.
|
||||||
- `model`
|
- Resolve `content_file` relative to the prompt file's directory inside the same source.
|
||||||
- `session_id`
|
- Keep the existing filesystem repository API; it may delegate to the `fs.FS` implementation.
|
||||||
- `messages`
|
|
||||||
- `temperature`
|
|
||||||
- `max_tokens`
|
|
||||||
- `top_p`
|
|
||||||
- `service_tier`
|
|
||||||
- `reasoning_effort`
|
|
||||||
- `response_format`
|
|
||||||
5. Reject empty `extra_params` keys.
|
|
||||||
6. Ensure each `extra_params` value can be marshaled as JSON. If marshaling fails, return `ErrInvalidRequest` with context.
|
|
||||||
7. Keep existing request behavior unchanged when `reasoning_effort` and `extra_params` are unset.
|
|
||||||
|
|
||||||
### Recommended Implementation Shape
|
2. Add public prompt source wiring.
|
||||||
|
- `Config.PromptDir` wires the filesystem repository.
|
||||||
|
- `WithPromptFS` wires the `fs.FS` repository.
|
||||||
|
- `WithPromptFile(path)` wires a single-file source and must still select by prompt YAML `id`.
|
||||||
|
|
||||||
Use a custom marshal path for the outbound chat request rather than string manipulation.
|
3. Add public profile source wiring.
|
||||||
|
- Reuse the Stage 1 profile `fs.FS` repository.
|
||||||
|
- `WithProfileFS` and `WithProfileFile` become overlay primaries above built-ins.
|
||||||
|
- Empty profile source still means built-ins only.
|
||||||
|
|
||||||
One acceptable shape:
|
4. Add schema `fs.FS` support.
|
||||||
|
- Extend or wrap the standard validator so schema files can be loaded from an `fs.FS` source.
|
||||||
|
- `WithSchemaFS` should preserve existing `schema_path` semantics.
|
||||||
|
- `WithSchemaFile(path)` should expose the file by its base name; prompts using it should set `schema_path` to that base name.
|
||||||
|
|
||||||
- Add `ReasoningEffort string` and `ExtraParams map[string]any` to the internal `openAIChatRequest`.
|
5. Keep adapter scope narrow.
|
||||||
- Add a helper that converts `openAIChatRequest` into `map[string]any`, inserts first-class fields when set, then inserts `ExtraParams` after collision validation.
|
- This stage is for the public package and shared repositories/validators.
|
||||||
- Marshal that map with `encoding/json`.
|
- Do not change CLI/HTTP request shapes for prompt or schema `fs.FS` sources.
|
||||||
|
|
||||||
Do not construct outbound JSON with manual string concatenation.
|
|
||||||
|
|
||||||
### Tests
|
### Tests
|
||||||
|
|
||||||
Update `internal/llm/openai_compatible_client_test.go`.
|
Required coverage:
|
||||||
|
|
||||||
Required test coverage:
|
- Public `Prepare` works with prompt definitions from `embed.FS`.
|
||||||
|
- `content_file` references resolve relative to the prompt file in `embed.FS`.
|
||||||
- outbound JSON includes `reasoning_effort` when set;
|
- Public `Prepare` works with `WithPromptFile`.
|
||||||
- outbound JSON omits `reasoning_effort` when unset;
|
- Public `Run` or `Prepare` works with `WithProfileFS` over built-ins.
|
||||||
- outbound JSON includes string, number, boolean, object, and array `extra_params`;
|
- Public `Run` or `Prepare` works with `WithProfileFile` over built-ins.
|
||||||
- reserved `extra_params` keys fail before provider call;
|
- Public structured-output schema validation works with `WithSchemaFS`.
|
||||||
- empty `extra_params` keys fail before provider call;
|
- `WithSchemaFile` works when the prompt's `schema_path` is the schema file base name.
|
||||||
- existing message, cache-control, service-tier, response-format, and usage parsing tests continue to pass.
|
- Explicit source options override `Config` directory fields.
|
||||||
|
- Invalid/nil source options return `ErrInvalidConfig`.
|
||||||
|
- Existing filesystem prompt/profile/schema behavior remains unchanged.
|
||||||
|
|
||||||
### Verification
|
### Verification
|
||||||
|
|
||||||
Run:
|
Run:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
go test ./internal/promptdef ./internal/profile ./internal/validate .
|
||||||
go test ./...
|
go test ./...
|
||||||
```
|
```
|
||||||
|
|
||||||
## Stage 4: Documentation And Examples
|
## Stage 5: Public In-Memory Profiles And Profile Templates
|
||||||
|
|
||||||
Goal: move implemented behavior from roadmap to canonical docs after code is complete.
|
Goal: allow library callers to provide typed profile values and construct stable OpenAI-compatible profile templates without generating YAML.
|
||||||
|
|
||||||
Update only after Stages 1 through 3 are implemented.
|
### Public API
|
||||||
|
|
||||||
### Required Docs
|
Add a public profile facade aligned with the profile YAML contract:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type Profile struct {
|
||||||
|
ID string
|
||||||
|
Endpoint string
|
||||||
|
Model string
|
||||||
|
Temperature float64
|
||||||
|
MaxTokens int
|
||||||
|
TopP float64
|
||||||
|
TimeoutSeconds int
|
||||||
|
ServiceTier string
|
||||||
|
ReasoningEffort string
|
||||||
|
APIKeyRequired bool
|
||||||
|
ExtraParams map[string]any
|
||||||
|
}
|
||||||
|
|
||||||
|
func WithProfiles(profiles ...Profile) Option
|
||||||
|
```
|
||||||
|
|
||||||
|
Add an OpenAI-compatible template constructor:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type OpenAICompatibleProfileConfig struct {
|
||||||
|
ID string
|
||||||
|
Endpoint string
|
||||||
|
Model string
|
||||||
|
APIKeyRequired bool
|
||||||
|
Temperature float64
|
||||||
|
MaxTokens int
|
||||||
|
TopP float64
|
||||||
|
TimeoutSeconds int
|
||||||
|
ServiceTier string
|
||||||
|
ReasoningEffort string
|
||||||
|
ExtraParams map[string]any
|
||||||
|
}
|
||||||
|
|
||||||
|
func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile
|
||||||
|
```
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
- `WithProfiles` profiles override built-ins with the same ID.
|
||||||
|
- If `WithProfiles` and a file/FS profile source are both configured, in-memory profiles have highest precedence, then file/FS profiles, then built-ins.
|
||||||
|
- Public profile values must not include raw API-key fields.
|
||||||
|
- Use `APIKeyRequired` to indicate whether `RunRequest.APIKey` is required for the public path. Internal conversion may map this to the existing auth-required/profile credential model without exposing raw keys.
|
||||||
|
|
||||||
|
### Implementation Steps
|
||||||
|
|
||||||
|
1. Add a profile repository for public in-memory `Profile` values.
|
||||||
|
- Validate with the same effective rules as YAML profiles.
|
||||||
|
- Reject duplicate IDs within the provided values.
|
||||||
|
- Deep-copy `ExtraParams` across public/internal boundaries.
|
||||||
|
|
||||||
|
2. Compose profile sources in public `NewEngine`.
|
||||||
|
- Highest: in-memory public profiles.
|
||||||
|
- Next: configured profile file/FS/directory source.
|
||||||
|
- Fallback: built-in profiles.
|
||||||
|
|
||||||
|
3. Add template constructor conversion.
|
||||||
|
- `OpenAICompatibleProfile` should be a convenience constructor only.
|
||||||
|
- It should not register global state.
|
||||||
|
- It should not maintain a broad model catalog.
|
||||||
|
|
||||||
|
4. Ensure direct API-key behavior works with in-memory/template profiles.
|
||||||
|
- Profiles marked `APIKeyRequired` should require `RunRequest.APIKey` for public default LLM execution.
|
||||||
|
- Profiles not marked `APIKeyRequired` should not require an API key.
|
||||||
|
|
||||||
|
### Tests
|
||||||
|
|
||||||
|
Required coverage:
|
||||||
|
|
||||||
|
- Public `Prepare`/`Run` uses `WithProfiles` without any profile files.
|
||||||
|
- In-memory profiles override built-ins.
|
||||||
|
- In-memory profiles override file/FS profile sources when IDs collide.
|
||||||
|
- Duplicate in-memory profile IDs return `ErrInvalidConfig`.
|
||||||
|
- Template-created profiles execute through the same path as normal profiles.
|
||||||
|
- Template-created profiles requiring an API key work with `RunRequest.APIKey`.
|
||||||
|
- Template-created profiles that do not require an API key work without one.
|
||||||
|
- `ExtraParams` in public profiles are deep-copied and isolated from caller mutation.
|
||||||
|
|
||||||
|
### Verification
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go test .
|
||||||
|
go test ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
## Stage 6: Public Documentation And Examples
|
||||||
|
|
||||||
|
Goal: document implemented behavior in canonical locations after code exists.
|
||||||
|
|
||||||
|
### Documentation
|
||||||
|
|
||||||
Update:
|
Update:
|
||||||
|
|
||||||
- `docs/config.md`
|
- `README.md`: add a short pointer to library usage and built-in profiles without turning the README into a manual.
|
||||||
- `docs/cli.md`
|
- `docs/config.md`: document optional `profile_dir`, built-in override behavior, and implemented built-in profile IDs.
|
||||||
- `docs/integrations/http-api.md`
|
- `docs/cli.md`: document optional `--profile-dir` and built-in profile selection.
|
||||||
- `docs/integrations/openai-compatible-chat.md`
|
- `docs/internal/adapters.md`: document repository composition and public library adapter surface.
|
||||||
- `docs/internal/runner.md`
|
- `docs/consumers/api.md`: describe public consumer surfaces at a high level.
|
||||||
- `docs/internal/adapters.md`
|
- `docs/consumers/pkg-scriptorium.md`: document root package usage, source options, direct API-key value, errors, and examples.
|
||||||
|
- `docs/integrations/openai-compatible-chat.md`: update only if direct API-key plumbing changes provider-call semantics.
|
||||||
|
|
||||||
Required documentation content:
|
Do not document unimplemented roadmap behavior outside `docs/roadmap/`.
|
||||||
|
|
||||||
- `reasoning_effort` is serialized outbound when set.
|
|
||||||
- `extra_params` serializes as provider-specific top-level outbound JSON fields.
|
|
||||||
- `extra_params` supports JSON-compatible values.
|
|
||||||
- reserved `extra_params` fields are rejected.
|
|
||||||
- per-request numeric overrides distinguish omitted values from explicit zero values.
|
|
||||||
- CLI explicit zero behavior for existing numeric flags.
|
|
||||||
- HTTP explicit zero behavior for model override numeric fields.
|
|
||||||
- no raw API-key values are accepted or emitted.
|
|
||||||
|
|
||||||
### Examples
|
### Examples
|
||||||
|
|
||||||
Update examples only if needed to keep them accurate and runnable.
|
Add or update copyable examples that require no real provider credentials:
|
||||||
|
|
||||||
If adding an `extra_params` example, keep it secret-free and simple. Prefer a harmless provider-routing example over a vendor-specific feature that requires special credentials.
|
- prepare with built-in profile and no `profile_dir`;
|
||||||
|
- public library prepare from `embed.FS`;
|
||||||
|
- public library run with injected fake LLM;
|
||||||
|
- public library run with typed/template profile and direct API key, using a fake/local provider path so no real secret is required.
|
||||||
|
|
||||||
### Verification
|
Examples must be secret-free.
|
||||||
|
|
||||||
Run:
|
### Tests And Smoke Commands
|
||||||
|
|
||||||
|
Required verification:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
go test ./...
|
go test ./...
|
||||||
@@ -251,12 +426,26 @@ go run ./cmd/scriptorium render \
|
|||||||
--format json
|
--format json
|
||||||
```
|
```
|
||||||
|
|
||||||
## Final Checks
|
Also run smoke commands for any new examples, such as:
|
||||||
|
|
||||||
Before considering the feature complete:
|
```bash
|
||||||
|
go run ./examples/go-library/prepare
|
||||||
|
```
|
||||||
|
|
||||||
1. Confirm `git diff` contains only intended code, test, doc, and example changes.
|
If an example relies on a specific built-in profile ID, use an ID from `docs/roadmap/profiles/` and include that smoke command in the implementing change.
|
||||||
2. Confirm all non-roadmap docs describe implemented behavior only.
|
|
||||||
3. Confirm no output path exposes raw API key values.
|
## Final Completion Checklist
|
||||||
4. Confirm `go test ./...` passes.
|
|
||||||
5. Confirm the render smoke command passes.
|
Before marking the combined feature complete:
|
||||||
|
|
||||||
|
1. `go test ./...` passes.
|
||||||
|
2. Existing CLI render smoke command passes.
|
||||||
|
3. At least one smoke command proves built-in profile lookup works without `profile_dir`.
|
||||||
|
4. Public package examples compile/run without real provider credentials.
|
||||||
|
5. `profile_dir` is optional in CLI, HTTP, and public engine construction.
|
||||||
|
6. Custom profiles override built-ins with the same ID.
|
||||||
|
7. Malformed selected custom profiles do not fall back to built-ins.
|
||||||
|
8. Built-in profiles use the same validation rules as file profiles.
|
||||||
|
9. Public `RunRequest.APIKey` is the documented library credential method.
|
||||||
|
10. Raw API keys do not appear in YAML/JSON config, prepared output, run results, logs, hashes, or examples.
|
||||||
|
11. Non-roadmap docs describe only implemented behavior.
|
||||||
|
|||||||
176
docs/roadmap/library.md
Normal file
176
docs/roadmap/library.md
Normal file
@@ -0,0 +1,176 @@
|
|||||||
|
# Library API Production Roadmap
|
||||||
|
|
||||||
|
This roadmap defines the target state for making Scriptorium's public Go package production-ready for downstream applications while preserving the existing CLI and HTTP behavior.
|
||||||
|
|
||||||
|
The library remains an additional adapter surface. It should not replace the subprocess, CLI, or HTTP contracts that already exist.
|
||||||
|
|
||||||
|
## Motivation
|
||||||
|
|
||||||
|
Many Go applications can use Scriptorium more cleanly as an imported package than as a subprocess. A downstream developer should be able to keep prompt assets in standard Scriptorium format, pass application data through a small adapter, and receive a typed response without reimplementing prompt rendering, profile resolution, validation, or OpenAI-compatible request construction.
|
||||||
|
|
||||||
|
The core consumer story is:
|
||||||
|
|
||||||
|
- the downstream app owns one or more `prompt.yml` files in standard Scriptorium format;
|
||||||
|
- those prompts may use inline content, `content_file` references, variables, cache-control markers, and structured-output schemas;
|
||||||
|
- the app may provide its own `profile.yml`, or select a standard built-in/profile-template configuration;
|
||||||
|
- the app supplies an API key as a normal Go value when the selected profile requires one;
|
||||||
|
- the app calls the public Go package to prepare or run the request and receives typed results.
|
||||||
|
|
||||||
|
## Current State
|
||||||
|
|
||||||
|
The public package already provides the first library facade:
|
||||||
|
|
||||||
|
- root package import;
|
||||||
|
- typed engine construction;
|
||||||
|
- typed prepare/run requests and results;
|
||||||
|
- file and inline artifact references;
|
||||||
|
- execution overrides;
|
||||||
|
- custom LLM injection for testing or alternate execution;
|
||||||
|
- public error categories that map internal failures to stable caller-facing errors.
|
||||||
|
|
||||||
|
The remaining production-readiness gaps are mostly about consumer ergonomics and asset sourcing:
|
||||||
|
|
||||||
|
- callers are still oriented around filesystem prompt/profile/schema directories;
|
||||||
|
- embedded prompt/profile/schema assets are not a first-class public use case;
|
||||||
|
- standard or built-in profile selection is not yet available;
|
||||||
|
- library credential supply is still tightly coupled to environment-variable lookup rather than direct API-key values;
|
||||||
|
- public documentation and examples need to show the intended downstream app adapter pattern.
|
||||||
|
|
||||||
|
## Target State
|
||||||
|
|
||||||
|
The public package should let a downstream Go application use standard Scriptorium assets without temporary directories, subprocess invocation, or internal package imports.
|
||||||
|
|
||||||
|
### Prompt Assets
|
||||||
|
|
||||||
|
The library should support prompt definitions from:
|
||||||
|
|
||||||
|
- existing prompt directories;
|
||||||
|
- a single prompt file;
|
||||||
|
- `fs.FS`, including `embed.FS`.
|
||||||
|
|
||||||
|
Prompt syntax should remain the standard Scriptorium prompt YAML format. `content_file` references should continue to be supported and should resolve relative to the prompt definition's source location within the same asset source.
|
||||||
|
|
||||||
|
The public API should not introduce a separate in-code prompt DSL as the primary path. YAML remains the canonical authoring format so prompts can be shared between CLI, HTTP, subprocess, and library usage.
|
||||||
|
|
||||||
|
### Schema Assets
|
||||||
|
|
||||||
|
Structured-output schemas should be loadable from the same kinds of sources as prompt definitions:
|
||||||
|
|
||||||
|
- existing schema directories;
|
||||||
|
- a single schema file where appropriate;
|
||||||
|
- `fs.FS`, including `embed.FS`.
|
||||||
|
|
||||||
|
Schema references should retain the existing prompt-format semantics. A schema referenced by a prompt should resolve through the configured schema source, not through ad hoc caller code.
|
||||||
|
|
||||||
|
### Profile Assets
|
||||||
|
|
||||||
|
The library should support both custom and standard profile configuration:
|
||||||
|
|
||||||
|
- existing profile directories;
|
||||||
|
- a single profile file;
|
||||||
|
- `fs.FS`, including `embed.FS`;
|
||||||
|
- direct public profile values for applications that already have profile configuration in memory;
|
||||||
|
- built-in/profile-template helpers for common OpenAI-compatible targets.
|
||||||
|
|
||||||
|
Custom profiles and built-in/template profiles should flow through the same internal profile resolution and request-construction path. The built-in path should not become a separate execution mode.
|
||||||
|
|
||||||
|
### Built-In Profile Templates
|
||||||
|
|
||||||
|
Built-in support should favor stable profile templates over a large registry of fixed model IDs.
|
||||||
|
|
||||||
|
For example, the public package should make it easy to construct or select an OpenAI-compatible profile by supplying the durable parts of the profile:
|
||||||
|
|
||||||
|
- profile ID or name;
|
||||||
|
- base URL;
|
||||||
|
- model;
|
||||||
|
- whether the profile requires an API key;
|
||||||
|
- default numeric parameters where desired;
|
||||||
|
- structured-output and extra-parameter behavior consistent with normal profiles.
|
||||||
|
|
||||||
|
The package may include a small set of named helpers for common OpenAI-compatible services, but those helpers should avoid hard-coding a broad and fast-changing list of model names.
|
||||||
|
|
||||||
|
### Credentials
|
||||||
|
|
||||||
|
The public package must keep raw API keys out of prompt/profile YAML, prepared-run output, run results, logs, and examples.
|
||||||
|
|
||||||
|
For the public library API, the single supported credential-supply method should be a direct API-key value passed by the consuming Go application. The consuming application is responsible for loading and managing its own secrets before calling Scriptorium.
|
||||||
|
|
||||||
|
This may be exposed as a field such as `Config.APIKey`, an option such as `WithAPIKey`, or an equivalent request/engine-level value that is easy to pass through an application adapter. The exact API should avoid accidental serialization in prepared output, run results, logs, and examples.
|
||||||
|
|
||||||
|
The public package should not encourage raw API-key storage in prompt/profile YAML. Existing CLI behavior may continue to use environment-variable references for compatibility, but the production library path should not introduce a separate credential resolver or secret-manager abstraction.
|
||||||
|
|
||||||
|
### Public API Shape
|
||||||
|
|
||||||
|
The public API should remain narrow, idiomatic, and stable. Recommended additions include:
|
||||||
|
|
||||||
|
- engine options for prompt/profile/schema directories;
|
||||||
|
- engine options for prompt/profile/schema `fs.FS` sources;
|
||||||
|
- engine options for single prompt/profile/schema files where useful;
|
||||||
|
- public profile/template constructors that map to internal profile definitions;
|
||||||
|
- a direct API-key value for profiles that require authentication;
|
||||||
|
- examples showing `embed.FS`, custom profile files, template profile selection, and fake LLM testing.
|
||||||
|
|
||||||
|
The public package should continue to expose facade types rather than exporting internal package types. Internal package layout should remain free to evolve.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
In scope:
|
||||||
|
|
||||||
|
- first-class `fs.FS` support for public library prompt, profile, and schema sources;
|
||||||
|
- ergonomic single-file asset options where they reduce caller boilerplate;
|
||||||
|
- built-in/profile-template helpers for common OpenAI-compatible usage;
|
||||||
|
- in-memory public profile values where appropriate;
|
||||||
|
- direct API-key value support for the public library path;
|
||||||
|
- consumer-facing examples under `examples/`;
|
||||||
|
- consumer package documentation under `docs/consumers/` once behavior is implemented;
|
||||||
|
- tests proving library behavior matches existing CLI/use-case behavior.
|
||||||
|
|
||||||
|
Out of scope:
|
||||||
|
|
||||||
|
- changing standard prompt, profile, or schema file formats;
|
||||||
|
- exposing internal packages as public API;
|
||||||
|
- replacing or removing CLI, HTTP, or subprocess support;
|
||||||
|
- adding a multi-step workflow engine;
|
||||||
|
- adding non-Go bindings;
|
||||||
|
- maintaining a comprehensive provider/model catalog;
|
||||||
|
- accepting raw API keys in serialized YAML/JSON configuration;
|
||||||
|
- adding a credential resolver or secret-manager abstraction.
|
||||||
|
|
||||||
|
## Acceptance Criteria
|
||||||
|
|
||||||
|
- A Go caller can import the root package and run a standard Scriptorium prompt without invoking a subprocess.
|
||||||
|
- A Go caller can use prompt definitions from `embed.FS`, including prompts with `content_file` references.
|
||||||
|
- A Go caller can use structured-output schemas from `embed.FS` or filesystem sources.
|
||||||
|
- A Go caller can provide a custom profile from filesystem, `fs.FS`, or public in-memory profile values.
|
||||||
|
- A Go caller can select a standard OpenAI-compatible profile template without writing a full profile file.
|
||||||
|
- A Go caller can supply an API key as a normal Go value without raw secrets appearing in serialized config, prepared output, or results.
|
||||||
|
- Public library behavior remains consistent with CLI/HTTP semantics for rendering, validation, profile resolution, runtime overrides, structured output, cache control, and LLM invocation.
|
||||||
|
- Existing CLI and HTTP behavior remains unchanged.
|
||||||
|
- Library tests use fake or local LLM boundaries and do not require real provider credentials.
|
||||||
|
- Public docs outside `docs/roadmap/` describe only implemented behavior after the feature is built.
|
||||||
|
|
||||||
|
## Design Decisions
|
||||||
|
|
||||||
|
### Asset Source API
|
||||||
|
|
||||||
|
Add first-class `fs.FS` options for prompt, profile, and schema sources while keeping existing directory-based configuration. Also add single-file convenience options where they remove meaningful caller boilerplate. Resolve `content_file` references relative to the prompt file's location inside the same source.
|
||||||
|
|
||||||
|
Reasoning:
|
||||||
|
|
||||||
|
This is the most idiomatic path for production Go libraries because it supports `embed.FS`, `os.DirFS`, tests, and in-memory fixture files through the same abstraction. It also avoids requiring downstream applications to unpack embedded assets into temporary directories.
|
||||||
|
|
||||||
|
### Built-In Profile Strategy
|
||||||
|
|
||||||
|
Provide stable profile-template helpers for OpenAI-compatible endpoints rather than a broad registry of fixed provider/model profiles. Let callers choose the model and endpoint where those values are service-specific or fast-changing.
|
||||||
|
|
||||||
|
Reasoning:
|
||||||
|
|
||||||
|
Endpoint shape and credential mechanics are relatively stable; model catalogs change frequently. Templates give consumers a short, correct path without making Scriptorium responsible for tracking every provider's model list.
|
||||||
|
|
||||||
|
### In-Memory Profile Values
|
||||||
|
|
||||||
|
Expose a small public profile facade type for in-memory profile configuration and map it to internal profile definitions. Keep it intentionally aligned with the existing profile YAML contract.
|
||||||
|
|
||||||
|
Reasoning:
|
||||||
|
|
||||||
|
Many applications already hold configuration in typed structs and should not need to generate YAML files just to call Scriptorium. A public facade keeps internal types private while making the library practical for production use.
|
||||||
@@ -1,96 +0,0 @@
|
|||||||
# Runtime Parameter Feature Roadmap
|
|
||||||
|
|
||||||
This roadmap defines the target behavior for runtime model parameters.
|
|
||||||
|
|
||||||
Current behavior has two limitations:
|
|
||||||
|
|
||||||
- `reasoning_effort` and `extra_params` are parsed into effective execution settings but are not serialized into outbound OpenAI-compatible chat-completions requests.
|
|
||||||
- Per-request numeric execution overrides use zero-value merge semantics, so callers cannot reliably override a profile value with an explicit zero such as `temperature: 0`.
|
|
||||||
|
|
||||||
The implementation plan for this feature lives in `docs/roadmap/implementation.md`.
|
|
||||||
|
|
||||||
## Target State
|
|
||||||
|
|
||||||
Scriptorium should preserve the existing separation between prompt definitions, execution profiles, and per-request execution overrides while making runtime parameter behavior explicit and predictable.
|
|
||||||
|
|
||||||
Expected end state:
|
|
||||||
|
|
||||||
- Effective execution settings remain visible in prepared-run output, run metadata, and HTTP metadata without exposing raw secret values.
|
|
||||||
- `reasoning_effort` is treated as a first-class effective execution setting and is serialized to the outbound OpenAI-compatible request when set.
|
|
||||||
- `extra_params` supports provider-specific OpenAI-compatible request fields.
|
|
||||||
- `extra_params` is serialized as additional top-level outbound JSON fields.
|
|
||||||
- `extra_params` values support JSON-compatible scalar, object, and array values.
|
|
||||||
- `extra_params` cannot override first-class outbound request fields.
|
|
||||||
- Per-request numeric overrides preserve caller intent, including explicit zero values.
|
|
||||||
- Omitted per-request numeric overrides continue to inherit the selected profile and built-in defaults.
|
|
||||||
- External decoding remains strict for config, prompt, profile, and HTTP request payloads.
|
|
||||||
|
|
||||||
## Policy Decisions
|
|
||||||
|
|
||||||
### `extra_params`
|
|
||||||
|
|
||||||
`extra_params` should serialize as additional top-level outbound JSON fields in the OpenAI-compatible chat-completions request.
|
|
||||||
|
|
||||||
Reasoning:
|
|
||||||
|
|
||||||
Most OpenAI-compatible providers expose vendor-specific chat-completions parameters as top-level fields. This keeps Scriptorium's adapter compatible with that ecosystem without adding first-class fields for every provider option.
|
|
||||||
|
|
||||||
`extra_params` must not silently override Scriptorium-owned fields. Reserved outbound fields include at least:
|
|
||||||
|
|
||||||
- `model`
|
|
||||||
- `session_id`
|
|
||||||
- `messages`
|
|
||||||
- `temperature`
|
|
||||||
- `max_tokens`
|
|
||||||
- `top_p`
|
|
||||||
- `service_tier`
|
|
||||||
- `reasoning_effort`
|
|
||||||
- `response_format`
|
|
||||||
|
|
||||||
If a caller supplies a reserved key through `extra_params`, Scriptorium should fail before making the outbound HTTP request.
|
|
||||||
|
|
||||||
`extra_params` should use JSON-compatible values rather than only strings.
|
|
||||||
|
|
||||||
Reasoning:
|
|
||||||
|
|
||||||
Provider-specific parameters commonly need booleans, numbers, objects, or arrays. String-only values would force awkward encoding and would likely require a later compatibility break.
|
|
||||||
|
|
||||||
### Presence-Aware Overrides
|
|
||||||
|
|
||||||
Per-request execution overrides should use a presence-aware type with pointer fields for optional numeric values.
|
|
||||||
|
|
||||||
Reasoning:
|
|
||||||
|
|
||||||
The resolved execution target should remain a concrete value used by prepared runs, generated requests, and metadata. Optionality matters at the request boundary, not after the runner has resolved the effective target.
|
|
||||||
|
|
||||||
This keeps adapter and merge logic precise while avoiding nil checks in formatter, metadata, and LLM serialization paths.
|
|
||||||
|
|
||||||
## Scope
|
|
||||||
|
|
||||||
In scope:
|
|
||||||
|
|
||||||
- Runtime merge behavior for per-request execution overrides.
|
|
||||||
- HTTP model override decoding for explicit zero numeric values.
|
|
||||||
- CLI execution override handling for explicit zero numeric flags.
|
|
||||||
- Outbound serialization of `reasoning_effort`.
|
|
||||||
- Outbound serialization of JSON-compatible `extra_params`.
|
|
||||||
- Tests and documentation for the changed implemented behavior.
|
|
||||||
|
|
||||||
Out of scope:
|
|
||||||
|
|
||||||
- Expanding the HTTP API beyond `POST /v1/runs`.
|
|
||||||
- Adding built-in HTTP authentication or authorization.
|
|
||||||
- Adding durable run state, run history, or multi-step orchestration.
|
|
||||||
- Adding broad provider-specific adapter packages.
|
|
||||||
- Adding new CLI flags for every provider-specific parameter.
|
|
||||||
|
|
||||||
## Acceptance Criteria
|
|
||||||
|
|
||||||
- A profile containing `reasoning_effort: medium` produces an outbound request with `reasoning_effort`.
|
|
||||||
- HTTP callers can pass `reasoning_effort` through the existing `model` override object and have it appear outbound.
|
|
||||||
- A profile or HTTP request containing JSON-compatible `extra_params` produces outbound top-level JSON fields according to the reserved-field policy.
|
|
||||||
- Reserved `extra_params` collisions fail before the outbound provider call.
|
|
||||||
- CLI callers can pass `--temperature 0` and observe `temperature: 0` in rendered/effective settings and outbound requests.
|
|
||||||
- HTTP callers can send `"temperature": 0` and observe the same behavior.
|
|
||||||
- Omitting `temperature` continues to preserve the selected profile/default value.
|
|
||||||
- Raw API key values remain unsupported in config, profiles, CLI flags, HTTP payloads, logs, and rendered output.
|
|
||||||
9
docs/roadmap/profiles/aion-labs/aion-2.yml
Normal file
9
docs/roadmap/profiles/aion-labs/aion-2.yml
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: aion-2
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: aion-labs/aion-2.0
|
||||||
|
temperature: 0.72
|
||||||
|
reasoning_effort: high
|
||||||
|
top_p: 0.95
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
7
docs/roadmap/profiles/anthropic/claude-fable-latest.yml
Normal file
7
docs/roadmap/profiles/anthropic/claude-fable-latest.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
id: claude-fable-latest
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "~anthropic/claude-fable-latest"
|
||||||
|
reasoning_effort: high
|
||||||
|
timeout_seconds: 600
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
7
docs/roadmap/profiles/anthropic/claude-haiku-latest.yml
Normal file
7
docs/roadmap/profiles/anthropic/claude-haiku-latest.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
id: claude-haiku-latest
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "~anthropic/claude-haiku-latest"
|
||||||
|
reasoning_effort: medium
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
7
docs/roadmap/profiles/anthropic/claude-opus-latest.yml
Normal file
7
docs/roadmap/profiles/anthropic/claude-opus-latest.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
id: claude-opus-latest
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "~anthropic/claude-opus-latest"
|
||||||
|
reasoning_effort: high
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
7
docs/roadmap/profiles/anthropic/claude-sonnet-latest.yml
Normal file
7
docs/roadmap/profiles/anthropic/claude-sonnet-latest.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
id: claude-sonnet-latest
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "~anthropic/claude-sonnet-latest"
|
||||||
|
reasoning_effort: high
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
7
docs/roadmap/profiles/deepseek/deepseek-3-2.yml
Normal file
7
docs/roadmap/profiles/deepseek/deepseek-3-2.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
id: deepseek-3-2
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: deepseek/deepseek-v3.2
|
||||||
|
reasoning_effort: high
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
7
docs/roadmap/profiles/deepseek/deepseek-4-pro.yml
Normal file
7
docs/roadmap/profiles/deepseek/deepseek-4-pro.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
id: deepseek-4-pro
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: deepseek/deepseek-v4-pro
|
||||||
|
reasoning_effort: high
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
9
docs/roadmap/profiles/google/gemini-2-flash-lite.yml
Normal file
9
docs/roadmap/profiles/google/gemini-2-flash-lite.yml
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: gemini-2-flash-lite
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "google/gemini-2.5-flash-lite"
|
||||||
|
#temperature: 0.15
|
||||||
|
reasoning_effort: high
|
||||||
|
#top_p: 0.98
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
9
docs/roadmap/profiles/google/gemini-2-flash.yml
Normal file
9
docs/roadmap/profiles/google/gemini-2-flash.yml
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: gemini-2-flash
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "google/gemini-2.5-flash"
|
||||||
|
#temperature: 0.15
|
||||||
|
reasoning_effort: high
|
||||||
|
#top_p: 0.98
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
9
docs/roadmap/profiles/google/gemini-2-pro.yml
Normal file
9
docs/roadmap/profiles/google/gemini-2-pro.yml
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: gemini-2-pro
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "google/gemini-2.5-pro"
|
||||||
|
#temperature: 0.15
|
||||||
|
reasoning_effort: high
|
||||||
|
#top_p: 0.98
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
9
docs/roadmap/profiles/google/gemini-3-flash-lite.yml
Normal file
9
docs/roadmap/profiles/google/gemini-3-flash-lite.yml
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: gemini-3-flash-lite
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "google/gemini-3.1-flash-lite"
|
||||||
|
#temperature: 0.15
|
||||||
|
reasoning_effort: high
|
||||||
|
#top_p: 0.98
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
9
docs/roadmap/profiles/google/gemini-flash-latest.yml
Normal file
9
docs/roadmap/profiles/google/gemini-flash-latest.yml
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: gemini-flash-latest
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "~google/gemini-flash-latest"
|
||||||
|
#temperature: 0.15
|
||||||
|
reasoning_effort: high
|
||||||
|
#top_p: 0.98
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
9
docs/roadmap/profiles/google/gemini-pro-latest.yml
Normal file
9
docs/roadmap/profiles/google/gemini-pro-latest.yml
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: gemini-pro-latest
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "~google/gemini-pro-latest"
|
||||||
|
#temperature: 0.15
|
||||||
|
reasoning_effort: high
|
||||||
|
#top_p: 0.98
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
9
docs/roadmap/profiles/google/gemma-4-31b.yml
Normal file
9
docs/roadmap/profiles/google/gemma-4-31b.yml
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: gemma-4-31b
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: google/gemma-4-31b-it:exacto
|
||||||
|
temperature: 0.15
|
||||||
|
reasoning_effort: high
|
||||||
|
top_p: 0.98
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
9
docs/roadmap/profiles/minimax/minimax-m2.yml
Normal file
9
docs/roadmap/profiles/minimax/minimax-m2.yml
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: minimax-m2
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: minimax/minimax-m2.5
|
||||||
|
temperature: 0.5
|
||||||
|
reasoning_effort: high
|
||||||
|
top_p: 0.95
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
9
docs/roadmap/profiles/minimax/minimax-m3.yml
Normal file
9
docs/roadmap/profiles/minimax/minimax-m3.yml
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
id: minimax-m3
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: minimax/minimax-m3
|
||||||
|
#temperature: 0.5
|
||||||
|
reasoning_effort: high
|
||||||
|
#top_p: 0.95
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
7
docs/roadmap/profiles/mistral/mistral-large-2512.yml
Normal file
7
docs/roadmap/profiles/mistral/mistral-large-2512.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
id: mistral-large-2512
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: mistralai/mistral-large-2512
|
||||||
|
temperature: 0.15
|
||||||
|
top_p: 0.98
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
8
docs/roadmap/profiles/mistral/mistral-medium-3-5.yml
Normal file
8
docs/roadmap/profiles/mistral/mistral-medium-3-5.yml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
id: mistral-medium-3-5
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: mistralai/mistral-medium-3-5
|
||||||
|
temperature: 0.15
|
||||||
|
reasoning_effort: high
|
||||||
|
top_p: 0.98
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
7
docs/roadmap/profiles/mistral/mistral-small-3.yml
Normal file
7
docs/roadmap/profiles/mistral/mistral-small-3.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
id: mistral-small-3
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: mistralai/mistral-small-3.2-24b-instruct
|
||||||
|
temperature: 0.05
|
||||||
|
top_p: 1.0
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
8
docs/roadmap/profiles/mistral/mistral-small-4.yml
Normal file
8
docs/roadmap/profiles/mistral/mistral-small-4.yml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
id: mistral-small-4
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: mistralai/mistral-small-2603
|
||||||
|
temperature: 0.1
|
||||||
|
reasoning_effort: high
|
||||||
|
top_p: 0.98
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
7
docs/roadmap/profiles/nvidia/nemotron-3-ultra.yml
Normal file
7
docs/roadmap/profiles/nvidia/nemotron-3-ultra.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
id: nemotron-3-ultra
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: nvidia/nemotron-3-ultra-550b-a55b
|
||||||
|
reasoning_effort: high
|
||||||
|
timeout_seconds: 180
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
7
docs/roadmap/profiles/openai/gpt-5-mini.yml
Normal file
7
docs/roadmap/profiles/openai/gpt-5-mini.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
id: gpt-5-mini
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "openai/gpt-5.4-mini"
|
||||||
|
reasoning_effort: high
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
7
docs/roadmap/profiles/openai/gpt-5-nano.yml
Normal file
7
docs/roadmap/profiles/openai/gpt-5-nano.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
id: gpt-5-nano
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
model: "openai/gpt-5.4-nano"
|
||||||
|
reasoning_effort: high
|
||||||
|
timeout_seconds: 240
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
service_tier: flex
|
||||||
141
engine.go
Normal file
141
engine.go
Normal file
@@ -0,0 +1,141 @@
|
|||||||
|
package scriptorium
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
artifactadapter "gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrInvalidConfig indicates invalid public engine configuration.
|
||||||
|
var ErrInvalidConfig = errors.New("invalid engine configuration")
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrInvalidRequest = errors.New("invalid run request")
|
||||||
|
ErrPromptNotFound = errors.New("prompt not found")
|
||||||
|
ErrProfileNotFound = errors.New("profile not found")
|
||||||
|
ErrPromptLoad = errors.New("failed to load prompt definition")
|
||||||
|
ErrProfileLoad = errors.New("failed to load execution profile")
|
||||||
|
ErrArtifactLoad = errors.New("failed to load artifact")
|
||||||
|
ErrPromptRender = errors.New("failed to render prompt")
|
||||||
|
ErrLLMGenerate = errors.New("failed to generate output")
|
||||||
|
ErrValidation = errors.New("failed to validate output")
|
||||||
|
)
|
||||||
|
|
||||||
|
// Engine prepares and runs Scriptorium prompt requests.
|
||||||
|
type Engine struct {
|
||||||
|
runner *usecase.Runner
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config configures a public Scriptorium engine.
|
||||||
|
type Config struct {
|
||||||
|
PromptDir string
|
||||||
|
ProfileDir string
|
||||||
|
SchemaDir string
|
||||||
|
Timeout time.Duration
|
||||||
|
HTTPClient *http.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
// Option customizes engine construction.
|
||||||
|
type Option func(*engineOptions) error
|
||||||
|
|
||||||
|
type engineOptions struct {
|
||||||
|
llmClient llm.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithLLMClient injects a custom LLM client for execution.
|
||||||
|
func WithLLMClient(client LLMClient) Option {
|
||||||
|
return func(options *engineOptions) error {
|
||||||
|
if client == nil {
|
||||||
|
return ErrInvalidConfig
|
||||||
|
}
|
||||||
|
options.llmClient = publicLLMClientAdapter{client: client}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewEngine constructs an Engine using the same default internal components as
|
||||||
|
// the CLI and HTTP adapters.
|
||||||
|
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||||
|
if strings.TrimSpace(cfg.PromptDir) == "" {
|
||||||
|
return nil, fmt.Errorf("%w: prompt directory is required", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(cfg.ProfileDir) == "" {
|
||||||
|
return nil, fmt.Errorf("%w: profile directory is required", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
var options engineOptions
|
||||||
|
for _, opt := range opts {
|
||||||
|
if opt == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := opt(&options); err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
schemaDir := cfg.SchemaDir
|
||||||
|
if strings.TrimSpace(schemaDir) == "" {
|
||||||
|
schemaDir = defaults.SchemaDirDefault
|
||||||
|
}
|
||||||
|
|
||||||
|
llmClient := options.llmClient
|
||||||
|
if llmClient == nil {
|
||||||
|
var err error
|
||||||
|
llmClient, err = llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
|
||||||
|
Timeout: cfg.Timeout,
|
||||||
|
HTTPClient: cfg.HTTPClient,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Engine{
|
||||||
|
runner: usecase.NewRunner(
|
||||||
|
promptdef.NewFilesystemRepository(cfg.PromptDir),
|
||||||
|
profile.NewFilesystemRepository(cfg.ProfileDir),
|
||||||
|
artifactadapter.NewCompositeReader(),
|
||||||
|
prompt.NewGoRenderer(),
|
||||||
|
llmClient,
|
||||||
|
validate.NewStandardValidator(schemaDir),
|
||||||
|
),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prepare resolves a prompt request without calling an LLM.
|
||||||
|
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
||||||
|
if e == nil || e.runner == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := e.runner.Prepare(ctx, toDomainRunRequest(req))
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return fromDomainPreparedRun(prepared), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run executes a prompt request and returns the generated artifact and metadata.
|
||||||
|
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||||
|
if e == nil || e.runner == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := e.runner.Run(ctx, toDomainRunRequest(req))
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return fromDomainRunResult(result), nil
|
||||||
|
}
|
||||||
559
engine_test.go
Normal file
559
engine_test.go
Normal file
@@ -0,0 +1,559 @@
|
|||||||
|
package scriptorium_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewEngineRejectsMissingPromptDir(t *testing.T) {
|
||||||
|
_, err := scriptorium.NewEngine(scriptorium.Config{ProfileDir: "./examples/profiles"})
|
||||||
|
if !errors.Is(err, scriptorium.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("expected ErrInvalidConfig, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewEngineRejectsMissingProfileDir(t *testing.T) {
|
||||||
|
_, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: "./examples/prompts"})
|
||||||
|
if !errors.Is(err, scriptorium.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("expected ErrInvalidConfig, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareWorksWithExampleDirectoriesAndFileInputs(t *testing.T) {
|
||||||
|
engine := newExampleEngine(t)
|
||||||
|
|
||||||
|
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||||
|
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected prepare to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
if prepared.PromptID != "generic.markdown_summary" {
|
||||||
|
t.Fatalf("unexpected prompt id: %q", prepared.PromptID)
|
||||||
|
}
|
||||||
|
if prepared.SelectedProfileID != "local-fast" {
|
||||||
|
t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.Model != "gpt-4o-mini" {
|
||||||
|
t.Fatalf("unexpected effective model: %q", prepared.EffectiveModelParams.Model)
|
||||||
|
}
|
||||||
|
if len(prepared.Messages) != 2 {
|
||||||
|
t.Fatalf("expected rendered messages, got %d", len(prepared.Messages))
|
||||||
|
}
|
||||||
|
if prepared.InputHashes["transcript"] == "" || prepared.InputHashes["glossary"] == "" {
|
||||||
|
t.Fatalf("expected input hashes, got %#v", prepared.InputHashes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareWorksWithInlineInputs(t *testing.T) {
|
||||||
|
engine := newExampleEngine(t)
|
||||||
|
|
||||||
|
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.Inline("Rin scouts the tower.\nKara lights a lantern."),
|
||||||
|
"glossary": scriptorium.InlineWithURI("memory://glossary.yml", "party:\n - Rin\n - Kara\n"),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected prepare to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
if len(prepared.Messages) != 2 {
|
||||||
|
t.Fatalf("expected rendered messages, got %d", len(prepared.Messages))
|
||||||
|
}
|
||||||
|
rendered := prepared.Messages[1].Content
|
||||||
|
if !strings.Contains(rendered, "Rin scouts the tower.") || !strings.Contains(rendered, "party:") {
|
||||||
|
t.Fatalf("expected inline inputs in rendered prompt, got %q", rendered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparedRunJSONDoesNotExposeSecretOrTargetPresence(t *testing.T) {
|
||||||
|
const envName = "SCRIPTORIUM_API_KEY"
|
||||||
|
const secret = "public-api-test-secret"
|
||||||
|
t.Setenv(envName, secret)
|
||||||
|
|
||||||
|
engine := newExampleEngine(t)
|
||||||
|
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.structured_events",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||||
|
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected prepare to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
payload, err := json.Marshal(prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected prepared run to marshal, got %v", err)
|
||||||
|
}
|
||||||
|
out := string(payload)
|
||||||
|
if strings.Contains(out, secret) {
|
||||||
|
t.Fatalf("prepared run JSON leaked raw API key value: %s", out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, envName) {
|
||||||
|
t.Fatalf("prepared run JSON should retain api_key_env name, got %s", out)
|
||||||
|
}
|
||||||
|
for _, forbidden := range []string{"TargetPresence", "target_presence"} {
|
||||||
|
if strings.Contains(out, forbidden) {
|
||||||
|
t.Fatalf("prepared run JSON exposed internal target presence metadata %q: %s", forbidden, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparePreservesExplicitZeroExecutionOverrides(t *testing.T) {
|
||||||
|
engine := newExampleEngine(t)
|
||||||
|
zeroFloat := 0.0
|
||||||
|
zeroInt := 0
|
||||||
|
|
||||||
|
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||||
|
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||||
|
},
|
||||||
|
Execution: &scriptorium.ExecutionTargetOverride{
|
||||||
|
Temperature: &zeroFloat,
|
||||||
|
MaxTokens: &zeroInt,
|
||||||
|
TopP: &zeroFloat,
|
||||||
|
TimeoutSeconds: &zeroInt,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected prepare to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
target := prepared.EffectiveModelParams
|
||||||
|
if target.Temperature != 0 || target.MaxTokens != 0 || target.TopP != 0 || target.TimeoutSeconds != 0 {
|
||||||
|
t.Fatalf("expected explicit zero overrides in effective target, got %+v", target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunSucceedsWithInjectedLLMClient(t *testing.T) {
|
||||||
|
const envName = "SCRIPTORIUM_API_KEY"
|
||||||
|
const secret = "run-secret-value"
|
||||||
|
t.Setenv(envName, secret)
|
||||||
|
|
||||||
|
fake := &fakeLLMClient{
|
||||||
|
response: &scriptorium.GenerateResponse{
|
||||||
|
Content: "# Summary\n\nDone.",
|
||||||
|
Usage: scriptorium.TokenUsage{
|
||||||
|
PromptTokens: 10,
|
||||||
|
CompletionTokens: 5,
|
||||||
|
TotalTokens: 15,
|
||||||
|
CachedTokens: 3,
|
||||||
|
CacheWriteTokens: 2,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake))
|
||||||
|
|
||||||
|
result, err := engine.Run(context.Background(), scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||||
|
"glossary": scriptorium.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
Execution: &scriptorium.ExecutionTargetOverride{
|
||||||
|
APIKeyEnv: envName,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected run to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
if result.RunID == "" {
|
||||||
|
t.Fatalf("expected run id")
|
||||||
|
}
|
||||||
|
if result.RawOutput != fake.response.Content {
|
||||||
|
t.Fatalf("unexpected raw output: %q", result.RawOutput)
|
||||||
|
}
|
||||||
|
if string(result.Artifact.Body) != fake.response.Content {
|
||||||
|
t.Fatalf("unexpected artifact body: %q", string(result.Artifact.Body))
|
||||||
|
}
|
||||||
|
if result.Artifact.ContentType != "text/markdown" {
|
||||||
|
t.Fatalf("unexpected artifact content type: %q", result.Artifact.ContentType)
|
||||||
|
}
|
||||||
|
if result.Validation.Status != scriptorium.ValidationPassed || !result.Validation.IsValid {
|
||||||
|
t.Fatalf("expected passed validation, got %+v", result.Validation)
|
||||||
|
}
|
||||||
|
if result.PromptID != "generic.markdown_summary" || result.SelectedProfileID != "local-fast" || result.ModelName != "gpt-4o-mini" {
|
||||||
|
t.Fatalf("unexpected run metadata: %+v", result)
|
||||||
|
}
|
||||||
|
if result.Usage.TotalTokens != 15 || result.Usage.CachedTokens != 3 || result.Usage.CacheWriteTokens != 2 {
|
||||||
|
t.Fatalf("unexpected usage: %+v", result.Usage)
|
||||||
|
}
|
||||||
|
|
||||||
|
payload, err := json.Marshal(result)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected run result to marshal, got %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(payload), secret) {
|
||||||
|
t.Fatalf("run result JSON leaked raw API key value: %s", payload)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) {
|
||||||
|
fake := &fakeLLMClient{
|
||||||
|
response: &scriptorium.GenerateResponse{Content: "ok"},
|
||||||
|
}
|
||||||
|
engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake))
|
||||||
|
zeroFloat := 0.0
|
||||||
|
zeroInt := 0
|
||||||
|
|
||||||
|
_, err := engine.Run(context.Background(), scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||||
|
"glossary": scriptorium.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
Execution: &scriptorium.ExecutionTargetOverride{
|
||||||
|
Temperature: &zeroFloat,
|
||||||
|
MaxTokens: &zeroInt,
|
||||||
|
TopP: &zeroFloat,
|
||||||
|
TimeoutSeconds: &zeroInt,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected run to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
if len(fake.requests) != 1 {
|
||||||
|
t.Fatalf("expected one generate request, got %d", len(fake.requests))
|
||||||
|
}
|
||||||
|
req := fake.requests[0]
|
||||||
|
if len(req.Prompt.Messages) != 2 || !strings.Contains(req.Prompt.Messages[1].Content, "Rin opens the gate.") {
|
||||||
|
t.Fatalf("expected rendered prompt in generate request, got %+v", req.Prompt)
|
||||||
|
}
|
||||||
|
if req.Target.Model != "gpt-4o-mini" || req.Target.Temperature != 0 || req.Target.MaxTokens != 0 || req.Target.TopP != 0 || req.Target.TimeoutSeconds != 0 {
|
||||||
|
t.Fatalf("unexpected effective target: %+v", req.Target)
|
||||||
|
}
|
||||||
|
if !req.TargetPresence.Temperature || !req.TargetPresence.MaxTokens || !req.TargetPresence.TopP || !req.TargetPresence.TimeoutSeconds {
|
||||||
|
t.Fatalf("expected explicit zero target presence, got %+v", req.TargetPresence)
|
||||||
|
}
|
||||||
|
if req.StructuredOutput != nil {
|
||||||
|
t.Fatalf("did not expect structured output for markdown prompt: %+v", req.StructuredOutput)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunValidationFailureReturnsResult(t *testing.T) {
|
||||||
|
fake := &fakeLLMClient{
|
||||||
|
response: &scriptorium.GenerateResponse{Content: ""},
|
||||||
|
}
|
||||||
|
engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake))
|
||||||
|
|
||||||
|
result, err := engine.Run(context.Background(), scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||||
|
"glossary": scriptorium.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected validation failure as successful result, got %v", err)
|
||||||
|
}
|
||||||
|
if result.Validation.Status != scriptorium.ValidationFailed || result.Validation.IsValid {
|
||||||
|
t.Fatalf("expected failed validation result, got %+v", result.Validation)
|
||||||
|
}
|
||||||
|
if len(result.Validation.Errors) == 0 {
|
||||||
|
t.Fatalf("expected validation errors")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPublicErrorsSupportErrorsIs(t *testing.T) {
|
||||||
|
llmErr := errors.New("llm failed")
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
req scriptorium.RunRequest
|
||||||
|
client scriptorium.LLMClient
|
||||||
|
schemaDir string
|
||||||
|
want error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "invalid request",
|
||||||
|
req: scriptorium.RunRequest{},
|
||||||
|
client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}},
|
||||||
|
want: scriptorium.ErrInvalidRequest,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "prompt not found",
|
||||||
|
req: scriptorium.RunRequest{PromptID: "missing.prompt"},
|
||||||
|
client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}},
|
||||||
|
want: scriptorium.ErrPromptNotFound,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "profile not found",
|
||||||
|
req: scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
ProfileID: "missing-profile",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}},
|
||||||
|
want: scriptorium.ErrProfileNotFound,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "artifact load",
|
||||||
|
req: scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.File("./examples/fixtures/does-not-exist.md"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}},
|
||||||
|
want: scriptorium.ErrArtifactLoad,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "prompt render",
|
||||||
|
req: scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
},
|
||||||
|
client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}},
|
||||||
|
want: scriptorium.ErrPromptRender,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "llm failure",
|
||||||
|
req: scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||||
|
"glossary": scriptorium.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
client: &fakeLLMClient{err: llmErr},
|
||||||
|
want: scriptorium.ErrLLMGenerate,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation runtime failure",
|
||||||
|
req: scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.structured_events",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: `{"events":[]}`}},
|
||||||
|
schemaDir: t.TempDir(),
|
||||||
|
want: scriptorium.ErrValidation,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Setenv("SCRIPTORIUM_API_KEY", "test-secret")
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
schemaDir := tc.schemaDir
|
||||||
|
if schemaDir == "" {
|
||||||
|
schemaDir = "./examples/schemas"
|
||||||
|
}
|
||||||
|
engine := newExampleEngineWithOptions(t, schemaDir, scriptorium.WithLLMClient(tc.client))
|
||||||
|
_, err := engine.Run(context.Background(), tc.req)
|
||||||
|
if !errors.Is(err, tc.want) {
|
||||||
|
t.Fatalf("expected errors.Is(%v), got %v", tc.want, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectedProfileRawAPIKeyMapsToProfileLoad(t *testing.T) {
|
||||||
|
profileDir := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(profileDir, "raw.yaml"), []byte(`
|
||||||
|
id: raw-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
api_key: secret
|
||||||
|
`), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||||
|
PromptDir: "./examples/prompts",
|
||||||
|
ProfileDir: profileDir,
|
||||||
|
SchemaDir: "./examples/schemas",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected engine construction to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
ProfileID: "raw-profile",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||||
|
"glossary": scriptorium.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if !errors.Is(err, scriptorium.ErrProfileLoad) {
|
||||||
|
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||||
|
}
|
||||||
|
if errors.Is(err, scriptorium.ErrPromptLoad) {
|
||||||
|
t.Fatalf("did not expect ErrPromptLoad, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectedProfileInvalidYAMLMapsToProfileLoad(t *testing.T) {
|
||||||
|
profileDir := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(profileDir, "broken.yaml"), []byte(`
|
||||||
|
id: broken-profile
|
||||||
|
unknown_field: true
|
||||||
|
`), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||||
|
PromptDir: "./examples/prompts",
|
||||||
|
ProfileDir: profileDir,
|
||||||
|
SchemaDir: "./examples/schemas",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected engine construction to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
ProfileID: "broken-profile",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||||
|
"glossary": scriptorium.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if !errors.Is(err, scriptorium.ErrProfileLoad) {
|
||||||
|
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||||
|
}
|
||||||
|
if errors.Is(err, scriptorium.ErrPromptLoad) {
|
||||||
|
t.Fatalf("did not expect ErrPromptLoad, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtraParamsTypedNestedValuesAreCopiedAcrossPublicBoundary(t *testing.T) {
|
||||||
|
fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}
|
||||||
|
engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake))
|
||||||
|
|
||||||
|
labels := map[string]string{"route": "primary"}
|
||||||
|
counts := map[string]int{"retry_budget": 2}
|
||||||
|
weights := []float64{0.25, 0.75}
|
||||||
|
ids := []int{1, 2, 3}
|
||||||
|
nested := map[string]any{
|
||||||
|
"labels": labels,
|
||||||
|
"counts": counts,
|
||||||
|
"weights": weights,
|
||||||
|
"ids": ids,
|
||||||
|
}
|
||||||
|
extraParams := map[string]any{
|
||||||
|
"labels": labels,
|
||||||
|
"counts": counts,
|
||||||
|
"nested": nested,
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := engine.Run(context.Background(), scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||||
|
"glossary": scriptorium.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
Execution: &scriptorium.ExecutionTargetOverride{ExtraParams: extraParams},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected run to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
if len(fake.requests) != 1 {
|
||||||
|
t.Fatalf("expected one generate request, got %d", len(fake.requests))
|
||||||
|
}
|
||||||
|
|
||||||
|
captured := fake.requests[0].Target.ExtraParams
|
||||||
|
labels["route"] = "mutated"
|
||||||
|
counts["retry_budget"] = 99
|
||||||
|
weights[0] = 9.9
|
||||||
|
ids[0] = 99
|
||||||
|
nested["added"] = "mutated"
|
||||||
|
extraParams["new_top_level"] = "mutated"
|
||||||
|
|
||||||
|
want := map[string]any{
|
||||||
|
"labels": map[string]string{"route": "primary"},
|
||||||
|
"counts": map[string]int{"retry_budget": 2},
|
||||||
|
"nested": map[string]any{
|
||||||
|
"labels": map[string]string{"route": "primary"},
|
||||||
|
"counts": map[string]int{"retry_budget": 2},
|
||||||
|
"weights": []float64{0.25, 0.75},
|
||||||
|
"ids": []int{1, 2, 3},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(captured, want) {
|
||||||
|
t.Fatalf("captured extra_params changed after mutating source:\ngot=%#v\nwant=%#v", captured, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWithLLMClientRejectsNilClient(t *testing.T) {
|
||||||
|
_, err := scriptorium.NewEngine(exampleConfig("./examples/schemas"), scriptorium.WithLLMClient(nil))
|
||||||
|
if !errors.Is(err, scriptorium.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("expected ErrInvalidConfig, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewEngineConstructsDefaultLLMClientWithoutCredentials(t *testing.T) {
|
||||||
|
if _, err := scriptorium.NewEngine(exampleConfig("./examples/schemas")); err != nil {
|
||||||
|
t.Fatalf("expected default engine construction without credentials to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newExampleEngine(t *testing.T) *scriptorium.Engine {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
for _, path := range []string{
|
||||||
|
"./examples/prompts",
|
||||||
|
"./examples/profiles",
|
||||||
|
"./examples/schemas",
|
||||||
|
} {
|
||||||
|
if _, err := os.Stat(path); err != nil {
|
||||||
|
t.Fatalf("expected example path %s to exist: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
engine, err := scriptorium.NewEngine(exampleConfig("./examples/schemas"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected engine construction to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
return engine
|
||||||
|
}
|
||||||
|
|
||||||
|
func newExampleEngineWithOptions(t *testing.T, schemaDir string, opts ...scriptorium.Option) *scriptorium.Engine {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
engine, err := scriptorium.NewEngine(exampleConfig(schemaDir), opts...)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected engine construction to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
return engine
|
||||||
|
}
|
||||||
|
|
||||||
|
func exampleConfig(schemaDir string) scriptorium.Config {
|
||||||
|
return scriptorium.Config{
|
||||||
|
PromptDir: "./examples/prompts",
|
||||||
|
ProfileDir: "./examples/profiles",
|
||||||
|
SchemaDir: schemaDir,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeLLMClient struct {
|
||||||
|
response *scriptorium.GenerateResponse
|
||||||
|
err error
|
||||||
|
requests []scriptorium.GenerateRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeLLMClient) Generate(_ context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
|
||||||
|
f.requests = append(f.requests, req)
|
||||||
|
if f.err != nil {
|
||||||
|
return nil, f.err
|
||||||
|
}
|
||||||
|
return f.response, nil
|
||||||
|
}
|
||||||
77
errors.go
Normal file
77
errors.go
Normal file
@@ -0,0 +1,77 @@
|
|||||||
|
package scriptorium
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||||
|
)
|
||||||
|
|
||||||
|
func mapPublicError(err error) error {
|
||||||
|
if err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if hasPublicError(err) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
publicErr := publicErrorFor(err)
|
||||||
|
if publicErr == nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%w: %w", publicErr, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasPublicError(err error) bool {
|
||||||
|
for _, publicErr := range []error{
|
||||||
|
ErrInvalidConfig,
|
||||||
|
ErrInvalidRequest,
|
||||||
|
ErrPromptNotFound,
|
||||||
|
ErrProfileNotFound,
|
||||||
|
ErrPromptLoad,
|
||||||
|
ErrProfileLoad,
|
||||||
|
ErrArtifactLoad,
|
||||||
|
ErrPromptRender,
|
||||||
|
ErrLLMGenerate,
|
||||||
|
ErrValidation,
|
||||||
|
} {
|
||||||
|
if errors.Is(err, publicErr) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func publicErrorFor(err error) error {
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, promptdef.ErrPromptDefinitionNotFound):
|
||||||
|
return ErrPromptNotFound
|
||||||
|
case errors.Is(err, profile.ErrProfileNotFound):
|
||||||
|
return ErrProfileNotFound
|
||||||
|
case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition):
|
||||||
|
return ErrPromptLoad
|
||||||
|
case isProfileLoadCause(err):
|
||||||
|
return ErrProfileLoad
|
||||||
|
case errors.Is(err, usecase.ErrArtifactLoad):
|
||||||
|
return ErrArtifactLoad
|
||||||
|
case errors.Is(err, usecase.ErrPromptRender):
|
||||||
|
return ErrPromptRender
|
||||||
|
case errors.Is(err, usecase.ErrLLMGenerate):
|
||||||
|
return ErrLLMGenerate
|
||||||
|
case errors.Is(err, usecase.ErrValidation):
|
||||||
|
return ErrValidation
|
||||||
|
case errors.Is(err, usecase.ErrInvalidRequest):
|
||||||
|
return ErrInvalidRequest
|
||||||
|
case errors.Is(err, usecase.ErrProfileLoad):
|
||||||
|
return ErrPromptLoad
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func isProfileLoadCause(err error) bool {
|
||||||
|
return errors.Is(err, profile.ErrInvalidYAML) ||
|
||||||
|
errors.Is(err, profile.ErrInvalidProfile) ||
|
||||||
|
errors.Is(err, profile.ErrRawAPIKeyNotAllowed)
|
||||||
|
}
|
||||||
50
examples/go-library/prepare/main.go
Normal file
50
examples/go-library/prepare/main.go
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||||
|
PromptDir: "./examples/prompts",
|
||||||
|
ProfileDir: "./examples/profiles",
|
||||||
|
SchemaDir: "./examples/schemas",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||||
|
PromptID: "generic.markdown_summary",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||||
|
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
summary := struct {
|
||||||
|
PromptID string `json:"prompt_id"`
|
||||||
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
|
Model string `json:"model"`
|
||||||
|
MessageCount int `json:"message_count"`
|
||||||
|
InputHashes map[string]string `json:"input_hashes"`
|
||||||
|
}{
|
||||||
|
PromptID: prepared.PromptID,
|
||||||
|
SelectedProfileID: prepared.SelectedProfileID,
|
||||||
|
Model: prepared.EffectiveModelParams.Model,
|
||||||
|
MessageCount: len(prepared.Messages),
|
||||||
|
InputHashes: prepared.InputHashes,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := json.NewEncoder(os.Stdout).Encode(summary); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
23
llm_adapter.go
Normal file
23
llm_adapter.go
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
package scriptorium
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
type publicLLMClientAdapter struct {
|
||||||
|
client LLMClient
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a publicLLMClientAdapter) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
|
resp, err := a.client.Generate(ctx, fromDomainGenerateRequest(req))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if resp == nil {
|
||||||
|
return nil, fmt.Errorf("%w: llm client returned nil response", ErrLLMGenerate)
|
||||||
|
}
|
||||||
|
return toDomainGenerateResponse(resp), nil
|
||||||
|
}
|
||||||
256
types.go
Normal file
256
types.go
Normal file
@@ -0,0 +1,256 @@
|
|||||||
|
package scriptorium
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ArtifactRefType defines how an artifact is referenced.
|
||||||
|
type ArtifactRefType string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ArtifactRefInline ArtifactRefType = "inline"
|
||||||
|
ArtifactRefFile ArtifactRefType = "file"
|
||||||
|
)
|
||||||
|
|
||||||
|
// OutputFormat defines the desired output format.
|
||||||
|
type OutputFormat string
|
||||||
|
|
||||||
|
const (
|
||||||
|
FormatText OutputFormat = "text"
|
||||||
|
FormatMarkdown OutputFormat = "markdown"
|
||||||
|
FormatJSON OutputFormat = "json"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ValidationMode defines the output validation strategy.
|
||||||
|
type ValidationMode string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ValidationNone ValidationMode = "none"
|
||||||
|
ValidationBasic ValidationMode = "basic"
|
||||||
|
ValidationJSON ValidationMode = "json"
|
||||||
|
ValidationJSONSchema ValidationMode = "json_schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ValidationStatus defines the result of a validation check.
|
||||||
|
type ValidationStatus string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ValidationPassed ValidationStatus = "passed"
|
||||||
|
ValidationFailed ValidationStatus = "failed"
|
||||||
|
ValidationSkipped ValidationStatus = "skipped"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CacheControlType defines provider cache behavior for prompt content.
|
||||||
|
type CacheControlType string
|
||||||
|
|
||||||
|
const (
|
||||||
|
CacheControlEphemeral CacheControlType = "ephemeral"
|
||||||
|
)
|
||||||
|
|
||||||
|
// StructuredOutputType identifies provider-level structured output modes.
|
||||||
|
type StructuredOutputType string
|
||||||
|
|
||||||
|
const (
|
||||||
|
StructuredOutputJSONSchema StructuredOutputType = "json_schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RunRequest represents a request to prepare or run a single prompt.
|
||||||
|
type RunRequest struct {
|
||||||
|
PromptID string
|
||||||
|
PromptVersion string
|
||||||
|
ProfileID string
|
||||||
|
Inputs map[string]ArtifactRef
|
||||||
|
Vars map[string]string
|
||||||
|
Execution *ExecutionTargetOverride
|
||||||
|
Validation *OutputContract
|
||||||
|
Metadata map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// PreparedRun contains prepared prompt execution state. It does not include
|
||||||
|
// resolved API key values, model output, validation results, or internal target
|
||||||
|
// presence metadata.
|
||||||
|
type PreparedRun struct {
|
||||||
|
PromptID string `json:"prompt_id"`
|
||||||
|
PromptVersion string `json:"prompt_version,omitempty"`
|
||||||
|
PromptHash string `json:"prompt_hash,omitempty"`
|
||||||
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
|
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||||
|
OutputContract OutputContract `json:"output_contract"`
|
||||||
|
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||||
|
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||||
|
SessionID string `json:"session_id,omitempty"`
|
||||||
|
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||||
|
Messages []RenderedMessage `json:"messages"`
|
||||||
|
StartTime time.Time `json:"start_time,omitempty"`
|
||||||
|
EndTime time.Time `json:"end_time,omitempty"`
|
||||||
|
DurationMS int64 `json:"duration_ms,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunResult contains generated output, validation state, and run metadata.
|
||||||
|
type RunResult struct {
|
||||||
|
RunID string `json:"run_id"`
|
||||||
|
Artifact Artifact `json:"artifact"`
|
||||||
|
RawOutput string `json:"raw_output"`
|
||||||
|
Validation ValidationResult `json:"validation"`
|
||||||
|
PromptID string `json:"prompt_id"`
|
||||||
|
PromptVersion string `json:"prompt_version,omitempty"`
|
||||||
|
PromptHash string `json:"prompt_hash,omitempty"`
|
||||||
|
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||||
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
|
ModelName string `json:"model_name"`
|
||||||
|
Endpoint string `json:"endpoint"`
|
||||||
|
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||||
|
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||||
|
Usage TokenUsage `json:"usage"`
|
||||||
|
StartTime time.Time `json:"start_time,omitempty"`
|
||||||
|
EndTime time.Time `json:"end_time,omitempty"`
|
||||||
|
Duration time.Duration `json:"duration,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ArtifactRef represents a reference to prompt input content.
|
||||||
|
type ArtifactRef struct {
|
||||||
|
Type ArtifactRefType
|
||||||
|
URI string
|
||||||
|
Body string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Artifact represents loaded artifact content.
|
||||||
|
type Artifact struct {
|
||||||
|
Name string
|
||||||
|
ContentType string
|
||||||
|
Body []byte
|
||||||
|
URI string
|
||||||
|
Size int64
|
||||||
|
Hash string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecutionTarget represents effective model runtime settings.
|
||||||
|
type ExecutionTarget struct {
|
||||||
|
Endpoint string `json:"endpoint"`
|
||||||
|
Model string `json:"model"`
|
||||||
|
Temperature float64 `json:"temperature"`
|
||||||
|
MaxTokens int `json:"max_tokens"`
|
||||||
|
TopP float64 `json:"top_p"`
|
||||||
|
TimeoutSeconds int `json:"timeout_seconds"`
|
||||||
|
ServiceTier string `json:"service_tier"`
|
||||||
|
ReasoningEffort string `json:"reasoning_effort"`
|
||||||
|
APIKeyEnv string `json:"api_key_env"`
|
||||||
|
ExtraParams map[string]any `json:"extra_params"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecutionTargetOverride represents per-request runtime setting overrides.
|
||||||
|
type ExecutionTargetOverride struct {
|
||||||
|
Endpoint string
|
||||||
|
Model string
|
||||||
|
Temperature *float64
|
||||||
|
MaxTokens *int
|
||||||
|
TopP *float64
|
||||||
|
TimeoutSeconds *int
|
||||||
|
ServiceTier string
|
||||||
|
ReasoningEffort string
|
||||||
|
APIKeyEnv string
|
||||||
|
ExtraParams map[string]any
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecutionTargetPresence tracks which numeric runtime settings were explicit
|
||||||
|
// request overrides.
|
||||||
|
type ExecutionTargetPresence struct {
|
||||||
|
Temperature bool
|
||||||
|
MaxTokens bool
|
||||||
|
TopP bool
|
||||||
|
TimeoutSeconds bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// OutputContract defines output and validation requirements.
|
||||||
|
type OutputContract struct {
|
||||||
|
Format OutputFormat `json:"format"`
|
||||||
|
ValidationMode ValidationMode `json:"validation_mode"`
|
||||||
|
SchemaPath string `json:"schema_path"`
|
||||||
|
RepairAttempts int `json:"repair_attempts"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidationResult represents output validation state.
|
||||||
|
type ValidationResult struct {
|
||||||
|
Status ValidationStatus `json:"status"`
|
||||||
|
Mode ValidationMode `json:"mode"`
|
||||||
|
Errors []string `json:"errors,omitempty"`
|
||||||
|
SchemaPath string `json:"schema_path,omitempty"`
|
||||||
|
RepairAttempts int `json:"repair_attempts"`
|
||||||
|
IsValid bool `json:"is_valid"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// TokenUsage tracks token consumption.
|
||||||
|
type TokenUsage struct {
|
||||||
|
PromptTokens int `json:"prompt_tokens"`
|
||||||
|
CompletionTokens int `json:"completion_tokens"`
|
||||||
|
TotalTokens int `json:"total_tokens"`
|
||||||
|
CachedTokens int `json:"cached_tokens"`
|
||||||
|
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RenderedPrompt is the fully rendered prompt passed to an LLM client.
|
||||||
|
type RenderedPrompt struct {
|
||||||
|
SessionID string `json:"session_id,omitempty"`
|
||||||
|
Messages []RenderedMessage `json:"messages"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RenderedMessage is a rendered chat message.
|
||||||
|
type RenderedMessage struct {
|
||||||
|
Role string `json:"role"`
|
||||||
|
Content string `json:"content"`
|
||||||
|
CacheControl *CacheControl `json:"cache_control,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CacheControl describes provider cache metadata attached to prompt content.
|
||||||
|
type CacheControl struct {
|
||||||
|
Type CacheControlType `json:"type"`
|
||||||
|
TTL string `json:"ttl,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// StructuredOutputSpec describes provider-level structured output.
|
||||||
|
type StructuredOutputSpec struct {
|
||||||
|
Type StructuredOutputType `json:"type"`
|
||||||
|
JSONSchema *StructuredOutputJSONSpec `json:"json_schema,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// StructuredOutputJSONSpec contains JSON Schema output constraints.
|
||||||
|
type StructuredOutputJSONSpec struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Strict bool `json:"strict"`
|
||||||
|
Schema any `json:"schema"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// LLMClient executes rendered prompts for Engine.Run.
|
||||||
|
type LLMClient interface {
|
||||||
|
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GenerateRequest is passed to an injected LLM client.
|
||||||
|
type GenerateRequest struct {
|
||||||
|
Prompt RenderedPrompt `json:"prompt"`
|
||||||
|
Target ExecutionTarget `json:"target"`
|
||||||
|
TargetPresence ExecutionTargetPresence `json:"target_presence"`
|
||||||
|
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// GenerateResponse is returned by an injected LLM client.
|
||||||
|
type GenerateResponse struct {
|
||||||
|
Content string `json:"content"`
|
||||||
|
Usage TokenUsage `json:"usage"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// File returns a file-backed artifact reference.
|
||||||
|
func File(path string) ArtifactRef {
|
||||||
|
return ArtifactRef{Type: ArtifactRefFile, URI: path}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Inline returns an inline artifact reference.
|
||||||
|
func Inline(body string) ArtifactRef {
|
||||||
|
return ArtifactRef{Type: ArtifactRefInline, Body: body}
|
||||||
|
}
|
||||||
|
|
||||||
|
// InlineWithURI returns an inline artifact reference with URI metadata.
|
||||||
|
func InlineWithURI(uri string, body string) ArtifactRef {
|
||||||
|
return ArtifactRef{Type: ArtifactRefInline, URI: uri, Body: body}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user