Compare commits
4 Commits
v0.10.0
...
03d4f27d2b
| Author | SHA1 | Date | |
|---|---|---|---|
| 03d4f27d2b | |||
| 4ac2038331 | |||
| 14a7e7e04c | |||
| 5e522bad8b |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1,6 +1,5 @@
|
||||
# ---> Codex
|
||||
.codex
|
||||
AGENTS.md
|
||||
|
||||
# ---> Go
|
||||
# If you prefer the allow list template instead of the deny list, see community template:
|
||||
|
||||
4
AGENTS.md
Normal file
4
AGENTS.md
Normal file
@@ -0,0 +1,4 @@
|
||||
Please carefully review the relevant documents in `docs/policy` before making any changes to this repository.
|
||||
- `development.md` defines the contributor workflow for this application.
|
||||
- `architecture.md` provides the canonical high-level architecture policy for this repository, and should be reviewed before writing or changing any code.
|
||||
- `documentation.md` provides the canonical documentation policy for this repository, and should be reviewed before writing or changing any documentation.
|
||||
@@ -25,6 +25,7 @@ This command renders the prepared prompt and effective runtime settings without
|
||||
- [Configuration reference](docs/config.md)
|
||||
- [Operations guide](docs/operations.md)
|
||||
- [Troubleshooting](docs/troubleshooting.md)
|
||||
- [Go library package](docs/consumers/pkg-scriptorium.md)
|
||||
- [HTTP API integration](docs/integrations/http-api.md)
|
||||
- [OpenAI-compatible chat integration](docs/integrations/openai-compatible-chat.md)
|
||||
- [Narratio subprocess integration](docs/integrations/narratio.md)
|
||||
@@ -34,3 +35,4 @@ This command renders the prepared prompt and effective runtime settings without
|
||||
|
||||
- `examples/render-markdown-summary.sh`
|
||||
- `examples/http-run.json`
|
||||
- `examples/go-library/prepare`
|
||||
|
||||
335
convert.go
Normal file
335
convert.go
Normal file
@@ -0,0 +1,335 @@
|
||||
package scriptorium
|
||||
|
||||
import "gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
|
||||
func toDomainRunRequest(req RunRequest) domain.RunRequest {
|
||||
return domain.RunRequest{
|
||||
PromptID: req.PromptID,
|
||||
PromptVersion: req.PromptVersion,
|
||||
ProfileID: req.ProfileID,
|
||||
Inputs: toDomainArtifactRefMap(req.Inputs),
|
||||
Vars: copyStringMap(req.Vars),
|
||||
Execution: toDomainExecutionTargetOverride(req.Execution),
|
||||
Validation: toDomainOutputContractPtr(req.Validation),
|
||||
Metadata: copyStringMap(req.Metadata),
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
||||
if prepared == nil {
|
||||
return nil
|
||||
}
|
||||
return &PreparedRun{
|
||||
PromptID: prepared.PromptID,
|
||||
PromptVersion: prepared.PromptVersion,
|
||||
PromptHash: prepared.PromptHash,
|
||||
SelectedProfileID: prepared.SelectedProfileID,
|
||||
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
|
||||
OutputContract: fromDomainOutputContract(prepared.OutputContract),
|
||||
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
|
||||
InputHashes: copyStringMap(prepared.InputHashes),
|
||||
SessionID: prepared.SessionID,
|
||||
RenderedPromptHash: prepared.RenderedPromptHash,
|
||||
Messages: fromDomainRenderedMessages(prepared.Messages),
|
||||
StartTime: prepared.StartTime,
|
||||
EndTime: prepared.EndTime,
|
||||
DurationMS: prepared.DurationMS,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
return &RunResult{
|
||||
RunID: result.RunID,
|
||||
Artifact: fromDomainArtifact(result.Artifact),
|
||||
RawOutput: result.RawOutput,
|
||||
Validation: fromDomainValidationResult(result.Validation),
|
||||
PromptID: result.PromptID,
|
||||
PromptVersion: result.PromptVersion,
|
||||
PromptHash: result.PromptHash,
|
||||
RenderedPromptHash: result.RenderedPromptHash,
|
||||
SelectedProfileID: result.SelectedProfileID,
|
||||
ModelName: result.ModelName,
|
||||
Endpoint: result.Endpoint,
|
||||
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
|
||||
InputHashes: copyStringMap(result.InputHashes),
|
||||
Usage: fromDomainTokenUsage(result.Usage),
|
||||
StartTime: result.StartTime,
|
||||
EndTime: result.EndTime,
|
||||
Duration: result.Duration,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainGenerateRequest(req domain.GenerateRequest) GenerateRequest {
|
||||
return GenerateRequest{
|
||||
Prompt: fromDomainRenderedPrompt(req.Prompt),
|
||||
Target: fromDomainExecutionTarget(req.Target),
|
||||
TargetPresence: fromDomainExecutionTargetPresence(req.TargetPresence),
|
||||
StructuredOutput: fromDomainStructuredOutputSpec(req.StructuredOutput),
|
||||
}
|
||||
}
|
||||
|
||||
func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse {
|
||||
if resp == nil {
|
||||
return nil
|
||||
}
|
||||
return &domain.GenerateResponse{
|
||||
Content: resp.Content,
|
||||
Usage: toDomainTokenUsage(resp.Usage),
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainRenderedPrompt(prompt domain.RenderedPrompt) RenderedPrompt {
|
||||
return RenderedPrompt{
|
||||
SessionID: prompt.SessionID,
|
||||
Messages: fromDomainRenderedMessages(prompt.Messages),
|
||||
}
|
||||
}
|
||||
|
||||
func toDomainArtifactRefMap(src map[string]ArtifactRef) map[string]domain.ArtifactRef {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]domain.ArtifactRef, len(src))
|
||||
for k, v := range src {
|
||||
out[k] = toDomainArtifactRef(v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func toDomainArtifactRef(ref ArtifactRef) domain.ArtifactRef {
|
||||
return domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefType(ref.Type),
|
||||
URI: ref.URI,
|
||||
Body: ref.Body,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainArtifact(artifact domain.Artifact) Artifact {
|
||||
return Artifact{
|
||||
Name: artifact.Name,
|
||||
ContentType: artifact.ContentType,
|
||||
Body: copyBytes(artifact.Body),
|
||||
URI: artifact.URI,
|
||||
Size: artifact.Size,
|
||||
Hash: artifact.Hash,
|
||||
}
|
||||
}
|
||||
|
||||
func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) *domain.ExecutionTargetOverride {
|
||||
if override == nil {
|
||||
return nil
|
||||
}
|
||||
return &domain.ExecutionTargetOverride{
|
||||
Endpoint: override.Endpoint,
|
||||
Model: override.Model,
|
||||
Temperature: copyFloat64Ptr(override.Temperature),
|
||||
MaxTokens: copyIntPtr(override.MaxTokens),
|
||||
TopP: copyFloat64Ptr(override.TopP),
|
||||
TimeoutSeconds: copyIntPtr(override.TimeoutSeconds),
|
||||
ServiceTier: override.ServiceTier,
|
||||
ReasoningEffort: override.ReasoningEffort,
|
||||
APIKeyEnv: override.APIKeyEnv,
|
||||
ExtraParams: copyAnyMap(override.ExtraParams),
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
||||
return ExecutionTarget{
|
||||
Endpoint: target.Endpoint,
|
||||
Model: target.Model,
|
||||
Temperature: target.Temperature,
|
||||
MaxTokens: target.MaxTokens,
|
||||
TopP: target.TopP,
|
||||
TimeoutSeconds: target.TimeoutSeconds,
|
||||
ServiceTier: target.ServiceTier,
|
||||
ReasoningEffort: target.ReasoningEffort,
|
||||
APIKeyEnv: target.APIKeyEnv,
|
||||
ExtraParams: copyAnyMap(target.ExtraParams),
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
||||
return ExecutionTargetPresence{
|
||||
Temperature: presence.Temperature,
|
||||
MaxTokens: presence.MaxTokens,
|
||||
TopP: presence.TopP,
|
||||
TimeoutSeconds: presence.TimeoutSeconds,
|
||||
}
|
||||
}
|
||||
|
||||
func toDomainOutputContractPtr(contract *OutputContract) *domain.OutputContract {
|
||||
if contract == nil {
|
||||
return nil
|
||||
}
|
||||
out := toDomainOutputContract(*contract)
|
||||
return &out
|
||||
}
|
||||
|
||||
func toDomainOutputContract(contract OutputContract) domain.OutputContract {
|
||||
return domain.OutputContract{
|
||||
Format: domain.OutputFormat(contract.Format),
|
||||
ValidationMode: domain.ValidationMode(contract.ValidationMode),
|
||||
SchemaPath: contract.SchemaPath,
|
||||
RepairAttempts: contract.RepairAttempts,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainOutputContract(contract domain.OutputContract) OutputContract {
|
||||
return OutputContract{
|
||||
Format: OutputFormat(contract.Format),
|
||||
ValidationMode: ValidationMode(contract.ValidationMode),
|
||||
SchemaPath: contract.SchemaPath,
|
||||
RepairAttempts: contract.RepairAttempts,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainValidationResult(result domain.ValidationResult) ValidationResult {
|
||||
return ValidationResult{
|
||||
Status: ValidationStatus(result.Status),
|
||||
Mode: ValidationMode(result.Mode),
|
||||
Errors: copyStringSlice(result.Errors),
|
||||
SchemaPath: result.SchemaPath,
|
||||
RepairAttempts: result.RepairAttempts,
|
||||
IsValid: result.IsValid,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainTokenUsage(usage domain.TokenUsage) TokenUsage {
|
||||
return TokenUsage{
|
||||
PromptTokens: usage.PromptTokens,
|
||||
CompletionTokens: usage.CompletionTokens,
|
||||
TotalTokens: usage.TotalTokens,
|
||||
CachedTokens: usage.CachedTokens,
|
||||
CacheWriteTokens: usage.CacheWriteTokens,
|
||||
}
|
||||
}
|
||||
|
||||
func toDomainTokenUsage(usage TokenUsage) domain.TokenUsage {
|
||||
return domain.TokenUsage{
|
||||
PromptTokens: usage.PromptTokens,
|
||||
CompletionTokens: usage.CompletionTokens,
|
||||
TotalTokens: usage.TotalTokens,
|
||||
CachedTokens: usage.CachedTokens,
|
||||
CacheWriteTokens: usage.CacheWriteTokens,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainRenderedMessages(messages []domain.RenderedMessage) []RenderedMessage {
|
||||
if messages == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]RenderedMessage, len(messages))
|
||||
for i, msg := range messages {
|
||||
out[i] = RenderedMessage{
|
||||
Role: msg.Role,
|
||||
Content: msg.Content,
|
||||
CacheControl: fromDomainCacheControl(msg.CacheControl),
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl {
|
||||
if cacheControl == nil {
|
||||
return nil
|
||||
}
|
||||
return &CacheControl{
|
||||
Type: CacheControlType(cacheControl.Type),
|
||||
TTL: cacheControl.TTL,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainStructuredOutputSpec(spec *domain.StructuredOutputSpec) *StructuredOutputSpec {
|
||||
if spec == nil {
|
||||
return nil
|
||||
}
|
||||
out := &StructuredOutputSpec{
|
||||
Type: StructuredOutputType(spec.Type),
|
||||
}
|
||||
if spec.JSONSchema != nil {
|
||||
out.JSONSchema = &StructuredOutputJSONSpec{
|
||||
Name: spec.JSONSchema.Name,
|
||||
Strict: spec.JSONSchema.Strict,
|
||||
Schema: copyAny(spec.JSONSchema.Schema),
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func copyStringMap(src map[string]string) map[string]string {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(src))
|
||||
for k, v := range src {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func copyAnyMap(src map[string]any) map[string]any {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(src))
|
||||
for k, v := range src {
|
||||
out[k] = copyAny(v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func copyAny(value any) any {
|
||||
switch v := value.(type) {
|
||||
case map[string]any:
|
||||
return copyAnyMap(v)
|
||||
case []any:
|
||||
out := make([]any, len(v))
|
||||
for i, item := range v {
|
||||
out[i] = copyAny(item)
|
||||
}
|
||||
return out
|
||||
case []string:
|
||||
return copyStringSlice(v)
|
||||
case []byte:
|
||||
return copyBytes(v)
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
func copyStringSlice(src []string) []string {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, len(src))
|
||||
copy(out, src)
|
||||
return out
|
||||
}
|
||||
|
||||
func copyBytes(src []byte) []byte {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]byte, len(src))
|
||||
copy(out, src)
|
||||
return out
|
||||
}
|
||||
|
||||
func copyFloat64Ptr(src *float64) *float64 {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
v := *src
|
||||
return &v
|
||||
}
|
||||
|
||||
func copyIntPtr(src *int) *int {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
v := *src
|
||||
return &v
|
||||
}
|
||||
13
docs/consumers/api.md
Normal file
13
docs/consumers/api.md
Normal file
@@ -0,0 +1,13 @@
|
||||
# Consumer API Overview
|
||||
|
||||
Scriptorium can be used by consumers through three implemented surfaces:
|
||||
|
||||
- CLI commands, documented in [CLI reference](../cli.md).
|
||||
- HTTP `POST /v1/runs`, documented in [HTTP API integration](../integrations/http-api.md).
|
||||
- Go package `gitea.maximumdirect.net/eric/scriptorium`, documented in [pkg-scriptorium](pkg-scriptorium.md).
|
||||
|
||||
The Go package is the typed in-process API. It prepares prompts, runs prompts, accepts file or inline artifacts, supports per-request execution overrides, and exposes stable public errors for `errors.Is`.
|
||||
|
||||
Use the Go package when the caller is a Go program that wants typed requests/results, context cancellation, repeated calls without subprocess overhead, or fake LLM injection for tests. Use the CLI or HTTP surfaces when process isolation, language neutrality, or an HTTP boundary is preferred.
|
||||
|
||||
Raw API key values are not accepted in public payloads and are not returned in prepared or run results. Execution profiles may reference an environment variable name through `api_key_env`.
|
||||
129
docs/consumers/pkg-scriptorium.md
Normal file
129
docs/consumers/pkg-scriptorium.md
Normal file
@@ -0,0 +1,129 @@
|
||||
# Package scriptorium
|
||||
|
||||
Import path:
|
||||
|
||||
```go
|
||||
import "gitea.maximumdirect.net/eric/scriptorium"
|
||||
```
|
||||
|
||||
The root package is a public facade over Scriptorium's prompt execution use case. It keeps `internal/*` packages private while exposing typed construction, preparation, execution, inputs, results, and errors.
|
||||
|
||||
## Construct An Engine
|
||||
|
||||
```go
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||
PromptDir: "./examples/prompts",
|
||||
ProfileDir: "./examples/profiles",
|
||||
SchemaDir: "./examples/schemas",
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
`PromptDir` and `ProfileDir` are required. `SchemaDir` defaults to the built-in schema directory. `Timeout` and `HTTPClient` configure the default OpenAI-compatible client used by `Run` when no custom LLM client is supplied.
|
||||
|
||||
## Prepare A Prompt
|
||||
|
||||
`Prepare` resolves the prompt definition, profile, inputs, variables, output contract, structured-output metadata, and rendered messages without calling an LLM.
|
||||
|
||||
```go
|
||||
prepared, err := engine.Prepare(ctx, scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = prepared.Messages
|
||||
```
|
||||
|
||||
Input helpers:
|
||||
|
||||
- `scriptorium.File(path)` loads an input artifact from a file.
|
||||
- `scriptorium.Inline(body)` passes inline input content.
|
||||
- `scriptorium.InlineWithURI(uri, body)` passes inline content with URI metadata.
|
||||
|
||||
## Run A Prompt
|
||||
|
||||
`Run` prepares the prompt, calls the configured LLM client, builds the output artifact, and validates the output.
|
||||
|
||||
```go
|
||||
result, err := engine.Run(ctx, scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = result.Artifact
|
||||
```
|
||||
|
||||
`RunResult` includes the run ID, output artifact, raw output, validation result, prompt/profile/model metadata, effective model parameters, input hashes, token/cache usage, and timing fields. Validation content failures return a successful `RunResult` with failed validation status. Runtime validation errors return `ErrValidation`.
|
||||
|
||||
## Inject An LLM Client
|
||||
|
||||
Use `WithLLMClient` for tests or custom model integrations:
|
||||
|
||||
```go
|
||||
type fakeLLM struct{}
|
||||
|
||||
func (fakeLLM) Generate(ctx context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
|
||||
return &scriptorium.GenerateResponse{
|
||||
Content: "generated text",
|
||||
Usage: scriptorium.TokenUsage{TotalTokens: 12},
|
||||
}, nil
|
||||
}
|
||||
|
||||
engine, err := scriptorium.NewEngine(cfg, scriptorium.WithLLMClient(fakeLLM{}))
|
||||
```
|
||||
|
||||
The injected client receives the rendered prompt, effective execution target, target presence metadata for explicit numeric overrides, and structured-output spec. `WithLLMClient(nil)` returns `ErrInvalidConfig`.
|
||||
|
||||
## Request Overrides
|
||||
|
||||
`RunRequest.Execution` accepts per-request overrides. Numeric override fields are pointers so explicit zero values are preserved:
|
||||
|
||||
```go
|
||||
zero := 0
|
||||
req.Execution = &scriptorium.ExecutionTargetOverride{
|
||||
MaxTokens: &zero,
|
||||
}
|
||||
```
|
||||
|
||||
## Errors
|
||||
|
||||
Public methods wrap context while preserving stable sentinel checks with `errors.Is`:
|
||||
|
||||
- `ErrInvalidConfig`
|
||||
- `ErrInvalidRequest`
|
||||
- `ErrPromptNotFound`
|
||||
- `ErrProfileNotFound`
|
||||
- `ErrPromptLoad`
|
||||
- `ErrProfileLoad`
|
||||
- `ErrArtifactLoad`
|
||||
- `ErrPromptRender`
|
||||
- `ErrLLMGenerate`
|
||||
- `ErrValidation`
|
||||
|
||||
Example:
|
||||
|
||||
```go
|
||||
if errors.Is(err, scriptorium.ErrPromptNotFound) {
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
Run the prepare-only example from the repository root:
|
||||
|
||||
```bash
|
||||
go run ./examples/go-library/prepare
|
||||
```
|
||||
@@ -8,6 +8,7 @@ This document describes implemented adapter/repository boundaries and their curr
|
||||
|
||||
- `internal/adapter/cli`: CLI command parsing, app wiring, stdout/stderr handling, exit codes.
|
||||
- `internal/adapter/http`: HTTP request/response mapping for `POST /v1/runs`.
|
||||
- root package `scriptorium`: public Go library facade for preparing and running prompt requests.
|
||||
- `internal/promptdef`: filesystem prompt-definition repository.
|
||||
- `internal/profile`: filesystem execution-profile repository.
|
||||
- `internal/artifact`: input artifact reader.
|
||||
@@ -30,6 +31,13 @@ HTTP adapter:
|
||||
- Output: JSON success/error body with mapped status codes.
|
||||
- Success metadata includes token usage plus cache usage counters.
|
||||
|
||||
Public library facade:
|
||||
|
||||
- Input: typed `scriptorium.RunRequest` values.
|
||||
- Output: typed `PreparedRun` and `RunResult` values plus public sentinel errors.
|
||||
- Custom LLM behavior is injected with `WithLLMClient`; otherwise the default OpenAI-compatible client is used.
|
||||
- Public types are facade types converted at the package boundary; internal domain types remain internal.
|
||||
|
||||
Filesystem repositories:
|
||||
|
||||
- Input: prompt/profile YAML files under configured directories.
|
||||
|
||||
@@ -2,12 +2,13 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
Project documentation must help four audiences:
|
||||
Project documentation must help five audiences:
|
||||
|
||||
1. users who need to run the application;
|
||||
2. administrators/operators who need to configure and operate it;
|
||||
3. developers who need to understand and change it safely;
|
||||
4. LLM coding agents that need clear scope, boundaries, and invariants.
|
||||
4. LLM coding agents that need clear scope, boundaries, and invariants;
|
||||
5. developers and LLM coding agents integrating this project from another codebase.
|
||||
|
||||
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
||||
|
||||
@@ -42,11 +43,14 @@ Canonical homes:
|
||||
|
||||
- project purpose and quickstart: `README.md`
|
||||
- development principles: `docs/policy/architecture.md`
|
||||
- public HTTP API reference: `docs/api.md`
|
||||
- configuration reference: `docs/config.md`
|
||||
- CLI reference: `docs/cli.md`
|
||||
- operations and recovery: `docs/operations.md`
|
||||
- troubleshooting: `docs/troubleshooting.md`
|
||||
- public API/package consumer guidance: `docs/consumers/`
|
||||
- implemented internals: `docs/internal/`
|
||||
- external protocol, service, and file-format contracts: `docs/integrations/`
|
||||
- future work: `docs/roadmap/`
|
||||
- contributor workflow: `docs/policy/development.md`
|
||||
- copyable examples: `examples/`
|
||||
@@ -106,7 +110,7 @@ Recommended:
|
||||
- `examples/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
### Modular, staged, service-oriented, or orchestration application
|
||||
### Modular, service-oriented, or orchestration application
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`, if CLI-based
|
||||
@@ -119,6 +123,31 @@ Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- validated examples under `examples/`
|
||||
|
||||
### Public HTTP API service
|
||||
|
||||
Required:
|
||||
- `docs/api.md`
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
- `docs/internal/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- `docs/consumers/`, for task-oriented client integration guides
|
||||
- `docs/integrations/`, for upstream/downstream service contracts
|
||||
- validated examples under `examples/`
|
||||
|
||||
### Project with public packages or consumer APIs
|
||||
|
||||
Required:
|
||||
- `docs/consumers/api.md`
|
||||
- one `docs/consumers/pkg-<name>.md` file per public package, if public packages exist
|
||||
|
||||
Recommended:
|
||||
- copyable consumer examples under `examples/`, if practical
|
||||
|
||||
## Required Documents
|
||||
|
||||
### README.md
|
||||
@@ -159,7 +188,35 @@ It should include:
|
||||
- architectural invariants;
|
||||
- explicit non-goals, if useful.
|
||||
|
||||
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
|
||||
Notably, this file should prescribe a core development *policy* that should remain unchanged as the application evolves. It is not a place for details (e.g., CLI flags) that could change over time.
|
||||
|
||||
The contents of `architecture.md` should be trim and concise. LLMs may be directed to review it routinely via AGENTS.md, CLAUDE.md, or similar.
|
||||
|
||||
### docs/api.md
|
||||
|
||||
**Audience:** external HTTP API consumers, developers, LLM coding agents integrating by HTTP
|
||||
|
||||
Required for projects whose primary public interface is HTTP.
|
||||
|
||||
`docs/api.md` is the canonical public HTTP API contract. It should be normative for external consumers and should not be duplicated by README, operations docs, consumer guides, or integration docs.
|
||||
|
||||
It should include:
|
||||
|
||||
1. base URL conventions;
|
||||
2. authentication and authorization behavior, if implemented;
|
||||
3. response envelope;
|
||||
4. supported media types and content negotiation behavior;
|
||||
5. shared query parameters;
|
||||
6. endpoint reference grouped by route family;
|
||||
7. request parameters and validation rules;
|
||||
8. response fields, units, nullability, and optionality;
|
||||
9. error response shape and status codes;
|
||||
10. pagination, caching, rate-limit, idempotency, and retry behavior, if implemented;
|
||||
11. compact request and response examples.
|
||||
|
||||
It must document only implemented endpoints and behavior. Planned endpoints, proposed fields, future filters, and experimental response shapes belong only under `docs/roadmap/`.
|
||||
|
||||
For HTTP API projects, `docs/consumers/` may provide task-oriented client integration guides, but those guides should link to `docs/api.md` for the authoritative endpoint contract.
|
||||
|
||||
### docs/policy/development.md
|
||||
|
||||
@@ -175,7 +232,7 @@ It should include:
|
||||
- dependency policy;
|
||||
- how to add config fields;
|
||||
- how to add CLI flags;
|
||||
- how to add stages/modules/adapters, if applicable;
|
||||
- how to add modules or adapters, if applicable;
|
||||
- how to update examples;
|
||||
- documentation update expectations.
|
||||
|
||||
@@ -216,7 +273,7 @@ Explain when commands are useful, not just their syntax.
|
||||
|
||||
**Audience:** administrators, operators
|
||||
|
||||
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
|
||||
Required for applications that maintain state, support resume behavior, run multi-step workflows, write durable artifacts, use remote storage, or require recovery procedures.
|
||||
|
||||
It should cover:
|
||||
|
||||
@@ -244,11 +301,40 @@ Each entry should include:
|
||||
- safe fix;
|
||||
- relevant links.
|
||||
|
||||
### docs/consumers/
|
||||
|
||||
**Audience:** developers and LLM coding agents integrating this project from another codebase
|
||||
|
||||
Required for projects with public packages, SDKs, client APIs, plugin APIs, or other application-facing integration surfaces.
|
||||
|
||||
This directory describes how an external codebase should consume the project's public API. It should be task-oriented and copyable where useful. It is not the place for internal implementation details or operator procedures.
|
||||
|
||||
For projects whose public API is HTTP, `docs/consumers/` is not required, and it should not duplicate the endpoint reference in `docs/api.md`. If present, it may provide practical integration workflows, client-specific examples, or migration notes that link back to `docs/api.md`.
|
||||
|
||||
`docs/consumers/api.md` should provide the consumer-facing overview and primary implementation workflow. It should include:
|
||||
|
||||
1. intended consumer audience and use cases;
|
||||
2. required inputs supplied by operators or deployment configuration;
|
||||
3. recommended public package or API workflow;
|
||||
4. minimal copyable example;
|
||||
5. consumer responsibilities and boundaries;
|
||||
6. retry, idempotency, or status behavior, if applicable;
|
||||
7. links to package-specific docs and canonical integration contracts.
|
||||
|
||||
Package-specific docs should be named `pkg-<name>.md` and should include:
|
||||
|
||||
1. import path;
|
||||
2. intended use cases;
|
||||
3. primary types and functions needed by consumers;
|
||||
4. minimal examples;
|
||||
5. validation, error, retry, and boundary behavior;
|
||||
6. links to canonical file-format or wire-protocol contracts.
|
||||
|
||||
### docs/internal/
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for modular, staged, service-oriented, or orchestration projects.
|
||||
Required for modular, service-oriented, or orchestration projects.
|
||||
|
||||
This directory describes implemented internal components. It is not the roadmap.
|
||||
|
||||
@@ -289,7 +375,9 @@ Roadmap docs should not be confused with current behavior.
|
||||
|
||||
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
||||
|
||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
|
||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses or exposes.
|
||||
|
||||
For public HTTP API services, `docs/integrations/` should document upstream, downstream, storage, protocol, or runtime contracts that the service depends on or bridges. It should not become a second copy of the public HTTP endpoint reference; that belongs in `docs/api.md`.
|
||||
|
||||
Use one file per integration where useful.
|
||||
|
||||
@@ -346,8 +434,10 @@ Before merging documentation changes, verify:
|
||||
|
||||
- README is concise and orientation-focused.
|
||||
- `docs/policy/architecture.md` describes development principles.
|
||||
- `docs/api.md` is the canonical HTTP contract for HTTP API services.
|
||||
- Future work appears only under `docs/roadmap/`.
|
||||
- User-facing docs avoid unnecessary internals.
|
||||
- Consumer-facing docs explain public APIs without duplicating HTTP endpoint or integration contracts.
|
||||
- Developer-facing docs preserve boundaries and invariants.
|
||||
- Config examples match the schema.
|
||||
- CLI examples match real commands and flags.
|
||||
|
||||
@@ -1,111 +1,144 @@
|
||||
# Runtime Parameter Implementation Plan
|
||||
# Library API Implementation Plan
|
||||
|
||||
This plan implements the target state in `docs/roadmap/params.md`.
|
||||
This plan implements the target state in `docs/roadmap/library.md`.
|
||||
|
||||
Audience: LLM coding agents implementing the feature in order. Follow `docs/policy/architecture.md`, `docs/policy/development.md`, and `docs/policy/documentation.md` before changing code.
|
||||
|
||||
## Constraints
|
||||
|
||||
- Keep adapters thin. CLI and HTTP should capture caller intent and map it into domain request types; merge decisions belong in `internal/usecase`.
|
||||
- Keep external decoding strict. Unknown YAML/JSON fields must continue to fail.
|
||||
- Do not accept or emit raw API key values.
|
||||
- Do not add dependencies unless there is a clear need. This feature should use the standard library plus existing dependencies.
|
||||
- Do not expand the HTTP API surface beyond `POST /v1/runs`.
|
||||
- Do not add provider-specific adapter packages.
|
||||
- Add a public root package named `scriptorium`; keep existing `internal/*` packages internal.
|
||||
- Define public facade types and convert to/from internal domain types. Do not alias internal domain types as the public API.
|
||||
- Do not rewire CLI or HTTP through the public facade in this implementation.
|
||||
- Preserve current CLI, HTTP, prompt/profile loading, validation, secret-handling, and outbound LLM behavior.
|
||||
- Do not add dependencies.
|
||||
- Keep each stage passing `go test ./...` before moving to the next stage.
|
||||
|
||||
## Stage 1: Presence-Aware Request Overrides
|
||||
## Stage 1: Public Types, Engine Construction, And Prepare
|
||||
|
||||
Goal: make per-request numeric execution overrides presence-aware while keeping resolved execution settings concrete.
|
||||
Goal: make prompt preparation usable from an imported root package without calling an LLM.
|
||||
|
||||
### Domain Changes
|
||||
### Public Package
|
||||
|
||||
1. In `internal/domain/domain.go`, add a request-only type:
|
||||
Create Go files at the module root using:
|
||||
|
||||
```go
|
||||
type ExecutionTargetOverride struct {
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||
ServiceTier string `json:"service_tier,omitempty"`
|
||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||
package scriptorium
|
||||
```
|
||||
|
||||
Expose:
|
||||
|
||||
```go
|
||||
type Engine struct { /* unexported fields */ }
|
||||
|
||||
type Config struct {
|
||||
PromptDir string
|
||||
ProfileDir string
|
||||
SchemaDir string
|
||||
Timeout time.Duration
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
type Option func(*engineOptions) error
|
||||
|
||||
func NewEngine(cfg Config, opts ...Option) (*Engine, error)
|
||||
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error)
|
||||
```
|
||||
|
||||
Construction rules:
|
||||
|
||||
- `PromptDir` and `ProfileDir` are required.
|
||||
- `SchemaDir` defaults to the same built-in default used by app config.
|
||||
- `Timeout`, when non-zero, configures the default OpenAI-compatible client timeout.
|
||||
- `HTTPClient`, when non-nil, is used by the default OpenAI-compatible client.
|
||||
- `NewEngine` wires the same internal components used by CLI/HTTP: filesystem prompt/profile repositories, composite artifact reader, Go template renderer, standard validator, and OpenAI-compatible LLM client.
|
||||
- Return public `ErrInvalidConfig` for invalid engine configuration.
|
||||
|
||||
### Public Types
|
||||
|
||||
Define public facade types with exported fields:
|
||||
|
||||
- `RunRequest`
|
||||
- `PreparedRun`
|
||||
- `ArtifactRef`
|
||||
- `Artifact`
|
||||
- `ExecutionTarget`
|
||||
- `ExecutionTargetOverride`
|
||||
- `ExecutionTargetPresence`
|
||||
- `OutputContract`
|
||||
- `ValidationResult`
|
||||
- `TokenUsage`
|
||||
- `RenderedMessage`
|
||||
- `CacheControl`
|
||||
- `StructuredOutputSpec`
|
||||
|
||||
Use the same enum string values as internal domain types for formats, validation modes, validation statuses, artifact ref types, cache-control type, and structured-output type.
|
||||
|
||||
Required request shape:
|
||||
|
||||
```go
|
||||
type RunRequest struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
Inputs map[string]ArtifactRef
|
||||
Vars map[string]string
|
||||
Execution *ExecutionTargetOverride
|
||||
Validation *OutputContract
|
||||
Metadata map[string]string
|
||||
}
|
||||
```
|
||||
|
||||
2. Change `domain.RunRequest.Execution` from `*ExecutionTarget` to `*ExecutionTargetOverride`.
|
||||
3. Change `ExecutionProfile.ExtraParams` and `ExecutionTarget.ExtraParams` from `map[string]string` to `map[string]any`.
|
||||
4. Keep `ExecutionTarget` concrete. It represents the resolved effective runtime target after defaults, profile, and request overrides are merged.
|
||||
`ExecutionTargetOverride` must preserve numeric override presence using pointer fields:
|
||||
|
||||
### Runner Changes
|
||||
```go
|
||||
Temperature *float64
|
||||
MaxTokens *int
|
||||
TopP *float64
|
||||
TimeoutSeconds *int
|
||||
```
|
||||
|
||||
1. Update `internal/usecase/runner.go` so profile values still merge over built-in defaults and request overrides merge over that result.
|
||||
2. Keep the existing concrete profile merge semantics for profile numeric fields.
|
||||
3. Add a separate request override merge path that uses pointer presence:
|
||||
- `nil` numeric pointer means omitted; preserve the current value.
|
||||
- non-nil numeric pointer means explicit override, even when the value is `0`.
|
||||
4. Validate request override numeric values before or during merge:
|
||||
- `temperature`: `0 <= value <= 2`
|
||||
- `max_tokens`: `value >= 0`
|
||||
- `top_p`: `0 <= value <= 1`
|
||||
- `timeout_seconds`: `value >= 0`
|
||||
5. Preserve existing validation after merge:
|
||||
- effective endpoint required
|
||||
- effective model required
|
||||
- `api_key_env`, when set, must name a non-empty environment variable
|
||||
6. Preserve secret handling. The resolved API key value must never be stored in `PreparedRun`, `RunResult`, logs, or HTTP responses.
|
||||
`PreparedRun` should include the same user-observable fields as internal `domain.PreparedRun`, but should not expose internal-only target presence metadata.
|
||||
|
||||
### CLI Changes
|
||||
### Input Helpers
|
||||
|
||||
1. Update `internal/adapter/cli/run.go` request construction to build `domain.ExecutionTargetOverride`.
|
||||
2. Use the existing `flagWasSet` booleans to populate numeric pointers only when the user provided the flag.
|
||||
3. Required behavior:
|
||||
- omitted `--temperature` preserves profile/default temperature;
|
||||
- `--temperature 0` explicitly sets temperature to zero;
|
||||
- omitted `--top-p` preserves profile/default top-p;
|
||||
- `--top-p 0` explicitly sets top-p to zero;
|
||||
- omitted `--max-tokens` preserves profile/default max tokens;
|
||||
- `--max-tokens 0` explicitly sets max tokens to zero;
|
||||
- omitted `--timeout` preserves profile/default timeout;
|
||||
- `--timeout 0s` explicitly sets timeout seconds to zero.
|
||||
4. Do not add new CLI flags in this stage.
|
||||
Expose:
|
||||
|
||||
### HTTP Changes
|
||||
```go
|
||||
func File(path string) ArtifactRef
|
||||
func Inline(body string) ArtifactRef
|
||||
func InlineWithURI(uri string, body string) ArtifactRef
|
||||
```
|
||||
|
||||
1. Update `internal/adapter/http/dto.go` so numeric model override fields are pointers:
|
||||
- `Temperature *float64`
|
||||
- `MaxTokens *int`
|
||||
- `TopP *float64`
|
||||
- `TimeoutSeconds *int`
|
||||
2. Update DTO mapping in `internal/adapter/http/handler.go` to build `domain.ExecutionTargetOverride`.
|
||||
3. Preserve strict JSON decoding and existing error mapping.
|
||||
4. Required behavior:
|
||||
- omitted numeric JSON fields preserve profile/default values;
|
||||
- explicit numeric zero JSON fields override profile/default values.
|
||||
Mapping:
|
||||
|
||||
- `File(path)` maps to artifact type `file` with `URI: path`.
|
||||
- `Inline(body)` maps to artifact type `inline` with `Body: body`.
|
||||
- `InlineWithURI(uri, body)` maps to artifact type `inline` with both fields set.
|
||||
|
||||
### Conversion Layer
|
||||
|
||||
Implement unexported conversion helpers in the public package:
|
||||
|
||||
- public run request to internal `domain.RunRequest`
|
||||
- internal `domain.PreparedRun` to public `PreparedRun`
|
||||
- internal artifacts/messages/contracts/validation/usage/structured-output to public equivalents
|
||||
- public execution override to internal `domain.ExecutionTargetOverride`
|
||||
|
||||
Conversions must deep-copy maps and slices that cross the public/internal boundary.
|
||||
|
||||
### Tests
|
||||
|
||||
Add or update tests in:
|
||||
Add root package tests.
|
||||
|
||||
- `internal/usecase/runner_test.go`
|
||||
- `internal/adapter/cli/run_test.go`
|
||||
- `internal/adapter/http/handler_test.go`
|
||||
Required tests:
|
||||
|
||||
Required test coverage:
|
||||
|
||||
- Runner preserves profile value when request numeric override is omitted.
|
||||
- Runner applies explicit zero request override for `temperature`.
|
||||
- Runner applies explicit zero request override for `top_p`.
|
||||
- Runner applies explicit zero request override for `max_tokens`.
|
||||
- Runner applies explicit zero request override for `timeout_seconds`.
|
||||
- Invalid request override ranges fail as invalid request errors.
|
||||
- CLI `--temperature 0` reaches effective settings as zero.
|
||||
- HTTP `"temperature": 0` reaches effective settings as zero.
|
||||
- HTTP omitted `temperature` preserves profile/default value.
|
||||
- `NewEngine` rejects missing `PromptDir`.
|
||||
- `NewEngine` rejects missing `ProfileDir`.
|
||||
- `Prepare` works with `examples/config.yml` directories when passed directly through `Config`.
|
||||
- `Prepare` works with `File` input refs.
|
||||
- `Prepare` works with `Inline` input refs.
|
||||
- `Prepare` output does not expose raw API-key values or internal target presence metadata in JSON.
|
||||
- Explicit zero execution overrides survive into prepared effective settings.
|
||||
|
||||
### Verification
|
||||
|
||||
@@ -115,85 +148,109 @@ Run:
|
||||
go test ./...
|
||||
```
|
||||
|
||||
## Stage 2: JSON-Compatible `extra_params`
|
||||
## Stage 2: Run, LLM Injection, And Public Errors
|
||||
|
||||
Goal: allow provider-specific parameters to carry JSON-compatible values throughout profile, HTTP, prepared output, metadata, and LLM request construction.
|
||||
Goal: make full execution usable and testable without real provider credentials.
|
||||
|
||||
### Domain And Loader Changes
|
||||
### Public Run Method
|
||||
|
||||
1. Complete all compile fixes from changing `ExtraParams` to `map[string]any`.
|
||||
2. Ensure `internal/profile/filesystem_repository.go` continues to decode profiles strictly while allowing nested JSON-compatible values under `extra_params`.
|
||||
3. Add profile repository tests for `extra_params` containing:
|
||||
- string
|
||||
- number
|
||||
- boolean
|
||||
- nested object or array
|
||||
4. Ensure formatter output remains deterministic:
|
||||
- keep sorting `extra_params` keys in `internal/format/prepared_run.go`;
|
||||
- render non-string values with stable JSON encoding in text output.
|
||||
5. Preserve JSON formatter behavior through normal `encoding/json` output.
|
||||
Expose:
|
||||
|
||||
### HTTP Changes
|
||||
|
||||
1. Change HTTP model override `ExtraParams` to `map[string]any`.
|
||||
2. Add handler tests proving HTTP accepts JSON-compatible `extra_params` values.
|
||||
3. Preserve strict rejection of unknown fields and raw API-key payload fields.
|
||||
|
||||
### Verification
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./...
|
||||
```go
|
||||
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error)
|
||||
```
|
||||
|
||||
## Stage 3: Outbound Serialization
|
||||
`RunResult` should expose:
|
||||
|
||||
Goal: serialize `reasoning_effort` and `extra_params` to the OpenAI-compatible chat-completions request.
|
||||
- run ID
|
||||
- artifact
|
||||
- raw output
|
||||
- validation result
|
||||
- prompt/profile/model metadata
|
||||
- effective model params
|
||||
- input hashes
|
||||
- token/cache usage
|
||||
- start/end/duration timing
|
||||
|
||||
### LLM Adapter Changes
|
||||
Do not expose raw API-key values.
|
||||
|
||||
1. In `internal/llm/openai_compatible_client.go`, add first-class outbound support for `reasoning_effort`.
|
||||
2. Add `extra_params` support by flattening `domain.ExecutionTarget.ExtraParams` into additional top-level JSON request fields.
|
||||
3. Implement reserved-field collision checks before the HTTP request is made.
|
||||
4. Reserved keys must include:
|
||||
- `model`
|
||||
- `session_id`
|
||||
- `messages`
|
||||
- `temperature`
|
||||
- `max_tokens`
|
||||
- `top_p`
|
||||
- `service_tier`
|
||||
- `reasoning_effort`
|
||||
- `response_format`
|
||||
5. Reject empty `extra_params` keys.
|
||||
6. Ensure each `extra_params` value can be marshaled as JSON. If marshaling fails, return `ErrInvalidRequest` with context.
|
||||
7. Keep existing request behavior unchanged when `reasoning_effort` and `extra_params` are unset.
|
||||
### Public LLM Injection
|
||||
|
||||
### Recommended Implementation Shape
|
||||
Expose:
|
||||
|
||||
Use a custom marshal path for the outbound chat request rather than string manipulation.
|
||||
```go
|
||||
type LLMClient interface {
|
||||
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
|
||||
}
|
||||
|
||||
One acceptable shape:
|
||||
func WithLLMClient(client LLMClient) Option
|
||||
```
|
||||
|
||||
- Add `ReasoningEffort string` and `ExtraParams map[string]any` to the internal `openAIChatRequest`.
|
||||
- Add a helper that converts `openAIChatRequest` into `map[string]any`, inserts first-class fields when set, then inserts `ExtraParams` after collision validation.
|
||||
- Marshal that map with `encoding/json`.
|
||||
Public `GenerateRequest` must include:
|
||||
|
||||
Do not construct outbound JSON with manual string concatenation.
|
||||
- rendered prompt
|
||||
- effective execution target
|
||||
- execution target presence
|
||||
- structured-output spec
|
||||
|
||||
Public `GenerateResponse` must include:
|
||||
|
||||
- content
|
||||
- token usage
|
||||
|
||||
Implementation rule:
|
||||
|
||||
- `WithLLMClient` wraps the public client in an unexported adapter that satisfies `internal/llm.Client`.
|
||||
- The adapter converts internal generate requests to public generate requests and converts public generate responses back to internal responses.
|
||||
- A nil client passed to `WithLLMClient` returns `ErrInvalidConfig`.
|
||||
|
||||
Default behavior:
|
||||
|
||||
- If no custom LLM client is supplied, `NewEngine` uses `internal/llm.NewOpenAICompatibleClient`.
|
||||
- `Config.Timeout` and `Config.HTTPClient` apply only to the default OpenAI-compatible client.
|
||||
|
||||
### Public Errors
|
||||
|
||||
Define public sentinel errors:
|
||||
|
||||
- `ErrInvalidConfig`
|
||||
- `ErrInvalidRequest`
|
||||
- `ErrPromptNotFound`
|
||||
- `ErrProfileNotFound`
|
||||
- `ErrPromptLoad`
|
||||
- `ErrProfileLoad`
|
||||
- `ErrArtifactLoad`
|
||||
- `ErrPromptRender`
|
||||
- `ErrLLMGenerate`
|
||||
- `ErrValidation`
|
||||
|
||||
Public methods must map internal errors to public sentinels while preserving wrapped context. Callers must be able to use `errors.Is`.
|
||||
|
||||
Mapping rules:
|
||||
|
||||
- missing/invalid public engine config -> `ErrInvalidConfig`
|
||||
- internal `usecase.ErrInvalidRequest` -> `ErrInvalidRequest`
|
||||
- internal prompt not found -> `ErrPromptNotFound`
|
||||
- internal profile not found -> `ErrProfileNotFound`
|
||||
- internal prompt load errors -> `ErrPromptLoad`
|
||||
- internal profile load errors -> `ErrProfileLoad`
|
||||
- internal artifact load errors -> `ErrArtifactLoad`
|
||||
- internal prompt render errors -> `ErrPromptRender`
|
||||
- internal LLM generate errors -> `ErrLLMGenerate`
|
||||
- internal validation runtime errors -> `ErrValidation`
|
||||
|
||||
Do not expose internal sentinel values as public API.
|
||||
|
||||
### Tests
|
||||
|
||||
Update `internal/llm/openai_compatible_client_test.go`.
|
||||
Required tests:
|
||||
|
||||
Required test coverage:
|
||||
|
||||
- outbound JSON includes `reasoning_effort` when set;
|
||||
- outbound JSON omits `reasoning_effort` when unset;
|
||||
- outbound JSON includes string, number, boolean, object, and array `extra_params`;
|
||||
- reserved `extra_params` keys fail before provider call;
|
||||
- empty `extra_params` keys fail before provider call;
|
||||
- existing message, cache-control, service-tier, response-format, and usage parsing tests continue to pass.
|
||||
- `Run` succeeds with `WithLLMClient` fake and returns typed artifact, raw output, validation, metadata, and usage.
|
||||
- `Run` passes rendered prompt, effective execution target, and target presence to the injected LLM client.
|
||||
- `Run` validation failure returns a successful result with failed validation, not an error.
|
||||
- public errors support `errors.Is` for invalid request, prompt not found, profile not found, artifact load, render failure, LLM failure, and validation runtime failure.
|
||||
- nil `WithLLMClient(nil)` returns `ErrInvalidConfig`.
|
||||
- default OpenAI-compatible client can still be constructed without real provider credentials.
|
||||
|
||||
### Verification
|
||||
|
||||
@@ -203,39 +260,39 @@ Run:
|
||||
go test ./...
|
||||
```
|
||||
|
||||
## Stage 4: Documentation And Examples
|
||||
## Stage 3: Public Documentation And Consumer Examples
|
||||
|
||||
Goal: move implemented behavior from roadmap to canonical docs after code is complete.
|
||||
Goal: document implemented library behavior in canonical public-consumer docs.
|
||||
|
||||
Update only after Stages 1 through 3 are implemented.
|
||||
### Docs
|
||||
|
||||
### Required Docs
|
||||
After Stages 1 and 2 are implemented, update:
|
||||
|
||||
Update:
|
||||
- `README.md`: add a short link to library usage without turning the README into a manual.
|
||||
- `docs/internal/adapters.md`: list the public library facade as an implemented adapter surface.
|
||||
- `docs/consumers/api.md`: describe the public consumer API at a high level.
|
||||
- `docs/consumers/pkg-scriptorium.md`: document the root package usage, types, errors, and examples.
|
||||
|
||||
- `docs/config.md`
|
||||
- `docs/cli.md`
|
||||
- `docs/integrations/http-api.md`
|
||||
- `docs/integrations/openai-compatible-chat.md`
|
||||
- `docs/internal/runner.md`
|
||||
- `docs/internal/adapters.md`
|
||||
Create `docs/consumers/` if it does not exist.
|
||||
|
||||
Required documentation content:
|
||||
|
||||
- `reasoning_effort` is serialized outbound when set.
|
||||
- `extra_params` serializes as provider-specific top-level outbound JSON fields.
|
||||
- `extra_params` supports JSON-compatible values.
|
||||
- reserved `extra_params` fields are rejected.
|
||||
- per-request numeric overrides distinguish omitted values from explicit zero values.
|
||||
- CLI explicit zero behavior for existing numeric flags.
|
||||
- HTTP explicit zero behavior for model override numeric fields.
|
||||
- no raw API-key values are accepted or emitted.
|
||||
Do not document unimplemented future library features outside `docs/roadmap/`.
|
||||
|
||||
### Examples
|
||||
|
||||
Update examples only if needed to keep them accurate and runnable.
|
||||
Add copyable library examples only if they can be tested without real credentials.
|
||||
|
||||
If adding an `extra_params` example, keep it secret-free and simple. Prefer a harmless provider-routing example over a vendor-specific feature that requires special credentials.
|
||||
Recommended example:
|
||||
|
||||
- `examples/go-library/prepare/main.go` or equivalent prepare-only example using `examples/` prompt/profile/fixture assets.
|
||||
|
||||
If adding a run example, it must use an injected fake LLM client and must not require provider credentials.
|
||||
|
||||
### Tests
|
||||
|
||||
Required tests:
|
||||
|
||||
- doc/example smoke coverage for any added Go example using `go test` or `go test ./...`.
|
||||
- existing CLI/HTTP tests continue to pass unchanged.
|
||||
|
||||
### Verification
|
||||
|
||||
@@ -255,8 +312,10 @@ go run ./cmd/scriptorium render \
|
||||
|
||||
Before considering the feature complete:
|
||||
|
||||
1. Confirm `git diff` contains only intended code, test, doc, and example changes.
|
||||
2. Confirm all non-roadmap docs describe implemented behavior only.
|
||||
3. Confirm no output path exposes raw API key values.
|
||||
4. Confirm `go test ./...` passes.
|
||||
5. Confirm the render smoke command passes.
|
||||
1. Confirm the root package can be imported as `gitea.maximumdirect.net/eric/scriptorium`.
|
||||
2. Confirm public package tests do not require real provider credentials.
|
||||
3. Confirm `go test ./...` passes.
|
||||
4. Confirm the render smoke command passes.
|
||||
5. Confirm non-roadmap docs describe only implemented behavior.
|
||||
6. Confirm no public result or rendered/prepared output exposes raw API-key values.
|
||||
7. Confirm `git diff` does not include unrelated CLI/HTTP behavior changes.
|
||||
|
||||
134
docs/roadmap/library.md
Normal file
134
docs/roadmap/library.md
Normal file
@@ -0,0 +1,134 @@
|
||||
# Library API Roadmap
|
||||
|
||||
This roadmap defines the target behavior for making Scriptorium usable as an imported Go library while retaining the current standalone CLI and HTTP application behavior.
|
||||
|
||||
The implementation plan for this feature lives in `docs/roadmap/implementation.md`.
|
||||
|
||||
## Motivation
|
||||
|
||||
Scriptorium is currently optimized for subprocess use by other applications. That contract remains useful because it is language-neutral, operationally simple, and process-isolated.
|
||||
|
||||
For Go callers, an imported library should provide:
|
||||
|
||||
- typed requests and results instead of stdout/stderr parsing;
|
||||
- direct `context.Context` cancellation;
|
||||
- lower overhead for repeated calls;
|
||||
- easier test integration through injected clients or fixtures;
|
||||
- direct access to prepared-run data without process management;
|
||||
- fewer integration points where secrets or output metadata can be mishandled.
|
||||
|
||||
The library is an additional adapter surface, not a replacement for the CLI or HTTP API.
|
||||
|
||||
## Target State
|
||||
|
||||
Scriptorium should expose a small public Go API suitable for common embedding use cases:
|
||||
|
||||
- construct an engine from app-level settings such as prompt, profile, and schema directories;
|
||||
- prepare a prompt request without calling an LLM;
|
||||
- run a prompt request and receive a typed result;
|
||||
- pass file and inline artifacts;
|
||||
- apply profile selection, runtime overrides, vars, validation behavior, cache-control behavior, and structured-output behavior consistently with CLI/HTTP;
|
||||
- inject a custom LLM client or HTTP client where needed;
|
||||
- preserve existing CLI and HTTP behavior by continuing to route all entry paths through the same use-case layer.
|
||||
|
||||
The public library API should be stable, narrow, and intentionally higher-level than the current `internal/*` package layout.
|
||||
|
||||
## Public Package Policy
|
||||
|
||||
The public package should be the module root:
|
||||
|
||||
```go
|
||||
import "gitea.maximumdirect.net/eric/scriptorium"
|
||||
```
|
||||
|
||||
Recommended usage shape:
|
||||
|
||||
```go
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||
PromptDir: "./prompts",
|
||||
ProfileDir: "./profiles",
|
||||
SchemaDir: "./schemas",
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
prepared, err := engine.Prepare(ctx, scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./transcript.md"),
|
||||
},
|
||||
})
|
||||
|
||||
result, err := engine.Run(ctx, scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./transcript.md"),
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
## Policy Decisions
|
||||
|
||||
### Public Package Scope
|
||||
|
||||
Expose a narrow root facade package and keep existing `internal/*` packages internal.
|
||||
|
||||
Reasoning:
|
||||
|
||||
This gives callers the workflow they need without freezing the internal architecture as public API. It also preserves the current package-boundary policy and keeps future refactoring possible.
|
||||
|
||||
### Public Type Strategy
|
||||
|
||||
Define public facade types and map them to internal domain types.
|
||||
|
||||
Reasoning:
|
||||
|
||||
Public types can be designed around caller needs and long-term stability. Internal types can continue to evolve with implementation details such as adapter metadata, validation internals, and provider-specific behavior.
|
||||
|
||||
### CLI And HTTP Reuse
|
||||
|
||||
Keep CLI and HTTP on current internal wiring for the initial library release. Consider migrating them to the public facade only after the facade proves stable.
|
||||
|
||||
Reasoning:
|
||||
|
||||
This minimizes risk to the existing subprocess and HTTP contracts while adding the new API. It also avoids forcing the first public facade to satisfy every adapter edge case immediately.
|
||||
|
||||
### Error Surface
|
||||
|
||||
Expose public sentinel errors or typed error categories and map internal errors to them while preserving wrapped context.
|
||||
|
||||
Reasoning:
|
||||
|
||||
Library callers need stable, idiomatic error checks. Mapping internal errors avoids exposing internal package paths as public compatibility promises.
|
||||
|
||||
## Scope
|
||||
|
||||
In scope:
|
||||
|
||||
- Public facade package for library consumers.
|
||||
- Public request, result, prepared-run, artifact reference, execution override, validation, and config types.
|
||||
- Public constructors for common file and inline input references.
|
||||
- Public engine methods for `Prepare` and `Run`.
|
||||
- Optional dependency injection for LLM behavior and HTTP behavior.
|
||||
- Stable error behavior suitable for `errors.Is` and `errors.As`.
|
||||
- Tests proving public API behavior matches CLI/use-case behavior.
|
||||
- Documentation and examples for library usage after implementation.
|
||||
|
||||
Out of scope for the first library release:
|
||||
|
||||
- Making every `internal/*` package public.
|
||||
- Replacing or rewiring the CLI or HTTP adapters.
|
||||
- Adding a durable run store or workflow engine.
|
||||
- Adding broad provider-specific SDK surfaces.
|
||||
- Adding non-Go language bindings.
|
||||
- Adding global mutable configuration.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- A Go caller can import the root module and run a prompt without invoking a subprocess.
|
||||
- A Go caller can prepare a prompt without invoking an LLM.
|
||||
- Public library behavior matches current CLI/HTTP use-case semantics for prompt/profile loading, artifact reading, rendering, validation, and model invocation.
|
||||
- Existing CLI and HTTP behavior remains unchanged.
|
||||
- Library tests use injected/fake LLM behavior and do not require real provider credentials.
|
||||
- Public documentation is concise and limited to implemented behavior once code exists.
|
||||
@@ -1,96 +0,0 @@
|
||||
# Runtime Parameter Feature Roadmap
|
||||
|
||||
This roadmap defines the target behavior for runtime model parameters.
|
||||
|
||||
Current behavior has two limitations:
|
||||
|
||||
- `reasoning_effort` and `extra_params` are parsed into effective execution settings but are not serialized into outbound OpenAI-compatible chat-completions requests.
|
||||
- Per-request numeric execution overrides use zero-value merge semantics, so callers cannot reliably override a profile value with an explicit zero such as `temperature: 0`.
|
||||
|
||||
The implementation plan for this feature lives in `docs/roadmap/implementation.md`.
|
||||
|
||||
## Target State
|
||||
|
||||
Scriptorium should preserve the existing separation between prompt definitions, execution profiles, and per-request execution overrides while making runtime parameter behavior explicit and predictable.
|
||||
|
||||
Expected end state:
|
||||
|
||||
- Effective execution settings remain visible in prepared-run output, run metadata, and HTTP metadata without exposing raw secret values.
|
||||
- `reasoning_effort` is treated as a first-class effective execution setting and is serialized to the outbound OpenAI-compatible request when set.
|
||||
- `extra_params` supports provider-specific OpenAI-compatible request fields.
|
||||
- `extra_params` is serialized as additional top-level outbound JSON fields.
|
||||
- `extra_params` values support JSON-compatible scalar, object, and array values.
|
||||
- `extra_params` cannot override first-class outbound request fields.
|
||||
- Per-request numeric overrides preserve caller intent, including explicit zero values.
|
||||
- Omitted per-request numeric overrides continue to inherit the selected profile and built-in defaults.
|
||||
- External decoding remains strict for config, prompt, profile, and HTTP request payloads.
|
||||
|
||||
## Policy Decisions
|
||||
|
||||
### `extra_params`
|
||||
|
||||
`extra_params` should serialize as additional top-level outbound JSON fields in the OpenAI-compatible chat-completions request.
|
||||
|
||||
Reasoning:
|
||||
|
||||
Most OpenAI-compatible providers expose vendor-specific chat-completions parameters as top-level fields. This keeps Scriptorium's adapter compatible with that ecosystem without adding first-class fields for every provider option.
|
||||
|
||||
`extra_params` must not silently override Scriptorium-owned fields. Reserved outbound fields include at least:
|
||||
|
||||
- `model`
|
||||
- `session_id`
|
||||
- `messages`
|
||||
- `temperature`
|
||||
- `max_tokens`
|
||||
- `top_p`
|
||||
- `service_tier`
|
||||
- `reasoning_effort`
|
||||
- `response_format`
|
||||
|
||||
If a caller supplies a reserved key through `extra_params`, Scriptorium should fail before making the outbound HTTP request.
|
||||
|
||||
`extra_params` should use JSON-compatible values rather than only strings.
|
||||
|
||||
Reasoning:
|
||||
|
||||
Provider-specific parameters commonly need booleans, numbers, objects, or arrays. String-only values would force awkward encoding and would likely require a later compatibility break.
|
||||
|
||||
### Presence-Aware Overrides
|
||||
|
||||
Per-request execution overrides should use a presence-aware type with pointer fields for optional numeric values.
|
||||
|
||||
Reasoning:
|
||||
|
||||
The resolved execution target should remain a concrete value used by prepared runs, generated requests, and metadata. Optionality matters at the request boundary, not after the runner has resolved the effective target.
|
||||
|
||||
This keeps adapter and merge logic precise while avoiding nil checks in formatter, metadata, and LLM serialization paths.
|
||||
|
||||
## Scope
|
||||
|
||||
In scope:
|
||||
|
||||
- Runtime merge behavior for per-request execution overrides.
|
||||
- HTTP model override decoding for explicit zero numeric values.
|
||||
- CLI execution override handling for explicit zero numeric flags.
|
||||
- Outbound serialization of `reasoning_effort`.
|
||||
- Outbound serialization of JSON-compatible `extra_params`.
|
||||
- Tests and documentation for the changed implemented behavior.
|
||||
|
||||
Out of scope:
|
||||
|
||||
- Expanding the HTTP API beyond `POST /v1/runs`.
|
||||
- Adding built-in HTTP authentication or authorization.
|
||||
- Adding durable run state, run history, or multi-step orchestration.
|
||||
- Adding broad provider-specific adapter packages.
|
||||
- Adding new CLI flags for every provider-specific parameter.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- A profile containing `reasoning_effort: medium` produces an outbound request with `reasoning_effort`.
|
||||
- HTTP callers can pass `reasoning_effort` through the existing `model` override object and have it appear outbound.
|
||||
- A profile or HTTP request containing JSON-compatible `extra_params` produces outbound top-level JSON fields according to the reserved-field policy.
|
||||
- Reserved `extra_params` collisions fail before the outbound provider call.
|
||||
- CLI callers can pass `--temperature 0` and observe `temperature: 0` in rendered/effective settings and outbound requests.
|
||||
- HTTP callers can send `"temperature": 0` and observe the same behavior.
|
||||
- Omitting `temperature` continues to preserve the selected profile/default value.
|
||||
- Raw API key values remain unsupported in config, profiles, CLI flags, HTTP payloads, logs, and rendered output.
|
||||
141
engine.go
Normal file
141
engine.go
Normal file
@@ -0,0 +1,141 @@
|
||||
package scriptorium
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
artifactadapter "gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
||||
)
|
||||
|
||||
// ErrInvalidConfig indicates invalid public engine configuration.
|
||||
var ErrInvalidConfig = errors.New("invalid engine configuration")
|
||||
|
||||
var (
|
||||
ErrInvalidRequest = errors.New("invalid run request")
|
||||
ErrPromptNotFound = errors.New("prompt not found")
|
||||
ErrProfileNotFound = errors.New("profile not found")
|
||||
ErrPromptLoad = errors.New("failed to load prompt definition")
|
||||
ErrProfileLoad = errors.New("failed to load execution profile")
|
||||
ErrArtifactLoad = errors.New("failed to load artifact")
|
||||
ErrPromptRender = errors.New("failed to render prompt")
|
||||
ErrLLMGenerate = errors.New("failed to generate output")
|
||||
ErrValidation = errors.New("failed to validate output")
|
||||
)
|
||||
|
||||
// Engine prepares and runs Scriptorium prompt requests.
|
||||
type Engine struct {
|
||||
runner *usecase.Runner
|
||||
}
|
||||
|
||||
// Config configures a public Scriptorium engine.
|
||||
type Config struct {
|
||||
PromptDir string
|
||||
ProfileDir string
|
||||
SchemaDir string
|
||||
Timeout time.Duration
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
// Option customizes engine construction.
|
||||
type Option func(*engineOptions) error
|
||||
|
||||
type engineOptions struct {
|
||||
llmClient llm.Client
|
||||
}
|
||||
|
||||
// WithLLMClient injects a custom LLM client for execution.
|
||||
func WithLLMClient(client LLMClient) Option {
|
||||
return func(options *engineOptions) error {
|
||||
if client == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.llmClient = publicLLMClientAdapter{client: client}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// NewEngine constructs an Engine using the same default internal components as
|
||||
// the CLI and HTTP adapters.
|
||||
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
if strings.TrimSpace(cfg.PromptDir) == "" {
|
||||
return nil, fmt.Errorf("%w: prompt directory is required", ErrInvalidConfig)
|
||||
}
|
||||
if strings.TrimSpace(cfg.ProfileDir) == "" {
|
||||
return nil, fmt.Errorf("%w: profile directory is required", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
var options engineOptions
|
||||
for _, opt := range opts {
|
||||
if opt == nil {
|
||||
continue
|
||||
}
|
||||
if err := opt(&options); err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
}
|
||||
|
||||
schemaDir := cfg.SchemaDir
|
||||
if strings.TrimSpace(schemaDir) == "" {
|
||||
schemaDir = defaults.SchemaDirDefault
|
||||
}
|
||||
|
||||
llmClient := options.llmClient
|
||||
if llmClient == nil {
|
||||
var err error
|
||||
llmClient, err = llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
|
||||
Timeout: cfg.Timeout,
|
||||
HTTPClient: cfg.HTTPClient,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
}
|
||||
|
||||
return &Engine{
|
||||
runner: usecase.NewRunner(
|
||||
promptdef.NewFilesystemRepository(cfg.PromptDir),
|
||||
profile.NewFilesystemRepository(cfg.ProfileDir),
|
||||
artifactadapter.NewCompositeReader(),
|
||||
prompt.NewGoRenderer(),
|
||||
llmClient,
|
||||
validate.NewStandardValidator(schemaDir),
|
||||
),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Prepare resolves a prompt request without calling an LLM.
|
||||
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
prepared, err := e.runner.Prepare(ctx, toDomainRunRequest(req))
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return fromDomainPreparedRun(prepared), nil
|
||||
}
|
||||
|
||||
// Run executes a prompt request and returns the generated artifact and metadata.
|
||||
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
result, err := e.runner.Run(ctx, toDomainRunRequest(req))
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return fromDomainRunResult(result), nil
|
||||
}
|
||||
429
engine_test.go
Normal file
429
engine_test.go
Normal file
@@ -0,0 +1,429 @@
|
||||
package scriptorium_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
func TestNewEngineRejectsMissingPromptDir(t *testing.T) {
|
||||
_, err := scriptorium.NewEngine(scriptorium.Config{ProfileDir: "./examples/profiles"})
|
||||
if !errors.Is(err, scriptorium.ErrInvalidConfig) {
|
||||
t.Fatalf("expected ErrInvalidConfig, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewEngineRejectsMissingProfileDir(t *testing.T) {
|
||||
_, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: "./examples/prompts"})
|
||||
if !errors.Is(err, scriptorium.ErrInvalidConfig) {
|
||||
t.Fatalf("expected ErrInvalidConfig, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareWorksWithExampleDirectoriesAndFileInputs(t *testing.T) {
|
||||
engine := newExampleEngine(t)
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected prepare to succeed, got %v", err)
|
||||
}
|
||||
if prepared.PromptID != "generic.markdown_summary" {
|
||||
t.Fatalf("unexpected prompt id: %q", prepared.PromptID)
|
||||
}
|
||||
if prepared.SelectedProfileID != "local-fast" {
|
||||
t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID)
|
||||
}
|
||||
if prepared.EffectiveModelParams.Model != "gpt-4o-mini" {
|
||||
t.Fatalf("unexpected effective model: %q", prepared.EffectiveModelParams.Model)
|
||||
}
|
||||
if len(prepared.Messages) != 2 {
|
||||
t.Fatalf("expected rendered messages, got %d", len(prepared.Messages))
|
||||
}
|
||||
if prepared.InputHashes["transcript"] == "" || prepared.InputHashes["glossary"] == "" {
|
||||
t.Fatalf("expected input hashes, got %#v", prepared.InputHashes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareWorksWithInlineInputs(t *testing.T) {
|
||||
engine := newExampleEngine(t)
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline("Rin scouts the tower.\nKara lights a lantern."),
|
||||
"glossary": scriptorium.InlineWithURI("memory://glossary.yml", "party:\n - Rin\n - Kara\n"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected prepare to succeed, got %v", err)
|
||||
}
|
||||
if len(prepared.Messages) != 2 {
|
||||
t.Fatalf("expected rendered messages, got %d", len(prepared.Messages))
|
||||
}
|
||||
rendered := prepared.Messages[1].Content
|
||||
if !strings.Contains(rendered, "Rin scouts the tower.") || !strings.Contains(rendered, "party:") {
|
||||
t.Fatalf("expected inline inputs in rendered prompt, got %q", rendered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedRunJSONDoesNotExposeSecretOrTargetPresence(t *testing.T) {
|
||||
const envName = "SCRIPTORIUM_API_KEY"
|
||||
const secret = "public-api-test-secret"
|
||||
t.Setenv(envName, secret)
|
||||
|
||||
engine := newExampleEngine(t)
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "generic.structured_events",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected prepare to succeed, got %v", err)
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("expected prepared run to marshal, got %v", err)
|
||||
}
|
||||
out := string(payload)
|
||||
if strings.Contains(out, secret) {
|
||||
t.Fatalf("prepared run JSON leaked raw API key value: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, envName) {
|
||||
t.Fatalf("prepared run JSON should retain api_key_env name, got %s", out)
|
||||
}
|
||||
for _, forbidden := range []string{"TargetPresence", "target_presence"} {
|
||||
if strings.Contains(out, forbidden) {
|
||||
t.Fatalf("prepared run JSON exposed internal target presence metadata %q: %s", forbidden, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparePreservesExplicitZeroExecutionOverrides(t *testing.T) {
|
||||
engine := newExampleEngine(t)
|
||||
zeroFloat := 0.0
|
||||
zeroInt := 0
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||
},
|
||||
Execution: &scriptorium.ExecutionTargetOverride{
|
||||
Temperature: &zeroFloat,
|
||||
MaxTokens: &zeroInt,
|
||||
TopP: &zeroFloat,
|
||||
TimeoutSeconds: &zeroInt,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected prepare to succeed, got %v", err)
|
||||
}
|
||||
target := prepared.EffectiveModelParams
|
||||
if target.Temperature != 0 || target.MaxTokens != 0 || target.TopP != 0 || target.TimeoutSeconds != 0 {
|
||||
t.Fatalf("expected explicit zero overrides in effective target, got %+v", target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSucceedsWithInjectedLLMClient(t *testing.T) {
|
||||
const envName = "SCRIPTORIUM_API_KEY"
|
||||
const secret = "run-secret-value"
|
||||
t.Setenv(envName, secret)
|
||||
|
||||
fake := &fakeLLMClient{
|
||||
response: &scriptorium.GenerateResponse{
|
||||
Content: "# Summary\n\nDone.",
|
||||
Usage: scriptorium.TokenUsage{
|
||||
PromptTokens: 10,
|
||||
CompletionTokens: 5,
|
||||
TotalTokens: 15,
|
||||
CachedTokens: 3,
|
||||
CacheWriteTokens: 2,
|
||||
},
|
||||
},
|
||||
}
|
||||
engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake))
|
||||
|
||||
result, err := engine.Run(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||
"glossary": scriptorium.Inline("gate: A guarded passage."),
|
||||
},
|
||||
Execution: &scriptorium.ExecutionTargetOverride{
|
||||
APIKeyEnv: envName,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected run to succeed, got %v", err)
|
||||
}
|
||||
if result.RunID == "" {
|
||||
t.Fatalf("expected run id")
|
||||
}
|
||||
if result.RawOutput != fake.response.Content {
|
||||
t.Fatalf("unexpected raw output: %q", result.RawOutput)
|
||||
}
|
||||
if string(result.Artifact.Body) != fake.response.Content {
|
||||
t.Fatalf("unexpected artifact body: %q", string(result.Artifact.Body))
|
||||
}
|
||||
if result.Artifact.ContentType != "text/markdown" {
|
||||
t.Fatalf("unexpected artifact content type: %q", result.Artifact.ContentType)
|
||||
}
|
||||
if result.Validation.Status != scriptorium.ValidationPassed || !result.Validation.IsValid {
|
||||
t.Fatalf("expected passed validation, got %+v", result.Validation)
|
||||
}
|
||||
if result.PromptID != "generic.markdown_summary" || result.SelectedProfileID != "local-fast" || result.ModelName != "gpt-4o-mini" {
|
||||
t.Fatalf("unexpected run metadata: %+v", result)
|
||||
}
|
||||
if result.Usage.TotalTokens != 15 || result.Usage.CachedTokens != 3 || result.Usage.CacheWriteTokens != 2 {
|
||||
t.Fatalf("unexpected usage: %+v", result.Usage)
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
t.Fatalf("expected run result to marshal, got %v", err)
|
||||
}
|
||||
if strings.Contains(string(payload), secret) {
|
||||
t.Fatalf("run result JSON leaked raw API key value: %s", payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) {
|
||||
fake := &fakeLLMClient{
|
||||
response: &scriptorium.GenerateResponse{Content: "ok"},
|
||||
}
|
||||
engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake))
|
||||
zeroFloat := 0.0
|
||||
zeroInt := 0
|
||||
|
||||
_, err := engine.Run(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||
"glossary": scriptorium.Inline("gate: A guarded passage."),
|
||||
},
|
||||
Execution: &scriptorium.ExecutionTargetOverride{
|
||||
Temperature: &zeroFloat,
|
||||
MaxTokens: &zeroInt,
|
||||
TopP: &zeroFloat,
|
||||
TimeoutSeconds: &zeroInt,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected run to succeed, got %v", err)
|
||||
}
|
||||
if len(fake.requests) != 1 {
|
||||
t.Fatalf("expected one generate request, got %d", len(fake.requests))
|
||||
}
|
||||
req := fake.requests[0]
|
||||
if len(req.Prompt.Messages) != 2 || !strings.Contains(req.Prompt.Messages[1].Content, "Rin opens the gate.") {
|
||||
t.Fatalf("expected rendered prompt in generate request, got %+v", req.Prompt)
|
||||
}
|
||||
if req.Target.Model != "gpt-4o-mini" || req.Target.Temperature != 0 || req.Target.MaxTokens != 0 || req.Target.TopP != 0 || req.Target.TimeoutSeconds != 0 {
|
||||
t.Fatalf("unexpected effective target: %+v", req.Target)
|
||||
}
|
||||
if !req.TargetPresence.Temperature || !req.TargetPresence.MaxTokens || !req.TargetPresence.TopP || !req.TargetPresence.TimeoutSeconds {
|
||||
t.Fatalf("expected explicit zero target presence, got %+v", req.TargetPresence)
|
||||
}
|
||||
if req.StructuredOutput != nil {
|
||||
t.Fatalf("did not expect structured output for markdown prompt: %+v", req.StructuredOutput)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunValidationFailureReturnsResult(t *testing.T) {
|
||||
fake := &fakeLLMClient{
|
||||
response: &scriptorium.GenerateResponse{Content: ""},
|
||||
}
|
||||
engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake))
|
||||
|
||||
result, err := engine.Run(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||
"glossary": scriptorium.Inline("gate: A guarded passage."),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected validation failure as successful result, got %v", err)
|
||||
}
|
||||
if result.Validation.Status != scriptorium.ValidationFailed || result.Validation.IsValid {
|
||||
t.Fatalf("expected failed validation result, got %+v", result.Validation)
|
||||
}
|
||||
if len(result.Validation.Errors) == 0 {
|
||||
t.Fatalf("expected validation errors")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicErrorsSupportErrorsIs(t *testing.T) {
|
||||
llmErr := errors.New("llm failed")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
req scriptorium.RunRequest
|
||||
client scriptorium.LLMClient
|
||||
schemaDir string
|
||||
want error
|
||||
}{
|
||||
{
|
||||
name: "invalid request",
|
||||
req: scriptorium.RunRequest{},
|
||||
client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}},
|
||||
want: scriptorium.ErrInvalidRequest,
|
||||
},
|
||||
{
|
||||
name: "prompt not found",
|
||||
req: scriptorium.RunRequest{PromptID: "missing.prompt"},
|
||||
client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}},
|
||||
want: scriptorium.ErrPromptNotFound,
|
||||
},
|
||||
{
|
||||
name: "profile not found",
|
||||
req: scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
ProfileID: "missing-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||
},
|
||||
},
|
||||
client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}},
|
||||
want: scriptorium.ErrProfileNotFound,
|
||||
},
|
||||
{
|
||||
name: "artifact load",
|
||||
req: scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./examples/fixtures/does-not-exist.md"),
|
||||
},
|
||||
},
|
||||
client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}},
|
||||
want: scriptorium.ErrArtifactLoad,
|
||||
},
|
||||
{
|
||||
name: "prompt render",
|
||||
req: scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
},
|
||||
client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}},
|
||||
want: scriptorium.ErrPromptRender,
|
||||
},
|
||||
{
|
||||
name: "llm failure",
|
||||
req: scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||
"glossary": scriptorium.Inline("gate: A guarded passage."),
|
||||
},
|
||||
},
|
||||
client: &fakeLLMClient{err: llmErr},
|
||||
want: scriptorium.ErrLLMGenerate,
|
||||
},
|
||||
{
|
||||
name: "validation runtime failure",
|
||||
req: scriptorium.RunRequest{
|
||||
PromptID: "generic.structured_events",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline("Rin opens the gate."),
|
||||
},
|
||||
},
|
||||
client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: `{"events":[]}`}},
|
||||
schemaDir: t.TempDir(),
|
||||
want: scriptorium.ErrValidation,
|
||||
},
|
||||
}
|
||||
|
||||
t.Setenv("SCRIPTORIUM_API_KEY", "test-secret")
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
schemaDir := tc.schemaDir
|
||||
if schemaDir == "" {
|
||||
schemaDir = "./examples/schemas"
|
||||
}
|
||||
engine := newExampleEngineWithOptions(t, schemaDir, scriptorium.WithLLMClient(tc.client))
|
||||
_, err := engine.Run(context.Background(), tc.req)
|
||||
if !errors.Is(err, tc.want) {
|
||||
t.Fatalf("expected errors.Is(%v), got %v", tc.want, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithLLMClientRejectsNilClient(t *testing.T) {
|
||||
_, err := scriptorium.NewEngine(exampleConfig("./examples/schemas"), scriptorium.WithLLMClient(nil))
|
||||
if !errors.Is(err, scriptorium.ErrInvalidConfig) {
|
||||
t.Fatalf("expected ErrInvalidConfig, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewEngineConstructsDefaultLLMClientWithoutCredentials(t *testing.T) {
|
||||
if _, err := scriptorium.NewEngine(exampleConfig("./examples/schemas")); err != nil {
|
||||
t.Fatalf("expected default engine construction without credentials to succeed, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func newExampleEngine(t *testing.T) *scriptorium.Engine {
|
||||
t.Helper()
|
||||
|
||||
for _, path := range []string{
|
||||
"./examples/prompts",
|
||||
"./examples/profiles",
|
||||
"./examples/schemas",
|
||||
} {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("expected example path %s to exist: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
engine, err := scriptorium.NewEngine(exampleConfig("./examples/schemas"))
|
||||
if err != nil {
|
||||
t.Fatalf("expected engine construction to succeed, got %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func newExampleEngineWithOptions(t *testing.T, schemaDir string, opts ...scriptorium.Option) *scriptorium.Engine {
|
||||
t.Helper()
|
||||
|
||||
engine, err := scriptorium.NewEngine(exampleConfig(schemaDir), opts...)
|
||||
if err != nil {
|
||||
t.Fatalf("expected engine construction to succeed, got %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func exampleConfig(schemaDir string) scriptorium.Config {
|
||||
return scriptorium.Config{
|
||||
PromptDir: "./examples/prompts",
|
||||
ProfileDir: "./examples/profiles",
|
||||
SchemaDir: schemaDir,
|
||||
}
|
||||
}
|
||||
|
||||
type fakeLLMClient struct {
|
||||
response *scriptorium.GenerateResponse
|
||||
err error
|
||||
requests []scriptorium.GenerateRequest
|
||||
}
|
||||
|
||||
func (f *fakeLLMClient) Generate(_ context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
|
||||
f.requests = append(f.requests, req)
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return f.response, nil
|
||||
}
|
||||
71
errors.go
Normal file
71
errors.go
Normal file
@@ -0,0 +1,71 @@
|
||||
package scriptorium
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
||||
)
|
||||
|
||||
func mapPublicError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if hasPublicError(err) {
|
||||
return err
|
||||
}
|
||||
publicErr := publicErrorFor(err)
|
||||
if publicErr == nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("%w: %w", publicErr, err)
|
||||
}
|
||||
|
||||
func hasPublicError(err error) bool {
|
||||
for _, publicErr := range []error{
|
||||
ErrInvalidConfig,
|
||||
ErrInvalidRequest,
|
||||
ErrPromptNotFound,
|
||||
ErrProfileNotFound,
|
||||
ErrPromptLoad,
|
||||
ErrProfileLoad,
|
||||
ErrArtifactLoad,
|
||||
ErrPromptRender,
|
||||
ErrLLMGenerate,
|
||||
ErrValidation,
|
||||
} {
|
||||
if errors.Is(err, publicErr) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func publicErrorFor(err error) error {
|
||||
switch {
|
||||
case errors.Is(err, promptdef.ErrPromptDefinitionNotFound):
|
||||
return ErrPromptNotFound
|
||||
case errors.Is(err, profile.ErrProfileNotFound):
|
||||
return ErrProfileNotFound
|
||||
case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition):
|
||||
return ErrPromptLoad
|
||||
case errors.Is(err, profile.ErrInvalidYAML), errors.Is(err, profile.ErrInvalidProfile):
|
||||
return ErrProfileLoad
|
||||
case errors.Is(err, usecase.ErrArtifactLoad):
|
||||
return ErrArtifactLoad
|
||||
case errors.Is(err, usecase.ErrPromptRender):
|
||||
return ErrPromptRender
|
||||
case errors.Is(err, usecase.ErrLLMGenerate):
|
||||
return ErrLLMGenerate
|
||||
case errors.Is(err, usecase.ErrValidation):
|
||||
return ErrValidation
|
||||
case errors.Is(err, usecase.ErrInvalidRequest):
|
||||
return ErrInvalidRequest
|
||||
case errors.Is(err, usecase.ErrProfileLoad):
|
||||
return ErrPromptLoad
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
50
examples/go-library/prepare/main.go
Normal file
50
examples/go-library/prepare/main.go
Normal file
@@ -0,0 +1,50 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
func main() {
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||
PromptDir: "./examples/prompts",
|
||||
ProfileDir: "./examples/profiles",
|
||||
SchemaDir: "./examples/schemas",
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
summary := struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
Model string `json:"model"`
|
||||
MessageCount int `json:"message_count"`
|
||||
InputHashes map[string]string `json:"input_hashes"`
|
||||
}{
|
||||
PromptID: prepared.PromptID,
|
||||
SelectedProfileID: prepared.SelectedProfileID,
|
||||
Model: prepared.EffectiveModelParams.Model,
|
||||
MessageCount: len(prepared.Messages),
|
||||
InputHashes: prepared.InputHashes,
|
||||
}
|
||||
|
||||
if err := json.NewEncoder(os.Stdout).Encode(summary); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
23
llm_adapter.go
Normal file
23
llm_adapter.go
Normal file
@@ -0,0 +1,23 @@
|
||||
package scriptorium
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
||||
)
|
||||
|
||||
type publicLLMClientAdapter struct {
|
||||
client LLMClient
|
||||
}
|
||||
|
||||
func (a publicLLMClientAdapter) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
resp, err := a.client.Generate(ctx, fromDomainGenerateRequest(req))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp == nil {
|
||||
return nil, fmt.Errorf("%w: llm client returned nil response", ErrLLMGenerate)
|
||||
}
|
||||
return toDomainGenerateResponse(resp), nil
|
||||
}
|
||||
256
types.go
Normal file
256
types.go
Normal file
@@ -0,0 +1,256 @@
|
||||
package scriptorium
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ArtifactRefType defines how an artifact is referenced.
|
||||
type ArtifactRefType string
|
||||
|
||||
const (
|
||||
ArtifactRefInline ArtifactRefType = "inline"
|
||||
ArtifactRefFile ArtifactRefType = "file"
|
||||
)
|
||||
|
||||
// OutputFormat defines the desired output format.
|
||||
type OutputFormat string
|
||||
|
||||
const (
|
||||
FormatText OutputFormat = "text"
|
||||
FormatMarkdown OutputFormat = "markdown"
|
||||
FormatJSON OutputFormat = "json"
|
||||
)
|
||||
|
||||
// ValidationMode defines the output validation strategy.
|
||||
type ValidationMode string
|
||||
|
||||
const (
|
||||
ValidationNone ValidationMode = "none"
|
||||
ValidationBasic ValidationMode = "basic"
|
||||
ValidationJSON ValidationMode = "json"
|
||||
ValidationJSONSchema ValidationMode = "json_schema"
|
||||
)
|
||||
|
||||
// ValidationStatus defines the result of a validation check.
|
||||
type ValidationStatus string
|
||||
|
||||
const (
|
||||
ValidationPassed ValidationStatus = "passed"
|
||||
ValidationFailed ValidationStatus = "failed"
|
||||
ValidationSkipped ValidationStatus = "skipped"
|
||||
)
|
||||
|
||||
// CacheControlType defines provider cache behavior for prompt content.
|
||||
type CacheControlType string
|
||||
|
||||
const (
|
||||
CacheControlEphemeral CacheControlType = "ephemeral"
|
||||
)
|
||||
|
||||
// StructuredOutputType identifies provider-level structured output modes.
|
||||
type StructuredOutputType string
|
||||
|
||||
const (
|
||||
StructuredOutputJSONSchema StructuredOutputType = "json_schema"
|
||||
)
|
||||
|
||||
// RunRequest represents a request to prepare or run a single prompt.
|
||||
type RunRequest struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
Inputs map[string]ArtifactRef
|
||||
Vars map[string]string
|
||||
Execution *ExecutionTargetOverride
|
||||
Validation *OutputContract
|
||||
Metadata map[string]string
|
||||
}
|
||||
|
||||
// PreparedRun contains prepared prompt execution state. It does not include
|
||||
// resolved API key values, model output, validation results, or internal target
|
||||
// presence metadata.
|
||||
type PreparedRun struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
EndTime time.Time `json:"end_time,omitempty"`
|
||||
DurationMS int64 `json:"duration_ms,omitempty"`
|
||||
}
|
||||
|
||||
// RunResult contains generated output, validation state, and run metadata.
|
||||
type RunResult struct {
|
||||
RunID string `json:"run_id"`
|
||||
Artifact Artifact `json:"artifact"`
|
||||
RawOutput string `json:"raw_output"`
|
||||
Validation ValidationResult `json:"validation"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
ModelName string `json:"model_name"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
Usage TokenUsage `json:"usage"`
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
EndTime time.Time `json:"end_time,omitempty"`
|
||||
Duration time.Duration `json:"duration,omitempty"`
|
||||
}
|
||||
|
||||
// ArtifactRef represents a reference to prompt input content.
|
||||
type ArtifactRef struct {
|
||||
Type ArtifactRefType
|
||||
URI string
|
||||
Body string
|
||||
}
|
||||
|
||||
// Artifact represents loaded artifact content.
|
||||
type Artifact struct {
|
||||
Name string
|
||||
ContentType string
|
||||
Body []byte
|
||||
URI string
|
||||
Size int64
|
||||
Hash string
|
||||
}
|
||||
|
||||
// ExecutionTarget represents effective model runtime settings.
|
||||
type ExecutionTarget struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Model string `json:"model"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
TopP float64 `json:"top_p"`
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
ServiceTier string `json:"service_tier"`
|
||||
ReasoningEffort string `json:"reasoning_effort"`
|
||||
APIKeyEnv string `json:"api_key_env"`
|
||||
ExtraParams map[string]any `json:"extra_params"`
|
||||
}
|
||||
|
||||
// ExecutionTargetOverride represents per-request runtime setting overrides.
|
||||
type ExecutionTargetOverride struct {
|
||||
Endpoint string
|
||||
Model string
|
||||
Temperature *float64
|
||||
MaxTokens *int
|
||||
TopP *float64
|
||||
TimeoutSeconds *int
|
||||
ServiceTier string
|
||||
ReasoningEffort string
|
||||
APIKeyEnv string
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
|
||||
// ExecutionTargetPresence tracks which numeric runtime settings were explicit
|
||||
// request overrides.
|
||||
type ExecutionTargetPresence struct {
|
||||
Temperature bool
|
||||
MaxTokens bool
|
||||
TopP bool
|
||||
TimeoutSeconds bool
|
||||
}
|
||||
|
||||
// OutputContract defines output and validation requirements.
|
||||
type OutputContract struct {
|
||||
Format OutputFormat `json:"format"`
|
||||
ValidationMode ValidationMode `json:"validation_mode"`
|
||||
SchemaPath string `json:"schema_path"`
|
||||
RepairAttempts int `json:"repair_attempts"`
|
||||
}
|
||||
|
||||
// ValidationResult represents output validation state.
|
||||
type ValidationResult struct {
|
||||
Status ValidationStatus `json:"status"`
|
||||
Mode ValidationMode `json:"mode"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
SchemaPath string `json:"schema_path,omitempty"`
|
||||
RepairAttempts int `json:"repair_attempts"`
|
||||
IsValid bool `json:"is_valid"`
|
||||
}
|
||||
|
||||
// TokenUsage tracks token consumption.
|
||||
type TokenUsage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||
}
|
||||
|
||||
// RenderedPrompt is the fully rendered prompt passed to an LLM client.
|
||||
type RenderedPrompt struct {
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
}
|
||||
|
||||
// RenderedMessage is a rendered chat message.
|
||||
type RenderedMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
CacheControl *CacheControl `json:"cache_control,omitempty"`
|
||||
}
|
||||
|
||||
// CacheControl describes provider cache metadata attached to prompt content.
|
||||
type CacheControl struct {
|
||||
Type CacheControlType `json:"type"`
|
||||
TTL string `json:"ttl,omitempty"`
|
||||
}
|
||||
|
||||
// StructuredOutputSpec describes provider-level structured output.
|
||||
type StructuredOutputSpec struct {
|
||||
Type StructuredOutputType `json:"type"`
|
||||
JSONSchema *StructuredOutputJSONSpec `json:"json_schema,omitempty"`
|
||||
}
|
||||
|
||||
// StructuredOutputJSONSpec contains JSON Schema output constraints.
|
||||
type StructuredOutputJSONSpec struct {
|
||||
Name string `json:"name"`
|
||||
Strict bool `json:"strict"`
|
||||
Schema any `json:"schema"`
|
||||
}
|
||||
|
||||
// LLMClient executes rendered prompts for Engine.Run.
|
||||
type LLMClient interface {
|
||||
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
|
||||
}
|
||||
|
||||
// GenerateRequest is passed to an injected LLM client.
|
||||
type GenerateRequest struct {
|
||||
Prompt RenderedPrompt `json:"prompt"`
|
||||
Target ExecutionTarget `json:"target"`
|
||||
TargetPresence ExecutionTargetPresence `json:"target_presence"`
|
||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||
}
|
||||
|
||||
// GenerateResponse is returned by an injected LLM client.
|
||||
type GenerateResponse struct {
|
||||
Content string `json:"content"`
|
||||
Usage TokenUsage `json:"usage"`
|
||||
}
|
||||
|
||||
// File returns a file-backed artifact reference.
|
||||
func File(path string) ArtifactRef {
|
||||
return ArtifactRef{Type: ArtifactRefFile, URI: path}
|
||||
}
|
||||
|
||||
// Inline returns an inline artifact reference.
|
||||
func Inline(body string) ArtifactRef {
|
||||
return ArtifactRef{Type: ArtifactRefInline, Body: body}
|
||||
}
|
||||
|
||||
// InlineWithURI returns an inline artifact reference with URI metadata.
|
||||
func InlineWithURI(uri string, body string) ArtifactRef {
|
||||
return ArtifactRef{Type: ArtifactRefInline, URI: uri, Body: body}
|
||||
}
|
||||
Reference in New Issue
Block a user