Compare commits
30 Commits
68cd90c657
...
v0.13.0
| Author | SHA1 | Date | |
|---|---|---|---|
| f33af20524 | |||
| 85d14784ca | |||
| edca9fbbc1 | |||
| 6c2541c1ee | |||
| b9b98b2aec | |||
| 8936ca7c18 | |||
| 3e5f5c5198 | |||
| 68ebe9ee50 | |||
| 36c7c5a358 | |||
| 157209f097 | |||
| 06b37c2bae | |||
| 1a0f15e210 | |||
| 5f946a5a1f | |||
| 1806df9888 | |||
| c35ac5a0cb | |||
| 40f8a1c628 | |||
| 27d7ad5057 | |||
| 1f08a1a94c | |||
| 2f42bdde39 | |||
| 9a00f30c7b | |||
| f71d2bbb73 | |||
| 6f64947e42 | |||
| 7bb4cf35b9 | |||
| fb0b21c51d | |||
| 5a00ca81a2 | |||
| e13610481d | |||
| 309fe9b7ea | |||
| adfd08ffe2 | |||
| c7263ab2a8 | |||
| 532c31c09a |
3
.gitignore
vendored
3
.gitignore
vendored
@@ -56,6 +56,8 @@ mono_crash.*
|
|||||||
[Dd]ebugPublic/
|
[Dd]ebugPublic/
|
||||||
[Rr]elease/
|
[Rr]elease/
|
||||||
[Rr]eleases/
|
[Rr]eleases/
|
||||||
|
!docs/releases/
|
||||||
|
!docs/releases/*.md
|
||||||
x64/
|
x64/
|
||||||
x86/
|
x86/
|
||||||
[Ww][Ii][Nn]32/
|
[Ww][Ii][Nn]32/
|
||||||
@@ -433,4 +435,3 @@ FodyWeavers.xsd
|
|||||||
|
|
||||||
# JetBrains Rider
|
# JetBrains Rider
|
||||||
*.sln.iml
|
*.sln.iml
|
||||||
|
|
||||||
|
|||||||
@@ -11,9 +11,16 @@ steps:
|
|||||||
version="$CI_COMMIT_TAG"
|
version="$CI_COMMIT_TAG"
|
||||||
dist="dist"
|
dist="dist"
|
||||||
pkg="gitea.maximumdirect.net/eric/scriptorium/cmd/scriptorium"
|
pkg="gitea.maximumdirect.net/eric/scriptorium/cmd/scriptorium"
|
||||||
|
notes="docs/releases/$version.md"
|
||||||
|
|
||||||
|
if [ ! -f "$notes" ]; then
|
||||||
|
printf 'release notes not found: %s\n' "$notes" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
rm -rf "$dist"
|
rm -rf "$dist"
|
||||||
mkdir -p "$dist"
|
mkdir -p "$dist"
|
||||||
|
cp "$notes" "$dist/RELEASE_NOTES.md"
|
||||||
|
|
||||||
build_binary() {
|
build_binary() {
|
||||||
goos="$1"
|
goos="$1"
|
||||||
@@ -22,7 +29,7 @@ steps:
|
|||||||
output="$dist/scriptorium-$version-$goos-$goarch$suffix"
|
output="$dist/scriptorium-$version-$goos-$goarch$suffix"
|
||||||
|
|
||||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
|
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
|
||||||
go build -trimpath -ldflags "-s -w -X gitea.maximumdirect.net/eric/scriptorium/internal/buildinfo.Version=$version" \
|
go build -trimpath -ldflags "-s -w" \
|
||||||
-o "$output" "$pkg"
|
-o "$output" "$pkg"
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -38,6 +45,7 @@ steps:
|
|||||||
from_secret: GITEA_RELEASE_TOKEN
|
from_secret: GITEA_RELEASE_TOKEN
|
||||||
files:
|
files:
|
||||||
- dist/scriptorium-*
|
- dist/scriptorium-*
|
||||||
|
note: dist/RELEASE_NOTES.md
|
||||||
checksum: sha256
|
checksum: sha256
|
||||||
checksum-file: SHA256SUMS
|
checksum-file: SHA256SUMS
|
||||||
checksum-flatten: true
|
checksum-flatten: true
|
||||||
|
|||||||
34
README.md
34
README.md
@@ -1,12 +1,15 @@
|
|||||||
# scriptorium
|
# Scriptorium
|
||||||
|
|
||||||
Scriptorium is a narrow prompt-execution application for rendering prompt
|
Scriptorium is a prompt-execution application with a command-line interface and
|
||||||
requests, running them against OpenAI-compatible chat-completions endpoints, and
|
an HTTP service. It prepares prompt requests, runs them against
|
||||||
serving the same run workflow over HTTP.
|
OpenAI-compatible model endpoints, and returns generated output with validation
|
||||||
|
metadata.
|
||||||
|
|
||||||
It keeps prompt definitions, execution profiles, schemas, and input artifacts as
|
The application uses
|
||||||
separate files so prompts can be reviewed and reused without baking model
|
[Promptkit v0.9.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/)
|
||||||
runtime settings into application code.
|
for prompt, profile, schema, preparation, generation, and validation behavior.
|
||||||
|
Scriptorium owns executable configuration, CLI and HTTP mapping, process
|
||||||
|
behavior, output presentation, and HTTP artifact-containment policy.
|
||||||
|
|
||||||
## Quickstart
|
## Quickstart
|
||||||
|
|
||||||
@@ -21,8 +24,9 @@ go run ./cmd/scriptorium render \
|
|||||||
--format json
|
--format json
|
||||||
```
|
```
|
||||||
|
|
||||||
This command renders the prepared prompt and effective runtime settings without calling an LLM.
|
This renders the prepared prompt and effective runtime settings without calling
|
||||||
For complete invocation and output behavior, see the [CLI reference](docs/cli.md).
|
a model. For complete invocation and output behavior, see the
|
||||||
|
[CLI reference](docs/cli.md).
|
||||||
|
|
||||||
## Documentation
|
## Documentation
|
||||||
|
|
||||||
@@ -31,15 +35,17 @@ For complete invocation and output behavior, see the [CLI reference](docs/cli.md
|
|||||||
- [HTTP API reference](docs/api.md)
|
- [HTTP API reference](docs/api.md)
|
||||||
- [Operations guide](docs/operations.md)
|
- [Operations guide](docs/operations.md)
|
||||||
- [Consumer integration overview](docs/consumers/api.md)
|
- [Consumer integration overview](docs/consumers/api.md)
|
||||||
- [Go library package](docs/consumers/pkg-scriptorium.md)
|
- [Migration from the former Go package](docs/consumers/migrating-to-promptkit.md)
|
||||||
- [Subprocess integration](docs/integrations/subprocess.md)
|
- [Subprocess integration](docs/integrations/subprocess.md)
|
||||||
- [OpenAI-compatible chat integration](docs/integrations/openai-compatible-chat.md)
|
|
||||||
- [Architecture policy](docs/policy/architecture.md)
|
- [Architecture policy](docs/policy/architecture.md)
|
||||||
|
- [Promptkit framework formats](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md)
|
||||||
|
- [Promptkit Go consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md)
|
||||||
|
|
||||||
## Examples
|
## Examples
|
||||||
|
|
||||||
- [Minimal configuration](examples/config.yml) and [complete configuration](examples/config.full.yml)
|
- [Minimal configuration](examples/config.yml) and
|
||||||
- [Prompt definitions](examples/prompts/), [execution profiles](examples/profiles/), [schemas](examples/schemas/), and [synthetic input fixtures](examples/fixtures/)
|
[complete configuration](examples/config.full.yml)
|
||||||
|
- [Prompt definitions](examples/prompts/), [execution profiles](examples/profiles/),
|
||||||
|
[schemas](examples/schemas/), and [synthetic input fixtures](examples/fixtures/)
|
||||||
- [Render script](examples/render-markdown-summary.sh)
|
- [Render script](examples/render-markdown-summary.sh)
|
||||||
- [HTTP request](examples/http-run.json)
|
- [HTTP request](examples/http-run.json)
|
||||||
- [Go library example](examples/go-library/prepare/main.go)
|
|
||||||
|
|||||||
@@ -1,39 +0,0 @@
|
|||||||
package scriptorium
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
|
|
||||||
artifactadapter "gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
var errNilArtifactReaderResponse = errors.New("artifact reader returned nil artifact without error")
|
|
||||||
|
|
||||||
type publicArtifactReaderAdapter struct {
|
|
||||||
reader ArtifactReader
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ artifactadapter.Reader = publicArtifactReaderAdapter{}
|
|
||||||
|
|
||||||
func (a publicArtifactReaderAdapter) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
|
||||||
artifact, err := a.reader.Read(ctx, ArtifactRef{
|
|
||||||
Type: ArtifactRefType(ref.Type),
|
|
||||||
URI: ref.URI,
|
|
||||||
Body: ref.Body,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if artifact == nil {
|
|
||||||
return nil, errNilArtifactReaderResponse
|
|
||||||
}
|
|
||||||
return &domain.Artifact{
|
|
||||||
Name: artifact.Name,
|
|
||||||
ContentType: artifact.ContentType,
|
|
||||||
Body: copyBytes(artifact.Body),
|
|
||||||
URI: artifact.URI,
|
|
||||||
Size: artifact.Size,
|
|
||||||
Hash: artifact.Hash,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
@@ -1,36 +0,0 @@
|
|||||||
package scriptorium
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestPublicArtifactReaderAdapterCopiesBody(t *testing.T) {
|
|
||||||
reader := internalArtifactReaderFake{
|
|
||||||
artifact: &Artifact{Body: []byte("original")},
|
|
||||||
}
|
|
||||||
adapter := publicArtifactReaderAdapter{reader: &reader}
|
|
||||||
|
|
||||||
artifact, err := adapter.Read(context.Background(), domain.ArtifactRef{
|
|
||||||
Type: domain.ArtifactRefInline,
|
|
||||||
URI: "memory://input",
|
|
||||||
Body: "input",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read artifact: %v", err)
|
|
||||||
}
|
|
||||||
artifact.Body[0] = 'X'
|
|
||||||
if got := string(reader.artifact.Body); got != "original" {
|
|
||||||
t.Fatalf("reader artifact body was mutated: %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type internalArtifactReaderFake struct {
|
|
||||||
artifact *Artifact
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *internalArtifactReaderFake) Read(context.Context, ArtifactRef) (*Artifact, error) {
|
|
||||||
return r.artifact, nil
|
|
||||||
}
|
|
||||||
406
convert.go
406
convert.go
@@ -1,406 +0,0 @@
|
|||||||
package scriptorium
|
|
||||||
|
|
||||||
import (
|
|
||||||
"reflect"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
|
||||||
execution, err := toDomainExecutionTargetOverride(req.Execution)
|
|
||||||
if err != nil {
|
|
||||||
return domain.RunRequest{}, err
|
|
||||||
}
|
|
||||||
return domain.RunRequest{
|
|
||||||
PromptID: req.PromptID,
|
|
||||||
PromptVersion: req.PromptVersion,
|
|
||||||
ProfileID: req.ProfileID,
|
|
||||||
APIKey: req.APIKey,
|
|
||||||
Inputs: toDomainArtifactRefMap(req.Inputs),
|
|
||||||
Vars: copyStringMap(req.Vars),
|
|
||||||
Execution: execution,
|
|
||||||
Validation: toDomainOutputContractPtr(req.Validation),
|
|
||||||
Metadata: copyStringMap(req.Metadata),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
|
||||||
if prepared == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return &PreparedRun{
|
|
||||||
PromptID: prepared.PromptID,
|
|
||||||
PromptVersion: prepared.PromptVersion,
|
|
||||||
PromptHash: prepared.PromptHash,
|
|
||||||
SelectedProfileID: prepared.SelectedProfileID,
|
|
||||||
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
|
|
||||||
OutputContract: fromDomainOutputContract(prepared.OutputContract),
|
|
||||||
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
|
|
||||||
InputHashes: copyStringMap(prepared.InputHashes),
|
|
||||||
SessionID: prepared.SessionID,
|
|
||||||
RenderedPromptHash: prepared.RenderedPromptHash,
|
|
||||||
Messages: fromDomainRenderedMessages(prepared.Messages),
|
|
||||||
StartTime: prepared.StartTime,
|
|
||||||
EndTime: prepared.EndTime,
|
|
||||||
DurationMS: prepared.DurationMS,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
|
||||||
if result == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return &RunResult{
|
|
||||||
RunID: result.RunID,
|
|
||||||
Artifact: fromDomainArtifact(result.Artifact),
|
|
||||||
RawOutput: result.RawOutput,
|
|
||||||
Validation: fromDomainValidationResult(result.Validation),
|
|
||||||
PromptID: result.PromptID,
|
|
||||||
PromptVersion: result.PromptVersion,
|
|
||||||
PromptHash: result.PromptHash,
|
|
||||||
RenderedPromptHash: result.RenderedPromptHash,
|
|
||||||
SelectedProfileID: result.SelectedProfileID,
|
|
||||||
ModelName: result.ModelName,
|
|
||||||
Endpoint: result.Endpoint,
|
|
||||||
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
|
|
||||||
InputHashes: copyStringMap(result.InputHashes),
|
|
||||||
Usage: fromDomainTokenUsage(result.Usage),
|
|
||||||
StartTime: result.StartTime,
|
|
||||||
EndTime: result.EndTime,
|
|
||||||
Duration: result.Duration,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func fromDomainGenerateRequest(req domain.GenerateRequest) GenerateRequest {
|
|
||||||
return GenerateRequest{
|
|
||||||
Prompt: fromDomainRenderedPrompt(req.Prompt),
|
|
||||||
Target: fromDomainExecutionTarget(req.Target),
|
|
||||||
TargetPresence: fromDomainExecutionTargetPresence(req.TargetPresence),
|
|
||||||
StructuredOutput: fromDomainStructuredOutputSpec(req.StructuredOutput),
|
|
||||||
APIKey: req.Target.APIKey,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse {
|
|
||||||
if resp == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return &domain.GenerateResponse{
|
|
||||||
Content: resp.Content,
|
|
||||||
Usage: toDomainTokenUsage(resp.Usage),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func fromDomainRenderedPrompt(prompt domain.RenderedPrompt) RenderedPrompt {
|
|
||||||
return RenderedPrompt{
|
|
||||||
SessionID: prompt.SessionID,
|
|
||||||
Messages: fromDomainRenderedMessages(prompt.Messages),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func toDomainArtifactRefMap(src map[string]ArtifactRef) map[string]domain.ArtifactRef {
|
|
||||||
if src == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := make(map[string]domain.ArtifactRef, len(src))
|
|
||||||
for k, v := range src {
|
|
||||||
out[k] = toDomainArtifactRef(v)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func toDomainArtifactRef(ref ArtifactRef) domain.ArtifactRef {
|
|
||||||
return domain.ArtifactRef{
|
|
||||||
Type: domain.ArtifactRefType(ref.Type),
|
|
||||||
URI: ref.URI,
|
|
||||||
Body: ref.Body,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func fromDomainArtifact(artifact domain.Artifact) Artifact {
|
|
||||||
return Artifact{
|
|
||||||
Name: artifact.Name,
|
|
||||||
ContentType: artifact.ContentType,
|
|
||||||
Body: copyBytes(artifact.Body),
|
|
||||||
URI: artifact.URI,
|
|
||||||
Size: artifact.Size,
|
|
||||||
Hash: artifact.Hash,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain.ExecutionTargetOverride, error) {
|
|
||||||
if override == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
extraParams, err := copyPublicJSONMap(override.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &domain.ExecutionTargetOverride{
|
|
||||||
Endpoint: override.Endpoint,
|
|
||||||
Model: override.Model,
|
|
||||||
Temperature: copyFloat64Ptr(override.Temperature),
|
|
||||||
MaxTokens: copyIntPtr(override.MaxTokens),
|
|
||||||
TopP: copyFloat64Ptr(override.TopP),
|
|
||||||
TimeoutSeconds: copyIntPtr(override.TimeoutSeconds),
|
|
||||||
ServiceTier: override.ServiceTier,
|
|
||||||
ReasoningEffort: override.ReasoningEffort,
|
|
||||||
APIKeyEnv: override.APIKeyEnv,
|
|
||||||
ExtraParams: extraParams,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
|
||||||
return ExecutionTarget{
|
|
||||||
Endpoint: target.Endpoint,
|
|
||||||
Model: target.Model,
|
|
||||||
Temperature: target.Temperature,
|
|
||||||
MaxTokens: target.MaxTokens,
|
|
||||||
TopP: target.TopP,
|
|
||||||
TimeoutSeconds: target.TimeoutSeconds,
|
|
||||||
ServiceTier: target.ServiceTier,
|
|
||||||
ReasoningEffort: target.ReasoningEffort,
|
|
||||||
APIKeyEnv: target.APIKeyEnv,
|
|
||||||
ExtraParams: copyAnyMap(target.ExtraParams),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
|
||||||
return ExecutionTargetPresence{
|
|
||||||
Temperature: presence.Temperature,
|
|
||||||
MaxTokens: presence.MaxTokens,
|
|
||||||
TopP: presence.TopP,
|
|
||||||
TimeoutSeconds: presence.TimeoutSeconds,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func toDomainOutputContractPtr(contract *OutputContract) *domain.OutputContract {
|
|
||||||
if contract == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := toDomainOutputContract(*contract)
|
|
||||||
return &out
|
|
||||||
}
|
|
||||||
|
|
||||||
func toDomainOutputContract(contract OutputContract) domain.OutputContract {
|
|
||||||
return domain.OutputContract{
|
|
||||||
Format: domain.OutputFormat(contract.Format),
|
|
||||||
ValidationMode: domain.ValidationMode(contract.ValidationMode),
|
|
||||||
SchemaPath: contract.SchemaPath,
|
|
||||||
RepairAttempts: contract.RepairAttempts,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func fromDomainOutputContract(contract domain.OutputContract) OutputContract {
|
|
||||||
return OutputContract{
|
|
||||||
Format: OutputFormat(contract.Format),
|
|
||||||
ValidationMode: ValidationMode(contract.ValidationMode),
|
|
||||||
SchemaPath: contract.SchemaPath,
|
|
||||||
RepairAttempts: contract.RepairAttempts,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func fromDomainValidationResult(result domain.ValidationResult) ValidationResult {
|
|
||||||
return ValidationResult{
|
|
||||||
Status: ValidationStatus(result.Status),
|
|
||||||
Mode: ValidationMode(result.Mode),
|
|
||||||
Errors: copyStringSlice(result.Errors),
|
|
||||||
SchemaPath: result.SchemaPath,
|
|
||||||
RepairAttempts: result.RepairAttempts,
|
|
||||||
IsValid: result.IsValid,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func fromDomainTokenUsage(usage domain.TokenUsage) TokenUsage {
|
|
||||||
return TokenUsage{
|
|
||||||
PromptTokens: usage.PromptTokens,
|
|
||||||
CompletionTokens: usage.CompletionTokens,
|
|
||||||
TotalTokens: usage.TotalTokens,
|
|
||||||
CachedTokens: usage.CachedTokens,
|
|
||||||
CacheWriteTokens: usage.CacheWriteTokens,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func toDomainTokenUsage(usage TokenUsage) domain.TokenUsage {
|
|
||||||
return domain.TokenUsage{
|
|
||||||
PromptTokens: usage.PromptTokens,
|
|
||||||
CompletionTokens: usage.CompletionTokens,
|
|
||||||
TotalTokens: usage.TotalTokens,
|
|
||||||
CachedTokens: usage.CachedTokens,
|
|
||||||
CacheWriteTokens: usage.CacheWriteTokens,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func fromDomainRenderedMessages(messages []domain.RenderedMessage) []RenderedMessage {
|
|
||||||
if messages == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := make([]RenderedMessage, len(messages))
|
|
||||||
for i, msg := range messages {
|
|
||||||
out[i] = RenderedMessage{
|
|
||||||
Role: msg.Role,
|
|
||||||
Content: msg.Content,
|
|
||||||
CacheControl: fromDomainCacheControl(msg.CacheControl),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl {
|
|
||||||
if cacheControl == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return &CacheControl{
|
|
||||||
Type: CacheControlType(cacheControl.Type),
|
|
||||||
TTL: cacheControl.TTL,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func fromDomainStructuredOutputSpec(spec *domain.StructuredOutputSpec) *StructuredOutputSpec {
|
|
||||||
if spec == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := &StructuredOutputSpec{
|
|
||||||
Type: StructuredOutputType(spec.Type),
|
|
||||||
}
|
|
||||||
if spec.JSONSchema != nil {
|
|
||||||
out.JSONSchema = &StructuredOutputJSONSpec{
|
|
||||||
Name: spec.JSONSchema.Name,
|
|
||||||
Strict: spec.JSONSchema.Strict,
|
|
||||||
Schema: copyAny(spec.JSONSchema.Schema),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyStringMap(src map[string]string) map[string]string {
|
|
||||||
if src == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := make(map[string]string, len(src))
|
|
||||||
for k, v := range src {
|
|
||||||
out[k] = v
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyAnyMap(src map[string]any) map[string]any {
|
|
||||||
if src == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := make(map[string]any, len(src))
|
|
||||||
for k, v := range src {
|
|
||||||
out[k] = copyAny(v)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyAny(value any) any {
|
|
||||||
if value == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
switch v := value.(type) {
|
|
||||||
case map[string]any:
|
|
||||||
return copyAnyMap(v)
|
|
||||||
case []any:
|
|
||||||
out := make([]any, len(v))
|
|
||||||
for i, item := range v {
|
|
||||||
out[i] = copyAny(item)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case []string:
|
|
||||||
return copyStringSlice(v)
|
|
||||||
case []byte:
|
|
||||||
return copyBytes(v)
|
|
||||||
default:
|
|
||||||
return copyReflectValue(reflect.ValueOf(value)).Interface()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyReflectValue(value reflect.Value) reflect.Value {
|
|
||||||
if !value.IsValid() {
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
|
|
||||||
switch value.Kind() {
|
|
||||||
case reflect.Interface:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
copied := copyReflectValue(value.Elem())
|
|
||||||
if copied.IsValid() && copied.Type().AssignableTo(value.Type()) {
|
|
||||||
return copied
|
|
||||||
}
|
|
||||||
out := reflect.New(value.Type()).Elem()
|
|
||||||
out.Set(copied)
|
|
||||||
return out
|
|
||||||
case reflect.Pointer:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
out := reflect.New(value.Type().Elem())
|
|
||||||
out.Elem().Set(copyReflectValue(value.Elem()))
|
|
||||||
return out
|
|
||||||
case reflect.Map:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
out := reflect.MakeMapWithSize(value.Type(), value.Len())
|
|
||||||
iter := value.MapRange()
|
|
||||||
for iter.Next() {
|
|
||||||
out.SetMapIndex(copyReflectValue(iter.Key()), copyReflectValue(iter.Value()))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case reflect.Slice:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap())
|
|
||||||
for i := 0; i < value.Len(); i++ {
|
|
||||||
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case reflect.Array:
|
|
||||||
out := reflect.New(value.Type()).Elem()
|
|
||||||
for i := 0; i < value.Len(); i++ {
|
|
||||||
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
default:
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyStringSlice(src []string) []string {
|
|
||||||
if src == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := make([]string, len(src))
|
|
||||||
copy(out, src)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyBytes(src []byte) []byte {
|
|
||||||
if src == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := make([]byte, len(src))
|
|
||||||
copy(out, src)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyFloat64Ptr(src *float64) *float64 {
|
|
||||||
if src == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
v := *src
|
|
||||||
return &v
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyIntPtr(src *int) *int {
|
|
||||||
if src == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
v := *src
|
|
||||||
return &v
|
|
||||||
}
|
|
||||||
112
docs/adr/0004-definition-boundary.md
Normal file
112
docs/adr/0004-definition-boundary.md
Normal file
@@ -0,0 +1,112 @@
|
|||||||
|
# Preserve Definition Compatibility And A Simple Execution Model
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Accepted.
|
||||||
|
|
||||||
|
## Date
|
||||||
|
|
||||||
|
2026-08-29.
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Scriptorium is a user-facing application built on Promptkit. Promptkit is a Go
|
||||||
|
library with both declarative framework formats and public APIs intended for
|
||||||
|
applications that need to assemble specialized integrations or workflows.
|
||||||
|
|
||||||
|
Scriptorium needs a durable rule for deciding which upstream capabilities it
|
||||||
|
must support. Treating every new Promptkit public API as an application feature
|
||||||
|
would steadily expand Scriptorium into a workflow framework. Selecting an
|
||||||
|
arbitrary subset of prompt or profile fields would instead make valid Promptkit
|
||||||
|
definitions unexpectedly unusable through Scriptorium.
|
||||||
|
|
||||||
|
The intended product is narrower: Scriptorium provides simple executable
|
||||||
|
interfaces that select and execute predefined prompts and return their results
|
||||||
|
in a consistent, repeatable form. It is not intended to own conversations,
|
||||||
|
modify prompts dynamically, coordinate follow-up messages, or manage complex
|
||||||
|
multi-step workflows.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Scriptorium will aim to support the complete set of features expressible in
|
||||||
|
valid Promptkit prompt and profile definitions for the Promptkit version it
|
||||||
|
selects. This includes the schema, backend, source, and request plumbing needed
|
||||||
|
to prepare and execute those definitions without Scriptorium imposing a
|
||||||
|
narrower field-level format contract.
|
||||||
|
|
||||||
|
Promptkit remains the canonical parser and validator for its definitions.
|
||||||
|
Scriptorium will pass its configured framework sources and mapped request
|
||||||
|
values through the supported Promptkit public facade rather than copying
|
||||||
|
Promptkit formats, defaults, or validation rules.
|
||||||
|
|
||||||
|
Promptkit library features that are not required to use valid prompt or profile
|
||||||
|
definitions are considered individually. They may be incorporated when they
|
||||||
|
serve Scriptorium's simple execution model and fit its application-owned CLI,
|
||||||
|
HTTP, configuration, presentation, security, and process boundaries. A public
|
||||||
|
Promptkit declaration does not by itself require a corresponding Scriptorium
|
||||||
|
interface.
|
||||||
|
|
||||||
|
Each execution request will remain self-contained from the caller's
|
||||||
|
perspective. A request must not depend on conversation or workflow state
|
||||||
|
retained from a completed request, and Scriptorium will not become a general
|
||||||
|
workflow or conversation system. Application-managed prompt modification,
|
||||||
|
appended or follow-up message flows, conversation state, prepared-handle
|
||||||
|
coordination, multi-step orchestration, and similar higher-level workflows are
|
||||||
|
outside its intended scope.
|
||||||
|
|
||||||
|
Scriptorium may, and when necessary must, maintain process-scoped operational
|
||||||
|
state shared by concurrent requests. This includes the configured Promptkit
|
||||||
|
engine and backend registry, capacity admission and queue accounting,
|
||||||
|
in-flight request coordination, cancellation, and ordinary server lifecycle
|
||||||
|
resources. The state must be scoped so configured backend limits apply across
|
||||||
|
all relevant in-flight requests. But this is operational implementation state,
|
||||||
|
not durable user workflow or conversation state.
|
||||||
|
|
||||||
|
## Alternatives Considered
|
||||||
|
|
||||||
|
### Expose Every Promptkit Public Feature
|
||||||
|
|
||||||
|
This would make the application surface track the library API closely, but it
|
||||||
|
would blur the boundary between a reusable Go library and a simple executable
|
||||||
|
consumer. Library-oriented lifecycle, source-construction, extension, and
|
||||||
|
workflow primitives would add commands, wire contracts, configuration, and
|
||||||
|
state without necessarily improving predefined prompt execution.
|
||||||
|
|
||||||
|
### Support A Curated Subset Of Definition Fields
|
||||||
|
|
||||||
|
This would keep the application small in the short term, but valid Promptkit
|
||||||
|
definitions could fail or behave differently solely because they were invoked
|
||||||
|
through Scriptorium. Maintaining a parallel field-level compatibility list
|
||||||
|
would also duplicate upstream format ownership and create recurring drift.
|
||||||
|
|
||||||
|
### Expand Scriptorium Into A Workflow Service
|
||||||
|
|
||||||
|
Scriptorium could own conversations, prompt changes, follow-up messages,
|
||||||
|
retries, checkpoints, and multi-step execution. That is a different product
|
||||||
|
with durable state, lifecycle, recovery, privacy, and operational requirements
|
||||||
|
that are not justified by Scriptorium's current purpose.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
- Promptkit upgrades require an audit of the complete prompt and profile format
|
||||||
|
contracts, not only a source-compatibility build.
|
||||||
|
- Scriptorium adapters must not add unconditional request requirements that
|
||||||
|
reject a definition Promptkit can validly prepare or execute.
|
||||||
|
- Application configuration may need new plumbing, such as custom backend
|
||||||
|
registration, when it is necessary to use a valid profile feature.
|
||||||
|
- Definition parsing, validation, precedence, inheritance, defaults, and
|
||||||
|
built-in catalogs remain Promptkit responsibilities.
|
||||||
|
- A running HTTP server must share its appropriately scoped Promptkit engine
|
||||||
|
across requests so the engine's backend admission and concurrency limits
|
||||||
|
cannot be bypassed by per-request construction.
|
||||||
|
- Process-scoped counters, queues, in-flight request records, cancellation, and
|
||||||
|
lifecycle resources are permitted implementation state. They must be
|
||||||
|
concurrency-safe and do not create a durable resume or conversation
|
||||||
|
contract.
|
||||||
|
- Library-only source constructors, injected collaborators, opaque lifecycle
|
||||||
|
handles, and request-composition primitives remain optional Scriptorium
|
||||||
|
features rather than automatic requirements.
|
||||||
|
- New application features must be evaluated against the simple,
|
||||||
|
request-independent predefined-prompt execution model.
|
||||||
|
- Work that implements this decision remains tracked in roadmaps until it is
|
||||||
|
delivered and incorporated into current-state contracts.
|
||||||
44
docs/api.md
44
docs/api.md
@@ -32,14 +32,11 @@ returns `413 response_too_large`.
|
|||||||
### Request Body
|
### Request Body
|
||||||
|
|
||||||
The maintained [request example](../examples/http-run.json) is a complete
|
The maintained [request example](../examples/http-run.json) is a complete
|
||||||
copyable shape. The smallest valid shape is:
|
copyable shape. At the HTTP adapter boundary, the smallest valid shape is:
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"prompt_id": "generic.markdown_summary",
|
"prompt_id": "generic.markdown_summary"
|
||||||
"inputs": {
|
|
||||||
"transcript": {"type": "inline", "body": "Source text"}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -48,7 +45,8 @@ copyable shape. The smallest valid shape is:
|
|||||||
| `prompt_id` | yes | Non-blank prompt ID. |
|
| `prompt_id` | yes | Non-blank prompt ID. |
|
||||||
| `prompt_version` | no | Prompt version filter. |
|
| `prompt_version` | no | Prompt version filter. |
|
||||||
| `profile_id` | no | Execution-profile ID; otherwise the prompt must set `default_profile`. |
|
| `profile_id` | no | Execution-profile ID; otherwise the prompt must set `default_profile`. |
|
||||||
| `inputs` | yes | Non-empty object mapping input names to references. |
|
| `session_id` | no | Optional direct, non-secret session identifier. |
|
||||||
|
| `inputs` | no | Optional object mapping input names to references. Promptkit decides whether the selected definition needs them. |
|
||||||
| `vars` | no | Object mapping template-variable names to strings. |
|
| `vars` | no | Object mapping template-variable names to strings. |
|
||||||
| `model` | no | Runtime model-override object. |
|
| `model` | no | Runtime model-override object. |
|
||||||
| `include_raw_output` | no | Include `raw_model_output` when true. |
|
| `include_raw_output` | no | Include `raw_model_output` when true. |
|
||||||
@@ -66,20 +64,32 @@ untrusted writers.
|
|||||||
The optional `model` object accepts `endpoint`, `model`, `temperature`,
|
The optional `model` object accepts `endpoint`, `model`, `temperature`,
|
||||||
`max_tokens`, `top_p`, `timeout_seconds`, `service_tier`,
|
`max_tokens`, `top_p`, `timeout_seconds`, `service_tier`,
|
||||||
`reasoning_effort`, `api_key_env`, and `extra_params`. Numeric ranges and
|
`reasoning_effort`, `api_key_env`, and `extra_params`. Numeric ranges and
|
||||||
credential supply are defined by the [configuration reference](config.md).
|
framework credential semantics are defined by the
|
||||||
|
[Promptkit format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md).
|
||||||
Explicit zero values for the numeric fields are overrides; zero
|
Explicit zero values for the numeric fields are overrides; zero
|
||||||
`timeout_seconds` disables the per-generation deadline only, retaining the
|
`timeout_seconds` disables the per-generation deadline only, retaining the
|
||||||
request context and configured transport cap. The timeout layers are defined in
|
request context and configured transport cap. The timeout layers are defined in
|
||||||
the [outbound integration contract](integrations/openai-compatible-chat.md#authentication-and-timeout).
|
the [Promptkit outbound integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/integrations/openai-compatible-chat.md#timeout-and-cancellation).
|
||||||
|
|
||||||
Raw API-key values are not accepted. `api_key` and any other unknown model
|
Raw API-key values are not accepted. `api_key` and any other unknown model
|
||||||
field cause `400 invalid_json`.
|
field cause `400 invalid_json`.
|
||||||
|
|
||||||
|
`model.reasoning_effort` is an optional JSON string with three states: omission
|
||||||
|
inherits the selected profile, a non-empty string replaces its value, and an
|
||||||
|
empty string explicitly clears it. JSON `null` is treated as omission.
|
||||||
|
|
||||||
|
`session_id` is passed directly to Promptkit. A nonblank value replaces a
|
||||||
|
definition-rendered session ID; omission or a blank value lets the definition
|
||||||
|
provide one. Promptkit trims direct values and limits them to 256 Unicode code
|
||||||
|
points. Session IDs are not credentials and may be included in prepared data,
|
||||||
|
results, and provider-facing requests, so use stable non-sensitive identifiers.
|
||||||
|
|
||||||
### Strict JSON
|
### Strict JSON
|
||||||
|
|
||||||
Request decoding rejects malformed JSON, unknown fields at every request level,
|
Request decoding rejects malformed JSON, unknown fields at every request level,
|
||||||
and trailing JSON tokens with `400 invalid_json`. A blank `prompt_id` or
|
and trailing JSON tokens with `400 invalid_json`. A blank `prompt_id` returns
|
||||||
empty `inputs` object returns `400 invalid_request`.
|
`400 invalid_request`. Omitted or empty `inputs` are passed to Promptkit, which
|
||||||
|
reports any definition-required or template-referenced inputs.
|
||||||
|
|
||||||
### Success Response
|
### Success Response
|
||||||
|
|
||||||
@@ -90,17 +100,26 @@ validation contract. The response contains:
|
|||||||
optional `uri`;
|
optional `uri`;
|
||||||
- `validation`: `status`, `mode`, `repair_attempts`, `is_valid`, plus
|
- `validation`: `status`, `mode`, `repair_attempts`, `is_valid`, plus
|
||||||
optional `errors` and `schema_path`;
|
optional `errors` and `schema_path`;
|
||||||
- `metadata`: run, prompt, rendered-prompt, profile, model, input-hash, usage,
|
- `metadata`: run, prompt, rendered-prompt, profile, optional backend identity, model, input-hash, usage,
|
||||||
timing, validation, and repair-attempt metadata; and
|
timing, validation, and repair-attempt metadata; and
|
||||||
- optional `raw_model_output` when requested.
|
- optional `raw_model_output` when requested.
|
||||||
|
|
||||||
`metadata.model_params` has `endpoint`, `model`, `temperature`,
|
`metadata.model_params` has `endpoint`, `model`, `temperature`,
|
||||||
`max_tokens`, `top_p`, and `timeout_seconds`, plus optional
|
`max_tokens`, `top_p`, and `timeout_seconds`, plus optional
|
||||||
`service_tier`, `reasoning_effort`, `api_key_env`, and `extra_params`.
|
`backend_id`, `service_tier`, `reasoning_effort`, `api_key_env`, and `extra_params`.
|
||||||
`metadata.usage` always includes `prompt_tokens`, `completion_tokens`,
|
`metadata.usage` always includes `prompt_tokens`, `completion_tokens`,
|
||||||
`total_tokens`, `cached_tokens`, and `cache_write_tokens`; unavailable
|
`total_tokens`, `cached_tokens`, and `cache_write_tokens`; unavailable
|
||||||
cache usage is reported as zero.
|
cache usage is reported as zero.
|
||||||
|
|
||||||
|
When Promptkit resolves a direct or definition-rendered session ID,
|
||||||
|
`metadata.session_id` contains that effective result value. It is omitted when
|
||||||
|
no effective session ID exists.
|
||||||
|
|
||||||
|
`metadata.selected_backend_id` and `metadata.model_params.backend_id` report
|
||||||
|
the corresponding Promptkit result fields independently when present. Both are
|
||||||
|
omitted for an endpoint-only profile; Scriptorium does not infer backend
|
||||||
|
identity from an endpoint.
|
||||||
|
|
||||||
A validation failure has `validation.status: "failed"`, `is_valid: false`,
|
A validation failure has `validation.status: "failed"`, `is_valid: false`,
|
||||||
and any available diagnostic errors, while still returning the artifact and
|
and any available diagnostic errors, while still returning the artifact and
|
||||||
metadata.
|
metadata.
|
||||||
@@ -136,6 +155,7 @@ Messages are concise and do not expose wrapped internal causes.
|
|||||||
| `500` | `validation_runtime_failed` | Schema or validator runtime failure. |
|
| `500` | `validation_runtime_failed` | Schema or validator runtime failure. |
|
||||||
| `500` | `internal_error` | Unclassified server failure. |
|
| `500` | `internal_error` | Unclassified server failure. |
|
||||||
| `502` | `llm_failed` | Outbound model request failed. |
|
| `502` | `llm_failed` | Outbound model request failed. |
|
||||||
|
| `503` | `capacity_exceeded` | The selected model backend has no admission capacity. No retry timing is supplied. |
|
||||||
|
|
||||||
## Retry And Idempotency
|
## Retry And Idempotency
|
||||||
|
|
||||||
|
|||||||
74
docs/cli.md
74
docs/cli.md
@@ -1,7 +1,7 @@
|
|||||||
# CLI Reference
|
# CLI Reference
|
||||||
|
|
||||||
This is the canonical contract for invoking Scriptorium. Configuration discovery,
|
This is the canonical contract for invoking Scriptorium. Configuration discovery,
|
||||||
precedence, directories, profiles, and schemas are defined in the
|
precedence, application source locations, and server settings are defined in the
|
||||||
[configuration reference](config.md). The [HTTP API reference](api.md) owns
|
[configuration reference](config.md). The [HTTP API reference](api.md) owns
|
||||||
service request and response behavior.
|
service request and response behavior.
|
||||||
|
|
||||||
@@ -23,6 +23,8 @@ go run ./cmd/scriptorium render \
|
|||||||
generated artifact.
|
generated artifact.
|
||||||
- `scriptorium render`: prepare a prompt and write prepared-run output.
|
- `scriptorium render`: prepare a prompt and write prepared-run output.
|
||||||
- `scriptorium serve`: start the HTTP server.
|
- `scriptorium serve`: start the HTTP server.
|
||||||
|
- `scriptorium inspect prompt`: inspect one prompt definition without model execution.
|
||||||
|
- `scriptorium inspect profile`: inspect one effective profile without model execution.
|
||||||
|
|
||||||
All commands accept `--config <path>` and reject positional arguments. An
|
All commands accept `--config <path>` and reject positional arguments. An
|
||||||
effective `prompt_dir` is required for every command. Supply it through the
|
effective `prompt_dir` is required for every command. Supply it through the
|
||||||
@@ -39,7 +41,6 @@ Required flags:
|
|||||||
| Flag | Meaning |
|
| Flag | Meaning |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `--prompt <id>` | Prompt ID to execute. |
|
| `--prompt <id>` | Prompt ID to execute. |
|
||||||
| `--input name=path` | Input file mapping; repeat or use comma-separated mappings. |
|
|
||||||
|
|
||||||
Optional flags:
|
Optional flags:
|
||||||
|
|
||||||
@@ -49,7 +50,10 @@ Optional flags:
|
|||||||
| `--prompt-dir <dir>` | Prompt-definition directory override. |
|
| `--prompt-dir <dir>` | Prompt-definition directory override. |
|
||||||
| `--profile-dir <dir>` | Custom profile-directory override. |
|
| `--profile-dir <dir>` | Custom profile-directory override. |
|
||||||
| `--schema-dir <dir>` | Schema base-directory override. |
|
| `--schema-dir <dir>` | Schema base-directory override. |
|
||||||
|
| `--prompt-version <version>` | Optional prompt-definition version selector. |
|
||||||
| `--profile <id>` | Execution-profile override. |
|
| `--profile <id>` | Execution-profile override. |
|
||||||
|
| `--session-id <id>` | Optional direct session identifier. |
|
||||||
|
| `--input name=path` | Optional input file mapping; repeat or use comma-separated mappings. |
|
||||||
| `--var name=value` | Template-variable mapping; repeat or use comma-separated mappings. |
|
| `--var name=value` | Template-variable mapping; repeat or use comma-separated mappings. |
|
||||||
| `--out <path>` | Write generated content to this file instead of stdout. |
|
| `--out <path>` | Write generated content to this file instead of stdout. |
|
||||||
| `--llm-base-url <url>` | Runtime endpoint override. |
|
| `--llm-base-url <url>` | Runtime endpoint override. |
|
||||||
@@ -58,6 +62,7 @@ Optional flags:
|
|||||||
| `--temperature <float>` | Runtime temperature override. |
|
| `--temperature <float>` | Runtime temperature override. |
|
||||||
| `--max-tokens <int>` | Runtime maximum-token override. |
|
| `--max-tokens <int>` | Runtime maximum-token override. |
|
||||||
| `--top-p <float>` | Runtime top-p override. |
|
| `--top-p <float>` | Runtime top-p override. |
|
||||||
|
| `--reasoning-effort <value>` | Runtime reasoning-effort override. |
|
||||||
| `--timeout <duration>` | Runtime timeout override using Go duration syntax. |
|
| `--timeout <duration>` | Runtime timeout override using Go duration syntax. |
|
||||||
|
|
||||||
Deprecated aliases: `--prompt-id` for `--prompt`, and `--profile-id` for
|
Deprecated aliases: `--prompt-id` for `--prompt`, and `--profile-id` for
|
||||||
@@ -69,25 +74,44 @@ only; the caller context and configured transport cap remain active. CLI
|
|||||||
durations are converted to whole seconds by truncation toward zero, so any
|
durations are converted to whole seconds by truncation toward zero, so any
|
||||||
duration whose absolute value is below one second becomes an explicit
|
duration whose absolute value is below one second becomes an explicit
|
||||||
zero-second override. The timeout layers are defined in the
|
zero-second override. The timeout layers are defined in the
|
||||||
[outbound integration contract](integrations/openai-compatible-chat.md#authentication-and-timeout).
|
[Promptkit outbound integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/integrations/openai-compatible-chat.md#timeout-and-cancellation).
|
||||||
|
|
||||||
There is no raw API-key flag. Use `--api-key-env`.
|
There is no raw API-key flag. Use `--api-key-env`.
|
||||||
|
|
||||||
|
`--prompt-version` is passed directly to Promptkit. When it is omitted, the
|
||||||
|
selected prompt ID must have exactly one available version. `--input` is
|
||||||
|
optional at the CLI boundary: Promptkit decides whether the selected definition
|
||||||
|
requires declared inputs or template-referenced values.
|
||||||
|
|
||||||
|
`--session-id` supplies a direct, non-secret session identifier. A nonblank
|
||||||
|
value replaces a definition-rendered session ID; an omitted or blank value lets
|
||||||
|
the definition supply one. Promptkit trims direct values and limits them to 256
|
||||||
|
Unicode code points. Use stable, non-sensitive identifiers because effective
|
||||||
|
session IDs may appear in prepared output, run metadata, and provider-facing
|
||||||
|
requests.
|
||||||
|
|
||||||
|
`--reasoning-effort` is presence-aware: omitting it inherits the selected
|
||||||
|
profile value, a nonblank value replaces that value, and
|
||||||
|
`--reasoning-effort=` explicitly clears inherited reasoning. Promptkit treats
|
||||||
|
nonblank values as provider-specific opaque strings.
|
||||||
|
|
||||||
## `scriptorium render`
|
## `scriptorium render`
|
||||||
|
|
||||||
```text
|
```text
|
||||||
scriptorium render [flags]
|
scriptorium render [flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
`--prompt <id>` and at least one `--input name=path` are required. The
|
`--prompt <id>` is required. The following optional flags are supported:
|
||||||
following optional flags are supported: `--config`, `--prompt-dir`,
|
`--config`, `--prompt-dir`, `--profile-dir`, `--prompt-version`, `--profile`,
|
||||||
`--profile-dir`, `--profile`, `--var`, `--out`, `--llm-base-url`,
|
`--input`, `--var`, `--out`, `--llm-base-url`,
|
||||||
`--model`, `--api-key-env`, `--temperature`, `--max-tokens`, `--top-p`,
|
`--model`, `--api-key-env`, `--temperature`, `--max-tokens`, `--top-p`,
|
||||||
`--timeout`, and `--format text|json`. Their meanings match the corresponding
|
`--reasoning-effort`, `--session-id`, `--timeout`, and `--format text|json`. Their meanings match the corresponding
|
||||||
`run` flags; `--format` selects prepared-run output and otherwise uses
|
`run` flags; `--format` selects prepared-run output and otherwise uses
|
||||||
`defaults.render_format`.
|
`defaults.render_format`.
|
||||||
|
|
||||||
The same deprecated aliases and numeric/timeout behavior as `run` apply.
|
The same deprecated aliases and numeric/timeout behavior as `run` apply.
|
||||||
|
The same session and reasoning inheritance, replacement, and clearing behavior
|
||||||
|
also applies.
|
||||||
`render` does not accept `--schema-dir`; configure `schema_dir` through the
|
`render` does not accept `--schema-dir`; configure `schema_dir` through the
|
||||||
configuration file. It resolves profiles and schemas as part of preparation but
|
configuration file. It resolves profiles and schemas as part of preparation but
|
||||||
does not call an LLM.
|
does not call an LLM.
|
||||||
@@ -115,6 +139,31 @@ Optional flags:
|
|||||||
`serve` accepts no runtime model override flags. HTTP request fields, response
|
`serve` accepts no runtime model override flags. HTTP request fields, response
|
||||||
schemas, and error codes are defined in the [HTTP API reference](api.md).
|
schemas, and error codes are defined in the [HTTP API reference](api.md).
|
||||||
|
|
||||||
|
## `scriptorium inspect prompt`
|
||||||
|
|
||||||
|
```text
|
||||||
|
scriptorium inspect prompt --prompt ID [--prompt-version VERSION]
|
||||||
|
[--config PATH] [--prompt-dir DIR] [--format text|json] [--out PATH]
|
||||||
|
```
|
||||||
|
|
||||||
|
`--prompt` is required. Inspection uses normal configuration discovery and a
|
||||||
|
`--prompt-dir` override, defaults to text regardless of `defaults.render_format`,
|
||||||
|
and writes to stdout unless `--out` is supplied. It loads and normalizes the
|
||||||
|
selected definition but does not resolve a profile, load a schema, render a
|
||||||
|
template, reserve backend capacity, or call a model.
|
||||||
|
|
||||||
|
## `scriptorium inspect profile`
|
||||||
|
|
||||||
|
```text
|
||||||
|
scriptorium inspect profile --profile ID
|
||||||
|
[--config PATH] [--profile-dir DIR] [--format text|json] [--out PATH]
|
||||||
|
```
|
||||||
|
|
||||||
|
`--profile` is required; built-in profiles need no prompt directory. Inspection
|
||||||
|
resolves profile inheritance and backend defaults, but never reads a credential
|
||||||
|
value, loads a prompt, reserves capacity, or calls a model. Unset provider
|
||||||
|
controls are shown as zero values when Promptkit leaves them unspecified.
|
||||||
|
|
||||||
## Input And Variable Syntax
|
## Input And Variable Syntax
|
||||||
|
|
||||||
`--input name=path` maps an input name to a local file; `--var name=value`
|
`--input name=path` maps an input name to a local file; `--var name=value`
|
||||||
@@ -128,9 +177,12 @@ CLI inputs are file references. HTTP inline inputs are defined by the
|
|||||||
## Output And Exit Behavior
|
## Output And Exit Behavior
|
||||||
|
|
||||||
- `run` writes generated content to stdout, or to `--out` when supplied, and
|
- `run` writes generated content to stdout, or to `--out` when supplied, and
|
||||||
writes a concise summary to stderr.
|
writes a concise summary to stderr. The summary includes `backend=<id>` when
|
||||||
|
Promptkit selected a backend; endpoint-only profiles omit it.
|
||||||
- `render` writes prepared-run output to stdout, or to `--out` when supplied,
|
- `render` writes prepared-run output to stdout, or to `--out` when supplied,
|
||||||
without a success summary.
|
without a success summary. Text output includes `selected_backend_id` after
|
||||||
|
`selected_profile_id` when Promptkit selected one; endpoint-only profiles
|
||||||
|
omit it.
|
||||||
- `serve` writes startup and server errors to stderr.
|
- `serve` writes startup and server errors to stderr.
|
||||||
|
|
||||||
Exit statuses:
|
Exit statuses:
|
||||||
@@ -141,6 +193,10 @@ Exit statuses:
|
|||||||
| `1` | Parse, configuration, loading, rendering, generation, output-write, or other runtime error. |
|
| `1` | Parse, configuration, loading, rendering, generation, output-write, or other runtime error. |
|
||||||
| `2` | `run` generated and wrote output, but validation failed. |
|
| `2` | `run` generated and wrote output, but validation failed. |
|
||||||
|
|
||||||
|
A backend admission rejection is a runtime error and prints `run error: model
|
||||||
|
backend capacity is exhausted`. The HTTP capacity response is defined in the
|
||||||
|
[HTTP API reference](api.md).
|
||||||
|
|
||||||
## Workflows And Examples
|
## Workflows And Examples
|
||||||
|
|
||||||
The [maintained render script](../examples/render-markdown-summary.sh) is a
|
The [maintained render script](../examples/render-markdown-summary.sh) is a
|
||||||
|
|||||||
203
docs/config.md
203
docs/config.md
@@ -1,15 +1,17 @@
|
|||||||
# Configuration Reference
|
# Configuration Reference
|
||||||
|
|
||||||
This is the canonical reference for Scriptorium application settings and the
|
This is the canonical reference for Scriptorium application settings. Prompt,
|
||||||
prompt, profile, and schema files those settings select. For command syntax,
|
profile, schema, execution-setting, built-in profile, and framework credential
|
||||||
see the [CLI reference](cli.md); for HTTP request shapes, limits, and outcomes,
|
semantics are defined by the
|
||||||
see the [HTTP API reference](api.md).
|
[Promptkit v0.9.0 format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md).
|
||||||
|
For command syntax, see the [CLI reference](cli.md); for HTTP request shapes and
|
||||||
|
outcomes, see the [HTTP API reference](api.md).
|
||||||
|
|
||||||
## Discovery And Precedence
|
## Discovery And Precedence
|
||||||
|
|
||||||
Application settings are resolved in this order:
|
Application settings are resolved in this order:
|
||||||
|
|
||||||
1. built-in defaults;
|
1. built-in Scriptorium defaults;
|
||||||
2. a configuration file; then
|
2. a configuration file; then
|
||||||
3. CLI overrides.
|
3. CLI overrides.
|
||||||
|
|
||||||
@@ -19,7 +21,7 @@ If neither exists, it uses built-in defaults. An explicit `--config` path must
|
|||||||
exist and decode successfully.
|
exist and decode successfully.
|
||||||
|
|
||||||
The maintained [minimal configuration](../examples/config.yml) and
|
The maintained [minimal configuration](../examples/config.yml) and
|
||||||
[full configuration](../examples/config.full.yml) are copyable examples.
|
[complete configuration](../examples/config.full.yml) are copyable examples.
|
||||||
|
|
||||||
## Application Configuration File
|
## Application Configuration File
|
||||||
|
|
||||||
@@ -28,146 +30,97 @@ do not override a prior value. Raw API-key fields are not accepted.
|
|||||||
|
|
||||||
| Field | Default | Meaning |
|
| Field | Default | Meaning |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `prompt_dir` | unset | Directory containing prompt-definition YAML. `run`, `render`, and `serve` require an effective value. |
|
| `prompt_dir` | unset | Promptkit prompt-definition source directory. `run`, `render`, and `serve` require an effective value. |
|
||||||
| `profile_dir` | unset | Directory containing custom profile YAML. Built-in profiles remain available. |
|
| `profile_dir` | unset | Optional custom Promptkit profile source directory overlaid on Promptkit built-ins. |
|
||||||
| `schema_dir` | `.` | Base directory for relative JSON Schema paths. |
|
| `schema_dir` | `.` | Promptkit schema source directory for relative schema paths. |
|
||||||
| `server.addr` | `:8080` | Address used by `serve`. |
|
| `server.addr` | `:8080` | Address used by `serve`. |
|
||||||
| `server.artifact_root` | unset | Root that enables HTTP `file` input references. |
|
| `server.artifact_root` | unset | Root that enables HTTP `file` input references. |
|
||||||
| `server.max_request_bytes` | `16777216` | Maximum encoded HTTP request body bytes; `0` disables the limit. |
|
| `server.max_request_bytes` | `16777216` | Maximum encoded HTTP request-body bytes; `0` disables the limit. |
|
||||||
| `server.max_artifact_bytes` | `16777216` | Maximum HTTP file-input artifact bytes; `0` disables the limit. |
|
| `server.max_artifact_bytes` | `16777216` | Maximum HTTP file-input artifact bytes; `0` disables the limit. |
|
||||||
| `server.max_response_bytes` | `16777216` | Maximum encoded HTTP response bytes; `0` disables the limit. |
|
| `server.max_response_bytes` | `16777216` | Maximum encoded HTTP response bytes; `0` disables the limit. |
|
||||||
| `defaults.render_format` | `text` | Default `render` output format: `text` or `json`. |
|
| `defaults.render_format` | `text` | Default prepared-run output format: `text` or `json`. |
|
||||||
|
| `backends` | unset | Optional mapping of custom Promptkit backend IDs to engine-scoped connection and capacity settings. |
|
||||||
|
|
||||||
The three size fields must be zero or greater. The HTTP contract defines how
|
The size fields must be zero or greater. The [HTTP API](api.md) defines how
|
||||||
each limit is enforced and reported. `server.artifact_root` configures the
|
each limit is enforced and reported. `server.artifact_root` configures an HTTP
|
||||||
deployment boundary; see the [HTTP API reference](api.md) for request-path and
|
deployment boundary; see [operations](operations.md) for deployment handling.
|
||||||
containment behavior, and [operations](operations.md) for deployment handling.
|
|
||||||
|
|
||||||
## Prompt Definition Files
|
## Custom Backends
|
||||||
|
|
||||||
Prompt definitions are strict YAML files anywhere below `prompt_dir`. A prompt
|
Use `backends` when a profile selects an application-defined backend ID:
|
||||||
is selected by its YAML `id`, not by file path; nested directories are only for
|
|
||||||
organization. See [maintained prompt examples](../examples/prompts/).
|
|
||||||
|
|
||||||
| Field | Required | Meaning |
|
```yaml
|
||||||
| --- | --- | --- |
|
backends:
|
||||||
| `id` | yes | Prompt identifier. |
|
local-gpu:
|
||||||
| `version` | yes | Prompt version. |
|
endpoint: http://localhost:11434/v1
|
||||||
| `default_profile` | no | Profile used when a request omits a profile ID. |
|
api_key_env: LOCAL_GPU_API_KEY
|
||||||
| `description` | no | Human-readable description. |
|
extra_params:
|
||||||
| `session_id` | no | Go-template string rendered from request variables and sent to a compatible provider when non-empty. |
|
provider_option: enabled
|
||||||
| `inputs` | no | Declared input metadata. |
|
concurrency_limit: 2
|
||||||
| `messages` | yes | Chat-message templates. |
|
queue_capacity: 0
|
||||||
| `output` | yes | Output format and validation contract. |
|
```
|
||||||
|
|
||||||
### Inputs And Messages
|
Each mapping key is the case-sensitive backend ID. `endpoint` is required;
|
||||||
|
`api_key_env`, `extra_params`, `concurrency_limit`, and `queue_capacity` are
|
||||||
|
optional. `concurrency_limit: 0` leaves the backend unlimited. Omitting
|
||||||
|
`queue_capacity` lets Promptkit use its default for a limited backend, while
|
||||||
|
an explicit `queue_capacity: 0` disables queueing.
|
||||||
|
|
||||||
Each `inputs` item has a required `name` and optional `required`,
|
Configuration strictly owns the YAML shape and rejects unknown fields. Promptkit
|
||||||
`content_type`, and `description` fields. Input names must be unique.
|
validates backend IDs, endpoints, environment-variable names, extra parameters,
|
||||||
|
and capacity relationships when Scriptorium constructs its engine. There are no
|
||||||
|
backend command-line overrides. Store only an environment-variable name in
|
||||||
|
`api_key_env`; raw API-key fields are not accepted.
|
||||||
|
|
||||||
Each message has a required `role`, exactly one of `content` or `content_file`,
|
## Framework Source Mapping
|
||||||
and optional `cache_control`. A `content_file` path is relative to the prompt
|
|
||||||
file. `cache_control.type` must be `ephemeral`; its optional `ttl` is `1h`.
|
|
||||||
|
|
||||||
`session_id` uses the same template variables as messages. Empty rendered
|
Scriptorium passes `prompt_dir`, `profile_dir`, and `schema_dir` to Promptkit
|
||||||
values are omitted. A rendered value may contain at most 256 Unicode code
|
when constructing its engine. Scriptorium does not redefine or independently
|
||||||
points.
|
parse those framework file formats.
|
||||||
|
|
||||||
### Output Contract
|
- Prompt selection, versions, message templates, inputs, output contracts, and
|
||||||
|
session IDs are Promptkit contracts.
|
||||||
|
- Profile fields, numeric ranges, execution defaults, overlay precedence,
|
||||||
|
built-in profiles, and credential rules are Promptkit contracts.
|
||||||
|
- Schema path behavior and generated-content validation are Promptkit
|
||||||
|
contracts.
|
||||||
|
|
||||||
| Field | Required | Values or behavior |
|
See the
|
||||||
| --- | --- | --- |
|
[tagged Promptkit format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md)
|
||||||
| `format` | yes | `text`, `markdown`, or `json`. |
|
for all of those definitions. The files under
|
||||||
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema`. |
|
[`examples/prompts`](../examples/prompts/),
|
||||||
| `schema_path` | for `json_schema` | Schema path, relative to `schema_dir` unless absolute. |
|
[`examples/profiles`](../examples/profiles/), and
|
||||||
| `repair_attempts` | no | Integer greater than or equal to `0`; omitted means `0`. |
|
[`examples/schemas`](../examples/schemas/) are maintained Scriptorium
|
||||||
|
application inputs using that tagged format.
|
||||||
|
|
||||||
## Profile Definition Files
|
## Credentials And Outbound Behavior
|
||||||
|
|
||||||
Profiles are strict YAML files anywhere below `profile_dir`. A profile is
|
Scriptorium maps `--api-key-env` and HTTP `model.api_key_env` into Promptkit
|
||||||
selected by YAML `id`; nested directories are organizational. See the
|
request overrides. Keep secret values in environment variables and store only
|
||||||
[maintained profile examples](../examples/profiles/).
|
their names in configuration or framework source files. Do not place raw keys
|
||||||
|
in configuration, prompts, profiles, CLI arguments, examples, or HTTP
|
||||||
|
payloads.
|
||||||
|
|
||||||
| Field | Required | Meaning |
|
An optional `api_key_env` whose environment value is absent or empty can result
|
||||||
| --- | --- | --- |
|
in an unauthenticated provider request. A profile that declares credentials
|
||||||
| `id` | yes | Profile identifier. |
|
required still fails as an invalid request when no credential source is
|
||||||
| `endpoint` | yes | OpenAI-compatible base URL, including its API version path when needed. |
|
selected, and fails with `ErrAPIKeyEnvMissing` when its selected environment
|
||||||
| `model` | yes | Provider model name. |
|
source is absent or empty. Scriptorium never reads or emits the environment
|
||||||
| `temperature` | no | Number from `0` through `2`. |
|
value itself.
|
||||||
| `max_tokens` | no | Integer zero or greater. |
|
|
||||||
| `top_p` | no | Number from `0` through `1`. |
|
|
||||||
| `timeout_seconds` | no | Per-generation-call deadline in whole seconds; integer zero or greater. |
|
|
||||||
| `service_tier` | no | Non-empty provider-specific request tier. |
|
|
||||||
| `reasoning_effort` | no | Non-empty provider-specific reasoning setting. |
|
|
||||||
| `api_key_env` | no | Environment-variable name containing the API key. |
|
|
||||||
| `extra_params` | no | JSON-compatible provider-specific outbound request fields. |
|
|
||||||
|
|
||||||
Execution defaults before profile and request overrides are `temperature: 0`,
|
Unset optional provider controls are omitted from compatible provider requests.
|
||||||
`max_tokens: 0`, `top_p: 1`, and `timeout_seconds: 600`. Profile numeric values
|
A positive Promptkit `repair_attempts` budget can add provider calls, latency,
|
||||||
merge by non-zero value. Request overrides preserve presence, so an explicit
|
token use, and cost; see the tagged format reference for its permitted values
|
||||||
zero can override a profile value. For `timeout_seconds`, explicit request zero
|
and validation-mode requirements.
|
||||||
disables the generation deadline while retaining the caller context and the
|
|
||||||
built-in client's transport cap. See the
|
|
||||||
[OpenAI-compatible integration contract](integrations/openai-compatible-chat.md#authentication-and-timeout)
|
|
||||||
for the complete timeout interaction.
|
|
||||||
|
|
||||||
Custom profiles take precedence over built-ins with the same ID. Invalid custom
|
Promptkit's
|
||||||
profiles are errors; they do not fall back to a built-in profile. Raw `api_key`
|
[OpenAI-compatible integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/integrations/openai-compatible-chat.md)
|
||||||
is rejected. Use `api_key_env`, or the public Go package's request-scoped key
|
defines outbound authentication, provider request mapping, transport limits,
|
||||||
mechanism described in the [package contract](consumers/pkg-scriptorium.md).
|
and timeout layering.
|
||||||
|
|
||||||
`extra_params` keys must be non-empty and cannot be `model`, `session_id`,
|
|
||||||
`messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`,
|
|
||||||
`reasoning_effort`, or `response_format`.
|
|
||||||
|
|
||||||
### Built-In Profile Catalog
|
|
||||||
|
|
||||||
Each embedded profile uses `OPENROUTER_API_KEY`.
|
|
||||||
|
|
||||||
| Provider | ID | Model |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| aion-labs | `aion-2` | `aion-labs/aion-2.0` |
|
|
||||||
| anthropic | `claude-fable-latest` | `~anthropic/claude-fable-latest` |
|
|
||||||
| anthropic | `claude-haiku-latest` | `~anthropic/claude-haiku-latest` |
|
|
||||||
| anthropic | `claude-opus-latest` | `~anthropic/claude-opus-latest` |
|
|
||||||
| anthropic | `claude-sonnet-latest` | `~anthropic/claude-sonnet-latest` |
|
|
||||||
| deepseek | `deepseek-3-2` | `deepseek/deepseek-v3.2` |
|
|
||||||
| deepseek | `deepseek-4-flash` | `deepseek/deepseek-v4-flash` |
|
|
||||||
| deepseek | `deepseek-4-pro` | `deepseek/deepseek-v4-pro` |
|
|
||||||
| google | `gemini-2-flash` | `google/gemini-2.5-flash` |
|
|
||||||
| google | `gemini-2-flash-lite` | `google/gemini-2.5-flash-lite` |
|
|
||||||
| google | `gemini-2-pro` | `google/gemini-2.5-pro` |
|
|
||||||
| google | `gemini-3-flash-lite` | `google/gemini-3.1-flash-lite` |
|
|
||||||
| google | `gemini-flash-latest` | `~google/gemini-flash-latest` |
|
|
||||||
| google | `gemini-pro-latest` | `~google/gemini-pro-latest` |
|
|
||||||
| google | `gemma-4-31b` | `google/gemma-4-31b-it:exacto` |
|
|
||||||
| minimax | `minimax-m2` | `minimax/minimax-m2.5` |
|
|
||||||
| minimax | `minimax-m3` | `minimax/minimax-m3` |
|
|
||||||
| mistral | `mistral-large-2512` | `mistralai/mistral-large-2512` |
|
|
||||||
| mistral | `mistral-medium-3-5` | `mistralai/mistral-medium-3-5` |
|
|
||||||
| mistral | `mistral-small-3` | `mistralai/mistral-small-3.2-24b-instruct` |
|
|
||||||
| mistral | `mistral-small-4` | `mistralai/mistral-small-2603` |
|
|
||||||
| nvidia | `nemotron-3-ultra` | `nvidia/nemotron-3-ultra-550b-a55b` |
|
|
||||||
| openai | `gpt-5-mini` | `openai/gpt-5.4-mini` |
|
|
||||||
| openai | `gpt-5-nano` | `openai/gpt-5.4-nano` |
|
|
||||||
|
|
||||||
## Schemas
|
|
||||||
|
|
||||||
Schemas are JSON files, normally below `schema_dir`. `json_schema` output
|
|
||||||
requires a `schema_path`. Relative paths resolve from `schema_dir`; absolute
|
|
||||||
paths are used directly. Referenced nested schemas use relative paths and are
|
|
||||||
not discovered by basename. An unreadable or invalid schema is a runtime
|
|
||||||
validation error; generated content that fails JSON or schema validation is a
|
|
||||||
validation result.
|
|
||||||
|
|
||||||
## Credentials
|
|
||||||
|
|
||||||
Keep secrets in environment variables. Store only an environment-variable name
|
|
||||||
in `api_key_env`; do not place raw keys in configuration, prompt or profile
|
|
||||||
files, CLI arguments, examples, or HTTP payloads.
|
|
||||||
|
|
||||||
## Related References
|
## Related References
|
||||||
|
|
||||||
- [CLI reference](cli.md)
|
- [CLI reference](cli.md)
|
||||||
- [HTTP API reference](api.md)
|
- [HTTP API reference](api.md)
|
||||||
- [OpenAI-compatible outbound contract](integrations/openai-compatible-chat.md)
|
- [Operations guide](operations.md)
|
||||||
|
- [Promptkit framework formats](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md)
|
||||||
|
|||||||
@@ -1,45 +1,23 @@
|
|||||||
# Consumer Integration Overview
|
# Consumer Integration Overview
|
||||||
|
|
||||||
This guide helps applications choose a Scriptorium interface and understand
|
Scriptorium exposes executable interfaces. Choose between a local subprocess
|
||||||
their responsibilities. The linked contracts own interface syntax and wire
|
and the HTTP service according to the boundary your application needs.
|
||||||
semantics.
|
|
||||||
|
|
||||||
| Interface | Use when |
|
| Interface | Use when |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| Go package | The consumer is Go and needs typed requests, results, or an injected LLM client. |
|
| CLI subprocess | The consumer needs a synchronous local process boundary or prepared output. |
|
||||||
| CLI subprocess | The consumer needs process isolation or is not written in Go. |
|
|
||||||
| HTTP API | The consumer needs a service boundary or remote access. |
|
| HTTP API | The consumer needs a service boundary or remote access. |
|
||||||
|
|
||||||
- Go package: [package contract](pkg-scriptorium.md)
|
|
||||||
- CLI subprocess: [subprocess integration](../integrations/subprocess.md)
|
- CLI subprocess: [subprocess integration](../integrations/subprocess.md)
|
||||||
- HTTP service: [HTTP API reference](../api.md)
|
- HTTP service: [HTTP API reference](../api.md)
|
||||||
- Prompt, profile, schema, and credential configuration: [configuration reference](../config.md)
|
- Application configuration: [configuration reference](../config.md)
|
||||||
|
|
||||||
## Minimal Go Use
|
Go applications that need an in-process prompt framework should import
|
||||||
|
Promptkit directly. The tagged
|
||||||
```go
|
[Promptkit Go consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md)
|
||||||
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
owns that interface; Scriptorium does not provide a Go library package.
|
||||||
PromptDir: "./examples/prompts",
|
Consumers arriving from the former Scriptorium Go API should follow the
|
||||||
ProfileDir: "./examples/profiles",
|
[migration guide](migrating-to-promptkit.md).
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
prepared, err := engine.Prepare(ctx, scriptorium.RunRequest{
|
|
||||||
PromptID: "generic.markdown_summary",
|
|
||||||
Inputs: map[string]scriptorium.ArtifactRef{
|
|
||||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
_ = prepared
|
|
||||||
```
|
|
||||||
|
|
||||||
For a maintained program, see
|
|
||||||
[`examples/go-library/prepare`](../../examples/go-library/prepare).
|
|
||||||
|
|
||||||
## Consumer Responsibilities
|
## Consumer Responsibilities
|
||||||
|
|
||||||
@@ -47,13 +25,14 @@ Consumers are responsible for:
|
|||||||
|
|
||||||
- selecting and deploying prompt, profile, and schema assets;
|
- selecting and deploying prompt, profile, and schema assets;
|
||||||
- supplying required inputs and template variables;
|
- supplying required inputs and template variables;
|
||||||
- supplying credentials through the applicable interface;
|
- supplying credentials through the chosen interface;
|
||||||
- protecting rendered prompts and generated artifacts as potentially sensitive;
|
- protecting rendered prompts and generated artifacts as potentially
|
||||||
|
sensitive;
|
||||||
- deciding whether validation-failed output is usable; and
|
- deciding whether validation-failed output is usable; and
|
||||||
- retrying only when another model call is acceptable.
|
- retrying only when another model call is acceptable.
|
||||||
|
|
||||||
Scriptorium does not persist run state. A retry can produce different output and
|
Scriptorium does not persist run state. A retry can produce different output
|
||||||
can incur another provider request. CLI exit behavior belongs to the
|
and can incur another provider request. CLI exits belong to the
|
||||||
[CLI reference](../cli.md); HTTP status behavior belongs to the
|
[CLI reference](../cli.md), HTTP status behavior belongs to the
|
||||||
[HTTP API reference](../api.md); package errors and results belong to the
|
[HTTP API reference](../api.md), and framework semantics belong to
|
||||||
[package contract](pkg-scriptorium.md).
|
[Promptkit v0.9.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md).
|
||||||
|
|||||||
109
docs/consumers/migrating-to-promptkit.md
Normal file
109
docs/consumers/migrating-to-promptkit.md
Normal file
@@ -0,0 +1,109 @@
|
|||||||
|
# Migrate From Scriptorium To Promptkit
|
||||||
|
|
||||||
|
## Supported Migration Boundary
|
||||||
|
|
||||||
|
Scriptorium `v0.11.1` at
|
||||||
|
`gitea.maximumdirect.net/eric/scriptorium` is the final release that provides
|
||||||
|
the former in-process Go framework. Promptkit `v0.9.0` at
|
||||||
|
`gitea.maximumdirect.net/eric/promptkit` is the destination for that framework
|
||||||
|
API. Scriptorium `v0.12.0` and later provide the CLI and HTTP application only.
|
||||||
|
|
||||||
|
There is no Scriptorium compatibility facade, alias package, forwarding
|
||||||
|
package, or deprecated wrapper. A consumer that cannot migrate may remain
|
||||||
|
pinned to Scriptorium `v0.11.1`, but that framework-bearing line does not
|
||||||
|
provide the slim application release.
|
||||||
|
|
||||||
|
## Update A Go Consumer
|
||||||
|
|
||||||
|
Start from a clean consumer checkout and review the pending diff before
|
||||||
|
committing it. Add the published Promptkit module:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.9.0
|
||||||
|
```
|
||||||
|
|
||||||
|
For an ordinary consumer that imports the former root package under its
|
||||||
|
default name, replace the exact import and package qualifier, then format the
|
||||||
|
changed Go files:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git grep -l \
|
||||||
|
'"gitea.maximumdirect.net/eric/scriptorium"' \
|
||||||
|
-- '*.go' |
|
||||||
|
while IFS= read -r go_file
|
||||||
|
do
|
||||||
|
perl -pi -e \
|
||||||
|
's{"gitea.maximumdirect.net/eric/scriptorium"}{"gitea.maximumdirect.net/eric/promptkit"}g; s{\bscriptorium\.}{promptkit.}g' \
|
||||||
|
"$go_file"
|
||||||
|
gofmt -w "$go_file"
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
Inspect the resulting diff. Consumers that used an import alias should retain
|
||||||
|
or deliberately rename that alias instead of applying the qualifier
|
||||||
|
replacement mechanically.
|
||||||
|
|
||||||
|
Remove the now-unused Scriptorium requirement through module tidiness and run
|
||||||
|
the consumer's complete tests:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go mod tidy
|
||||||
|
go test ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
Confirm that `go.mod` selects Promptkit `v0.9.0` and that no Go file imports
|
||||||
|
the former Scriptorium package:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
test "$(
|
||||||
|
go list -m -f '{{.Path}}@{{.Version}}' \
|
||||||
|
gitea.maximumdirect.net/eric/promptkit
|
||||||
|
)" = 'gitea.maximumdirect.net/eric/promptkit@v0.9.0'
|
||||||
|
if git grep -n \
|
||||||
|
'gitea.maximumdirect.net/eric/scriptorium' \
|
||||||
|
-- '*.go'
|
||||||
|
then
|
||||||
|
printf '%s\n' 'a former Scriptorium Go import remains' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
```
|
||||||
|
|
||||||
|
## Compatibility And Additions
|
||||||
|
|
||||||
|
Promptkit preserves the established engine, request, result, profile,
|
||||||
|
source-option, model-client, artifact, validation-value, and public-error
|
||||||
|
shapes where practical. Exact declarations and current behavior belong to the
|
||||||
|
tagged [Promptkit consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md)
|
||||||
|
and Go source.
|
||||||
|
|
||||||
|
Promptkit also includes migration-relevant public contracts that were not in
|
||||||
|
Scriptorium `v0.11.1`:
|
||||||
|
|
||||||
|
- [`WithArtifactReader`](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/engine.go#L182)
|
||||||
|
and the
|
||||||
|
[`ArtifactReader` declaration](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/types.go#L304)
|
||||||
|
provide the artifact-reading extension described by the tagged
|
||||||
|
[extension-interface guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md#extension-interfaces).
|
||||||
|
- [`ErrProfileRequired` and `ErrAPIKeyEnvMissing`](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/engine.go#L46-L61)
|
||||||
|
provide the specific identities described by the tagged
|
||||||
|
[error guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md#errors).
|
||||||
|
|
||||||
|
Use those tagged owners for exact signatures, wrapping guarantees, and
|
||||||
|
extension behavior.
|
||||||
|
|
||||||
|
## Verify Consumer Behavior
|
||||||
|
|
||||||
|
Source compatibility is only the first check. Exercise the behavior the
|
||||||
|
consumer actually relies upon, especially:
|
||||||
|
|
||||||
|
- prompt, profile, and schema source selection;
|
||||||
|
- direct and environment-based credentials;
|
||||||
|
- caller, generation, and transport timeout layering;
|
||||||
|
- output validation and validation-failure handling;
|
||||||
|
- injected model-client and artifact-reader extensions; and
|
||||||
|
- every `errors.Is` branch used for recovery or classification.
|
||||||
|
|
||||||
|
Also verify any serialized values, redaction expectations, filesystem policy,
|
||||||
|
and provider integration behavior that crosses the consumer's own boundary.
|
||||||
|
Promptkit owns the in-process framework contract; Scriptorium owns only its
|
||||||
|
executable CLI and HTTP application interfaces.
|
||||||
@@ -1,203 +0,0 @@
|
|||||||
# Package `scriptorium`
|
|
||||||
|
|
||||||
Import path:
|
|
||||||
|
|
||||||
```go
|
|
||||||
import "gitea.maximumdirect.net/eric/scriptorium"
|
|
||||||
```
|
|
||||||
|
|
||||||
This is the canonical public Go contract for in-process prompt preparation and
|
|
||||||
execution. Prompt, profile, and schema file formats are defined in the
|
|
||||||
[configuration reference](../config.md).
|
|
||||||
|
|
||||||
## Engine Construction
|
|
||||||
|
|
||||||
`NewEngine(Config, ...Option)` constructs an engine. `Config` has these
|
|
||||||
fields:
|
|
||||||
|
|
||||||
| Field | Meaning |
|
|
||||||
| --- | --- |
|
|
||||||
| `PromptDir` | Prompt-definition directory, required unless a prompt source option is supplied. |
|
|
||||||
| `ProfileDir` | Optional custom profile directory over built-ins. |
|
|
||||||
| `SchemaDir` | Schema directory; empty uses `.`. |
|
|
||||||
| `Timeout` | Transport-wide safety cap for the built-in OpenAI-compatible client when `HTTPClient` is absent or has a non-positive timeout. A non-positive value uses the internal ten-minute default. |
|
|
||||||
| `HTTPClient` | Optional HTTP client for that built-in client. It is cloned; a positive `Timeout` on it is the transport cap and takes precedence over `Config.Timeout`. A non-positive client timeout is treated as unset. |
|
|
||||||
|
|
||||||
Nil options are ignored. Invalid construction, including
|
|
||||||
`WithLLMClient(nil)` and `WithArtifactReader(nil)`, returns an error matching
|
|
||||||
`ErrInvalidConfig`.
|
|
||||||
|
|
||||||
Profile and request `timeout_seconds` values select a per-generation-call
|
|
||||||
deadline independently of the transport cap. An explicit request override of
|
|
||||||
zero disables that generation deadline only. The complete interaction with the
|
|
||||||
caller context is defined in the
|
|
||||||
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md#authentication-and-timeout).
|
|
||||||
|
|
||||||
Source options replace their matching directory source:
|
|
||||||
|
|
||||||
- prompts: `WithPromptFS(fsys, root)`, `WithPromptFile(path)`;
|
|
||||||
- profiles: `WithProfileFS(fsys, root)`, `WithProfileFile(path)`, and
|
|
||||||
`WithProfiles(profiles...)`;
|
|
||||||
- schemas: `WithSchemaFS(fsys, root)`, `WithSchemaFile(path)`; and
|
|
||||||
- LLM client: `WithLLMClient(client)`; and
|
|
||||||
- artifact reader: `WithArtifactReader(reader)`.
|
|
||||||
|
|
||||||
`fs.FS` prompt-content and schema paths stay inside their configured roots.
|
|
||||||
Single-file prompt and profile sources are selected by their YAML `id`, not
|
|
||||||
their file names. `WithPromptFile` resolves relative `content_file` paths from
|
|
||||||
the prompt file's directory. `WithSchemaFile` exposes its schema by the schema
|
|
||||||
file's base name. In-memory profiles take precedence over an explicit or
|
|
||||||
directory-backed profile source, which in turn takes precedence over built-ins.
|
|
||||||
File and filesystem sources use the format and credential rules in the
|
|
||||||
[configuration reference](../config.md).
|
|
||||||
|
|
||||||
## Prepare And Run
|
|
||||||
|
|
||||||
`Prepare(ctx, request)` resolves the prompt, profile, input artifacts,
|
|
||||||
validation contract, and rendered messages without calling an LLM.
|
|
||||||
`Run(ctx, request)` performs that preparation, calls the configured client,
|
|
||||||
and validates generated content.
|
|
||||||
|
|
||||||
```go
|
|
||||||
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
|
||||||
PromptDir: "./examples/prompts",
|
|
||||||
ProfileDir: "./examples/profiles",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
prepared, err := engine.Prepare(ctx, scriptorium.RunRequest{
|
|
||||||
PromptID: "generic.markdown_summary",
|
|
||||||
Inputs: map[string]scriptorium.ArtifactRef{
|
|
||||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
_ = prepared.Messages
|
|
||||||
```
|
|
||||||
|
|
||||||
The maintained package example is
|
|
||||||
[`examples/go-library/prepare`](../../examples/go-library/prepare).
|
|
||||||
|
|
||||||
`PreparedRun` exposes prompt, selected-profile, effective-model, output
|
|
||||||
contract, structured-output, input-hash, rendered-message, and timing
|
|
||||||
information. It does not include a resolved API key, model output, validation
|
|
||||||
result, or target-presence metadata.
|
|
||||||
|
|
||||||
`RunResult` adds run ID, artifact, raw output, validation, model metadata,
|
|
||||||
usage, and duration. Generated-content validation failures return a result with
|
|
||||||
`Validation.Status == ValidationFailed`; schema or validator runtime failures
|
|
||||||
return an error matching `ErrValidation`.
|
|
||||||
|
|
||||||
## Public Values
|
|
||||||
|
|
||||||
`ArtifactRef` has `Type`, `URI`, and `Body`; `Artifact` has `Name`,
|
|
||||||
`ContentType`, `Body`, `URI`, `Size`, and `Hash`. `ExecutionTarget` exposes the
|
|
||||||
effective endpoint, model, numeric settings, credential-environment name,
|
|
||||||
service tier, reasoning effort, and extra parameters. `ValidationResult`
|
|
||||||
contains status, mode, errors, schema path, repair attempts, and validity.
|
|
||||||
|
|
||||||
The exported constants define these serialized values:
|
|
||||||
|
|
||||||
- artifact types: `inline` and `file`;
|
|
||||||
- output formats: `text`, `markdown`, and `json`;
|
|
||||||
- validation modes: `none`, `basic`, `json`, and `json_schema`; and
|
|
||||||
- validation statuses: `passed`, `failed`, and `skipped`.
|
|
||||||
|
|
||||||
`TokenUsage` reports prompt, completion, total, cached, and cache-write token
|
|
||||||
counts. `RenderedPrompt`, `RenderedMessage`, `CacheControl`, and
|
|
||||||
`StructuredOutputSpec` are the public shapes used by injected LLM clients.
|
|
||||||
|
|
||||||
`ArtifactReader` implements
|
|
||||||
`Read(context.Context, ArtifactRef) (*Artifact, error)`. Supplying it through
|
|
||||||
`WithArtifactReader` replaces, rather than extends, the engine's default inline
|
|
||||||
and file reader for every input. Omitting the option retains that default;
|
|
||||||
`WithArtifactReader(nil)` makes engine construction fail with
|
|
||||||
`ErrInvalidConfig`.
|
|
||||||
|
|
||||||
Reader failures are surfaced as errors matching `ErrArtifactLoad` while
|
|
||||||
preserving the reader's original error identity for `errors.Is`. A `(nil, nil)`
|
|
||||||
reader response is also an artifact-load failure. Readers are responsible for
|
|
||||||
artifact metadata, although the engine assigns the input-map name when the
|
|
||||||
returned name is empty; readers should not retain or mutate caller values.
|
|
||||||
|
|
||||||
## Requests, Inputs, And Overrides
|
|
||||||
|
|
||||||
`RunRequest` fields are `PromptID`, `PromptVersion`, `ProfileID`,
|
|
||||||
`APIKey`, `Inputs`, `Vars`, `Execution`, `Validation`, and
|
|
||||||
`Metadata`.
|
|
||||||
|
|
||||||
Input helpers are:
|
|
||||||
|
|
||||||
- `File(path)` for a file-backed artifact;
|
|
||||||
- `Inline(body)` for inline content; and
|
|
||||||
- `InlineWithURI(uri, body)` for inline content with URI metadata.
|
|
||||||
|
|
||||||
Required declared inputs must be supplied. Template rendering must also resolve
|
|
||||||
every input name the prompt actually references. Extra entries in `Inputs`
|
|
||||||
are not rejected solely because they are undeclared.
|
|
||||||
|
|
||||||
`ExecutionTargetOverride` supplies endpoint, model, credential-environment,
|
|
||||||
service-tier, reasoning-effort, and extra-parameter overrides. Its numeric
|
|
||||||
fields (`Temperature`, `MaxTokens`, `TopP`, and `TimeoutSeconds`) are
|
|
||||||
pointers so explicit zero values are preserved. `OutputContract` supplies
|
|
||||||
`Format`, `ValidationMode`, `SchemaPath`, and `RepairAttempts`.
|
|
||||||
|
|
||||||
`ExtraParams` accepts JSON-compatible values: strings, booleans, finite
|
|
||||||
numbers, objects with string keys, arrays or slices, and nil. Unsupported
|
|
||||||
values, non-string map keys, non-finite floats, and cycles return
|
|
||||||
`ErrInvalidConfig` for profiles or `ErrInvalidRequest` for request
|
|
||||||
overrides.
|
|
||||||
|
|
||||||
## Profiles And Credentials
|
|
||||||
|
|
||||||
`OpenAICompatibleProfile(OpenAICompatibleProfileConfig)` creates an
|
|
||||||
in-memory `Profile`. Its public fields are `ID`, `Endpoint`, `Model`,
|
|
||||||
`Temperature`, `MaxTokens`, `TopP`, `TimeoutSeconds`, `ServiceTier`,
|
|
||||||
`ReasoningEffort`, `APIKeyRequired`, and `ExtraParams`.
|
|
||||||
`WithProfiles` rejects duplicate IDs in one call.
|
|
||||||
|
|
||||||
A direct `RunRequest.APIKey` is request-scoped and takes precedence over
|
|
||||||
`api_key_env` for the built-in client. It is excluded from JSON output and
|
|
||||||
from `PreparedRun` and `RunResult`. The package's `String` and
|
|
||||||
`GoString` methods report only whether a direct key is set. Do not use
|
|
||||||
reflection-based dumps of request structs, which can bypass that redaction.
|
|
||||||
|
|
||||||
## Injected LLM Clients
|
|
||||||
|
|
||||||
`LLMClient` implements:
|
|
||||||
|
|
||||||
```go
|
|
||||||
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
|
|
||||||
```
|
|
||||||
|
|
||||||
Injected clients receive the rendered prompt, effective execution target, numeric
|
|
||||||
target-presence metadata, optional structured-output specification, and direct
|
|
||||||
request API key. `GenerateResponse` returns content and `TokenUsage`.
|
|
||||||
Custom clients should avoid logging raw prompts or credentials.
|
|
||||||
|
|
||||||
## Errors
|
|
||||||
|
|
||||||
Public methods preserve these sentinel checks through `errors.Is`:
|
|
||||||
|
|
||||||
- `ErrInvalidConfig`
|
|
||||||
- `ErrInvalidRequest`
|
|
||||||
- `ErrPromptNotFound`
|
|
||||||
- `ErrProfileNotFound`
|
|
||||||
- `ErrProfileRequired`
|
|
||||||
- `ErrPromptLoad`
|
|
||||||
- `ErrProfileLoad`
|
|
||||||
- `ErrAPIKeyEnvMissing`
|
|
||||||
- `ErrArtifactLoad`
|
|
||||||
- `ErrPromptRender`
|
|
||||||
- `ErrLLMGenerate`
|
|
||||||
- `ErrValidation`
|
|
||||||
|
|
||||||
`ErrProfileRequired` and `ErrAPIKeyEnvMissing` each also match
|
|
||||||
`ErrInvalidRequest`, so callers can select either the broad request category or
|
|
||||||
the specific condition.
|
|
||||||
|
|
||||||
For the HTTP interface, see the [HTTP API reference](../api.md).
|
|
||||||
@@ -1,63 +1,56 @@
|
|||||||
# Development
|
# Development
|
||||||
|
|
||||||
This is the contributor entry point for Scriptorium. Use the task-specific
|
This is the contributor entry point for Scriptorium. Scriptorium is an
|
||||||
reading guide below before making changes. Canonical architecture, contracts,
|
application that consumes the public
|
||||||
component behavior, and policies remain in their owning documents.
|
[Promptkit v0.9.0 package](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md);
|
||||||
|
framework implementation work belongs in Promptkit.
|
||||||
|
|
||||||
## Initial Orientation
|
## Initial Orientation
|
||||||
|
|
||||||
Before starting work:
|
Before starting work:
|
||||||
|
|
||||||
1. inspect the working tree and preserve unrelated changes;
|
1. inspect the working tree and preserve unrelated changes;
|
||||||
2. read the architecture policy for code or design work;
|
2. read the [architecture policy](policy/architecture.md);
|
||||||
3. read the policy, contract, and internal documents listed for the task;
|
3. follow the task-specific contracts and internal documents below; and
|
||||||
4. inspect the relevant implementation and tests before deciding how to change
|
4. inspect the relevant implementation and tests before changing them.
|
||||||
them.
|
|
||||||
|
|
||||||
Start with:
|
Also read the [documentation policy](policy/documentation.md) before changing
|
||||||
|
documentation and the [testing policy](policy/testing.md) before changing
|
||||||
- [Architecture policy](policy/architecture.md) for system boundaries,
|
tests.
|
||||||
invariants, and non-goals;
|
|
||||||
- [Internal component overview](internal/overview.md) for the current package
|
|
||||||
and component map;
|
|
||||||
- [Documentation policy](policy/documentation.md) before changing
|
|
||||||
documentation;
|
|
||||||
- [Testing policy](policy/testing.md) before adding, rewriting, or deleting
|
|
||||||
tests.
|
|
||||||
|
|
||||||
## Task-Specific Reading Guide
|
## Task-Specific Reading Guide
|
||||||
|
|
||||||
| Task | Read before changing |
|
| Task | Read before changing |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| Repository orientation or component responsibility | [Internal component overview](internal/overview.md) and [architecture policy](policy/architecture.md) |
|
| Repository orientation or component responsibility | [Internal component overview](internal/overview.md) and [architecture policy](policy/architecture.md) |
|
||||||
| Public Go package or engine behavior | [Go package consumer contract](consumers/pkg-scriptorium.md), [internal component overview](internal/overview.md), [runner internals](internal/runner.md), [adapter internals](internal/adapters.md), and [source internals](internal/sources.md) |
|
| CLI commands, flags, output, or exit behavior | [CLI contract](cli.md) and [adapter internals](internal/adapters.md) |
|
||||||
| CLI commands, flags, output, or exit behavior | [CLI contract](cli.md), [internal component overview](internal/overview.md), and [adapter internals](internal/adapters.md) |
|
| HTTP routes, DTOs, limits, status mapping, or artifact policy | [HTTP API contract](api.md), [adapter internals](internal/adapters.md), and [source internals](internal/sources.md) |
|
||||||
| HTTP routes, DTOs, limits, or status mapping | [HTTP API contract](api.md), [internal component overview](internal/overview.md), [adapter internals](internal/adapters.md), and [source internals](internal/sources.md) |
|
| Application configuration or precedence | [Configuration contract](config.md), [adapter internals](internal/adapters.md), and [source internals](internal/sources.md) |
|
||||||
| Application configuration | [Configuration contract](config.md), [internal component overview](internal/overview.md), [adapter internals](internal/adapters.md), and [source internals](internal/sources.md) |
|
| Prepared-run presentation | [CLI contract](cli.md), [adapter internals](internal/adapters.md), and `internal/format` |
|
||||||
| Prompt, profile, schema, or artifact loading | [Configuration contract](config.md), [internal component overview](internal/overview.md), and [source internals](internal/sources.md) |
|
| Prompt, profile, schema, generation, or validation semantics | [Promptkit framework formats](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md) and the [Promptkit consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md) |
|
||||||
| Runner orchestration, rendering, validation, or repair | [Runner internals](internal/runner.md) and [source internals](internal/sources.md) |
|
| OpenAI-compatible outbound behavior or timeout layering | [Promptkit integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/integrations/openai-compatible-chat.md) |
|
||||||
| OpenAI-compatible request or response behavior | [OpenAI-compatible integration](integrations/openai-compatible-chat.md), [LLM internals](internal/llm.md), [runner internals](internal/runner.md), and [adapter internals](internal/adapters.md) |
|
|
||||||
| Subprocess behavior | [Subprocess integration](integrations/subprocess.md) and [CLI contract](cli.md) |
|
| Subprocess behavior | [Subprocess integration](integrations/subprocess.md) and [CLI contract](cli.md) |
|
||||||
| Runtime operation or recovery | [Operations](operations.md) |
|
| Runtime operation or recovery | [Operations](operations.md) |
|
||||||
| Examples or copyable assets | The owning contract for the demonstrated behavior and the related files under `examples/` |
|
| Release packaging or publication | The [release procedure](release.md), [hosted release workflow](../.woodpecker/release.yml), and [architecture policy](policy/architecture.md) |
|
||||||
| Architecture decisions or future work | The [documentation policy](policy/documentation.md), relevant accepted ADRs such as [ADR 0001](adr/0001-adopt-canonical-documentation-ownership.md), and relevant roadmap documents under `roadmap/` |
|
| Examples or copyable assets | The owning Scriptorium contract, the relevant [Promptkit format contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md), and the related files under `examples/` |
|
||||||
|
| Architecture decisions or future work | The [documentation policy](policy/documentation.md), relevant accepted ADRs, and relevant roadmap documents |
|
||||||
|
|
||||||
For cross-cutting changes, follow every applicable row. Internal component
|
Cross-project changes land and release in Promptkit before Scriptorium adopts
|
||||||
documents own detailed subsystem change recipes.
|
the tagged version. Do not commit a Go workspace, local replacement, vendored
|
||||||
|
Promptkit source, or an import of a Promptkit `internal` package.
|
||||||
|
|
||||||
## Baseline Validation
|
## Baseline Validation
|
||||||
|
|
||||||
Use focused checks while iterating, then run validation proportionate to the
|
For code changes, run:
|
||||||
change and the risks described by the testing policy.
|
|
||||||
|
|
||||||
The repository-level baseline for code changes is:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
go test ./...
|
go test ./...
|
||||||
|
go test -race ./...
|
||||||
go vet ./...
|
go vet ./...
|
||||||
go build ./cmd/scriptorium
|
go build ./cmd/scriptorium
|
||||||
```
|
```
|
||||||
|
|
||||||
Documentation-only work does not require the full Go suite unless it changes
|
Check formatting with `gofmt`, run `git diff --check`, and validate affected
|
||||||
commands, examples, generated output, or another behavior that the suite
|
examples and documentation links. Documentation-only work does not require
|
||||||
validates. Always check changed links, paths, examples, and canonical ownership.
|
unrelated new tests, but commands and examples changed by documentation must be
|
||||||
|
run.
|
||||||
|
|||||||
@@ -1,121 +0,0 @@
|
|||||||
# OpenAI-Compatible Chat Integration
|
|
||||||
|
|
||||||
This is the outbound wire contract for Scriptorium's OpenAI-compatible
|
|
||||||
chat-completions client.
|
|
||||||
|
|
||||||
## Endpoint And Method
|
|
||||||
|
|
||||||
Scriptorium uses the request endpoint override when present; otherwise it uses
|
|
||||||
the configured client base URL. It removes a trailing slash and sends
|
|
||||||
`POST /chat/completions`.
|
|
||||||
|
|
||||||
For example, `http://localhost:8000/v1` becomes
|
|
||||||
`http://localhost:8000/v1/chat/completions`.
|
|
||||||
|
|
||||||
## Request Payload
|
|
||||||
|
|
||||||
The payload always contains `model` and rendered `messages`. It additionally
|
|
||||||
contains these fields when applicable:
|
|
||||||
|
|
||||||
| Field | Inclusion |
|
|
||||||
| --- | --- |
|
|
||||||
| `session_id` | Non-empty rendered prompt session ID. |
|
|
||||||
| `temperature` | Non-zero effective value or an explicit zero override. |
|
|
||||||
| `max_tokens` | Non-zero effective value or an explicit zero override. |
|
|
||||||
| `top_p` | Non-zero effective value or an explicit zero override. |
|
|
||||||
| `service_tier` | Any non-empty configured value. |
|
|
||||||
| `reasoning_effort` | Any non-empty configured value. |
|
|
||||||
| `response_format` | Structured output is requested. |
|
|
||||||
| provider-specific fields | Flattened from `extra_params`. |
|
|
||||||
|
|
||||||
`service_tier` and `reasoning_effort` are forwarded without a provider value
|
|
||||||
catalog; the selected backend decides which values it supports.
|
|
||||||
|
|
||||||
`extra_params` are top-level JSON fields, not a nested object. Keys cannot be
|
|
||||||
empty or collide with `model`, `session_id`, `messages`, `temperature`,
|
|
||||||
`max_tokens`, `top_p`, `service_tier`, `reasoning_effort`, or
|
|
||||||
`response_format`. Values must be JSON-serializable.
|
|
||||||
|
|
||||||
A rendered `session_id` is sent as a top-level JSON field, not as a header.
|
|
||||||
Empty values are omitted. The maximum length is 256 Unicode code points.
|
|
||||||
|
|
||||||
Messages without cache control use string `content`. A message with cache
|
|
||||||
control uses one text block:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"role": "system",
|
|
||||||
"content": [{
|
|
||||||
"type": "text",
|
|
||||||
"text": "rendered text",
|
|
||||||
"cache_control": {"type": "ephemeral", "ttl": "1h"}
|
|
||||||
}]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
When the prompt omits cache-control `ttl`, the payload omits `ttl`.
|
|
||||||
Structured JSON Schema output is sent as:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"response_format": {
|
|
||||||
"type": "json_schema",
|
|
||||||
"json_schema": {
|
|
||||||
"name": "schema name",
|
|
||||||
"strict": true,
|
|
||||||
"schema": {"type": "object"}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Authentication And Timeout
|
|
||||||
|
|
||||||
When a direct request API key is present, Scriptorium sends
|
|
||||||
`Authorization: Bearer <key>` and does not read `api_key_env`. Otherwise, it
|
|
||||||
resolves the configured non-empty `api_key_env` at request time and sends the
|
|
||||||
same header. If neither mechanism supplies a key, it sends no
|
|
||||||
`Authorization` header.
|
|
||||||
|
|
||||||
The transport-wide safety cap is chosen at client construction. A positive
|
|
||||||
timeout on a supplied `http.Client` takes precedence over a positive
|
|
||||||
`Config.Timeout`; if neither is positive, the internal ten-minute default is
|
|
||||||
used. The supplied client is cloned, and zero or negative timeout values are
|
|
||||||
treated as unset.
|
|
||||||
|
|
||||||
Separately, a positive effective `timeout_seconds` creates a deadline for each
|
|
||||||
outbound generation call. Its value follows the execution-setting hierarchy:
|
|
||||||
an explicit request override, then a non-zero profile value, then the
|
|
||||||
600-second framework default. An explicit request override of zero disables
|
|
||||||
only this generation deadline. Negative values are rejected before a request
|
|
||||||
is sent.
|
|
||||||
|
|
||||||
The complete observable rule is that the earliest caller-context deadline,
|
|
||||||
transport cap, or positive generation deadline terminates the call. Transport
|
|
||||||
and cancellation failures retain the generation-error classification.
|
|
||||||
|
|
||||||
## Response Subset And Failures
|
|
||||||
|
|
||||||
A successful provider response must supply non-empty
|
|
||||||
`choices[0].message.content`. Scriptorium reads these optional or required
|
|
||||||
usage fields when present:
|
|
||||||
|
|
||||||
- `usage.prompt_tokens`
|
|
||||||
- `usage.completion_tokens`
|
|
||||||
- `usage.total_tokens`
|
|
||||||
- `usage.prompt_tokens_details.cached_tokens`
|
|
||||||
- `usage.cache_write_tokens`
|
|
||||||
|
|
||||||
Missing cache usage is reported as zero. Invalid JSON, an empty choices array,
|
|
||||||
or empty first-choice content is a malformed provider response. Network and
|
|
||||||
request-construction failures, non-2xx responses, and malformed responses fail
|
|
||||||
the outbound call. Provider response bodies are not exposed by this client.
|
|
||||||
|
|
||||||
The client does not implement built-in retries, tool calls, top-level
|
|
||||||
`cache_control`, or multi-request payload modes.
|
|
||||||
|
|
||||||
## Related References
|
|
||||||
|
|
||||||
Prompt schema preparation and runner orchestration are described in
|
|
||||||
[runner internals](../internal/runner.md). Prompt and profile configuration is
|
|
||||||
defined by the [configuration reference](../config.md).
|
|
||||||
@@ -2,115 +2,101 @@
|
|||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
Adapters translate external inputs into public engine requests and translate
|
Scriptorium adapters translate executable inputs into Promptkit public requests
|
||||||
public results or errors back to their interface. They own IO and presentation
|
and translate Promptkit results or errors back to CLI or HTTP behavior. They
|
||||||
mechanics; use-case decisions remain behind the root `scriptorium` facade.
|
own IO and presentation mechanics, not framework decisions.
|
||||||
|
|
||||||
External contracts are canonical in the [CLI reference](../cli.md), [HTTP API
|
External contracts are canonical in the [CLI reference](../cli.md) and
|
||||||
reference](../api.md), and [Go package contract](../consumers/pkg-scriptorium.md).
|
[HTTP API reference](../api.md). Promptkit's public engine contract is
|
||||||
|
described by its tagged
|
||||||
|
[Go consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md).
|
||||||
|
|
||||||
## Components And Collaborators
|
## Components And Collaborators
|
||||||
|
|
||||||
- `cmd/scriptorium` passes process arguments and streams to
|
- `cmd/scriptorium` passes process arguments and streams to
|
||||||
`internal/adapter/cli`.
|
`internal/adapter/cli`.
|
||||||
- `internal/adapter/cli` parses commands, resolves application settings through
|
- `internal/adapter/cli` resolves settings through `internal/config`,
|
||||||
`internal/config`, constructs the public engine, and owns process output
|
constructs `promptkit.Engine`, maps CLI values to `promptkit.RunRequest`,
|
||||||
handling.
|
and owns output files, summaries, and exit codes.
|
||||||
- `internal/adapter/http` decodes DTOs, maps them to public run requests,
|
- `internal/adapter/http` strictly decodes request DTOs, maps them to Promptkit
|
||||||
calls its local public `Runner` interface, and maps public errors and results
|
public values, calls its adapter-owned `Runner` interface, and maps results
|
||||||
to HTTP DTOs.
|
and errors to HTTP DTOs.
|
||||||
- The root `scriptorium` package maps its public types and options to internal
|
- `internal/format` renders `promptkit.PreparedRun` values as deterministic text
|
||||||
collaborators and maps selected internal errors to public sentinels.
|
or JSON.
|
||||||
- `internal/format` formats public prepared runs for the CLI.
|
|
||||||
|
|
||||||
## Wiring Flows
|
## Wiring Flows
|
||||||
|
|
||||||
### CLI
|
### CLI
|
||||||
|
|
||||||
The CLI resolves configuration before constructing the public engine. `run`
|
`run` calls `promptkit.Engine.Run`; `render` calls
|
||||||
calls `Engine.Run` with a public request and `render` calls `Engine.Prepare`
|
`promptkit.Engine.Prepare`. Both share request mapping for prompt ID/version
|
||||||
with the same request mapping. `serve` constructs the HTTP-owned restricted
|
and profile selection, optional file inputs, variables, and presence-aware
|
||||||
artifact reader, injects it with `WithArtifactReader`, passes the resulting
|
execution overrides. Omitted framework settings remain zero values so Promptkit
|
||||||
engine directly to the HTTP handler, and starts the server.
|
resolves its own defaults and definition-required inputs.
|
||||||
|
|
||||||
Parser state records whether numeric runtime values were explicitly supplied.
|
`inspect prompt` maps the selected ID and optional version to
|
||||||
That presence is carried into `scriptorium.ExecutionTargetOverride`, allowing
|
`promptkit.Engine.InspectPrompt`, then formats Scriptorium-owned inspection
|
||||||
the engine to distinguish omitted values from explicit zero overrides.
|
data. It constructs the same configuration-aware engine but does not perform
|
||||||
|
preparation or generation.
|
||||||
|
|
||||||
|
`inspect profile` maps an explicit profile ID to `Engine.InspectProfile` and
|
||||||
|
formats a safe application-owned effective profile view. It intentionally does
|
||||||
|
not require prompt, schema, or artifact sources.
|
||||||
|
|
||||||
|
`serve` constructs Scriptorium's restricted HTTP artifact reader, injects it
|
||||||
|
with `promptkit.WithArtifactReader`, passes the engine through the HTTP
|
||||||
|
adapter's consumer-owned `Runner` interface, and starts the server.
|
||||||
|
|
||||||
|
All three CLI paths assemble the engine from the same resolved prompt, profile,
|
||||||
|
and schema directories plus configured custom backends. Each backend is mapped
|
||||||
|
to Promptkit's public `Backend` value and registered during engine construction,
|
||||||
|
so one constructed server engine retains one immutable backend registry and its
|
||||||
|
associated capacity state.
|
||||||
|
|
||||||
### HTTP
|
### HTTP
|
||||||
|
|
||||||
The handler first enforces transport limits, strict JSON decoding, and the
|
The handler enforces transport limits and strict JSON decoding before mapping
|
||||||
minimal request shape. It maps DTO values to public types without deciding
|
DTOs into `promptkit.RunRequest`, `promptkit.ArtifactRef`, and
|
||||||
prompt selection, source behavior, or validation semantics. On success it maps
|
`promptkit.ExecutionTargetOverride`. On success it reads Promptkit artifact,
|
||||||
the public result to the response DTO; on failure it uses `errors.Is` over
|
validation, model, usage, and metadata values directly.
|
||||||
public framework errors and HTTP-local artifact-policy errors to choose the
|
|
||||||
public error mapping.
|
|
||||||
|
|
||||||
The [HTTP API reference](../api.md) owns the route, DTO schema, status codes,
|
Failure mapping uses `errors.Is` against Promptkit's public sentinels and the
|
||||||
and externally observable limit behavior.
|
HTTP reader's Scriptorium-owned containment and size errors. Wrapped reader
|
||||||
|
errors preserve their identity through Promptkit's artifact-load boundary.
|
||||||
### Public Go Facade
|
|
||||||
|
|
||||||
`NewEngine` applies public options, selects filesystem, `fs.FS`, single-file,
|
|
||||||
or in-memory dependencies, and constructs a runner. The conversion functions
|
|
||||||
copy maps and slices across the boundary so callers do not receive internal
|
|
||||||
domain values. The facade maps selected internal errors to the public sentinel
|
|
||||||
set and keeps direct request API keys out of public results.
|
|
||||||
|
|
||||||
## Package-Local Guarantees
|
## Package-Local Guarantees
|
||||||
|
|
||||||
- Adapters do not embed framework orchestration or source-loading decisions.
|
- Adapters contain no copied framework types or orchestration.
|
||||||
- Configuration is resolved before adapter dependency composition.
|
- Configuration is resolved before Promptkit engine construction.
|
||||||
- CLI and HTTP consume the public engine without a repairer; a repairer remains
|
- Explicit numeric overrides preserve presence, including zero.
|
||||||
available only through explicit internal runner construction.
|
- HTTP DTO and error mapping remains stable and transport-owned.
|
||||||
- DTO conversion preserves explicit numeric-override presence.
|
- Resolved secrets are not serialized or printed.
|
||||||
- Error mapping matches error identities, not error text.
|
- No adapter creates durable run state.
|
||||||
- No adapter creates durable run state; caller-selected output files are not
|
|
||||||
application state.
|
|
||||||
|
|
||||||
## Failure And Verification Boundaries
|
## Verification
|
||||||
|
|
||||||
Keep external error payloads concise, preserve strict external decoding, and do
|
Inspect:
|
||||||
not serialize resolved secret values. Validation content failures remain result
|
|
||||||
state; runtime failures remain errors for the relevant adapter to map.
|
|
||||||
|
|
||||||
Inspect focused tests when changing this area:
|
|
||||||
|
|
||||||
- `internal/adapter/cli/run_test.go`
|
- `internal/adapter/cli/run_test.go`
|
||||||
- `internal/adapter/http/handler_test.go`
|
- `internal/adapter/http/handler_test.go`
|
||||||
- `engine_test.go`
|
- `internal/adapter/http/artifact_reader_test.go`
|
||||||
- `internal/format/prepared_run_test.go`
|
- `internal/format/prepared_run_test.go`
|
||||||
|
- `internal/adapter/dependency_test.go`
|
||||||
|
|
||||||
Run the affected adapter package tests and recheck the relevant canonical
|
The adapter tests protect parsing, configuration mapping, output, status
|
||||||
contract. The [testing policy](../policy/testing.md) owns global test
|
mapping, restricted artifacts, and representative real Promptkit-engine
|
||||||
sufficiency guidance.
|
workflows. The dependency test protects the repository boundary.
|
||||||
|
|
||||||
## Change Recipes
|
## Change Recipes
|
||||||
|
|
||||||
### Application Configuration Fields
|
For a CLI or HTTP change:
|
||||||
|
|
||||||
1. Add the field to the relevant `internal/config` shape and default handling.
|
1. identify the Scriptorium-owned external contract;
|
||||||
2. Parse and validate it, then preserve configuration and CLI-override
|
2. map through Promptkit public values without copying framework semantics;
|
||||||
precedence while wiring it through its consuming adapter.
|
3. add or update the narrow application-owned test;
|
||||||
3. Add focused configuration and adapter tests for parsing, mapping, and
|
4. update the canonical Scriptorium contract; and
|
||||||
effective behavior.
|
5. coordinate and tag Promptkit first if a required public capability is
|
||||||
4. Update the [configuration contract](../config.md) and any affected external
|
genuinely absent.
|
||||||
contract.
|
|
||||||
|
|
||||||
### CLI Flags
|
Update [source internals](sources.md) when application source locations or HTTP
|
||||||
|
artifact containment changes.
|
||||||
1. Add the flag to the relevant parser in `internal/adapter/cli/run.go`.
|
|
||||||
2. Keep command scope and application-configuration precedence intentional.
|
|
||||||
3. Add or update parser and command tests in
|
|
||||||
`internal/adapter/cli/run_test.go`.
|
|
||||||
4. Update the [CLI contract](../cli.md) and affected maintained examples.
|
|
||||||
|
|
||||||
### Adapter Capabilities
|
|
||||||
|
|
||||||
1. Define or reuse an adapter-local consumer interface with public facade
|
|
||||||
types when a test seam is needed.
|
|
||||||
2. Implement translation and IO behavior without moving framework decisions out
|
|
||||||
of the public engine.
|
|
||||||
3. Add focused mapping, parsing, and error-behavior tests.
|
|
||||||
4. Update this document and the affected public or integration contract. Update
|
|
||||||
[source internals](sources.md) when source-loading behavior changes.
|
|
||||||
|
|||||||
@@ -1,86 +0,0 @@
|
|||||||
# LLM Internals
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
`internal/llm` defines the provider-neutral `Client` interface and the
|
|
||||||
OpenAI-compatible client implementation. The [OpenAI-compatible integration
|
|
||||||
contract](../integrations/openai-compatible-chat.md) owns the outbound HTTP wire
|
|
||||||
format and protocol behavior.
|
|
||||||
|
|
||||||
## Construction
|
|
||||||
|
|
||||||
`NewOpenAICompatibleClient` validates a non-empty configured base URL, records
|
|
||||||
an optional default model, and resolves one transport cap. A supplied client
|
|
||||||
with a positive timeout supplies that cap; otherwise a positive configured
|
|
||||||
timeout is used, then the internal default.
|
|
||||||
|
|
||||||
When callers supply an `http.Client`, construction clones it rather than
|
|
||||||
mutating the caller's instance. A supplied client with a zero or negative
|
|
||||||
timeout receives the resolved transport cap in the clone. The client stores the
|
|
||||||
trimmed base URL, default model, and cloned client.
|
|
||||||
|
|
||||||
## Generate Flow
|
|
||||||
|
|
||||||
`Generate` receives a `domain.GenerateRequest` from the runner:
|
|
||||||
|
|
||||||
1. validate the effective timeout and choose the request endpoint;
|
|
||||||
2. map the domain request to the internal wire-request representation;
|
|
||||||
3. validate and flatten extra parameters and encode JSON;
|
|
||||||
4. derive a child context when the effective generation timeout is positive,
|
|
||||||
then create the HTTP request with that context;
|
|
||||||
5. prefer a direct API key, otherwise resolve the configured key environment
|
|
||||||
variable;
|
|
||||||
6. execute with the construction-time HTTP client, reject non-success status
|
|
||||||
responses without returning
|
|
||||||
provider response bodies; and
|
|
||||||
7. decode the response subset into `domain.GenerateResponse`.
|
|
||||||
|
|
||||||
`openAIChatRequestFromGenerateRequest` is the conversion boundary for effective
|
|
||||||
model defaults, explicit numeric-presence state, rendered messages, structured
|
|
||||||
output, and session-ID validation. `openAIChatRequestPayload` protects reserved
|
|
||||||
fields and JSON encoding before an HTTP call. The external payload shape is
|
|
||||||
defined only in the [integration contract](../integrations/openai-compatible-chat.md).
|
|
||||||
|
|
||||||
## Error Categories
|
|
||||||
|
|
||||||
The package uses these internal sentinels:
|
|
||||||
|
|
||||||
- `ErrInvalidConfig` for invalid client construction;
|
|
||||||
- `ErrInvalidRequest` for invalid effective generation input;
|
|
||||||
- `ErrRequestFailed` for request construction or transport failures;
|
|
||||||
- `ErrUnexpectedStatus` for non-success HTTP responses; and
|
|
||||||
- `ErrMalformedResponse` for invalid or incomplete successful-response data.
|
|
||||||
|
|
||||||
The runner maps an invalid LLM request to its invalid-request category and
|
|
||||||
other LLM failures to its generation category. Adapters then apply their public
|
|
||||||
error contracts.
|
|
||||||
|
|
||||||
## Package-Local Guarantees
|
|
||||||
|
|
||||||
- The default-model fallback happens before wire encoding.
|
|
||||||
- Per-generation timeout handling derives a request context; it never replaces
|
|
||||||
or mutates the configured HTTP client's transport cap.
|
|
||||||
- Direct API keys take precedence over environment lookup within this client.
|
|
||||||
- Provider response bodies are discarded for non-success status responses.
|
|
||||||
- The client does not implement retries, tool calls, or a stateful session
|
|
||||||
store.
|
|
||||||
|
|
||||||
## Verification And Change Recipe
|
|
||||||
|
|
||||||
Inspect:
|
|
||||||
|
|
||||||
- `internal/llm/openai_compatible_client_test.go`
|
|
||||||
- `internal/usecase/runner_test.go`
|
|
||||||
- `internal/adapter/http/handler_test.go`
|
|
||||||
|
|
||||||
When changing the client:
|
|
||||||
|
|
||||||
1. keep domain-to-wire mapping inside `internal/llm` and preserve the `Client`
|
|
||||||
interface;
|
|
||||||
2. test construction, timeout selection, mapping, and error categorization;
|
|
||||||
3. update the [OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
|
|
||||||
for any observable wire or protocol change; and
|
|
||||||
4. update [runner internals](runner.md) if the client boundary or structured
|
|
||||||
output handoff changes.
|
|
||||||
|
|
||||||
The [testing policy](../policy/testing.md) owns global test sufficiency.
|
|
||||||
@@ -1,48 +1,18 @@
|
|||||||
# Internal Component Overview
|
# Internal Component Overview
|
||||||
|
|
||||||
## Purpose
|
This is the complete inventory of Scriptorium's implemented Go components.
|
||||||
|
The [architecture policy](../policy/architecture.md) owns normative boundaries;
|
||||||
This is the inventory of Scriptorium's implemented components for contributors.
|
public behavior belongs in the linked contracts.
|
||||||
The [architecture policy](../policy/architecture.md) owns normative boundaries
|
|
||||||
and invariants; public behavior belongs in the linked contracts.
|
|
||||||
|
|
||||||
## Public And Command Entrypoints
|
|
||||||
|
|
||||||
| Component | Implemented responsibility | References |
|
| Component | Implemented responsibility | References |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Root package `scriptorium` | Public Go facade that constructs the engine, exposes request/result types and options, and maps internal errors. | [Go package contract](../consumers/pkg-scriptorium.md), [adapter internals](adapters.md) |
|
| `cmd/scriptorium` | Process entrypoint that delegates arguments and streams to the CLI adapter. | [CLI contract](../cli.md), [adapter internals](adapters.md) |
|
||||||
| `cmd/scriptorium` | Process entrypoint that delegates command execution to the CLI adapter. | [CLI contract](../cli.md), [adapter internals](adapters.md) |
|
| `internal/adapter/cli` | Parses commands, resolves application settings, constructs Promptkit engines, maps requests, and owns process output and exit behavior. | [CLI contract](../cli.md), [adapter internals](adapters.md) |
|
||||||
|
| `internal/adapter/http` | Owns routes, DTOs, strict decoding, limits, Promptkit request/result mapping, public error mapping, and restricted HTTP artifact reading. | [HTTP API](../api.md), [adapter internals](adapters.md), [source internals](sources.md) |
|
||||||
|
| `internal/config` | Discovers and strictly decodes application configuration and applies built-in and CLI precedence. | [configuration contract](../config.md), [adapter internals](adapters.md) |
|
||||||
|
| `internal/defaults` | Holds Scriptorium-owned application and HTTP defaults. | [configuration contract](../config.md) |
|
||||||
|
| `internal/format` | Formats Promptkit prepared-run and prompt/profile inspection values for CLI text or JSON output. | [CLI contract](../cli.md), [adapter internals](adapters.md) |
|
||||||
|
|
||||||
## Adapters, Domain, And Use Case
|
Framework implementation packages are provided by
|
||||||
|
[Promptkit v0.9.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md)
|
||||||
| Component | Implemented responsibility | References |
|
and are not part of this repository.
|
||||||
| --- | --- | --- |
|
|
||||||
| `internal/adapter/cli` | Parses CLI commands, constructs the public engine from application settings, and handles process input and output. | [CLI contract](../cli.md), [adapter internals](adapters.md) |
|
|
||||||
| `internal/adapter/http` | Maps HTTP requests and responses through public engine values, maps public errors, and owns restricted HTTP artifact policy. | [HTTP API contract](../api.md), [adapter internals](adapters.md) |
|
|
||||||
| `internal/domain` | Defines core request, result, output-contract, and LLM-boundary types. | [runner internals](runner.md) |
|
|
||||||
| `internal/usecase` | Implements `Runner` preparation, execution, validation coordination, and the repairer boundary. | [runner internals](runner.md) |
|
|
||||||
|
|
||||||
## Configuration And Sources
|
|
||||||
|
|
||||||
| Component | Implemented responsibility | References |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| `internal/config` | Loads application settings, applies defaults, and applies CLI overrides. | [configuration contract](../config.md), [adapter internals](adapters.md) |
|
|
||||||
| `internal/defaults` | Holds compile-time default values used when application settings are resolved. | [configuration contract](../config.md) |
|
|
||||||
| `internal/promptdef` | Loads prompt definitions from filesystem and `fs.FS` sources. | [configuration contract](../config.md), [source internals](sources.md) |
|
|
||||||
| `internal/profile` | Loads filesystem and `fs.FS` execution profiles and combines profile repositories. | [configuration contract](../config.md), [source internals](sources.md) |
|
|
||||||
| `internal/profile/builtin` | Provides embedded built-in execution profiles as a repository. | [configuration contract](../config.md), [source internals](sources.md) |
|
|
||||||
| `internal/filecatalog` | Provides shared YAML discovery and source-root helpers. | [source internals](sources.md) |
|
|
||||||
| `internal/artifact` | Provides the framework's ordinary inline and unrestricted file artifact reader. | [configuration contract](../config.md), [source internals](sources.md) |
|
|
||||||
| `internal/prompt` | Renders prompt templates into messages. | [runner internals](runner.md) |
|
|
||||||
|
|
||||||
## Formatting, Validation, And Model Access
|
|
||||||
|
|
||||||
| Component | Implemented responsibility | References |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| `internal/format` | Formats public prepared-run information for CLI output. | [CLI contract](../cli.md), [adapter internals](adapters.md) |
|
|
||||||
| `internal/validate` | Defines validation interfaces and provides standard filesystem and `fs.FS` schema validation. | [configuration contract](../config.md), [source internals](sources.md), [runner internals](runner.md) |
|
|
||||||
| `internal/llm` | Defines the provider-neutral LLM client boundary and its OpenAI-compatible implementation. | [OpenAI-compatible integration](../integrations/openai-compatible-chat.md), [LLM internals](llm.md), [runner internals](runner.md) |
|
|
||||||
|
|
||||||
Focused internal documents describe the components that have detailed
|
|
||||||
orchestration, adapter, or source behavior. Package tests live alongside the
|
|
||||||
implementation and are identified in those focused documents where relevant.
|
|
||||||
|
|||||||
@@ -1,120 +0,0 @@
|
|||||||
# Runner Internals
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
`internal/usecase.Runner` is the prompt-execution orchestrator. It prepares
|
|
||||||
domain requests, invokes an injected LLM client, validates output, and returns
|
|
||||||
domain results. Transport parsing, response mapping, and public type conversion
|
|
||||||
remain outside this package.
|
|
||||||
|
|
||||||
The [configuration reference](../config.md) owns prompt, profile, schema, and
|
|
||||||
runtime-setting definitions. Public error behavior is defined by the
|
|
||||||
[HTTP API](../api.md) and [Go package](../consumers/pkg-scriptorium.md)
|
|
||||||
contracts.
|
|
||||||
|
|
||||||
## Dependencies And Construction
|
|
||||||
|
|
||||||
`Runner` receives these collaborators:
|
|
||||||
|
|
||||||
- `promptdef.Repository`;
|
|
||||||
- `profile.Repository`;
|
|
||||||
- `artifact.Reader`;
|
|
||||||
- `prompt.Renderer`;
|
|
||||||
- `llm.Client`;
|
|
||||||
- `validate.Validator`; and
|
|
||||||
- an optional `OutputRepairer`.
|
|
||||||
|
|
||||||
`NewRunner` constructs a runner without a repairer. `NewRunnerWithRepairer`
|
|
||||||
accepts one explicitly. The public engine chooses concrete repositories and
|
|
||||||
readers; executable adapters reach the runner only through that engine. The
|
|
||||||
runner does not load application configuration.
|
|
||||||
|
|
||||||
## Prepare Flow
|
|
||||||
|
|
||||||
`Prepare` performs one deterministic preparation pass for a request:
|
|
||||||
|
|
||||||
1. validate the prompt ID and load the prompt definition;
|
|
||||||
2. hash the definition and select the explicit or default profile;
|
|
||||||
3. load the profile and resolve effective execution settings;
|
|
||||||
4. validate endpoint, model, and credential availability;
|
|
||||||
5. resolve the output contract and, for JSON Schema output, load a structured
|
|
||||||
schema document before model execution;
|
|
||||||
6. read and hash input artifacts;
|
|
||||||
7. render messages and the session ID; and
|
|
||||||
8. return a `PreparedRun` containing the effective state and rendered-prompt
|
|
||||||
hash.
|
|
||||||
|
|
||||||
Execution settings merge defaults, profile values, and a request override.
|
|
||||||
Numeric override presence is retained so explicit zero values are not confused
|
|
||||||
with omissions.
|
|
||||||
|
|
||||||
## Run And Validation Flow
|
|
||||||
|
|
||||||
`Run` creates a run ID and timestamps, then calls `Prepare` rather than
|
|
||||||
duplicating preparation. It sends the prepared prompt, effective target,
|
|
||||||
target-presence state, and optional structured-output specification to the LLM
|
|
||||||
client. It converts the returned content to an output artifact, validates it,
|
|
||||||
and returns the artifact, validation, hashes, usage, and timing metadata.
|
|
||||||
|
|
||||||
A validator can return a content result or an operational error. Content
|
|
||||||
failures stay in the result; schema loading, compilation, and validator
|
|
||||||
operational failures are returned as `ErrValidation`. The canonical distinction
|
|
||||||
for callers is documented by the public contracts.
|
|
||||||
|
|
||||||
## Repair Boundary
|
|
||||||
|
|
||||||
Repair is an internal optional loop. It starts only when a repairer is present,
|
|
||||||
the output contract permits one or more attempts, validation failed, and the
|
|
||||||
validation mode is JSON or JSON Schema. Each repair receives the previous
|
|
||||||
output, validation errors, effective target, structured-output specification,
|
|
||||||
and attempt metadata; every repaired result is validated again.
|
|
||||||
|
|
||||||
`NewDefaultOutputRepairer` delegates to the injected LLM client. The public
|
|
||||||
engine, and therefore CLI and HTTP, uses `NewRunner` and does not inject this
|
|
||||||
repairer.
|
|
||||||
|
|
||||||
## Error Translation
|
|
||||||
|
|
||||||
Runner sentinels identify failure categories for adapters:
|
|
||||||
|
|
||||||
- `ErrInvalidRequest`
|
|
||||||
- `ErrProfileRequired`
|
|
||||||
- `ErrAPIKeyEnvMissing` and `ErrAPIKeyRequired`
|
|
||||||
- `ErrPromptLoad`, `ErrProfileLoad`, and `ErrArtifactLoad`
|
|
||||||
- `ErrPromptRender`
|
|
||||||
- `ErrLLMGenerate`
|
|
||||||
- `ErrValidation`
|
|
||||||
|
|
||||||
Wrap errors with those sentinels and preserve their identities through
|
|
||||||
`errors.Is`; adapters must not classify errors by message text. The runner
|
|
||||||
passes direct keys only to the LLM boundary and never includes resolved key
|
|
||||||
values in prepared or run results.
|
|
||||||
|
|
||||||
## Package-Local Guarantees
|
|
||||||
|
|
||||||
- `Run` always reuses `Prepare`.
|
|
||||||
- Schema documents are loaded before the initial LLM call when structured output
|
|
||||||
is required.
|
|
||||||
- Output validation records attempts used, including repair attempts.
|
|
||||||
- Runner state is per request; the package does not create a durable run store
|
|
||||||
or manifest.
|
|
||||||
- Source, renderer, validator, and LLM implementations remain injected
|
|
||||||
boundaries.
|
|
||||||
|
|
||||||
## Verification And Change Recipe
|
|
||||||
|
|
||||||
Inspect:
|
|
||||||
|
|
||||||
- `internal/usecase/runner_test.go`
|
|
||||||
- `engine_test.go`
|
|
||||||
|
|
||||||
When changing orchestration:
|
|
||||||
|
|
||||||
1. identify the collaborator boundary and the affected `Prepare` or `Run` state;
|
|
||||||
2. preserve the `Run`-through-`Prepare` path and error identity;
|
|
||||||
3. add focused runner or integration tests for changed state transitions,
|
|
||||||
validation, or repair behavior; and
|
|
||||||
4. update the owning external contract and any affected source or LLM internal
|
|
||||||
document.
|
|
||||||
|
|
||||||
The [testing policy](../policy/testing.md) owns global test sufficiency.
|
|
||||||
@@ -2,98 +2,69 @@
|
|||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
This document describes how source packages load prompt definitions, profiles,
|
This document covers Scriptorium-owned source locations and the restricted HTTP
|
||||||
schemas, and artifacts. The [configuration reference](../config.md) owns their
|
artifact reader. Prompt, profile, schema, and ordinary artifact semantics are
|
||||||
user-facing formats and settings. The [HTTP API reference](../api.md) owns
|
owned by the tagged
|
||||||
HTTP-visible artifact outcomes; [operations](../operations.md) owns deployment
|
[Promptkit format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md).
|
||||||
handling.
|
|
||||||
|
|
||||||
## Prompt Definitions
|
## Application Source Locations
|
||||||
|
|
||||||
`internal/promptdef` provides filesystem and `fs.FS` repositories. Both use
|
`internal/config` resolves `prompt_dir`, `profile_dir`, and `schema_dir` from
|
||||||
`internal/filecatalog` for recursive YAML discovery, deterministic ordering,
|
Scriptorium defaults, configuration files, and CLI overrides. It also resolves
|
||||||
display paths, and root cleaning.
|
the application-owned `backends` mapping into sorted engine settings.
|
||||||
|
`internal/adapter/cli` passes the directories into `promptkit.Config` and maps
|
||||||
|
each configured backend to Promptkit's public engine registration when
|
||||||
|
constructing an engine shared by the command path.
|
||||||
|
|
||||||
Repositories select a prompt by YAML ID and optional version rather than by
|
Scriptorium does not search, parse, validate, or overlay framework source files
|
||||||
path. They decode through strict YAML handling, reject duplicate matching
|
itself. Promptkit owns prompt selection, profile built-ins and overlays, schema
|
||||||
definitions, and resolve `content_file` relative to the definition. The `fs.FS`
|
resolution, ordinary file artifacts, and the related error identities.
|
||||||
implementation resolves content paths inside its source root; absolute paths and
|
|
||||||
traversal outside that root are rejected before file access.
|
|
||||||
|
|
||||||
## Profiles And Built-Ins
|
The [configuration reference](../config.md) owns Scriptorium's source-location
|
||||||
|
fields and precedence. Maintained files under `examples/` are application
|
||||||
|
inputs that use Promptkit's tagged formats.
|
||||||
|
|
||||||
`internal/profile` provides filesystem, `fs.FS`, and overlay repositories.
|
## Restricted HTTP Artifact Reader
|
||||||
`internal/profile/builtin` exposes embedded assets through the same repository
|
|
||||||
interface.
|
|
||||||
|
|
||||||
An overlay asks its primary source first. It falls back only when the primary
|
`internal/adapter/http` implements `promptkit.ArtifactReader` for HTTP
|
||||||
reports `ErrProfileNotFound`; invalid YAML, duplicate IDs, validation failures,
|
requests. The `serve` path injects it with
|
||||||
and raw-key failures are returned rather than hidden by fallback. This makes a
|
`promptkit.WithArtifactReader`, replacing Promptkit's ordinary reader for
|
||||||
custom ID override a built-in ID while retaining errors in the custom source.
|
inbound HTTP inputs.
|
||||||
|
|
||||||
The public engine can overlay in-memory profiles ahead of both file-backed and
|
The reader:
|
||||||
built-in repositories. Profile field definitions, validation ranges, and the
|
|
||||||
built-in catalog remain in the [configuration reference](../config.md).
|
|
||||||
|
|
||||||
## Schemas
|
- accepts inline references without an artifact root;
|
||||||
|
- denies file references when no root is configured;
|
||||||
|
- resolves relative paths below the configured root;
|
||||||
|
- accepts absolute paths only when they are lexically within that root;
|
||||||
|
- rejects lexical traversal outside the root;
|
||||||
|
- applies the configured file byte limit, with zero meaning unlimited;
|
||||||
|
- preserves content type, body, size, hash, name, and URI metadata; and
|
||||||
|
- honors context cancellation.
|
||||||
|
|
||||||
`internal/validate` supplies `StandardValidator` for filesystem sources and
|
Containment is lexical and does not resolve symlinks. The operating system
|
||||||
`FSValidator` for `fs.FS` sources. Directory-backed validation loads the named
|
follows symlinks after the check. The [HTTP API](../api.md) owns observable
|
||||||
schema path; it does not search directories by basename. `fs.FS` schema paths
|
request outcomes, and [operations](../operations.md) owns safe deployment
|
||||||
are cleaned and checked against their configured root, while a single-file
|
permissions and root selection.
|
||||||
source matches its file base name.
|
|
||||||
|
|
||||||
The runner requests a schema document before generation when it needs
|
Reader errors remain identifiable after Promptkit wraps them as artifact-load
|
||||||
structured output. JSON and schema mismatches in generated content are
|
failures, allowing the HTTP adapter to preserve Scriptorium status and error
|
||||||
validation results; source access, decoding, registration, and compilation
|
codes.
|
||||||
failures are operational errors.
|
|
||||||
|
|
||||||
## Artifacts
|
|
||||||
|
|
||||||
`internal/artifact` owns the framework's ordinary inline and unrestricted file
|
|
||||||
reader. The public engine uses it by default and permits consumers to replace it
|
|
||||||
for every input through the public `ArtifactReader` extension. The
|
|
||||||
HTTP adapter owns its restricted reader for HTTP containment: `serve` injects
|
|
||||||
that reader into the public engine with `WithArtifactReader`.
|
|
||||||
|
|
||||||
The rooted reader cleans paths and applies lexical containment without resolving
|
|
||||||
symlinks. It checks relative references against the configured root and accepts
|
|
||||||
absolute references only when they remain inside that lexical root. The OS still
|
|
||||||
follows symlinks after that check. The public containment outcome is documented
|
|
||||||
by the [HTTP API reference](../api.md); deployment permissions belong in
|
|
||||||
[operations](../operations.md).
|
|
||||||
|
|
||||||
## Failure Boundaries
|
|
||||||
|
|
||||||
Source packages report repository, decoding, duplicate, validation, and read
|
|
||||||
failures to their callers. They do not select public status codes or response
|
|
||||||
schemas. The runner categorizes source failures and the public engine preserves
|
|
||||||
the corresponding public error identities; adapters map those identities to
|
|
||||||
their own external contract.
|
|
||||||
|
|
||||||
Source reads use current filesystem or `fs.FS` content for each request. These
|
|
||||||
packages create no manifests, checkpoints, or durable run state.
|
|
||||||
|
|
||||||
## Verification And Change Recipe
|
## Verification And Change Recipe
|
||||||
|
|
||||||
Inspect:
|
Inspect:
|
||||||
|
|
||||||
- `internal/promptdef/repository_test.go`
|
- `internal/config/config_test.go`
|
||||||
- `internal/profile/repository_test.go`
|
- `internal/adapter/cli/run_test.go`
|
||||||
- `internal/profile/builtin/repository_test.go`
|
|
||||||
- `internal/artifact/reader_test.go`
|
|
||||||
- `internal/adapter/http/artifact_reader_test.go`
|
- `internal/adapter/http/artifact_reader_test.go`
|
||||||
- `internal/validate/standard_validator_test.go`
|
- `internal/adapter/http/handler_test.go`
|
||||||
- `engine_test.go`
|
|
||||||
|
|
||||||
When updating prompt, profile, schema, or built-in assets:
|
When changing an application source location or HTTP artifact policy:
|
||||||
|
|
||||||
1. keep assets valid for the strict loader and the relevant source boundary;
|
1. preserve strict configuration precedence and the Promptkit public boundary;
|
||||||
2. update the [configuration reference](../config.md) when a file-format,
|
2. keep containment and size policy in Scriptorium;
|
||||||
catalog, or default changes;
|
3. update focused configuration, reader, and handler tests;
|
||||||
3. run focused source and integration tests, including the built-in repository
|
4. update the [configuration](../config.md), [HTTP](../api.md), and
|
||||||
test when embedded assets change; and
|
[operations](../operations.md) contracts as applicable; and
|
||||||
4. update this document when discovery, precedence, containment, or failure
|
5. do not duplicate Promptkit loaders, formats, or ordinary artifact behavior.
|
||||||
mechanics change.
|
|
||||||
|
|
||||||
The [testing policy](../policy/testing.md) owns global test sufficiency.
|
|
||||||
|
|||||||
@@ -7,8 +7,11 @@ recovery for Scriptorium. It does not redefine invocation syntax, configuration
|
|||||||
fields, or HTTP wire behavior.
|
fields, or HTTP wire behavior.
|
||||||
|
|
||||||
- [CLI reference](cli.md): commands, output destinations, and exit codes.
|
- [CLI reference](cli.md): commands, output destinations, and exit codes.
|
||||||
- [Configuration reference](config.md): configuration, prompt/profile/schema
|
- [Configuration reference](config.md): application settings, source
|
||||||
formats, defaults, and credentials.
|
locations, defaults, and credential mapping.
|
||||||
|
- [Promptkit framework formats](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md):
|
||||||
|
prompt, profile, schema, execution-setting, and framework credential
|
||||||
|
contracts.
|
||||||
- [HTTP API reference](api.md): route, request/response schema, status codes,
|
- [HTTP API reference](api.md): route, request/response schema, status codes,
|
||||||
limits, and HTTP artifact access.
|
limits, and HTTP artifact access.
|
||||||
- [Consumer integration overview](consumers/api.md): caller responsibilities.
|
- [Consumer integration overview](consumers/api.md): caller responsibilities.
|
||||||
@@ -26,10 +29,10 @@ responsibilities.
|
|||||||
|
|
||||||
## Deploy The Filesystem And Process
|
## Deploy The Filesystem And Process
|
||||||
|
|
||||||
Provide the process with readable prompt, profile, and schema sources. Keep
|
Provide the process with readable configured Promptkit prompt, profile, and
|
||||||
prompt templates adjacent to the prompt definitions that reference them. For an
|
schema sources that follow the tagged framework formats. For an HTTP deployment
|
||||||
HTTP deployment that accepts file artifacts, use a dedicated, narrow artifact
|
that accepts file artifacts, use a dedicated, narrow artifact directory rather
|
||||||
directory rather than a general-purpose or sensitive filesystem tree.
|
than a general-purpose or sensitive filesystem tree.
|
||||||
|
|
||||||
Run Scriptorium under an identity that can:
|
Run Scriptorium under an identity that can:
|
||||||
|
|
||||||
@@ -91,6 +94,14 @@ protect request bodies, HTTP file artifacts, and encoded responses; configure
|
|||||||
them through the [configuration reference](config.md) and rely on the
|
them through the [configuration reference](config.md) and rely on the
|
||||||
[HTTP API reference](api.md) for their response effects.
|
[HTTP API reference](api.md) for their response effects.
|
||||||
|
|
||||||
|
Configured backend concurrency and queue capacity are enforced per constructed
|
||||||
|
Promptkit engine. A `serve` process constructs one engine for its handler, so
|
||||||
|
concurrent HTTP requests share that transient admission state. Scriptorium does
|
||||||
|
not retain workflow state: capacity is neither durable nor a queue of resumable
|
||||||
|
runs. When admission is exhausted, HTTP returns `503 capacity_exceeded` without
|
||||||
|
retry timing; callers choose any retry policy that is safe for another model
|
||||||
|
call.
|
||||||
|
|
||||||
Before increasing a limit:
|
Before increasing a limit:
|
||||||
|
|
||||||
1. measure representative input, generated-output, and optional raw-output
|
1. measure representative input, generated-output, and optional raw-output
|
||||||
|
|||||||
@@ -1,83 +1,106 @@
|
|||||||
# Architecture
|
# Architecture
|
||||||
|
|
||||||
This document is the development architecture policy for Scriptorium.
|
This document defines Scriptorium's current application architecture and
|
||||||
|
durable development boundaries.
|
||||||
It is for developers and LLM coding agents. User-facing behavior belongs in `README.md` and the docs under `docs/` that target operators/users.
|
|
||||||
|
|
||||||
## System Shape
|
## System Shape
|
||||||
|
|
||||||
Scriptorium is a narrow prompt-execution application with three executable
|
Scriptorium is an executable application with four command entry paths: CLI
|
||||||
entry paths: CLI `run`, CLI `render`, and the HTTP service started by `serve`.
|
`run`, CLI `render`, CLI `inspect`, and the HTTP service started by `serve`.
|
||||||
It also provides a public Go package for in-process use. Executable adapters
|
The `inspect` command has prompt and profile modes. Scriptorium does not expose
|
||||||
consume framework behavior through that public facade; the facade continues to
|
a reusable root Go package.
|
||||||
compose the framework implementation inside this single repository. Its current
|
|
||||||
component inventory is maintained in the [internal overview](../internal/overview.md).
|
|
||||||
|
|
||||||
Domain behavior is centralized in `internal/usecase` and `internal/domain`.
|
The application consumes
|
||||||
|
[Promptkit v0.9.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md)
|
||||||
|
through its supported root package. Promptkit owns prompt execution,
|
||||||
|
preparation, source formats, built-in profiles, model-client behavior, and
|
||||||
|
validation. Scriptorium owns application configuration, executable adapters,
|
||||||
|
prepared-run and definition-inspection presentation, process behavior, and HTTP
|
||||||
|
deployment policy.
|
||||||
|
|
||||||
## Core Principles
|
The concrete package inventory is maintained in the
|
||||||
|
[internal overview](../internal/overview.md).
|
||||||
- Keep orchestration narrow: Scriptorium executes one prompt request; it is not a multi-step workflow engine.
|
|
||||||
- Keep adapter logic thin: adapters map external shapes to public engine
|
|
||||||
requests/results and should not hold framework decisions.
|
|
||||||
- Keep boundaries explicit: repositories/loaders/renderers/validators/LLM client stay behind package interfaces.
|
|
||||||
- Keep external decoding strict: configuration, prompt, and profile YAML and
|
|
||||||
HTTP JSON should reject unknown fields.
|
|
||||||
- Keep secrets out of payloads: raw API key values must not be accepted or emitted.
|
|
||||||
|
|
||||||
## Dependency Direction
|
## Dependency Direction
|
||||||
|
|
||||||
- Adapters translate external shapes and IO concerns; they do not make
|
```text
|
||||||
use-case decisions.
|
cmd/scriptorium
|
||||||
- Executable adapters and prepared-run formatting use the public facade for
|
|
|
||||||
framework behavior rather than importing framework implementation packages
|
v
|
||||||
directly.
|
CLI and HTTP adapters, configuration, defaults, and formatting
|
||||||
- Use-case and domain code depend on explicit repository, renderer, validator,
|
|
|
||||||
and LLM interfaces rather than adapter implementations.
|
v
|
||||||
- Source, rendering, validation, and LLM implementations remain behind their
|
gitea.maximumdirect.net/eric/promptkit
|
||||||
package boundaries.
|
```
|
||||||
- Dependency-specific types must not leak across unrelated package boundaries.
|
|
||||||
- Prefer the standard library; add an external dependency only when it
|
|
||||||
materially reduces risk or complexity.
|
|
||||||
|
|
||||||
## State And Persistence Policy
|
- Retained application packages may import Promptkit's root package.
|
||||||
|
- They must not import Promptkit `internal` packages.
|
||||||
|
- They must not import Promptkit catalog modules directly.
|
||||||
|
- They must not import the removed Scriptorium root facade or recreate former
|
||||||
|
framework package families.
|
||||||
|
- Adapter-owned interfaces use Promptkit public values when a consumer-side
|
||||||
|
substitution boundary is needed.
|
||||||
|
- Scriptorium passes omitted framework settings as zero values so Promptkit
|
||||||
|
applies its own defaults.
|
||||||
|
|
||||||
Scriptorium has no durable run-state store.
|
The repository architecture guard enforces these import and removal
|
||||||
|
invariants.
|
||||||
|
|
||||||
- No built-in resume/checkpoint/archive behavior.
|
## Retained Boundaries
|
||||||
- Recovery model is rerun after correcting inputs/config/environment.
|
|
||||||
|
|
||||||
## Contract Ownership
|
- `internal/adapter/cli` owns commands, flags, configuration precedence,
|
||||||
|
process streams, output files, summaries, and exit codes.
|
||||||
|
- `internal/adapter/http` owns routes, strict JSON DTOs, size limits, response
|
||||||
|
mapping, status mapping, and the restricted artifact reader.
|
||||||
|
- `internal/config` owns discovery and strict decoding of Scriptorium
|
||||||
|
application configuration.
|
||||||
|
- `internal/defaults` owns Scriptorium application and HTTP defaults only.
|
||||||
|
- `internal/format` owns deterministic prepared-run and definition-inspection
|
||||||
|
text and JSON presentation.
|
||||||
|
- Promptkit owns framework orchestration and contracts. Its
|
||||||
|
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md)
|
||||||
|
and
|
||||||
|
[outbound integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/integrations/openai-compatible-chat.md)
|
||||||
|
are canonical.
|
||||||
|
|
||||||
The [CLI](../cli.md), [configuration](../config.md), [HTTP API](../api.md),
|
## HTTP Artifact Security Boundary
|
||||||
[public Go package](../consumers/pkg-scriptorium.md), and
|
|
||||||
[integration](../integrations/) documents own their respective external
|
|
||||||
contracts. This policy keeps only the architectural boundaries that govern
|
|
||||||
their implementation.
|
|
||||||
|
|
||||||
## Error Handling And Logging
|
Ordinary CLI file loading is provided by Promptkit. Scriptorium's HTTP adapter
|
||||||
|
injects a restricted `promptkit.ArtifactReader` for inbound HTTP requests.
|
||||||
|
That reader denies file references without an artifact root, enforces the
|
||||||
|
configured byte limit, and applies Scriptorium's lexical root-containment rule.
|
||||||
|
The operating system still follows symlinks after the lexical check.
|
||||||
|
|
||||||
- Wrap errors with domain/operation context.
|
The [HTTP API](../api.md) owns observable request outcomes, and
|
||||||
- Map public error identities to adapter-appropriate statuses/codes without
|
[operations](../operations.md) owns deployment permissions and root selection.
|
||||||
leaking sensitive internals.
|
|
||||||
- Never emit raw secret values.
|
|
||||||
|
|
||||||
## Testing And Documentation
|
## State, Errors, And Secrets
|
||||||
|
|
||||||
Testing philosophy and change-validation expectations are defined by the
|
Scriptorium has no durable run-state store, checkpoint, cache, or resume
|
||||||
[testing policy](testing.md). Documentation ownership and maintenance rules are
|
mechanism. Recovery is a new request after correcting inputs, configuration, or
|
||||||
defined by the [documentation policy](documentation.md).
|
environment.
|
||||||
|
|
||||||
|
Adapters map Promptkit public error identities into CLI exits or HTTP statuses
|
||||||
|
without classifying by message text. Raw API keys are not accepted in
|
||||||
|
Scriptorium configuration, CLI arguments, or HTTP payloads, and resolved
|
||||||
|
secrets must not be emitted.
|
||||||
|
|
||||||
## Architectural Invariants
|
## Architectural Invariants
|
||||||
|
|
||||||
- `Runner.Run` reuses `Runner.Prepare` flow.
|
- External YAML and JSON decoding remains strict.
|
||||||
- Raw API key values must not be accepted through external configuration or
|
- CLI and HTTP behavior remains presentation and transport logic rather than
|
||||||
request payloads, and resolved secret values must not be emitted.
|
framework orchestration.
|
||||||
|
- Explicit numeric request overrides preserve presence, including zero.
|
||||||
|
- HTTP artifact containment and byte limits remain Scriptorium policy.
|
||||||
|
- No application package depends on Promptkit implementation packages.
|
||||||
|
|
||||||
## Non-Goals
|
## Non-Goals
|
||||||
|
|
||||||
- Do not move orchestration responsibilities from external callers into Scriptorium.
|
- Do not recreate an in-process Scriptorium framework API or compatibility
|
||||||
- Do not add adapter-specific business logic in `internal/adapter/*` packages.
|
facade.
|
||||||
- Do not bypass repository/renderer/validator/LLM boundaries by introducing cross-package coupling.
|
- Do not copy Promptkit types, defaults, built-in profiles, or implementation
|
||||||
|
into Scriptorium.
|
||||||
|
- Do not move CLI, inbound HTTP, process, or deployment policy into Promptkit.
|
||||||
|
- Do not add durable workflow, archive, or resume behavior.
|
||||||
|
|
||||||
Work that is not implemented belongs in `docs/roadmap/`.
|
Work that is not implemented belongs in `docs/roadmap/`.
|
||||||
|
|||||||
@@ -69,11 +69,13 @@ secret values.
|
|||||||
| Documentation organization | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
|
| Documentation organization | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
|
||||||
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression-test policy, and criteria for adding, rewriting, or deleting tests. | Subsystem behavior, application contracts, subsystem-specific test inventories, and implementation plans. |
|
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression-test policy, and criteria for adding, rewriting, or deleting tests. | Subsystem behavior, application contracts, subsystem-specific test inventories, and implementation plans. |
|
||||||
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
|
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
|
||||||
| Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. |
|
| Configuration contract | `docs/config.md` | Application discovery and precedence, source locations, server fields, render default, HTTP limits, and credential mapping. | Promptkit framework formats and defaults, complete example files, CLI syntax, runtime lifecycle, and implementation detail. |
|
||||||
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
|
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
|
||||||
|
| Release procedure | `docs/release.md` | Candidate validation, version and tag operations, hosted-workflow observation, and published-artifact verification. | Runtime operations, version-specific announcements, and complete application-interface contracts. |
|
||||||
|
| Version-specific release notes | `docs/releases/` | Immutable release summaries, compatibility notices, and migration announcements for one published version. | Complete CLI, HTTP, configuration, operations, or dependency contracts. |
|
||||||
| Public HTTP contract | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
|
| Public HTTP contract | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
|
||||||
| Consumer guidance | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
|
| Consumer guidance | `docs/consumers/` | Choosing between Scriptorium's executable interfaces and understanding consumer responsibilities. | HTTP wire semantics, CLI syntax, Promptkit's Go package, and internal implementation detail. |
|
||||||
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
| External and durable integration contracts | `docs/integrations/` | Scriptorium-owned process and executable integration contracts. | Promptkit framework formats and outbound provider protocols, physical runtime placement, internal transformations, CLI syntax, and configuration defaults. |
|
||||||
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
||||||
| Internal component behavior | Other files under `docs/internal/` | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
| Internal component behavior | Other files under `docs/internal/` | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
||||||
| Architectural decision history | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
|
| Architectural decision history | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
|
||||||
|
|||||||
@@ -77,7 +77,7 @@ Test through the narrowest stable boundary that expresses the behavior clearly.
|
|||||||
|
|
||||||
This is often the package API, but it may instead be:
|
This is often the package API, but it may instead be:
|
||||||
|
|
||||||
- a smaller pure function when dense domain logic is most clearly isolated there;
|
- a smaller pure function when dense application logic is most clearly isolated there;
|
||||||
- a package-level operation when several internal collaborators jointly produce the behavior; or
|
- a package-level operation when several internal collaborators jointly produce the behavior; or
|
||||||
- a larger integration boundary when correctness emerges from interaction with a real dependency.
|
- a larger integration boundary when correctness emerges from interaction with a real dependency.
|
||||||
|
|
||||||
@@ -162,13 +162,16 @@ Use a test-controlled limit and measure the behavior relative to that limit. Do
|
|||||||
|
|
||||||
Each behavior should have a clear test owner.
|
Each behavior should have a clear test owner.
|
||||||
|
|
||||||
- Parser tests own parsing cases.
|
- Configuration tests own application YAML, discovery, precedence, and
|
||||||
- Validator tests own validation rules.
|
application defaults.
|
||||||
- Domain tests own transformations and invariants.
|
- CLI tests own argument mapping, streams, summaries, exit behavior, and
|
||||||
- Adapter tests own external integration behavior.
|
representative command workflows.
|
||||||
- Orchestrator tests own coordination and failure propagation.
|
- HTTP tests own DTOs, strict decoding, limits, status mapping, and restricted
|
||||||
- CLI tests own argument and configuration mapping.
|
artifact policy.
|
||||||
- End-to-end tests prove that representative assembled workflows work.
|
- Formatter tests own prepared-run text and JSON presentation.
|
||||||
|
- Architecture tests own dependency direction and removal invariants.
|
||||||
|
- Promptkit owns framework parsing, orchestration, validation, profiles, and
|
||||||
|
model-client behavior.
|
||||||
|
|
||||||
Higher-level tests should not repeat every lower-level case. A single intentional policy change should not require unrelated edits across many test files.
|
Higher-level tests should not repeat every lower-level case. A single intentional policy change should not require unrelated edits across many test files.
|
||||||
|
|
||||||
@@ -221,7 +224,9 @@ Coverage is a diagnostic, not a target.
|
|||||||
|
|
||||||
Use it to find untested critical branches and unexpectedly weak packages. Do not write low-value tests solely to increase a percentage, and do not infer test quality from coverage alone.
|
Use it to find untested critical branches and unexpectedly weak packages. Do not write low-value tests solely to increase a percentage, and do not infer test quality from coverage alone.
|
||||||
|
|
||||||
Pure domain logic will often warrant higher coverage than CLI wiring or external adapters. Uneven coverage is acceptable when it reflects risk.
|
Security-sensitive HTTP containment and external mappings may warrant denser
|
||||||
|
coverage than straightforward process wiring. Uneven coverage is acceptable
|
||||||
|
when it reflects risk.
|
||||||
|
|
||||||
Increasing coverage is valuable only when the newly covered behavior protects a meaningful risk at an acceptable cost.
|
Increasing coverage is valuable only when the newly covered behavior protects a meaningful risk at an acceptable cost.
|
||||||
|
|
||||||
|
|||||||
377
docs/release.md
Normal file
377
docs/release.md
Normal file
@@ -0,0 +1,377 @@
|
|||||||
|
# Release Procedure
|
||||||
|
|
||||||
|
## Release Model And Status
|
||||||
|
|
||||||
|
Scriptorium publishes annotated semantic tags and tag-triggered Linux binary
|
||||||
|
releases. The hosted
|
||||||
|
[release workflow](../.woodpecker/release.yml) builds `amd64` and `arm64`
|
||||||
|
executables, publishes their SHA-256 checksums, and uses the matching file
|
||||||
|
under `docs/releases/` as the hosted release body.
|
||||||
|
|
||||||
|
`v0.12.0` is the first published application-only release. For each later
|
||||||
|
release, select a new `vMAJOR.MINOR.PATCH` version according to the intended
|
||||||
|
compatibility change. A selected version remains an unreleased candidate until
|
||||||
|
its annotated tag is published, the hosted workflow succeeds, and every
|
||||||
|
published artifact is verified.
|
||||||
|
|
||||||
|
Run this procedure from the Scriptorium repository root. A release must not
|
||||||
|
depend on a Go workspace, module replacement, vendor tree, sibling checkout,
|
||||||
|
unpublished dependency, or unpushed source commit.
|
||||||
|
|
||||||
|
## Establish The Candidate
|
||||||
|
|
||||||
|
Start a POSIX shell, choose a semantic version that has not been published, and
|
||||||
|
export it as `RELEASE_VERSION`. For example, if `v0.12.1` is the intended next
|
||||||
|
version and remains unpublished, select:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
export RELEASE_VERSION=v0.12.1
|
||||||
|
```
|
||||||
|
|
||||||
|
Use the version appropriate to the actual compatibility change rather than
|
||||||
|
assuming that the example is the next release. Then run the following guard in
|
||||||
|
that same shell:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
: "${RELEASE_VERSION:?export an unpublished vMAJOR.MINOR.PATCH version}"
|
||||||
|
if ! printf '%s\n' "$RELEASE_VERSION" |
|
||||||
|
grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$'
|
||||||
|
then
|
||||||
|
printf '%s\n' "invalid release version: $RELEASE_VERSION" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
RELEASE_COMMIT=$(git rev-parse --verify 'HEAD^{commit}')
|
||||||
|
export RELEASE_COMMIT
|
||||||
|
|
||||||
|
check_release_candidate() {
|
||||||
|
test "$(git branch --show-current)" = main
|
||||||
|
test -z "$(git status --porcelain)"
|
||||||
|
|
||||||
|
gowork_value=$(go env GOWORK)
|
||||||
|
case "$gowork_value" in
|
||||||
|
''|off) ;;
|
||||||
|
*)
|
||||||
|
printf '%s\n' "active Go workspace: $gowork_value" >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
test -z "$(git ls-files go.work go.work.sum)"
|
||||||
|
test ! -e vendor
|
||||||
|
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||||
|
then
|
||||||
|
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
git fetch origin main --tags
|
||||||
|
test "$RELEASE_COMMIT" = \
|
||||||
|
"$(git rev-parse --verify 'refs/remotes/origin/main^{commit}')"
|
||||||
|
|
||||||
|
if git show-ref --verify --quiet "refs/tags/$RELEASE_VERSION"
|
||||||
|
then
|
||||||
|
printf '%s\n' "local tag already exists: $RELEASE_VERSION" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if test -n "$(
|
||||||
|
git ls-remote --tags origin \
|
||||||
|
"refs/tags/$RELEASE_VERSION" \
|
||||||
|
"refs/tags/$RELEASE_VERSION^{}"
|
||||||
|
)"
|
||||||
|
then
|
||||||
|
printf '%s\n' "remote tag already exists: $RELEASE_VERSION" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
check_release_candidate
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not continue unless this guard succeeds. It deliberately requires the
|
||||||
|
candidate to be the exact clean commit already published at `origin/main`.
|
||||||
|
|
||||||
|
## Verify Modules And Repository Boundaries
|
||||||
|
|
||||||
|
Confirm the module path and declared Go version:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
test "$(
|
||||||
|
GOWORK=off go list -m -f '{{.Path}} {{.GoVersion}}'
|
||||||
|
)" = 'gitea.maximumdirect.net/eric/scriptorium 1.25.5'
|
||||||
|
```
|
||||||
|
|
||||||
|
Require Promptkit `v0.9.0` as both the direct module-graph edge and the selected
|
||||||
|
module version:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
direct_promptkit=$(
|
||||||
|
GOWORK=off go mod graph |
|
||||||
|
awk '
|
||||||
|
$1 == "gitea.maximumdirect.net/eric/scriptorium" &&
|
||||||
|
$2 ~ /^gitea\.maximumdirect\.net\/eric\/promptkit@/ {
|
||||||
|
print $2
|
||||||
|
}
|
||||||
|
'
|
||||||
|
)
|
||||||
|
test "$direct_promptkit" = \
|
||||||
|
'gitea.maximumdirect.net/eric/promptkit@v0.9.0'
|
||||||
|
test "$(
|
||||||
|
GOWORK=off go list -m -f '{{.Path}}@{{.Version}}' \
|
||||||
|
gitea.maximumdirect.net/eric/promptkit
|
||||||
|
)" = 'gitea.maximumdirect.net/eric/promptkit@v0.9.0'
|
||||||
|
GOWORK=off go list -m all
|
||||||
|
```
|
||||||
|
|
||||||
|
Require tidy module metadata and recheck the repository exclusions:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
GOWORK=off go mod tidy -diff
|
||||||
|
test -z "$(git ls-files go.work go.work.sum)"
|
||||||
|
test ! -e vendor
|
||||||
|
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||||
|
then
|
||||||
|
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
test -z "$(git status --porcelain)"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Validate The Application
|
||||||
|
|
||||||
|
Run the complete application validation:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
GOWORK=off go test ./...
|
||||||
|
GOWORK=off go test -race ./...
|
||||||
|
GOWORK=off go vet ./...
|
||||||
|
validation_build_dir=$(mktemp -d)
|
||||||
|
GOWORK=off go build \
|
||||||
|
-o "$validation_build_dir/scriptorium" \
|
||||||
|
./cmd/scriptorium
|
||||||
|
```
|
||||||
|
|
||||||
|
The ordinary test run includes the architecture guard that rejects a root Go
|
||||||
|
package, former framework package families, and imports of Promptkit internal
|
||||||
|
packages. Inspect the repository for generated binaries, credentials,
|
||||||
|
temporary output, sibling paths, and other files that do not belong in the
|
||||||
|
tracked release source.
|
||||||
|
|
||||||
|
Check every tracked Go file. This command must produce no output:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
unformatted=$(
|
||||||
|
git ls-files '*.go' |
|
||||||
|
while IFS= read -r go_file
|
||||||
|
do
|
||||||
|
gofmt -l "$go_file"
|
||||||
|
done
|
||||||
|
)
|
||||||
|
test -z "$unformatted"
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the maintained render and Promptkit v0.9 feature scripts, then smoke-test
|
||||||
|
both maintained configuration examples without a model call:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
GOWORK=off ./examples/render-markdown-summary.sh
|
||||||
|
GOWORK=off ./examples/render-v0.9-features.sh
|
||||||
|
for config_file in examples/config.yml examples/config.full.yml
|
||||||
|
do
|
||||||
|
GOWORK=off go run ./cmd/scriptorium render \
|
||||||
|
--config "$config_file" \
|
||||||
|
--prompt generic.markdown_summary \
|
||||||
|
--input transcript=./examples/fixtures/transcript.md \
|
||||||
|
--input glossary=./examples/fixtures/glossary.yml \
|
||||||
|
--format json >/dev/null
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
Exercise usage output and offline rendering with the temporary native
|
||||||
|
executable:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
usage_output="$validation_build_dir/usage.txt"
|
||||||
|
if "$validation_build_dir/scriptorium" >"$usage_output" 2>&1
|
||||||
|
then
|
||||||
|
printf '%s\n' 'expected an invocation without a command to fail' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
grep -F 'usage: scriptorium' "$usage_output"
|
||||||
|
"$validation_build_dir/scriptorium" render \
|
||||||
|
--config ./examples/config.yml \
|
||||||
|
--prompt generic.markdown_summary \
|
||||||
|
--input transcript=./examples/fixtures/transcript.md \
|
||||||
|
--input glossary=./examples/fixtures/glossary.yml \
|
||||||
|
--format text >/dev/null
|
||||||
|
```
|
||||||
|
|
||||||
|
Follow every maintained local, Promptkit-tagged, and other external Markdown
|
||||||
|
link. Confirm that all repository-relative link targets exist. Finish the
|
||||||
|
application checks with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git diff --check
|
||||||
|
test -z "$(git status --porcelain)"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Require Release Notes And Reproduce Packaging
|
||||||
|
|
||||||
|
The immutable release note must exist before tagging:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
release_notes="docs/releases/$RELEASE_VERSION.md"
|
||||||
|
test -f "$release_notes"
|
||||||
|
test -s "$release_notes"
|
||||||
|
```
|
||||||
|
|
||||||
|
Validate every local and currently published link in the note. For links
|
||||||
|
pinned to the candidate Scriptorium tag, confirm that the corresponding
|
||||||
|
repository-relative path exists even though its tag URL is not live yet.
|
||||||
|
|
||||||
|
Reproduce the hosted build flags, targets, and filenames in a temporary
|
||||||
|
directory:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
release_dist=$(mktemp -d)
|
||||||
|
release_package='gitea.maximumdirect.net/eric/scriptorium/cmd/scriptorium'
|
||||||
|
|
||||||
|
build_release_binary() {
|
||||||
|
target_os="$1"
|
||||||
|
target_arch="$2"
|
||||||
|
output="$release_dist/scriptorium-$RELEASE_VERSION-$target_os-$target_arch"
|
||||||
|
|
||||||
|
CGO_ENABLED=0 GOOS="$target_os" GOARCH="$target_arch" GOWORK=off \
|
||||||
|
go build -trimpath -ldflags '-s -w' \
|
||||||
|
-o "$output" "$release_package"
|
||||||
|
}
|
||||||
|
|
||||||
|
build_release_binary linux amd64
|
||||||
|
build_release_binary linux arm64
|
||||||
|
|
||||||
|
test -s "$release_dist/scriptorium-$RELEASE_VERSION-linux-amd64"
|
||||||
|
test -s "$release_dist/scriptorium-$RELEASE_VERSION-linux-arm64"
|
||||||
|
file "$release_dist/scriptorium-$RELEASE_VERSION-linux-amd64"
|
||||||
|
file "$release_dist/scriptorium-$RELEASE_VERSION-linux-arm64"
|
||||||
|
```
|
||||||
|
|
||||||
|
Require `file` to identify Linux executables for `x86-64` and `ARM aarch64`,
|
||||||
|
respectively. Inspect the
|
||||||
|
[hosted workflow](../.woodpecker/release.yml) and confirm that it uses the
|
||||||
|
same build flags and names, copies the selected release note to
|
||||||
|
`dist/RELEASE_NOTES.md`, publishes only `dist/scriptorium-*`, and keeps
|
||||||
|
checksum generation enabled.
|
||||||
|
|
||||||
|
## Create And Publish The Tag
|
||||||
|
|
||||||
|
Run the candidate guard again immediately before creating the tag:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
check_release_candidate
|
||||||
|
test -f "$release_notes"
|
||||||
|
test -s "$release_notes"
|
||||||
|
```
|
||||||
|
|
||||||
|
Create an annotated tag explicitly bound to the validated commit, using the
|
||||||
|
version-specific release note as its message:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git tag --annotate "$RELEASE_VERSION" \
|
||||||
|
--file "$release_notes" \
|
||||||
|
"$RELEASE_COMMIT"
|
||||||
|
```
|
||||||
|
|
||||||
|
Inspect the tag and require it to resolve to the validated source:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
test "$(git cat-file -t "refs/tags/$RELEASE_VERSION")" = tag
|
||||||
|
git show --no-patch --decorate "refs/tags/$RELEASE_VERSION"
|
||||||
|
test "$(
|
||||||
|
git rev-parse --verify "refs/tags/$RELEASE_VERSION^{commit}"
|
||||||
|
)" = "$RELEASE_COMMIT"
|
||||||
|
```
|
||||||
|
|
||||||
|
If inspection finds an error, delete only the unpublished local tag, correct
|
||||||
|
the candidate, and repeat the complete validation. Never move or recreate a
|
||||||
|
published tag.
|
||||||
|
|
||||||
|
Push only the selected tag ref:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git push origin \
|
||||||
|
"refs/tags/$RELEASE_VERSION:refs/tags/$RELEASE_VERSION"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Observe And Verify Publication
|
||||||
|
|
||||||
|
Open the hosted workflow run for the selected tag. Require
|
||||||
|
`build-release-assets` to succeed before `publish-release`, then require the
|
||||||
|
publication step and hosted release to succeed. A queued, running, failed, or
|
||||||
|
partially published workflow is not a verified release.
|
||||||
|
|
||||||
|
Compare the local and remote annotated-tag objects and their source commits:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
remote_tag=$(
|
||||||
|
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" |
|
||||||
|
awk 'NR == 1 { print $1 }'
|
||||||
|
)
|
||||||
|
remote_commit=$(
|
||||||
|
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION^{}" |
|
||||||
|
awk 'NR == 1 { print $1 }'
|
||||||
|
)
|
||||||
|
test -n "$remote_tag"
|
||||||
|
test "$remote_tag" = \
|
||||||
|
"$(git rev-parse --verify "refs/tags/$RELEASE_VERSION")"
|
||||||
|
test "$remote_commit" = "$RELEASE_COMMIT"
|
||||||
|
```
|
||||||
|
|
||||||
|
Download the hosted binaries and checksum file into a temporary directory:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
release_base="https://gitea.maximumdirect.net/eric/scriptorium/releases/download/$RELEASE_VERSION"
|
||||||
|
download_dir=$(mktemp -d)
|
||||||
|
(
|
||||||
|
cd "$download_dir"
|
||||||
|
for asset in \
|
||||||
|
"scriptorium-$RELEASE_VERSION-linux-amd64" \
|
||||||
|
"scriptorium-$RELEASE_VERSION-linux-arm64" \
|
||||||
|
SHA256SUMS
|
||||||
|
do
|
||||||
|
curl --fail --location --remote-name "$release_base/$asset"
|
||||||
|
done
|
||||||
|
|
||||||
|
test -s "scriptorium-$RELEASE_VERSION-linux-amd64"
|
||||||
|
test -s "scriptorium-$RELEASE_VERSION-linux-arm64"
|
||||||
|
test -s SHA256SUMS
|
||||||
|
sha256sum --check SHA256SUMS
|
||||||
|
test "$(wc -l < SHA256SUMS | tr -d ' ')" = 2
|
||||||
|
file "scriptorium-$RELEASE_VERSION-linux-amd64"
|
||||||
|
file "scriptorium-$RELEASE_VERSION-linux-arm64"
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Require the same Linux architectures observed in the local packaging check and
|
||||||
|
confirm that the hosted release contains no unexpected asset. On a compatible
|
||||||
|
Linux host, make the matching downloaded binary executable and repeat the
|
||||||
|
usage-output and offline-render smoke checks against it.
|
||||||
|
|
||||||
|
Only after the tag, workflow, release body, binaries, architectures, and
|
||||||
|
checksums all pass verification is the candidate a verified published release.
|
||||||
|
|
||||||
|
## Handle Failures
|
||||||
|
|
||||||
|
Before tag publication, correct the release commit or note and restart the
|
||||||
|
complete procedure. After tag publication, never delete, move, overwrite, or
|
||||||
|
recreate the tag. A transient hosted failure may be retried only against the
|
||||||
|
same immutable tag and commit and only when doing so cannot overwrite or
|
||||||
|
silently retain partial assets. A source, packaging, note, or artifact defect
|
||||||
|
requires a new corrective semantic version from a new validated commit.
|
||||||
|
|
||||||
|
Record the selected version, validated commit, tag object, workflow result,
|
||||||
|
artifact names, checksum result, and smoke-check outcome in the release
|
||||||
|
checkpoint. Keep temporary builds and downloaded assets outside the repository,
|
||||||
|
and require a clean `main` synchronized with `origin/main` when verification
|
||||||
|
is complete.
|
||||||
36
docs/releases/v0.12.0.md
Normal file
36
docs/releases/v0.12.0.md
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
# Scriptorium v0.12.0
|
||||||
|
|
||||||
|
## Breaking Project Boundary
|
||||||
|
|
||||||
|
Scriptorium is now an executable-only CLI and HTTP application. This is a
|
||||||
|
breaking change for Go consumers: the former root Go package is not included,
|
||||||
|
and no compatibility facade is provided.
|
||||||
|
|
||||||
|
Scriptorium `v0.11.1` was the final framework-bearing release. Former Go
|
||||||
|
consumers should follow the
|
||||||
|
[migration guide](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.12.0/docs/consumers/migrating-to-promptkit.md)
|
||||||
|
and adopt
|
||||||
|
[Promptkit `v0.1.0`](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/consumers/pkg-promptkit.md)
|
||||||
|
for in-process prompt preparation and execution.
|
||||||
|
|
||||||
|
## Application Interfaces
|
||||||
|
|
||||||
|
The Scriptorium command-line and HTTP application interfaces remain. Their
|
||||||
|
canonical documentation defines the supported commands, configuration,
|
||||||
|
requests, responses, operational behavior, and deployment responsibilities:
|
||||||
|
|
||||||
|
- [CLI reference](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.12.0/docs/cli.md)
|
||||||
|
- [HTTP API reference](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.12.0/docs/api.md)
|
||||||
|
- [Configuration reference](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.12.0/docs/config.md)
|
||||||
|
- [Operations guide](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.12.0/docs/operations.md)
|
||||||
|
|
||||||
|
## Framework Dependency And Consumers
|
||||||
|
|
||||||
|
The released Scriptorium binaries use Promptkit `v0.1.0` as their framework
|
||||||
|
dependency. Promptkit owns the reusable engine, source formats, profiles,
|
||||||
|
generation boundary, and validation contracts. See the
|
||||||
|
[Promptkit Go consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/consumers/pkg-promptkit.md)
|
||||||
|
for that supported API.
|
||||||
|
|
||||||
|
All known downstream Go consumers were migrated to Promptkit before this
|
||||||
|
release.
|
||||||
59
docs/releases/v0.13.0.md
Normal file
59
docs/releases/v0.13.0.md
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
# Scriptorium v0.13.0
|
||||||
|
|
||||||
|
## Promptkit v0.9 Adoption
|
||||||
|
|
||||||
|
Scriptorium now uses
|
||||||
|
[Promptkit `v0.9.0`](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/)
|
||||||
|
and supports the complete Promptkit v0.9 prompt and profile definition format
|
||||||
|
through its configured directory sources. This includes prompt versions,
|
||||||
|
optional inputs, all supported message roles, cache control, session templates,
|
||||||
|
output schemas and repair budgets, profile inheritance, backend selection,
|
||||||
|
provider controls, optional credential environment names, and JSON-compatible
|
||||||
|
extra parameters.
|
||||||
|
|
||||||
|
Promptkit remains the canonical owner of definition parsing and framework
|
||||||
|
validation. Scriptorium continues to own its CLI, HTTP, configuration,
|
||||||
|
presentation, and deployment contracts.
|
||||||
|
|
||||||
|
## Application Features
|
||||||
|
|
||||||
|
- `run` and `render` can select an explicit prompt version and can execute
|
||||||
|
prompts that do not declare or reference inputs.
|
||||||
|
- CLI and HTTP requests can provide a direct session ID. Reasoning controls can
|
||||||
|
inherit, replace, or explicitly clear the selected profile value.
|
||||||
|
- Application configuration can register custom OpenAI-compatible backends
|
||||||
|
with optional credential environment names, extra parameters, concurrency
|
||||||
|
limits, and queue policy.
|
||||||
|
- `inspect prompt` and `inspect profile` provide deterministic text or JSON
|
||||||
|
views without invoking a model. Profile inspection does not read credential
|
||||||
|
values or require a prompt definition.
|
||||||
|
- Prepared output, CLI summaries, and HTTP metadata report effective backend
|
||||||
|
identity when available.
|
||||||
|
- HTTP requests rejected by backend capacity policy return `503` with the
|
||||||
|
stable `capacity_exceeded` code. One server-scoped Promptkit engine enforces
|
||||||
|
those limits across concurrent requests.
|
||||||
|
|
||||||
|
See the versioned application contracts for exact behavior:
|
||||||
|
|
||||||
|
- [CLI reference](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.13.0/docs/cli.md)
|
||||||
|
- [HTTP API reference](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.13.0/docs/api.md)
|
||||||
|
- [Configuration reference](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.13.0/docs/config.md)
|
||||||
|
- [Operations guide](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.13.0/docs/operations.md)
|
||||||
|
- [Promptkit v0.9 format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md)
|
||||||
|
|
||||||
|
## Compatibility And Operations
|
||||||
|
|
||||||
|
There are no intentional removals from the v0.12 application interfaces. The
|
||||||
|
Promptkit upgrade does enforce its current definition-safety rules, including
|
||||||
|
supported message roles, absolute HTTP or HTTPS endpoints, contained content
|
||||||
|
and schema paths, bounded repair budgets, and JSON-compatible extra parameters.
|
||||||
|
Definitions rejected by these rules must be corrected before use.
|
||||||
|
|
||||||
|
Credential values remain outside configuration, definition files, CLI flags,
|
||||||
|
and HTTP payloads. Named environment sources are optional unless a selected
|
||||||
|
Promptkit profile explicitly requires a key. A positive repair budget permits
|
||||||
|
additional provider calls and can increase latency, token use, and cost.
|
||||||
|
|
||||||
|
The HTTP service retains only the shared operational state needed for backend
|
||||||
|
admission and in-flight request handling. Scriptorium does not retain durable
|
||||||
|
conversation, workflow, checkpoint, or resume state.
|
||||||
@@ -1,90 +1,601 @@
|
|||||||
# Step 5 Implementation Record
|
# Promptkit v0.9.0 Adoption Implementation Plan
|
||||||
|
|
||||||
## Status
|
## Status And Authority
|
||||||
|
|
||||||
Complete as of 2026-07-28.
|
Complete. Implement the stages below in numeric order. The accepted feature
|
||||||
|
scope and target state are defined by the
|
||||||
|
[Promptkit v0.9.0 adoption roadmap](promptkit-v0.9.0-adoption.md), and the
|
||||||
|
durable product boundary is defined by
|
||||||
|
[ADR 0004](../adr/0004-definition-boundary.md). If this plan and either source
|
||||||
|
conflict, the ADR governs product boundaries and the feature roadmap governs
|
||||||
|
scope.
|
||||||
|
|
||||||
This record summarizes the completed implementation of
|
Follow the architecture, documentation, and testing policies under
|
||||||
[Migration Step 5: Promptkit Repository Foundation](step5.md). The
|
`docs/policy/` throughout the work. In particular:
|
||||||
[main migration roadmap](migration.md) owns the continuing cross-repository
|
|
||||||
sequence.
|
|
||||||
|
|
||||||
## Completed Work
|
- use only Promptkit's public root package and its released v0.9.0 tag;
|
||||||
|
- do not add a workspace, module replacement, vendored Promptkit source,
|
||||||
|
direct catalog import, or Scriptorium reimplementation of Promptkit parsing
|
||||||
|
and validation;
|
||||||
|
- keep default tests deterministic, offline, and free of real credentials or
|
||||||
|
provider calls;
|
||||||
|
- update each current-state contract in the same stage that implements its
|
||||||
|
behavior; and
|
||||||
|
- prefer a small number of boundary and integration tests over duplicating
|
||||||
|
Promptkit's own framework test matrix.
|
||||||
|
|
||||||
### Validation And Release Decision
|
At the start of every stage, inspect the working tree and preserve unrelated
|
||||||
|
changes. At the end of every stage, run the focused tests named in that stage,
|
||||||
|
`gofmt` changed Go files, and run `git diff --check`. Do not proceed while a
|
||||||
|
stage's completion conditions are unmet.
|
||||||
|
|
||||||
- Added accepted
|
## Stage 1: Adopt The Promptkit v0.9.0 Compatibility Baseline
|
||||||
[ADR 0003](../adr/0003-use-maintainer-run-validation-and-tag-only-releases-for-promptkit.md)
|
|
||||||
for maintainer-run Promptkit validation and source-and-tag releases.
|
|
||||||
- Preserved accepted
|
|
||||||
[ADR 0002](../adr/0002-split-promptkit-from-scriptorium.md) and its repository
|
|
||||||
split, ownership, independent versioning, and release-order decisions.
|
|
||||||
- Reconciled the migration roadmap so Promptkit validation no longer depends on
|
|
||||||
hosted CI or binary packaging.
|
|
||||||
|
|
||||||
### Promptkit Module And License
|
**Completion: Complete.**
|
||||||
|
|
||||||
- Created module `gitea.maximumdirect.net/eric/promptkit` at Go `1.25.5`.
|
Make the released dependency the build baseline before adding new application
|
||||||
- Added the root `promptkit` package as the only implemented Go package.
|
surface area.
|
||||||
- Kept the module dependency-free and added no placeholder implementation.
|
|
||||||
- Preserved GPLv3 wording and attribution while changing the two template
|
|
||||||
example-notice project names to `Promptkit`.
|
|
||||||
- Confirmed the existing ignore rules cover Go output, editor and operating
|
|
||||||
system artifacts, `.env`, `go.work`, and `go.work.sum`.
|
|
||||||
|
|
||||||
### Policies And Architecture
|
### Implementation
|
||||||
|
|
||||||
- Replaced template documentation and architecture material with durable
|
1. Update `go.mod` to require
|
||||||
library-specific policies.
|
`gitea.maximumdirect.net/eric/promptkit v0.9.0`, then run `go mod tidy` so
|
||||||
- Assigned canonical documentation ownership, including conditional ownership
|
`go.mod` and `go.sum` contain the module graph selected by that tag. Accept
|
||||||
for public contracts, integrations, internal subsystems, ADRs, roadmaps, and
|
Promptkit's OpenRouter and Rakestrawhome catalog modules only as transitive
|
||||||
examples that may exist later.
|
dependencies; do not import them from Scriptorium.
|
||||||
- Defined the root facade, internal-package direction, consumer boundary,
|
2. In the HTTP request DTO, change `model.reasoning_effort` from `string` to
|
||||||
exported-API discipline, error boundary, dependency injection, and repository
|
`*string`, and pass the pointer directly to
|
||||||
independence constraints for future extraction.
|
`promptkit.ExecutionTargetOverride.ReasoningEffort`. This is required by the
|
||||||
- Updated testing policy for deterministic, offline, parallel-safe,
|
v0.9.0 API and preserves omitted versus explicit-empty input. Keep the JSON
|
||||||
behavior-focused, maintainer-run validation.
|
field itself a string when present.
|
||||||
- Recorded the root package as the sole implemented component.
|
3. Compile every Promptkit public struct literal against v0.9.0. Keep literals
|
||||||
|
keyed, do not restore the removed `RunRequest.Metadata` field, and do not
|
||||||
|
add compatibility wrappers around Promptkit.
|
||||||
|
4. Audit maintained prompt, profile, schema, configuration, and HTTP examples
|
||||||
|
against v0.9.0. Correct only actual incompatibilities: endpoints must be
|
||||||
|
absolute HTTP(S) URLs, roles must be `developer`, `system`, `user`, or
|
||||||
|
`assistant`, positive repair budgets must be at most three and paired with
|
||||||
|
`basic`, `json`, or `json_schema`, content and schema paths must remain
|
||||||
|
within their Promptkit source rules, and extra parameters must be
|
||||||
|
JSON-compatible and avoid reserved provider keys.
|
||||||
|
5. Replace v0.1.0 links and current-version statements in current-state
|
||||||
|
documentation with tagged v0.9.0 links. Do not rewrite historical version
|
||||||
|
statements in accepted ADRs. Update the owning documentation to state the
|
||||||
|
v0.9.0 credential and provider behavior: an optional missing environment
|
||||||
|
credential may result in an unauthenticated provider request, while a
|
||||||
|
profile that declares credentials required still fails as an invalid request
|
||||||
|
when no source is supplied and with `ErrAPIKeyEnvMissing` when its selected
|
||||||
|
environment source is empty; unset optional provider controls are omitted;
|
||||||
|
and a positive repair budget can add provider calls, latency, token use, and
|
||||||
|
cost.
|
||||||
|
6. Preserve the architecture guard for forbidden replacements, workspaces,
|
||||||
|
vendor trees, former facade packages, and Promptkit `internal` imports.
|
||||||
|
Extend it only if the dependency graph introduces a new realistic bypass
|
||||||
|
that the existing assertions do not cover.
|
||||||
|
|
||||||
### Orientation, Development, And Releases
|
### Verification
|
||||||
|
|
||||||
- Rewrote the Promptkit README without claiming unimplemented APIs.
|
- Update HTTP mapping tests for the pointer-valued reasoning field, including
|
||||||
- Replaced the contributor guide with task routing, complete local validation,
|
separate omitted and explicit-empty cases, and update `docs/api.md` with the
|
||||||
focused checks, and safe sibling-repository workflows.
|
HTTP three-state behavior in this stage. Stage 4 adds the corresponding CLI
|
||||||
- Added the canonical release procedure for annotated semantic Go module tags,
|
control and session-ID support.
|
||||||
validation evidence, publication verification, and consumer release ordering.
|
- Run `go test ./...`, `go test -race ./...`, `go vet ./...`, and
|
||||||
- Added no hosted CI, binary packaging, executable command, or release tag.
|
`go build ./cmd/scriptorium` against the tagged module with no local
|
||||||
|
replacement.
|
||||||
|
- Run the existing maintained render example and any example-validation tests.
|
||||||
|
- Confirm that a search for `v0.1.0` finds only legitimate historical material,
|
||||||
|
and that searches for `replace`, `go.work`, and Promptkit `/internal/`
|
||||||
|
imports do not reveal a prohibited dependency path.
|
||||||
|
|
||||||
### Cross-Repository Acceptance
|
### Completion Conditions
|
||||||
|
|
||||||
Promptkit passed, without an active workspace or local replacement:
|
The repository builds and all existing behavior tests pass on Promptkit v0.9.0;
|
||||||
|
current documentation identifies v0.9.0 as the framework contract; no local or
|
||||||
|
private Promptkit integration mechanism has been introduced.
|
||||||
|
|
||||||
```sh
|
## Stage 2: Remove Definition-Compatibility Restrictions In Run And Render
|
||||||
|
|
||||||
|
**Completion: Complete.**
|
||||||
|
|
||||||
|
Allow the executable adapters to address every valid directory-backed prompt
|
||||||
|
definition without imposing input requirements of their own.
|
||||||
|
|
||||||
|
### Implementation
|
||||||
|
|
||||||
|
1. Add `promptVersion string` to the shared run/render request configuration.
|
||||||
|
Register `--prompt-version <version>` for both `run` and `render`, include it
|
||||||
|
in usage text, and map it without local normalization to
|
||||||
|
`promptkit.RunRequest.PromptVersion`.
|
||||||
|
2. Make `--input` optional for `run` and `render`. Do not weaken
|
||||||
|
`parseMappings` for callers that actually supplied mappings; instead, skip
|
||||||
|
that parser when no input flag was supplied and pass a nil or empty input
|
||||||
|
map to Promptkit. Promptkit remains responsible for declared-required and
|
||||||
|
template-referenced input failures.
|
||||||
|
3. In `POST /v1/runs`, remove the adapter check that rejects a nil or empty
|
||||||
|
`inputs` object. Preserve strict JSON decoding and all validation of input
|
||||||
|
references that are present. Continue mapping the already-supported
|
||||||
|
`prompt_version` field exactly once into `RunRequest.PromptVersion`.
|
||||||
|
4. Keep extra supplied inputs legal and retain the CLI file-reader and HTTP
|
||||||
|
artifact containment and size policies.
|
||||||
|
5. Update `docs/cli.md` and `docs/api.md` in this stage: prompt ID remains
|
||||||
|
required, prompt version is optional, inputs are optional at the adapter
|
||||||
|
boundary, and Promptkit decides whether the selected definition requires
|
||||||
|
them. Update the shortest request examples only where necessary to avoid
|
||||||
|
implying that all prompts need inputs.
|
||||||
|
|
||||||
|
### Verification
|
||||||
|
|
||||||
|
- Add CLI behavior tests proving that an omitted version selects a sole prompt
|
||||||
|
definition, an explicit version selects the requested definition, and an
|
||||||
|
omitted version fails clearly when a prompt ID has multiple versions.
|
||||||
|
- Add CLI and HTTP adapter coverage for: a prompt with no declared inputs,
|
||||||
|
omitted optional inputs, a missing declared-required input, and a missing
|
||||||
|
template-referenced input. Use a capturing runner for pure request mapping
|
||||||
|
and one real Promptkit engine integration fixture for the definition-driven
|
||||||
|
failures; do not reproduce Promptkit's parser tests.
|
||||||
|
- Retain tests for malformed supplied mappings, invalid HTTP input references,
|
||||||
|
artifact containment, and request-size limits.
|
||||||
|
- Run `go test ./internal/adapter/cli ./internal/adapter/http` and the maintained
|
||||||
|
render workflow.
|
||||||
|
|
||||||
|
### Completion Conditions
|
||||||
|
|
||||||
|
Both executable interfaces can select a specific prompt version and can submit
|
||||||
|
no inputs; only Promptkit rejects missing definition-required data.
|
||||||
|
|
||||||
|
## Stage 3: Add Engine-Scoped Custom Backend Configuration
|
||||||
|
|
||||||
|
**Completion: Complete.**
|
||||||
|
|
||||||
|
Add the application-owned configuration needed for Promptkit profiles to select
|
||||||
|
custom backend IDs and capacity policies.
|
||||||
|
|
||||||
|
### Configuration Contract
|
||||||
|
|
||||||
|
Use this exact strict-YAML shape:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
backends:
|
||||||
|
local-gpu:
|
||||||
|
endpoint: http://localhost:11434/v1
|
||||||
|
api_key_env: LOCAL_GPU_API_KEY
|
||||||
|
extra_params:
|
||||||
|
provider_option: enabled
|
||||||
|
concurrency_limit: 2
|
||||||
|
queue_capacity: 0
|
||||||
|
```
|
||||||
|
|
||||||
|
The mapping key is the backend ID. `endpoint` is required by the effective
|
||||||
|
contract. `api_key_env`, `extra_params`, `concurrency_limit`, and
|
||||||
|
`queue_capacity` are optional. `concurrency_limit` uses zero as Promptkit's
|
||||||
|
unlimited value. Represent `queue_capacity` as `*int` so omission uses
|
||||||
|
Promptkit's default queue capacity and explicit zero disables queuing. There
|
||||||
|
are no backend CLI overrides and no raw API-key field.
|
||||||
|
|
||||||
|
### Implementation
|
||||||
|
|
||||||
|
1. Add `Backends map[string]BackendConfig` to the on-disk `config.Config` and a
|
||||||
|
`BackendConfig` with exactly the five YAML fields above. Add a resolved
|
||||||
|
`BackendSettings` value containing `ID` plus the corresponding fields, and
|
||||||
|
carry `[]BackendSettings` on `AppSettings`.
|
||||||
|
2. During config application, sort the YAML map keys and build the resolved
|
||||||
|
slice in that order. Preserve the `queue_capacity` pointer and the complete
|
||||||
|
`extra_params` value. The deterministic order makes multi-error diagnostics
|
||||||
|
and tests stable; it is not backend precedence.
|
||||||
|
3. Do not duplicate Promptkit's backend validators in `internal/config`.
|
||||||
|
Strict YAML decoding owns unknown-field and type errors. Promptkit
|
||||||
|
`NewEngine` owns blank/reserved/duplicate IDs, endpoint and environment-name
|
||||||
|
rules, reserved or invalid extra parameters, and capacity relationships.
|
||||||
|
4. Refactor CLI engine construction around a small shared engine-settings value
|
||||||
|
containing prompt, profile, and schema directories plus resolved backends.
|
||||||
|
Convert every `BackendSettings` to `promptkit.Backend`, register it with
|
||||||
|
`promptkit.WithBackend`, and use the same constructor for `run`, `render`,
|
||||||
|
and `serve`. Wrap construction failures with application-configuration and
|
||||||
|
engine-initialization context while preserving `errors.Is` identities.
|
||||||
|
5. Retain one immutable backend registry per constructed engine. Do not create
|
||||||
|
registries per HTTP request, and do not add direct imports of Promptkit's
|
||||||
|
catalog modules; Promptkit's built-ins remain available automatically.
|
||||||
|
6. Document `backends` in `docs/config.md`, including key identity,
|
||||||
|
queue-capacity presence, validation ownership, optional environment
|
||||||
|
credentials, and the raw-secret prohibition. Update
|
||||||
|
`docs/internal/adapters.md` and `docs/internal/sources.md` to describe the
|
||||||
|
implemented mapping and engine assembly. Add the exact shape to
|
||||||
|
`examples/config.full.yml`; keep the minimal config minimal.
|
||||||
|
|
||||||
|
### Verification
|
||||||
|
|
||||||
|
- In `internal/config`, test strict decoding of all fields, unknown and raw
|
||||||
|
secret-field rejection, deterministic resolved ordering, JSON-compatible
|
||||||
|
nested `extra_params`, and omitted versus explicit-zero `queue_capacity`.
|
||||||
|
- At the CLI/engine boundary, test a valid custom backend selected by a custom
|
||||||
|
profile through `render`, and representative Promptkit validation failures:
|
||||||
|
reserved backend ID, invalid endpoint, invalid capacity relationship, and a
|
||||||
|
reserved `extra_params` key. Assert the stable error identity or a small
|
||||||
|
diagnostic fragment, not Promptkit's full text.
|
||||||
|
- Test that endpoint-only profiles and built-in profiles still work without a
|
||||||
|
`backends` section.
|
||||||
|
- Run `go test ./internal/config ./internal/adapter/cli` and render with the
|
||||||
|
complete example without contacting a provider.
|
||||||
|
|
||||||
|
### Completion Conditions
|
||||||
|
|
||||||
|
Every command constructs its engine from the same resolved custom-backend
|
||||||
|
configuration, Promptkit remains the validator, and profiles can select custom,
|
||||||
|
built-in, or endpoint-only execution targets.
|
||||||
|
|
||||||
|
## Stage 4: Expose Session And Presence-Aware Reasoning Controls
|
||||||
|
|
||||||
|
**Completion: Complete.**
|
||||||
|
|
||||||
|
Complete the request mapping for direct session identifiers and the v0.9.0
|
||||||
|
reasoning override semantics.
|
||||||
|
|
||||||
|
### Implementation
|
||||||
|
|
||||||
|
1. Add `--session-id <id>` and `--reasoning-effort <value>` to both `run` and
|
||||||
|
`render`, with no aliases. Map the session value to
|
||||||
|
`RunRequest.SessionID`.
|
||||||
|
2. Track flag presence for `--reasoning-effort`. Omission leaves
|
||||||
|
`ExecutionTargetOverride.ReasoningEffort` nil, a nonblank value supplies a
|
||||||
|
pointer to that value, and `--reasoning-effort=` supplies a pointer to the
|
||||||
|
empty string. Include reasoning presence when deciding whether to allocate
|
||||||
|
the enclosing execution override.
|
||||||
|
3. Add optional `session_id` to the strict HTTP run-request DTO and map it to
|
||||||
|
`RunRequest.SessionID` without application-level normalization. Keep
|
||||||
|
`model.reasoning_effort` pointer-valued so omitted, nonblank, and empty JSON
|
||||||
|
strings remain distinct.
|
||||||
|
4. Add optional `session_id` to successful HTTP metadata and populate it from
|
||||||
|
the effective `RunResult.SessionID`, not directly from the request. Use
|
||||||
|
`omitempty`; absence means no effective session ID.
|
||||||
|
5. Continue to reject unknown HTTP fields and do not add `api_key`, appended
|
||||||
|
messages, or request-level output-contract fields.
|
||||||
|
6. Update `docs/cli.md` and `docs/api.md` with the exact flag/field names,
|
||||||
|
inheritance/replacement/clearing semantics, the non-secret nature and
|
||||||
|
Promptkit length rules of session IDs, and effective-session response
|
||||||
|
behavior.
|
||||||
|
|
||||||
|
### Verification
|
||||||
|
|
||||||
|
- CLI tests must distinguish an omitted reasoning flag, a nonblank replacement,
|
||||||
|
and an explicit empty clear by observing the resulting prepared run through
|
||||||
|
`render`. Add a mapping assertion that an explicit empty value still creates
|
||||||
|
an execution override.
|
||||||
|
- HTTP tests must capture and distinguish the same three reasoning states and
|
||||||
|
continue to reject non-string values under the strict DTO contract. Treat
|
||||||
|
JSON `null` like omission, matching ordinary pointer-field decoding; the
|
||||||
|
documented explicit clear remains the empty JSON string.
|
||||||
|
- CLI and HTTP tests must cover a direct session ID, a definition-rendered
|
||||||
|
session ID when no direct value is supplied, no effective session ID, and an
|
||||||
|
invalid overlong direct ID. Assert that successful HTTP metadata reports the
|
||||||
|
engine's effective result.
|
||||||
|
- Run `go test ./internal/adapter/cli ./internal/adapter/http ./internal/format`.
|
||||||
|
|
||||||
|
### Completion Conditions
|
||||||
|
|
||||||
|
CLI and HTTP callers can inherit, replace, or clear reasoning effort and can
|
||||||
|
supply a direct session ID, with Promptkit retaining normalization and
|
||||||
|
validation ownership.
|
||||||
|
|
||||||
|
## Stage 5: Present Backend Identity And Map Capacity Outcomes
|
||||||
|
|
||||||
|
**Completion: Complete.**
|
||||||
|
|
||||||
|
Expose Promptkit's selected routing identity and make overload behavior a
|
||||||
|
stable application contract.
|
||||||
|
|
||||||
|
### Implementation
|
||||||
|
|
||||||
|
1. In prepared-run text output, add `selected_backend_id` immediately after
|
||||||
|
`selected_profile_id` when it is nonempty. Preserve direct JSON marshaling of
|
||||||
|
`promptkit.PreparedRun`; v0.9.0 already supplies `selected_backend_id` and
|
||||||
|
`effective_model_params.backend_id` JSON fields.
|
||||||
|
2. In the CLI run summary, append `backend=<id>` when
|
||||||
|
`RunResult.SelectedBackendID` is nonempty. Omit the token for endpoint-only
|
||||||
|
profiles rather than deriving an ID from the endpoint.
|
||||||
|
3. Add `selected_backend_id,omitempty` to HTTP metadata and
|
||||||
|
`backend_id,omitempty` to `metadata.model_params`, mapping them from
|
||||||
|
`RunResult.SelectedBackendID` and `ExecutionTarget.BackendID` respectively.
|
||||||
|
Keep the two values independently mapped so an upstream inconsistency is not
|
||||||
|
hidden by Scriptorium.
|
||||||
|
4. In HTTP error mapping, classify `promptkit.ErrCapacityExceeded` before the
|
||||||
|
general generation failure and return HTTP 503 with code
|
||||||
|
`capacity_exceeded` and message `model backend capacity is exhausted`. Do
|
||||||
|
not emit `Retry-After` and do not expose `CapacityError` details.
|
||||||
|
5. In CLI `run`, classify the same identity as the existing runtime-error exit
|
||||||
|
status 1 and print the stable safe diagnostic
|
||||||
|
`run error: model backend capacity is exhausted`. Keep all other runtime
|
||||||
|
error handling unchanged.
|
||||||
|
6. Preserve `serveCommand`'s one-engine/one-handler construction outside the
|
||||||
|
request path. Do not add per-request engine construction; that would split
|
||||||
|
capacity state and violate ADR 0004.
|
||||||
|
7. Update `docs/cli.md`, `docs/api.md`, and `docs/operations.md` with backend
|
||||||
|
identity, endpoint-only omission, capacity status, lack of retry timing,
|
||||||
|
shared per-engine admission, and the distinction between transient
|
||||||
|
in-flight state and durable workflow state.
|
||||||
|
|
||||||
|
### Verification
|
||||||
|
|
||||||
|
- Formatter, CLI, and HTTP tests must cover a registered backend and an
|
||||||
|
endpoint-only profile. Assert supplied backend IDs and empty/omitted identity
|
||||||
|
rather than endpoint-derived guesses.
|
||||||
|
- Add HTTP error-table coverage proving capacity is 503
|
||||||
|
`capacity_exceeded`, provider failure remains 502 `llm_failed`, and neither
|
||||||
|
response leaks wrapped provider or capacity diagnostics.
|
||||||
|
- Add one offline concurrent HTTP integration test using a single real
|
||||||
|
Promptkit engine, a custom backend with `concurrency_limit: 1` and explicit
|
||||||
|
`queue_capacity: 0`, and a blocking fake LLM client. Hold the first request
|
||||||
|
in generation, issue a second request, assert the second receives the 503
|
||||||
|
contract, release the first, assert it succeeds, and assert observed peak
|
||||||
|
generation concurrency is one. This test owns the Scriptorium shared-engine
|
||||||
|
interaction; do not replicate Promptkit's FIFO or broader scheduler suite.
|
||||||
|
- Run `go test -race ./internal/adapter/http ./internal/adapter/cli ./internal/format`.
|
||||||
|
|
||||||
|
### Completion Conditions
|
||||||
|
|
||||||
|
All presentations report actual backend identity when present, endpoint-only
|
||||||
|
profiles remain identity-free, and one server instance enforces one engine's
|
||||||
|
backend capacity across concurrent requests.
|
||||||
|
|
||||||
|
## Stage 6: Add Stable Prompt Inspection Output And CLI
|
||||||
|
|
||||||
|
**Completion: Complete.**
|
||||||
|
|
||||||
|
Introduce reusable application-owned inspection presentation, then expose
|
||||||
|
prompt inspection without model execution.
|
||||||
|
|
||||||
|
### Presentation Contract
|
||||||
|
|
||||||
|
1. Generalize the internal prepared-run format selector to an internal
|
||||||
|
`OutputFormat` with `text` and `json` values, a text default, and one parser.
|
||||||
|
Update config and prepared-run call sites to use it without changing
|
||||||
|
existing prepared-run bytes or `defaults.render_format` behavior.
|
||||||
|
2. Define a Scriptorium-owned prompt-inspection DTO; never marshal
|
||||||
|
`promptkit.PromptInspection` directly. JSON contains these fields in this
|
||||||
|
shape, including empty strings and an empty array where applicable:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"prompt_id": "example",
|
||||||
|
"prompt_version": "1",
|
||||||
|
"prompt_hash": "opaque",
|
||||||
|
"default_profile_id": "",
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"name": "source",
|
||||||
|
"required": true,
|
||||||
|
"content_type": "text/plain",
|
||||||
|
"description": "Input text"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_contract": {
|
||||||
|
"format": "text",
|
||||||
|
"validation_mode": "none",
|
||||||
|
"schema_path": "",
|
||||||
|
"repair_attempts": 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
3. Preserve Promptkit's declared input order. Emit JSON with deterministic
|
||||||
|
indentation and one trailing newline. Text output uses the same field order,
|
||||||
|
renders `inputs: []` when empty, lists input records in declaration order,
|
||||||
|
quotes free-form descriptions safely, includes empty declared values, and
|
||||||
|
ends in one newline. The prompt hash is opaque and must not be interpreted.
|
||||||
|
|
||||||
|
### CLI Contract And Implementation
|
||||||
|
|
||||||
|
1. Add `inspect` to top-level dispatch and usage, with implementation in a new
|
||||||
|
focused CLI file rather than further expanding `run.go`.
|
||||||
|
2. Implement:
|
||||||
|
|
||||||
|
```text
|
||||||
|
scriptorium inspect prompt --prompt ID [--prompt-version VERSION]
|
||||||
|
[--config PATH] [--prompt-dir DIR] [--format text|json] [--out PATH]
|
||||||
|
```
|
||||||
|
|
||||||
|
`--prompt` is required. All other flags are optional. Positional arguments
|
||||||
|
and unknown inspection modes are errors. There are no deprecated aliases.
|
||||||
|
3. Apply ordinary config discovery and CLI-over-file precedence. Require an
|
||||||
|
effective prompt directory, pass the optional version directly to
|
||||||
|
`Engine.InspectPrompt`, and construct the engine through the shared settings
|
||||||
|
path so configured backends are validated consistently. Do not require a
|
||||||
|
profile or schema directory for this operation.
|
||||||
|
4. Default inspection format to text independently of
|
||||||
|
`defaults.render_format`; `--format` is the only inspection format override.
|
||||||
|
Reuse ordinary output-file handling. Write output only after successful
|
||||||
|
inspection and formatting. Use exit 0 for success and exit 1 for parsing,
|
||||||
|
configuration, load, formatting, or write failure.
|
||||||
|
5. Update `docs/cli.md`, `docs/internal/adapters.md`, and the internal format
|
||||||
|
component description. Make clear that inspection loads and normalizes a
|
||||||
|
definition but does not resolve a profile, load a schema, render templates,
|
||||||
|
reserve backend capacity, or call a model.
|
||||||
|
|
||||||
|
### Verification
|
||||||
|
|
||||||
|
- Formatter tests cover deterministic text and JSON, declared order, empty
|
||||||
|
inputs/default profile/schema path, all nonempty fields, and nil inspection
|
||||||
|
handling. Prefer structural JSON assertions plus a few stable text fragments
|
||||||
|
over large snapshots.
|
||||||
|
- CLI tests cover required flags, rejected positional arguments, config source
|
||||||
|
selection, `--prompt-dir` precedence, sole and explicit multi-version
|
||||||
|
selection, default and explicit formats, stdout versus `--out`, no partial
|
||||||
|
output on failure, and Promptkit prompt-not-found/load identities.
|
||||||
|
- Use a fake or deliberately failing model client to prove inspection performs
|
||||||
|
no generation; do not assert Promptkit's internal call sequence.
|
||||||
|
- Run `go test ./internal/format ./internal/adapter/cli`.
|
||||||
|
|
||||||
|
### Completion Conditions
|
||||||
|
|
||||||
|
`inspect prompt` produces stable Scriptorium text or JSON for every valid
|
||||||
|
directory-backed v0.9.0 prompt definition without inputs or model access, and
|
||||||
|
prepared-run presentation remains backward compatible.
|
||||||
|
|
||||||
|
## Stage 7: Add Stable Profile Inspection Output And CLI
|
||||||
|
|
||||||
|
**Completion: Complete.**
|
||||||
|
|
||||||
|
Complete the inspection command family with inherited, built-in, custom, and
|
||||||
|
endpoint-only profile support.
|
||||||
|
|
||||||
|
### Presentation Contract
|
||||||
|
|
||||||
|
1. Define a Scriptorium-owned profile-inspection DTO; never marshal
|
||||||
|
`promptkit.ProfileInspection` directly. JSON has the following complete
|
||||||
|
shape. Keep empty strings and zero numeric values because Promptkit reports
|
||||||
|
unresolved optional controls as zero, and normalize absent `extra_params`
|
||||||
|
to an empty object:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"profile_id": "example",
|
||||||
|
"effective_model_params": {
|
||||||
|
"backend_id": "",
|
||||||
|
"endpoint": "https://example.test/v1",
|
||||||
|
"model": "model-name",
|
||||||
|
"temperature": 0,
|
||||||
|
"max_tokens": 0,
|
||||||
|
"top_p": 0,
|
||||||
|
"timeout_seconds": 0,
|
||||||
|
"service_tier": "",
|
||||||
|
"reasoning_effort": "",
|
||||||
|
"api_key_env": "",
|
||||||
|
"extra_params": {}
|
||||||
|
},
|
||||||
|
"api_key_required": false
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Text output follows the same order, nests effective model parameters,
|
||||||
|
encodes extra-parameter values as deterministic compact JSON with sorted
|
||||||
|
keys, includes an empty `backend_id` for endpoint-only profiles, and ends in
|
||||||
|
one newline. Never read or print an environment-variable value or direct
|
||||||
|
API key.
|
||||||
|
|
||||||
|
### CLI Contract And Implementation
|
||||||
|
|
||||||
|
1. Implement the second mode:
|
||||||
|
|
||||||
|
```text
|
||||||
|
scriptorium inspect profile --profile ID
|
||||||
|
[--config PATH] [--profile-dir DIR] [--format text|json] [--out PATH]
|
||||||
|
```
|
||||||
|
|
||||||
|
`--profile` is required. The common inspection parse, format, output, and
|
||||||
|
exit rules from Stage 6 apply. No prompt, prompt directory, schema directory,
|
||||||
|
or input is required, so built-in profiles are inspectable with only their
|
||||||
|
ID.
|
||||||
|
2. Resolve config and `--profile-dir` with normal precedence, construct the
|
||||||
|
engine through the shared settings path so custom backends participate, and
|
||||||
|
call `Engine.InspectProfile` exactly once.
|
||||||
|
3. Update `docs/cli.md`, `docs/config.md`, and internal adapter/source docs.
|
||||||
|
Explain that profile inspection resolves inheritance and backend defaults
|
||||||
|
but does not read credentials, load a prompt, reserve capacity, or call a
|
||||||
|
model. Document that zero provider-control values may mean unspecified,
|
||||||
|
consistent with Promptkit v0.9.0.
|
||||||
|
|
||||||
|
### Verification
|
||||||
|
|
||||||
|
- Formatter tests cover deterministic text and JSON, registered and empty
|
||||||
|
backend identity, optional environment-variable names, `api_key_required`,
|
||||||
|
nested extra parameters, and the absence of credential values.
|
||||||
|
- CLI tests cover built-in, inherited custom, custom-backend, and endpoint-only
|
||||||
|
profiles; config and `--profile-dir` precedence; default and explicit format;
|
||||||
|
stdout and `--out`; missing and invalid profiles; no partial output; and no
|
||||||
|
requirement for `prompt_dir`.
|
||||||
|
- Include a sentinel environment secret in a test and assert it appears in no
|
||||||
|
inspection output or diagnostic while its variable name may appear.
|
||||||
|
- Run `go test ./internal/format ./internal/adapter/cli ./internal/config`.
|
||||||
|
|
||||||
|
### Completion Conditions
|
||||||
|
|
||||||
|
`inspect profile` reports the complete safe effective profile view for every
|
||||||
|
valid directory-backed v0.9.0 profile and configured backend combination
|
||||||
|
without requiring a prompt or contacting a model.
|
||||||
|
|
||||||
|
## Stage 8: Complete Cross-Feature Conformance And Release Validation
|
||||||
|
|
||||||
|
**Completion: Complete.**
|
||||||
|
|
||||||
|
Close the roadmap with representative definition-level integration coverage,
|
||||||
|
copyable examples, documentation alignment, and full validation. Do not defer
|
||||||
|
contract documentation from earlier stages to this stage; this is a final
|
||||||
|
cross-check.
|
||||||
|
|
||||||
|
### Implementation And Documentation
|
||||||
|
|
||||||
|
1. Maintain one compact shared fixture set under
|
||||||
|
`testdata/promptkit-v0.9/` with `prompts`, `profiles`, `schemas`, and any
|
||||||
|
message content files needed by CLI and HTTP integration tests. Through a
|
||||||
|
small set of composable definitions, cover:
|
||||||
|
|
||||||
|
- inline and file-backed messages;
|
||||||
|
- `developer`, `system`, `user`, and `assistant` roles and message cache
|
||||||
|
control;
|
||||||
|
- required and optional inputs plus template-referenced values;
|
||||||
|
- prompt versions, default profiles, and session-ID templates;
|
||||||
|
- text and JSON output plus `none`, `basic`, `json`, and `json_schema`
|
||||||
|
validation, a schema path, and a valid positive repair budget;
|
||||||
|
- standalone and inherited profiles;
|
||||||
|
- built-in, configured custom, and endpoint-only backends;
|
||||||
|
- model controls, service tier, reasoning effort, timeout, optional
|
||||||
|
credential environment names, and nested JSON-compatible extra
|
||||||
|
parameters.
|
||||||
|
|
||||||
|
Exercise these fixtures through Scriptorium's `render` and inspection
|
||||||
|
boundaries. Assert representative effective outcomes; do not independently
|
||||||
|
decode YAML or exhaustively retest every Promptkit field combination.
|
||||||
|
2. Add a copyable `examples/render-v0.9-features.sh` workflow that uses only
|
||||||
|
`render` and `inspect` operations to demonstrate prompt-version selection,
|
||||||
|
a direct session ID, reasoning replacement or clearing, a configured custom
|
||||||
|
backend, and both inspection modes without provider access. Keep
|
||||||
|
`examples/config.full.yml` complete and keep raw credentials out of every
|
||||||
|
example.
|
||||||
|
3. Review every canonical current-state owner named by the feature roadmap.
|
||||||
|
Remove stale claims that inputs are always required, that every command
|
||||||
|
needs `prompt_dir`, or that v0.1.0 semantics still apply. Ensure CLI, API,
|
||||||
|
configuration, operations, internal adapter/source, development, and
|
||||||
|
overview documents link instead of duplicating one another's contracts.
|
||||||
|
Change architecture policy or internal component boundaries only if the
|
||||||
|
implementation actually changed those durable responsibilities.
|
||||||
|
4. Verify that all explicit non-goals remain absent: appended messages,
|
||||||
|
retained prepared handles, HTTP inspection routes, alternative Promptkit
|
||||||
|
source kinds, request output-contract replacement, public provider details,
|
||||||
|
direct catalog imports, raw API keys, and durable workflow state.
|
||||||
|
5. After every completion criterion below passes, change the feature roadmap's
|
||||||
|
Status from `Planned` to `Complete`. Do not mark it complete for a partial
|
||||||
|
implementation.
|
||||||
|
|
||||||
|
### Final Validation
|
||||||
|
|
||||||
|
Run all of the following from the repository root:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
gofmt -w $(git ls-files '*.go')
|
||||||
go test ./...
|
go test ./...
|
||||||
|
go test -race ./...
|
||||||
go vet ./...
|
go vet ./...
|
||||||
go build ./...
|
go build ./cmd/scriptorium
|
||||||
gofmt -l $(git ls-files '*.go')
|
go run ./cmd/scriptorium render --config ./examples/config.yml --prompt generic.markdown_summary --input transcript=./examples/fixtures/transcript.md --input glossary=./examples/fixtures/glossary.yml
|
||||||
|
bash ./examples/render-v0.9-features.sh
|
||||||
git diff --check
|
git diff --check
|
||||||
```
|
```
|
||||||
|
|
||||||
Acceptance also verified exact module and package metadata, all maintained
|
Also verify the architecture guard, strict example decoding, documentation
|
||||||
Markdown links, clean repository hygiene, the Promptkit origin and tracked
|
links, executable permissions on shell examples, and absence of secrets. Review
|
||||||
default branch, the absence of dependencies and prohibited artifacts, and the
|
the suite for redundant upstream-semantic tests and retain only tests that
|
||||||
retained GPLv3 license.
|
protect Scriptorium mappings, presentation, configuration, concurrency
|
||||||
|
integration, or complete workflows.
|
||||||
|
|
||||||
Scriptorium acceptance confirmed that ADR 0003 and the migration roadmap
|
### Completion Conditions
|
||||||
distinguish Scriptorium's executable CI policy from Promptkit's maintainer-run
|
|
||||||
checks, preserve the first-tag release gate, and introduce no production code,
|
|
||||||
module dependency, workspace, or replacement change.
|
|
||||||
|
|
||||||
## Repository Boundaries
|
Every roadmap validation criterion is satisfied on the tagged dependency; all
|
||||||
|
current-state documentation and examples describe implemented behavior; the
|
||||||
|
full application supports every Promptkit v0.9.0 feature expressible in its
|
||||||
|
directory-backed prompt and profile definitions; and the ADR 0004 non-goals
|
||||||
|
remain intact.
|
||||||
|
|
||||||
Promptkit foundation changes remain in Promptkit's Git history. Scriptorium
|
## Open Questions
|
||||||
contains only the durable cross-project ADR and migration-roadmap records. No
|
|
||||||
tag was created and neither repository was pushed as part of this work.
|
|
||||||
|
|
||||||
## Next Work
|
None. The external syntax, application mappings, validation ownership,
|
||||||
|
inspection wire shapes, concurrency behavior, testing boundaries, and stage
|
||||||
Migration Step 6 may now extract and stabilize the application-neutral
|
order required for implementation are fixed above.
|
||||||
framework. It must not update Scriptorium to consume Promptkit until Promptkit
|
|
||||||
passes its documented validation and publishes its first versioned tag.
|
|
||||||
|
|||||||
@@ -1,321 +0,0 @@
|
|||||||
# Promptkit Migration Roadmap
|
|
||||||
|
|
||||||
## Status
|
|
||||||
|
|
||||||
Accepted plan. Steps 1 through 5 are complete. Steps 6 through 9 remain
|
|
||||||
proposed and are not yet implemented.
|
|
||||||
|
|
||||||
## Objective
|
|
||||||
|
|
||||||
Split the current repository into two projects:
|
|
||||||
|
|
||||||
- **Promptkit**: the reusable Go framework, public Go facade, execution engine,
|
|
||||||
source and validation support, OpenAI-compatible client, extension
|
|
||||||
interfaces, and built-in execution-profile registry.
|
|
||||||
- **Scriptorium**: a slim runnable application that imports Promptkit and
|
|
||||||
provides the CLI and HTTP interfaces.
|
|
||||||
|
|
||||||
Scriptorium will become another downstream Promptkit consumer rather than the
|
|
||||||
owner of the framework.
|
|
||||||
|
|
||||||
## Compatibility And Migration Policy
|
|
||||||
|
|
||||||
This is an intentionally breaking change.
|
|
||||||
|
|
||||||
- New and migrated Go consumers must import Promptkit instead of Scriptorium.
|
|
||||||
- Scriptorium will not retain type aliases, forwarding packages, deprecated
|
|
||||||
facade APIs, or other source-compatibility shims.
|
|
||||||
- Existing consumers may continue using a previously tagged Scriptorium module
|
|
||||||
version until they are migrated.
|
|
||||||
- The migration does not need to preserve compatibility between intermediate
|
|
||||||
development states. Each completed phase must instead leave the affected
|
|
||||||
repository internally consistent and tested.
|
|
||||||
- Promptkit should initially preserve the useful shape and behavior of the
|
|
||||||
current public Go facade where doing so reduces extraction risk. Broader API
|
|
||||||
redesign should follow the split unless required to establish the new
|
|
||||||
boundary.
|
|
||||||
|
|
||||||
## Target Ownership
|
|
||||||
|
|
||||||
Promptkit should own application-neutral framework behavior:
|
|
||||||
|
|
||||||
- public engine, request, result, option, extension, and error APIs;
|
|
||||||
- prompt-definition loading and rendering;
|
|
||||||
- execution profiles, overlays, and the built-in profile registry;
|
|
||||||
- artifact-loading interfaces and general-purpose `file` and `inline` support;
|
|
||||||
- schema loading and output validation;
|
|
||||||
- LLM client boundaries and the OpenAI-compatible implementation;
|
|
||||||
- preparation and execution orchestration;
|
|
||||||
- framework and execution defaults.
|
|
||||||
|
|
||||||
Scriptorium should own executable and transport concerns:
|
|
||||||
|
|
||||||
- the `scriptorium` command and its `run`, `render`, and `serve` interfaces;
|
|
||||||
- CLI parsing, output formatting, exit codes, and process behavior;
|
|
||||||
- application-config discovery and CLI precedence;
|
|
||||||
- HTTP routing, request and response DTOs, limits, and error/status mapping;
|
|
||||||
- HTTP artifact-root and deployment security policy;
|
|
||||||
- server and adapter defaults;
|
|
||||||
- executable examples, operations guidance, and transport documentation.
|
|
||||||
|
|
||||||
The intended dependency direction is:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Scriptorium CLI and HTTP adapters
|
|
||||||
|
|
|
||||||
v
|
|
||||||
Promptkit
|
|
||||||
|
|
|
||||||
v
|
|
||||||
consumer-supplied sources and clients
|
|
||||||
```
|
|
||||||
|
|
||||||
Scriptorium must use Promptkit's public API. It must not depend on Promptkit
|
|
||||||
implementation packages or reproduce framework orchestration.
|
|
||||||
|
|
||||||
## Migration Steps
|
|
||||||
|
|
||||||
### Step 1: Refresh And Synchronize Documentation
|
|
||||||
|
|
||||||
Perform a repository-wide documentation refresh before migration development.
|
|
||||||
|
|
||||||
At minimum:
|
|
||||||
|
|
||||||
- reconcile all current-behavior documentation with the code, tests, examples,
|
|
||||||
defaults, and current public contracts;
|
|
||||||
- introduce the planned documentation-policy updates;
|
|
||||||
- establish an architecture decision record policy and canonical ADR location;
|
|
||||||
- resolve stale, duplicated, or misplaced material;
|
|
||||||
- validate documentation links and maintained examples;
|
|
||||||
- leave future migration behavior in `docs/roadmap/` until implemented.
|
|
||||||
|
|
||||||
**Gate:** Do not begin architectural migration work until the documentation
|
|
||||||
refresh and policy updates are merged and the repository has an agreed,
|
|
||||||
accurate baseline.
|
|
||||||
|
|
||||||
**Gate status:** Complete as of 2026-07-26. The completed documentation
|
|
||||||
refresh, follow-up verification, and layered-timeout correction remain recorded
|
|
||||||
in repository history. Step 1 remains complete after that validation.
|
|
||||||
|
|
||||||
### Step 2: Record The Architectural Decision And Detailed Boundary
|
|
||||||
|
|
||||||
Create an ADR, under the policy established in Step 1, that records:
|
|
||||||
|
|
||||||
- the decision to split Promptkit from Scriptorium;
|
|
||||||
- the target ownership and dependency direction;
|
|
||||||
- the selected Promptkit repository and Go module paths;
|
|
||||||
- the breaking-change and versioning policy;
|
|
||||||
- ownership of configuration fields and defaults;
|
|
||||||
- artifact-reader and HTTP containment responsibilities;
|
|
||||||
- local multi-repository development and release coordination;
|
|
||||||
- documentation ownership after the split.
|
|
||||||
|
|
||||||
Use the ADR to resolve any remaining public-boundary decisions before code is
|
|
||||||
moved.
|
|
||||||
|
|
||||||
**Gate:** The ADR is accepted, and every existing package, public contract,
|
|
||||||
configuration category, and maintained asset has a target owner.
|
|
||||||
|
|
||||||
**Gate status:** Complete as of 2026-07-26.
|
|
||||||
[ADR 0002: Split Promptkit From Scriptorium](../adr/0002-split-promptkit-from-scriptorium.md)
|
|
||||||
is accepted and records the required ownership and coordination decisions.
|
|
||||||
|
|
||||||
### Step 3: Characterize Existing Framework Behavior
|
|
||||||
|
|
||||||
Strengthen or add contract-focused tests where needed so extraction can be
|
|
||||||
verified without relying on package placement.
|
|
||||||
|
|
||||||
The completed Step 3 gate records the accepted implementation scope and
|
|
||||||
intended completion state.
|
|
||||||
|
|
||||||
Preserve coverage of:
|
|
||||||
|
|
||||||
- `Prepare` and `Run` behavior;
|
|
||||||
- prompt, profile, execution-default, and request-override precedence;
|
|
||||||
- presence-aware numeric overrides;
|
|
||||||
- built-in profile fallback and custom-profile overlays;
|
|
||||||
- strict YAML and JSON decoding;
|
|
||||||
- prompt, profile, schema, and artifact source behavior;
|
|
||||||
- structured-output requests and output validation;
|
|
||||||
- validation failures versus validation runtime errors;
|
|
||||||
- secret handling and redaction;
|
|
||||||
- public error classification;
|
|
||||||
- HTTP artifact restrictions and transport mappings.
|
|
||||||
|
|
||||||
**Gate:** Current framework and adapter contracts are represented by passing
|
|
||||||
tests sufficient to detect behavioral regressions during the split.
|
|
||||||
|
|
||||||
**Gate status:** Complete as of 2026-07-27. The framework contract corpus,
|
|
||||||
public `Engine` characterization, ownership audit, full test and vet suites,
|
|
||||||
temporary executable build, and maintained offline examples passed. Step 4 is
|
|
||||||
next.
|
|
||||||
|
|
||||||
### Step 4: Make Scriptorium Adapters Consume The Public Facade
|
|
||||||
|
|
||||||
Within the current repository, refactor the CLI and HTTP adapters to use the
|
|
||||||
public framework facade rather than constructing or importing internal runner
|
|
||||||
components directly.
|
|
||||||
|
|
||||||
Add only the minimum public capabilities needed to support this boundary. These
|
|
||||||
may include:
|
|
||||||
|
|
||||||
- a small `Run`/`Prepare` consumer interface;
|
|
||||||
- injectable artifact-reading behavior for Scriptorium's restricted HTTP
|
|
||||||
policy;
|
|
||||||
- source options currently available only through internal constructors;
|
|
||||||
- prepared-run formatting based on public types;
|
|
||||||
- stable public error classification required by CLI and HTTP mappings.
|
|
||||||
|
|
||||||
Do not broadly export internal repositories, domain types, or use-case
|
|
||||||
implementations.
|
|
||||||
|
|
||||||
**Gate:** The CLI and HTTP adapters use only the public framework API for
|
|
||||||
framework behavior, and all tests and documented smoke commands pass.
|
|
||||||
|
|
||||||
**Gate status:** Complete as of 2026-07-28. CLI `run`, `render`, and `serve`,
|
|
||||||
the HTTP handler, and prepared-run formatting use the public facade; the
|
|
||||||
restricted HTTP reader is injected through the public extension point. The
|
|
||||||
post-implementation public-error, deterministic MIME, and recursive
|
|
||||||
dependency-guard corrections passed full tests, vet, build, race checks,
|
|
||||||
maintained examples, and configuration smoke checks. Step 5 repository
|
|
||||||
creation is next.
|
|
||||||
|
|
||||||
### Step 5: Create The Promptkit Repository
|
|
||||||
|
|
||||||
Promptkit was established as an independent repository and Go module through
|
|
||||||
the out-of-band workflow recorded in the
|
|
||||||
[Step 5 completion record](step5.md). Its foundation includes:
|
|
||||||
|
|
||||||
- confirmed repository access, governance, origin, and default-branch tracking;
|
|
||||||
- module `gitea.maximumdirect.net/eric/promptkit` at Go `1.25.5`;
|
|
||||||
- a minimal root `promptkit` public package boundary with no placeholder
|
|
||||||
framework packages;
|
|
||||||
- library-specific development, architecture, documentation, testing, and
|
|
||||||
release policies;
|
|
||||||
- documented maintainer-run test, vet, build, formatting, documentation-link,
|
|
||||||
and repository-hygiene validation;
|
|
||||||
- source-commit and semantic Go module tag releases without runnable binaries
|
|
||||||
or binary packaging; and
|
|
||||||
- temporary workspace and uncommitted replacement workflows for coordinated
|
|
||||||
development without committed repository coupling.
|
|
||||||
|
|
||||||
[ADR 0003](../adr/0003-use-maintainer-run-validation-and-tag-only-releases-for-promptkit.md)
|
|
||||||
records the controlling Promptkit validation and release decision.
|
|
||||||
|
|
||||||
**Gate status:** Complete as of 2026-07-28. Promptkit passed its documented
|
|
||||||
validation independently, all maintained links and repository-hygiene checks
|
|
||||||
passed, and no workspace, replacement, CI configuration, binary, tag, command,
|
|
||||||
or placeholder package was added. Framework extraction and stabilization in
|
|
||||||
Step 6 is the next gate.
|
|
||||||
|
|
||||||
### Step 6: Extract And Stabilize Promptkit
|
|
||||||
|
|
||||||
Move the application-neutral framework and built-in profile assets into
|
|
||||||
Promptkit. Preserve implementation packages as internal where practical.
|
|
||||||
|
|
||||||
The initial public API should remain focused on the established engine workflow
|
|
||||||
and the source and client extension points required by real consumers. Avoid
|
|
||||||
combining the extraction with unrelated API redesign.
|
|
||||||
|
|
||||||
Move or recreate the relevant:
|
|
||||||
|
|
||||||
- framework implementation;
|
|
||||||
- public package tests and framework contract tests;
|
|
||||||
- built-in profile assets and registry tests;
|
|
||||||
- Go consumer examples;
|
|
||||||
- framework, consumer, configuration-format, and integration documentation.
|
|
||||||
|
|
||||||
Verify that Promptkit can be built, tested, and consumed independently of the
|
|
||||||
Scriptorium repository.
|
|
||||||
|
|
||||||
**Gate:** Promptkit independently provides the agreed framework contract,
|
|
||||||
passes its documented validation, and has published its first versioned tag
|
|
||||||
before Scriptorium or another consumer adopts it, as required by
|
|
||||||
[ADR 0003](../adr/0003-use-maintainer-run-validation-and-tag-only-releases-for-promptkit.md).
|
|
||||||
|
|
||||||
### Step 7: Slim Scriptorium And Adopt Promptkit
|
|
||||||
|
|
||||||
Update Scriptorium to import the tagged Promptkit module and remove the
|
|
||||||
framework implementation and public Go facade that Promptkit replaces.
|
|
||||||
|
|
||||||
Retain only Scriptorium-owned executable and transport behavior. In particular:
|
|
||||||
|
|
||||||
- wire CLI and HTTP requests through Promptkit's public API;
|
|
||||||
- keep application config and transport defaults in Scriptorium;
|
|
||||||
- keep restricted HTTP artifact policy in Scriptorium while injecting it
|
|
||||||
through Promptkit's supported boundary;
|
|
||||||
- remove obsolete framework packages, tests, and documentation;
|
|
||||||
- update Scriptorium examples and docs to describe the CLI and HTTP application;
|
|
||||||
- direct Go framework consumers to Promptkit without providing compatibility
|
|
||||||
aliases or forwarding APIs.
|
|
||||||
|
|
||||||
**Gate:** Scriptorium builds and passes all tests using a tagged Promptkit
|
|
||||||
dependency, contains no duplicate framework implementation, and its current
|
|
||||||
documentation describes only the slimmed application.
|
|
||||||
|
|
||||||
### Step 8: Migrate Downstream Consumers To Promptkit
|
|
||||||
|
|
||||||
Inventory downstream Go consumers and migrate each from the Scriptorium package
|
|
||||||
to Promptkit. This work may occur in external repositories and must be tracked
|
|
||||||
explicitly.
|
|
||||||
|
|
||||||
For each consumer:
|
|
||||||
|
|
||||||
- update module imports and dependencies;
|
|
||||||
- adapt to any intentionally changed public API;
|
|
||||||
- run its tests and relevant integration or smoke checks;
|
|
||||||
- confirm configuration, source, validation, and error behavior;
|
|
||||||
- release or deploy the migrated consumer through its normal process.
|
|
||||||
|
|
||||||
Consumers that cannot migrate immediately may remain pinned to the last
|
|
||||||
framework-bearing Scriptorium tag. No compatibility work is required in the new
|
|
||||||
Scriptorium project for those consumers.
|
|
||||||
|
|
||||||
**Gate:** All in-scope downstream consumers are either migrated and verified or
|
|
||||||
explicitly recorded as remaining on the previous Scriptorium version with an
|
|
||||||
owner and follow-up plan. Do not declare the ecosystem migration complete until
|
|
||||||
the required out-of-band consumer changes are confirmed.
|
|
||||||
|
|
||||||
### Step 9: Complete Release And Documentation Cutover
|
|
||||||
|
|
||||||
Complete the coordinated project transition:
|
|
||||||
|
|
||||||
- publish Promptkit before dependent Scriptorium releases;
|
|
||||||
- release the breaking Scriptorium version against the tagged Promptkit
|
|
||||||
dependency;
|
|
||||||
- publish migration guidance that maps the former Scriptorium Go API to
|
|
||||||
Promptkit;
|
|
||||||
- update cross-project links, examples, package documentation, and release
|
|
||||||
notes;
|
|
||||||
- verify that no release artifact depends on local workspaces or replacements;
|
|
||||||
- archive completed roadmap material according to the documentation policy in
|
|
||||||
effect at that time.
|
|
||||||
|
|
||||||
**Gate:** Promptkit and Scriptorium are independently releasable, their
|
|
||||||
documentation has distinct and accurate ownership, and the migration status of
|
|
||||||
all identified downstream consumers is recorded.
|
|
||||||
|
|
||||||
## Cross-Cutting Constraints
|
|
||||||
|
|
||||||
- Preserve the invariant that execution orchestration remains narrow and
|
|
||||||
application-neutral.
|
|
||||||
- Keep adapter-specific decisions out of Promptkit.
|
|
||||||
- Keep Scriptorium dependent only on Promptkit's supported public API.
|
|
||||||
- Preserve strict external decoding, error classification, validation
|
|
||||||
semantics, and secret redaction throughout the migration.
|
|
||||||
- Keep each repository buildable and testable at merged phase boundaries.
|
|
||||||
- Coordinate cross-repository changes through tagged dependencies and explicit
|
|
||||||
gates rather than assuming atomic commits across repositories.
|
|
||||||
- Document only implemented behavior outside roadmap files.
|
|
||||||
|
|
||||||
## Completion Criteria
|
|
||||||
|
|
||||||
The migration is complete when:
|
|
||||||
|
|
||||||
- Promptkit is the independent owner of the reusable framework and built-in
|
|
||||||
profiles;
|
|
||||||
- Scriptorium is a slim CLI and HTTP consumer of Promptkit;
|
|
||||||
- Scriptorium no longer exposes or maintains the former public Go framework;
|
|
||||||
- all required downstream migrations and external repository work have been
|
|
||||||
completed or explicitly dispositioned;
|
|
||||||
- both repositories build, test, document, version, and release independently.
|
|
||||||
351
docs/roadmap/promptkit-v0.9.0-adoption.md
Normal file
351
docs/roadmap/promptkit-v0.9.0-adoption.md
Normal file
@@ -0,0 +1,351 @@
|
|||||||
|
# Promptkit v0.9.0 Adoption Roadmap
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Complete. This document records the delivered feature scope and targeted end
|
||||||
|
state. The completed delivery plan is maintained in
|
||||||
|
[implementation.md](implementation.md); current behavior remains defined by
|
||||||
|
the canonical application contracts linked below.
|
||||||
|
|
||||||
|
The durable product and integration boundary for this work is established by
|
||||||
|
[ADR 0004](../adr/0004-definition-boundary.md). This roadmap applies that
|
||||||
|
decision to Promptkit v0.9.0 and owns implementation scope and completion
|
||||||
|
status.
|
||||||
|
|
||||||
|
## Objective
|
||||||
|
|
||||||
|
Bring Scriptorium from Promptkit v0.1.0 to v0.9.0 and expose the newer
|
||||||
|
framework capabilities that fit Scriptorium's existing CLI, HTTP,
|
||||||
|
configuration, presentation, and process boundaries.
|
||||||
|
|
||||||
|
Scriptorium must accept the complete Promptkit v0.9.0 feature set expressed by
|
||||||
|
directory-backed prompt, profile, and schema definitions. Application adapters
|
||||||
|
must not narrow that format contract by requiring request values that the
|
||||||
|
selected definition does not require or by omitting selection values needed to
|
||||||
|
address a valid definition. Promptkit's tagged
|
||||||
|
[framework format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md)
|
||||||
|
remains canonical; Scriptorium must delegate parsing and framework validation
|
||||||
|
rather than reproduce those rules.
|
||||||
|
|
||||||
|
The completed application will retain Promptkit as the owner of reusable
|
||||||
|
prompt execution, source formats, profiles, backend definitions, capacity,
|
||||||
|
inspection, generation, and validation. Scriptorium will continue to own
|
||||||
|
application configuration, command and HTTP contracts, error and exit mapping,
|
||||||
|
output presentation, and deployment policy.
|
||||||
|
|
||||||
|
## Targeted End State
|
||||||
|
|
||||||
|
When this roadmap is complete:
|
||||||
|
|
||||||
|
- Scriptorium pins the released Promptkit v0.9.0 module without a workspace,
|
||||||
|
replacement, vendored source, direct catalog import, or Promptkit internal
|
||||||
|
import.
|
||||||
|
- Existing valid CLI, HTTP, configuration, prompt, profile, schema, and render
|
||||||
|
workflows remain supported subject to Promptkit's documented v0.2.0 through
|
||||||
|
v0.9.0 compatibility corrections.
|
||||||
|
- Every Promptkit v0.9.0 prompt and profile feature expressible in definitions
|
||||||
|
under Scriptorium's configured directory sources is usable through the
|
||||||
|
executable application, including prompts with no inputs and explicitly
|
||||||
|
selected prompt versions.
|
||||||
|
- CLI and HTTP requests can supply a direct non-secret session ID.
|
||||||
|
- HTTP requests and a presence-aware CLI flag can inherit, replace, or
|
||||||
|
explicitly clear reasoning effort.
|
||||||
|
- Prepared output, CLI run summaries, and HTTP result metadata expose effective
|
||||||
|
backend identity when Promptkit supplies one.
|
||||||
|
- HTTP capacity rejection has a stable Scriptorium-owned overload response.
|
||||||
|
- Application configuration can register reusable engine-scoped custom
|
||||||
|
backends, including bounded concurrency and queue policy, for selection by
|
||||||
|
Promptkit profiles.
|
||||||
|
- CLI users can inspect prompt and profile definitions without supplying
|
||||||
|
placeholder inputs or invoking a model.
|
||||||
|
- Scriptorium's contracts and examples point to Promptkit v0.9.0 and accurately
|
||||||
|
describe its current credential, validation-repair, provider-control,
|
||||||
|
message-role, backend, and source-validation semantics.
|
||||||
|
|
||||||
|
## Feature Scope
|
||||||
|
|
||||||
|
### Complete Prompt And Profile Definition Compatibility
|
||||||
|
|
||||||
|
Scriptorium will treat the tagged Promptkit v0.9.0 format contract as one
|
||||||
|
indivisible downstream compatibility boundary. This includes prompt identity
|
||||||
|
and version selection, optional and required input declarations, message and
|
||||||
|
content-file templates, supported roles, cache control, session templates,
|
||||||
|
default profiles, output contracts, schemas, and repair budgets. It also
|
||||||
|
includes standalone and inherited profiles, built-in and custom backend
|
||||||
|
selection, endpoint overrides, execution controls, optional credential
|
||||||
|
environment sources, and JSON-compatible extra parameters.
|
||||||
|
|
||||||
|
Most of this support is provided by passing configured prompt, profile, and
|
||||||
|
schema directories directly to Promptkit. Scriptorium will not independently
|
||||||
|
decode these framework definitions or maintain a field-level subset. Two
|
||||||
|
current adapter restrictions must be removed so the executable interfaces do
|
||||||
|
not reject definitions that Promptkit accepts:
|
||||||
|
|
||||||
|
- `run` and `render` will accept an optional prompt-version flag and map it to
|
||||||
|
`promptkit.RunRequest.PromptVersion`; and
|
||||||
|
- `run`, `render`, and `POST /v1/runs` will permit omitted or empty input maps.
|
||||||
|
Promptkit will decide whether an input is required by the selected
|
||||||
|
definition or referenced template.
|
||||||
|
|
||||||
|
The HTTP API already carries `prompt_version`; its mapping must remain covered
|
||||||
|
by adapter tests. CLI and HTTP callers may continue to supply extra inputs,
|
||||||
|
subject to their existing file, containment, and size policies. Removing the
|
||||||
|
application-level nonempty-input requirement does not weaken Promptkit's
|
||||||
|
declared-required-input or template-reference validation.
|
||||||
|
|
||||||
|
Profiles that name built-in backend IDs require no application registration.
|
||||||
|
Profiles that name other backend IDs become fully usable through the custom
|
||||||
|
backend configuration in this roadmap. Profile inheritance and all other
|
||||||
|
profile-field resolution remain Promptkit responsibilities.
|
||||||
|
|
||||||
|
This compatibility requirement concerns fields and behavior expressed in
|
||||||
|
Promptkit YAML definitions. Go-consumer construction alternatives such as
|
||||||
|
single-file sources, injected `fs.FS` sources, in-memory profiles, fallback
|
||||||
|
profile filesystems, and prepared-execution handles are library-integration
|
||||||
|
features and are not required merely to support the complete definition
|
||||||
|
format.
|
||||||
|
|
||||||
|
### Promptkit v0.9.0 Compatibility Baseline
|
||||||
|
|
||||||
|
Required dependency and source changes:
|
||||||
|
|
||||||
|
- update `go.mod` and `go.sum` to select Promptkit v0.9.0 and its selected
|
||||||
|
OpenRouter and Rakestrawhome catalog dependencies;
|
||||||
|
- change Scriptorium's inbound HTTP reasoning-effort representation to preserve
|
||||||
|
Promptkit's pointer semantics;
|
||||||
|
- retain keyed Promptkit public struct literals and confirm no removed
|
||||||
|
`RunRequest.Metadata` use exists;
|
||||||
|
- update tagged Promptkit links and version references throughout Scriptorium;
|
||||||
|
and
|
||||||
|
- preserve Scriptorium's architecture guard against replacements, workspaces,
|
||||||
|
vendored Promptkit code, former facade packages, and Promptkit internal
|
||||||
|
imports.
|
||||||
|
|
||||||
|
Required compatibility review:
|
||||||
|
|
||||||
|
- verify application and maintained example endpoints satisfy Promptkit's
|
||||||
|
absolute HTTP/HTTPS endpoint rules;
|
||||||
|
- verify prompt content paths, file artifacts, identities, JSON documents, and
|
||||||
|
JSON-compatible extra parameters satisfy the v0.6.0 safety boundaries;
|
||||||
|
- verify maintained prompt roles are limited to `developer`, `system`, `user`,
|
||||||
|
and `assistant`;
|
||||||
|
- verify every positive repair budget is no greater than three and is paired
|
||||||
|
with `basic`, `json`, or `json_schema` validation;
|
||||||
|
- document that positive repair budgets authorize additional provider calls,
|
||||||
|
latency, token use, and cost;
|
||||||
|
- document that unset optional provider controls are omitted and that a zero
|
||||||
|
effective value can represent an unspecified provider control unless an
|
||||||
|
explicit request override supplied it; and
|
||||||
|
- update credential guidance for Promptkit's optional environment lookup
|
||||||
|
behavior, profile `api_key_env` semantics, and Scriptorium's prohibition on
|
||||||
|
raw API-key inputs.
|
||||||
|
|
||||||
|
### Session And Reasoning Request Controls
|
||||||
|
|
||||||
|
The CLI `run` and `render` commands will accept a direct session-ID flag and map
|
||||||
|
it to `promptkit.RunRequest.SessionID`. The value remains non-secret
|
||||||
|
application correlation metadata and is subject to Promptkit's normalization
|
||||||
|
and length rules.
|
||||||
|
|
||||||
|
The HTTP run request will accept `session_id` and map it through the existing
|
||||||
|
strict DTO boundary. Successful HTTP metadata will report the effective
|
||||||
|
session ID when present.
|
||||||
|
|
||||||
|
The CLI `run` and `render` commands will also accept a presence-aware reasoning
|
||||||
|
effort flag. Omission inherits the selected profile, a nonblank value replaces
|
||||||
|
it, and an explicitly supplied empty value clears it. The existing HTTP
|
||||||
|
`model.reasoning_effort` field will gain the same three-state behavior while
|
||||||
|
remaining a JSON string when present.
|
||||||
|
|
||||||
|
These additions must not introduce raw direct API-key flags or fields.
|
||||||
|
|
||||||
|
### Backend Identity And Capacity Outcomes
|
||||||
|
|
||||||
|
Scriptorium presentation will expose `SelectedBackendID` and the effective
|
||||||
|
target's `BackendID` without deriving identity from endpoint text. Backend
|
||||||
|
identity will be included where applicable in:
|
||||||
|
|
||||||
|
- prepared-run text and JSON output;
|
||||||
|
- the CLI run summary; and
|
||||||
|
- HTTP response metadata and effective model parameters.
|
||||||
|
|
||||||
|
Endpoint-only profiles continue to have no backend ID, and empty identities
|
||||||
|
must remain distinguishable from registered built-in or custom backends.
|
||||||
|
|
||||||
|
The HTTP adapter will classify `promptkit.ErrCapacityExceeded` separately from
|
||||||
|
provider generation failures. The target public outcome is HTTP `503` with a
|
||||||
|
stable `capacity_exceeded` code and a generic message. Scriptorium will not
|
||||||
|
invent retry timing or expose an untrusted diagnostic. CLI capacity rejection
|
||||||
|
continues to be a runtime failure, with a clear safe diagnostic and the
|
||||||
|
existing runtime-error exit status.
|
||||||
|
|
||||||
|
The HTTP server will use one appropriately scoped Promptkit engine across its
|
||||||
|
requests so backend admission, active-generation limits, and queue capacity
|
||||||
|
apply across concurrent in-flight work. The server may retain the operational
|
||||||
|
state needed for those limits and for request cancellation and lifecycle
|
||||||
|
management. This does not create durable per-request, conversation, or resume
|
||||||
|
state.
|
||||||
|
|
||||||
|
Scriptorium will continue to return generic public model-generation failures.
|
||||||
|
Promptkit `GenerationError` provider code, type, and message values will not be
|
||||||
|
added to the public HTTP response because they are untrusted and potentially
|
||||||
|
sensitive.
|
||||||
|
|
||||||
|
### Custom Backend Configuration
|
||||||
|
|
||||||
|
The strict Scriptorium application configuration will gain an optional custom
|
||||||
|
backend collection. Each entry will support the application-owned mapping
|
||||||
|
needed to construct a public `promptkit.Backend`:
|
||||||
|
|
||||||
|
- backend ID;
|
||||||
|
- OpenAI-compatible endpoint;
|
||||||
|
- optional API-key environment-variable name;
|
||||||
|
- optional JSON-compatible request-wide extra parameters;
|
||||||
|
- non-negative concurrency limit; and
|
||||||
|
- queue capacity with presence preserved so omission and explicit zero remain
|
||||||
|
different.
|
||||||
|
|
||||||
|
Raw API-key values will remain invalid. Unknown fields will remain errors.
|
||||||
|
Configured backends will be registered through `promptkit.WithBackend` during
|
||||||
|
engine construction and will be immutable and engine-scoped. Promptkit will
|
||||||
|
remain the canonical validator for backend IDs, endpoints, reserved request
|
||||||
|
parameters, capacity bounds, duplicate or reserved IDs, environment-variable
|
||||||
|
names, and JSON-compatible values. Scriptorium will add context appropriate to
|
||||||
|
its configuration error boundary without copying those framework rules.
|
||||||
|
|
||||||
|
The collection will be configuration-file-owned. This roadmap does not add
|
||||||
|
per-backend CLI flags. Existing endpoint-only profiles remain valid, while
|
||||||
|
profiles may select configured custom backend IDs through Promptkit's existing
|
||||||
|
`backend` field. Promptkit's built-in OpenRouter and Rakestrawhome backends and
|
||||||
|
profiles remain available without Scriptorium registration.
|
||||||
|
|
||||||
|
The application configuration shape is a `backends` mapping whose keys are the
|
||||||
|
backend IDs registered with Promptkit:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
backends:
|
||||||
|
local-gpu:
|
||||||
|
endpoint: http://localhost:11434/v1
|
||||||
|
api_key_env: LOCAL_GPU_API_KEY
|
||||||
|
extra_params:
|
||||||
|
provider_option: enabled
|
||||||
|
concurrency_limit: 2
|
||||||
|
queue_capacity: 0
|
||||||
|
```
|
||||||
|
|
||||||
|
`endpoint` is required for each entry. The other fields are optional. Omitted
|
||||||
|
`queue_capacity` uses Promptkit's default queue policy when concurrency is
|
||||||
|
bounded, while an explicit zero disables queuing. The configuration contract
|
||||||
|
and complete maintained configuration example will document this shape when
|
||||||
|
the feature is implemented.
|
||||||
|
|
||||||
|
### Prompt And Profile Inspection CLI
|
||||||
|
|
||||||
|
Scriptorium will add a CLI inspection command with prompt and profile modes.
|
||||||
|
The intended command family is:
|
||||||
|
|
||||||
|
- `scriptorium inspect prompt`, backed by `Engine.InspectPrompt`; and
|
||||||
|
- `scriptorium inspect profile`, backed by `Engine.InspectProfile`.
|
||||||
|
|
||||||
|
Prompt inspection will select an ID and optional version and report the
|
||||||
|
normalized prompt identity, opaque prompt hash, declared default profile,
|
||||||
|
declared inputs, and normalized output contract. It will not require input
|
||||||
|
artifacts, resolve a profile, load a schema, render templates, reserve backend
|
||||||
|
capacity, or contact a model.
|
||||||
|
|
||||||
|
Profile inspection will select an explicit profile ID and report its resolved
|
||||||
|
effective target, backend identity, optional credential source name, and
|
||||||
|
whether a later request must provide credential configuration. It will not
|
||||||
|
read credential values, load a prompt, reserve capacity, or contact a model.
|
||||||
|
|
||||||
|
Inspection will support deterministic Scriptorium-owned text and JSON output
|
||||||
|
and normal output-file handling. Scriptorium will define explicit output DTOs
|
||||||
|
rather than treating Promptkit inspection structs as stable wire formats.
|
||||||
|
Inspection failures will use existing public Promptkit error identities and
|
||||||
|
Scriptorium-owned CLI diagnostics and exit behavior.
|
||||||
|
|
||||||
|
The prompt form accepts `--prompt`, optional `--prompt-version`, `--config`,
|
||||||
|
`--prompt-dir`, `--format text|json`, and `--out`. The profile form accepts
|
||||||
|
`--profile`, `--config`, `--profile-dir`, `--format text|json`, and `--out`.
|
||||||
|
Both forms reject positional arguments and default to text output independently
|
||||||
|
of `defaults.render_format`. Configuration discovery and CLI-over-file source
|
||||||
|
precedence match the existing command family. Prompt inspection requires an
|
||||||
|
effective prompt directory; profile inspection does not, because Promptkit's
|
||||||
|
built-in profiles remain inspectable without one.
|
||||||
|
|
||||||
|
## Documentation And Example Changes
|
||||||
|
|
||||||
|
Implementation of this roadmap requires coordinated updates to the canonical
|
||||||
|
owners of affected behavior:
|
||||||
|
|
||||||
|
- `docs/cli.md` for session and reasoning flags, inspection commands, output,
|
||||||
|
prompt-version selection, optional inputs, and exit behavior;
|
||||||
|
- `docs/api.md` for `session_id`, effective backend/session metadata, reasoning
|
||||||
|
presence semantics, optional inputs, and the capacity response;
|
||||||
|
- `docs/config.md` for custom backend configuration and current Promptkit
|
||||||
|
credential and provider-default semantics;
|
||||||
|
- `docs/operations.md` for capacity and custom-backend operational guidance
|
||||||
|
where deployment handling is affected;
|
||||||
|
- `docs/internal/adapters.md` and `docs/internal/sources.md` for implemented
|
||||||
|
mapping and engine-assembly behavior;
|
||||||
|
- `docs/policy/architecture.md` and `docs/internal/overview.md` only if concrete
|
||||||
|
component responsibilities or durable boundaries change; and
|
||||||
|
- maintained examples for valid v0.9.0 prompt/profile formats and custom
|
||||||
|
backend configuration.
|
||||||
|
|
||||||
|
Current-state documents must not describe these features as implemented until
|
||||||
|
the corresponding code lands.
|
||||||
|
|
||||||
|
## Validation And Completion Criteria
|
||||||
|
|
||||||
|
The roadmap is complete only when:
|
||||||
|
|
||||||
|
- ordinary and race-enabled tests pass against the tagged Promptkit v0.9.0
|
||||||
|
dependency;
|
||||||
|
- `go vet ./...` and `go build ./cmd/scriptorium` pass;
|
||||||
|
- Go formatting and `git diff --check` pass;
|
||||||
|
- the architecture guard passes with no workspace, replacement, vendor tree,
|
||||||
|
former facade, or Promptkit internal import;
|
||||||
|
- maintained render, custom-backend, session, reasoning, and inspection
|
||||||
|
examples execute successfully without real credentials or provider calls;
|
||||||
|
- CLI tests cover explicit and omitted prompt versions for prompt IDs with one
|
||||||
|
or multiple definitions;
|
||||||
|
- CLI and HTTP tests cover prompts with no declared inputs, omitted optional
|
||||||
|
inputs, missing required inputs, and template-referenced inputs;
|
||||||
|
- HTTP tests cover omitted, replacement, and clearing reasoning states;
|
||||||
|
- CLI and HTTP tests cover session propagation and invalid session handling;
|
||||||
|
- presentation tests cover registered and endpoint-only backend identity;
|
||||||
|
- HTTP tests cover capacity rejection independently from provider failure;
|
||||||
|
- concurrent HTTP tests demonstrate that one server's requests share backend
|
||||||
|
admission and cannot bypass capacity through per-request engine creation;
|
||||||
|
- configuration tests cover strict decoding, precedence, queue-capacity
|
||||||
|
presence, invalid backends, and secret-field rejection;
|
||||||
|
- inspection tests cover deterministic text and JSON, source selection,
|
||||||
|
output-file handling, safe credential presentation, and public error
|
||||||
|
mapping;
|
||||||
|
- all maintained prompt roles and repair budgets are valid under Promptkit
|
||||||
|
v0.9.0; and
|
||||||
|
- representative definition fixtures cover inline and file-backed messages,
|
||||||
|
cache control, session templates, every output validation mode, schemas,
|
||||||
|
profile inheritance, built-in and custom backends, execution controls,
|
||||||
|
credential environment names, and extra parameters without Scriptorium
|
||||||
|
independently parsing those fields; and
|
||||||
|
- affected documentation links and copyable examples validate successfully.
|
||||||
|
|
||||||
|
## Explicit Non-Goals
|
||||||
|
|
||||||
|
This roadmap does not include:
|
||||||
|
|
||||||
|
- appended request messages in either executable interface;
|
||||||
|
- retained or durable prepared-execution handles;
|
||||||
|
- HTTP prompt or profile inspection routes;
|
||||||
|
- embedded application fallback profiles;
|
||||||
|
- single-file, injected `fs.FS`, or in-memory prompt and profile source
|
||||||
|
configuration;
|
||||||
|
- request-level output-contract replacement through the executable interfaces;
|
||||||
|
- public disclosure of structured provider error details;
|
||||||
|
- direct imports or configuration of Promptkit's external catalog modules;
|
||||||
|
- raw API-key configuration, CLI flags, or HTTP fields;
|
||||||
|
- durable run state, conversations, retries, archives, checkpoints, or resume;
|
||||||
|
or
|
||||||
|
- changes to Promptkit itself.
|
||||||
@@ -1,94 +0,0 @@
|
|||||||
# Migration Step 5: Promptkit Repository Foundation
|
|
||||||
|
|
||||||
## Status
|
|
||||||
|
|
||||||
Complete as of 2026-07-28.
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Establish Promptkit as an independent, maintainable Go library repository ready
|
|
||||||
to receive framework extraction in Migration Step 6 without moving framework
|
|
||||||
behavior prematurely.
|
|
||||||
|
|
||||||
The [main migration roadmap](migration.md) owns the overall sequence.
|
|
||||||
[ADR 0002](../adr/0002-split-promptkit-from-scriptorium.md) owns the repository
|
|
||||||
boundary, module path, versioning direction, and cross-repository ordering.
|
|
||||||
[ADR 0003](../adr/0003-use-maintainer-run-validation-and-tag-only-releases-for-promptkit.md)
|
|
||||||
owns the maintainer-run validation and tag-only release decision.
|
|
||||||
|
|
||||||
## Completed Work
|
|
||||||
|
|
||||||
### Repository And Module Foundation
|
|
||||||
|
|
||||||
- Confirmed the Promptkit repository, maintainer governance, configured origin,
|
|
||||||
and `main` tracking of `origin/main`.
|
|
||||||
- Created dependency-free module
|
|
||||||
`gitea.maximumdirect.net/eric/promptkit` with Go `1.25.5`.
|
|
||||||
- Added the root `promptkit` package as the sole implemented Go package and
|
|
||||||
public package boundary.
|
|
||||||
- Added no framework implementation, exported API, internal package,
|
|
||||||
speculative package tree, command, workspace file, or local replacement.
|
|
||||||
- Retained GPLv3 and changed only the two template project names in its example
|
|
||||||
notices to `Promptkit`.
|
|
||||||
|
|
||||||
### Durable Repository Guidance
|
|
||||||
|
|
||||||
- Replaced application-template architecture, documentation, and testing
|
|
||||||
policies with library-specific policies.
|
|
||||||
- Recorded the root package as the only implemented component and kept future
|
|
||||||
framework boundaries explicitly unimplemented.
|
|
||||||
- Rewrote repository orientation and contributor workflow for the Promptkit
|
|
||||||
module.
|
|
||||||
- Documented deterministic, offline, parallel-safe, maintainer-run validation.
|
|
||||||
- Documented temporary sibling workspace and uncommitted replacement workflows
|
|
||||||
without committing machine-local coupling.
|
|
||||||
- Added a semantic Go module tag release procedure with no runnable binaries or
|
|
||||||
hosted CI.
|
|
||||||
|
|
||||||
### Decision Reconciliation
|
|
||||||
|
|
||||||
- Accepted ADR 0003 as the later controlling decision for Promptkit validation
|
|
||||||
while leaving ADR 0002 Accepted and its repository-split decision unchanged.
|
|
||||||
- Reconciled the main migration roadmap with maintainer-run validation,
|
|
||||||
tag-only releases, and the requirement that Promptkit publish its first
|
|
||||||
versioned tag before Scriptorium adopts it.
|
|
||||||
|
|
||||||
## Acceptance Record
|
|
||||||
|
|
||||||
From a clean Promptkit checkout with no active Go workspace or local
|
|
||||||
replacement, the maintainer successfully ran:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./...
|
|
||||||
go vet ./...
|
|
||||||
go build ./...
|
|
||||||
gofmt -l $(git ls-files '*.go')
|
|
||||||
git diff --check
|
|
||||||
```
|
|
||||||
|
|
||||||
The formatting command produced no paths. Acceptance also confirmed:
|
|
||||||
|
|
||||||
- module path `gitea.maximumdirect.net/eric/promptkit`, Go `1.25.5`, root
|
|
||||||
package `promptkit`, and matching import path;
|
|
||||||
- no external dependency, `go.sum`, `go.work`, `go.work.sum`, or `replace`
|
|
||||||
directive;
|
|
||||||
- no hosted CI configuration, command package, release artifact, version tag,
|
|
||||||
or placeholder framework package;
|
|
||||||
- every maintained Markdown link resolves;
|
|
||||||
- repository orientation, policies, inventory, contributor workflow, and
|
|
||||||
release procedure are internally consistent and free of template residue;
|
|
||||||
- the configured origin belongs to Promptkit and `main` tracks `origin/main`;
|
|
||||||
- GPLv3 wording and attribution remain unchanged apart from the two approved
|
|
||||||
Promptkit example-notice names; and
|
|
||||||
- Scriptorium contains only the decision and roadmap documentation required for
|
|
||||||
this foundation work, with no production code or module dependency change.
|
|
||||||
|
|
||||||
## Scope Boundary And Next Gate
|
|
||||||
|
|
||||||
No framework implementation or public framework API was extracted, no consumer
|
|
||||||
was migrated, and no `v0.1.0` tag was created. Promptkit remains an independently
|
|
||||||
valid repository foundation.
|
|
||||||
|
|
||||||
Migration Step 6 is now authorized to extract and stabilize the framework. It
|
|
||||||
must pass Promptkit's documented validation and publish the first versioned tag
|
|
||||||
before Scriptorium or another consumer adopts the module.
|
|
||||||
343
engine.go
343
engine.go
@@ -1,343 +0,0 @@
|
|||||||
package scriptorium
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io/fs"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
artifactadapter "gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile/builtin"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ErrInvalidConfig indicates invalid public engine configuration.
|
|
||||||
var ErrInvalidConfig = errors.New("invalid engine configuration")
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrInvalidRequest = errors.New("invalid run request")
|
|
||||||
ErrPromptNotFound = errors.New("prompt not found")
|
|
||||||
ErrProfileNotFound = errors.New("profile not found")
|
|
||||||
ErrProfileRequired = errors.New("profile selection is required")
|
|
||||||
ErrPromptLoad = errors.New("failed to load prompt definition")
|
|
||||||
ErrProfileLoad = errors.New("failed to load execution profile")
|
|
||||||
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
|
||||||
ErrArtifactLoad = errors.New("failed to load artifact")
|
|
||||||
ErrPromptRender = errors.New("failed to render prompt")
|
|
||||||
ErrLLMGenerate = errors.New("failed to generate output")
|
|
||||||
ErrValidation = errors.New("failed to validate output")
|
|
||||||
)
|
|
||||||
|
|
||||||
// Engine prepares and runs Scriptorium prompt requests.
|
|
||||||
type Engine struct {
|
|
||||||
runner *usecase.Runner
|
|
||||||
}
|
|
||||||
|
|
||||||
// Config configures a public Scriptorium engine.
|
|
||||||
type Config struct {
|
|
||||||
PromptDir string
|
|
||||||
ProfileDir string
|
|
||||||
SchemaDir string
|
|
||||||
// Timeout is the transport-wide safety cap for the built-in LLM client
|
|
||||||
// when HTTPClient is absent or has a non-positive timeout.
|
|
||||||
Timeout time.Duration
|
|
||||||
// HTTPClient is cloned for the built-in LLM client. Its positive Timeout
|
|
||||||
// takes precedence over Config.Timeout as the transport-wide safety cap.
|
|
||||||
HTTPClient *http.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
// Option customizes engine construction.
|
|
||||||
type Option interface {
|
|
||||||
apply(*engineOptions) error
|
|
||||||
}
|
|
||||||
|
|
||||||
type optionFunc func(*engineOptions) error
|
|
||||||
|
|
||||||
func (f optionFunc) apply(options *engineOptions) error {
|
|
||||||
return f(options)
|
|
||||||
}
|
|
||||||
|
|
||||||
type engineOptions struct {
|
|
||||||
llmClient llm.Client
|
|
||||||
artifactReader artifactadapter.Reader
|
|
||||||
promptDefs promptdef.Repository
|
|
||||||
profiles profile.Repository
|
|
||||||
memoryProfiles profile.Repository
|
|
||||||
validator validate.Validator
|
|
||||||
promptSource bool
|
|
||||||
profileSource bool
|
|
||||||
memorySource bool
|
|
||||||
validatorSource bool
|
|
||||||
artifactSource bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// WithLLMClient injects a custom LLM client for execution.
|
|
||||||
func WithLLMClient(client LLMClient) Option {
|
|
||||||
return optionFunc(func(options *engineOptions) error {
|
|
||||||
if client == nil {
|
|
||||||
return ErrInvalidConfig
|
|
||||||
}
|
|
||||||
options.llmClient = publicLLMClientAdapter{client: client}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// WithArtifactReader injects a reader for every input artifact reference.
|
|
||||||
func WithArtifactReader(reader ArtifactReader) Option {
|
|
||||||
return optionFunc(func(options *engineOptions) error {
|
|
||||||
if reader == nil {
|
|
||||||
return ErrInvalidConfig
|
|
||||||
}
|
|
||||||
options.artifactReader = publicArtifactReaderAdapter{reader: reader}
|
|
||||||
options.artifactSource = true
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// WithPromptFS loads prompt definitions from fsys under root.
|
|
||||||
//
|
|
||||||
// The source uses the same strict prompt YAML rules as configured prompt
|
|
||||||
// directories, and prompt content_file paths resolve within this source.
|
|
||||||
func WithPromptFS(fsys fs.FS, root string) Option {
|
|
||||||
return optionFunc(func(options *engineOptions) error {
|
|
||||||
if fsys == nil {
|
|
||||||
return ErrInvalidConfig
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(root) == "" {
|
|
||||||
return ErrInvalidConfig
|
|
||||||
}
|
|
||||||
options.promptDefs = promptdef.NewFSRepository(fsys, root)
|
|
||||||
options.promptSource = true
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// WithPromptFile loads prompt definitions from the single prompt file at path.
|
|
||||||
//
|
|
||||||
// Relative prompt content_file paths resolve from the file's directory.
|
|
||||||
func WithPromptFile(path string) Option {
|
|
||||||
return optionFunc(func(options *engineOptions) error {
|
|
||||||
fsys, root, err := fileSource(path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
options.promptDefs = promptdef.NewFSRepository(fsys, root)
|
|
||||||
options.promptSource = true
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// WithProfileFS loads execution profiles from fsys under root.
|
|
||||||
//
|
|
||||||
// Profiles from this source overlay built-in profiles. Profile YAML must use
|
|
||||||
// api_key_env for environment-based credentials; raw API keys are rejected.
|
|
||||||
func WithProfileFS(fsys fs.FS, root string) Option {
|
|
||||||
return optionFunc(func(options *engineOptions) error {
|
|
||||||
if fsys == nil {
|
|
||||||
return ErrInvalidConfig
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(root) == "" {
|
|
||||||
return ErrInvalidConfig
|
|
||||||
}
|
|
||||||
options.profiles = profile.NewFSRepository(fsys, root)
|
|
||||||
options.profileSource = true
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// WithProfileFile loads execution profiles from the single profile file at path.
|
|
||||||
//
|
|
||||||
// The profile overlays built-in profiles. Profile YAML must use api_key_env for
|
|
||||||
// environment-based credentials; raw API keys are rejected.
|
|
||||||
func WithProfileFile(path string) Option {
|
|
||||||
return optionFunc(func(options *engineOptions) error {
|
|
||||||
fsys, root, err := fileSource(path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
options.profiles = profile.NewFSRepository(fsys, root)
|
|
||||||
options.profileSource = true
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// WithProfiles configures in-memory profiles that take precedence over
|
|
||||||
// configured profile files and built-in profiles.
|
|
||||||
func WithProfiles(profiles ...Profile) Option {
|
|
||||||
return optionFunc(func(options *engineOptions) error {
|
|
||||||
repo, err := newMemoryProfileRepository(profiles)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
options.memoryProfiles = repo
|
|
||||||
options.memorySource = true
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// WithSchemaFS loads JSON Schema documents from fsys under root.
|
|
||||||
//
|
|
||||||
// Prompt schema_path values resolve within this source when schema validation
|
|
||||||
// or structured output is requested.
|
|
||||||
func WithSchemaFS(fsys fs.FS, root string) Option {
|
|
||||||
return optionFunc(func(options *engineOptions) error {
|
|
||||||
if fsys == nil {
|
|
||||||
return ErrInvalidConfig
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(root) == "" {
|
|
||||||
return ErrInvalidConfig
|
|
||||||
}
|
|
||||||
options.validator = validate.NewFSValidator(fsys, root)
|
|
||||||
options.validatorSource = true
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// WithSchemaFile loads JSON Schema documents from the single schema file at path.
|
|
||||||
//
|
|
||||||
// Prompt schema_path values refer to the file's base name.
|
|
||||||
func WithSchemaFile(path string) Option {
|
|
||||||
return optionFunc(func(options *engineOptions) error {
|
|
||||||
fsys, root, err := fileSource(path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
options.validator = validate.NewFSValidator(fsys, root)
|
|
||||||
options.validatorSource = true
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewEngine constructs an Engine using the same default internal components as
|
|
||||||
// the CLI and HTTP adapters.
|
|
||||||
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|
||||||
var options engineOptions
|
|
||||||
for _, opt := range opts {
|
|
||||||
if opt == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := opt.apply(&options); err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
promptDefs := options.promptDefs
|
|
||||||
if !options.promptSource {
|
|
||||||
if strings.TrimSpace(cfg.PromptDir) == "" {
|
|
||||||
return nil, fmt.Errorf("%w: prompt directory is required", ErrInvalidConfig)
|
|
||||||
}
|
|
||||||
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
|
|
||||||
}
|
|
||||||
|
|
||||||
profiles := builtin.NewRepositoryWithDirectory(cfg.ProfileDir)
|
|
||||||
if options.profileSource {
|
|
||||||
profiles = builtin.NewRepositoryWithPrimary(options.profiles)
|
|
||||||
}
|
|
||||||
if options.memorySource {
|
|
||||||
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
|
|
||||||
}
|
|
||||||
|
|
||||||
validator := options.validator
|
|
||||||
if !options.validatorSource {
|
|
||||||
schemaDir := cfg.SchemaDir
|
|
||||||
if strings.TrimSpace(schemaDir) == "" {
|
|
||||||
schemaDir = defaults.SchemaDirDefault
|
|
||||||
}
|
|
||||||
validator = validate.NewStandardValidator(schemaDir)
|
|
||||||
}
|
|
||||||
|
|
||||||
llmClient := options.llmClient
|
|
||||||
if llmClient == nil {
|
|
||||||
var err error
|
|
||||||
llmClient, err = llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
|
|
||||||
Timeout: cfg.Timeout,
|
|
||||||
HTTPClient: cfg.HTTPClient,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
artifacts := options.artifactReader
|
|
||||||
if !options.artifactSource {
|
|
||||||
artifacts = artifactadapter.NewCompositeReader()
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Engine{
|
|
||||||
runner: usecase.NewRunner(
|
|
||||||
promptDefs,
|
|
||||||
profiles,
|
|
||||||
artifacts,
|
|
||||||
prompt.NewGoRenderer(),
|
|
||||||
llmClient,
|
|
||||||
validator,
|
|
||||||
),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func fileSource(name string) (fs.FS, string, error) {
|
|
||||||
cleanName := strings.TrimSpace(name)
|
|
||||||
if cleanName == "" {
|
|
||||||
return nil, "", ErrInvalidConfig
|
|
||||||
}
|
|
||||||
dir := filepath.Dir(cleanName)
|
|
||||||
base := filepath.Base(cleanName)
|
|
||||||
if base == "." || base == string(filepath.Separator) || strings.TrimSpace(base) == "" {
|
|
||||||
return nil, "", ErrInvalidConfig
|
|
||||||
}
|
|
||||||
info, err := os.Stat(cleanName)
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, cleanName, err)
|
|
||||||
}
|
|
||||||
if info.IsDir() {
|
|
||||||
return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, cleanName)
|
|
||||||
}
|
|
||||||
return os.DirFS(dir), filepath.ToSlash(base), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Prepare resolves a prompt request without calling an LLM.
|
|
||||||
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
|
||||||
if e == nil || e.runner == nil {
|
|
||||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
|
||||||
}
|
|
||||||
|
|
||||||
domainReq, err := toDomainRunRequest(req)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
prepared, err := e.runner.Prepare(ctx, domainReq)
|
|
||||||
if err != nil {
|
|
||||||
return nil, mapPublicError(err)
|
|
||||||
}
|
|
||||||
return fromDomainPreparedRun(prepared), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run executes a prompt request and returns the generated artifact and metadata.
|
|
||||||
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
|
||||||
if e == nil || e.runner == nil {
|
|
||||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
|
||||||
}
|
|
||||||
|
|
||||||
domainReq, err := toDomainRunRequest(req)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := e.runner.Run(ctx, domainReq)
|
|
||||||
if err != nil {
|
|
||||||
return nil, mapPublicError(err)
|
|
||||||
}
|
|
||||||
return fromDomainRunResult(result), nil
|
|
||||||
}
|
|
||||||
2559
engine_test.go
2559
engine_test.go
File diff suppressed because it is too large
Load Diff
60
errors.go
60
errors.go
@@ -1,60 +0,0 @@
|
|||||||
package scriptorium
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
|
||||||
)
|
|
||||||
|
|
||||||
func mapPublicError(err error) error {
|
|
||||||
if err == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
publicErr := publicErrorFor(err)
|
|
||||||
if publicErr == nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return fmt.Errorf("%w: %w", publicErr, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func publicErrorFor(err error) error {
|
|
||||||
switch {
|
|
||||||
case errors.Is(err, promptdef.ErrPromptDefinitionNotFound):
|
|
||||||
return ErrPromptNotFound
|
|
||||||
case errors.Is(err, profile.ErrProfileNotFound):
|
|
||||||
return ErrProfileNotFound
|
|
||||||
case errors.Is(err, usecase.ErrProfileRequired):
|
|
||||||
return errors.Join(ErrInvalidRequest, ErrProfileRequired)
|
|
||||||
case errors.Is(err, usecase.ErrPromptLoad):
|
|
||||||
return ErrPromptLoad
|
|
||||||
case errors.Is(err, usecase.ErrProfileLoad):
|
|
||||||
return ErrProfileLoad
|
|
||||||
case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition):
|
|
||||||
return ErrPromptLoad
|
|
||||||
case isProfileLoadCause(err):
|
|
||||||
return ErrProfileLoad
|
|
||||||
case errors.Is(err, usecase.ErrAPIKeyEnvMissing):
|
|
||||||
return errors.Join(ErrInvalidRequest, ErrAPIKeyEnvMissing)
|
|
||||||
case errors.Is(err, usecase.ErrArtifactLoad):
|
|
||||||
return ErrArtifactLoad
|
|
||||||
case errors.Is(err, usecase.ErrPromptRender):
|
|
||||||
return ErrPromptRender
|
|
||||||
case errors.Is(err, usecase.ErrLLMGenerate):
|
|
||||||
return ErrLLMGenerate
|
|
||||||
case errors.Is(err, usecase.ErrValidation):
|
|
||||||
return ErrValidation
|
|
||||||
case errors.Is(err, usecase.ErrInvalidRequest):
|
|
||||||
return ErrInvalidRequest
|
|
||||||
default:
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func isProfileLoadCause(err error) bool {
|
|
||||||
return errors.Is(err, profile.ErrInvalidYAML) ||
|
|
||||||
errors.Is(err, profile.ErrInvalidProfile) ||
|
|
||||||
errors.Is(err, profile.ErrRawAPIKeyNotAllowed)
|
|
||||||
}
|
|
||||||
@@ -2,6 +2,15 @@ prompt_dir: ./examples/prompts
|
|||||||
profile_dir: ./examples/profiles
|
profile_dir: ./examples/profiles
|
||||||
schema_dir: ./examples/schemas
|
schema_dir: ./examples/schemas
|
||||||
|
|
||||||
|
backends:
|
||||||
|
local-gpu:
|
||||||
|
endpoint: http://localhost:11434/v1
|
||||||
|
api_key_env: LOCAL_GPU_API_KEY
|
||||||
|
extra_params:
|
||||||
|
provider_option: enabled
|
||||||
|
concurrency_limit: 2
|
||||||
|
queue_capacity: 0
|
||||||
|
|
||||||
server:
|
server:
|
||||||
addr: 127.0.0.1:8080
|
addr: 127.0.0.1:8080
|
||||||
artifact_root: .
|
artifact_root: .
|
||||||
|
|||||||
@@ -1,50 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium"
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
|
||||||
PromptDir: "./examples/prompts",
|
|
||||||
ProfileDir: "./examples/profiles",
|
|
||||||
SchemaDir: "./examples/schemas",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
|
||||||
PromptID: "generic.markdown_summary",
|
|
||||||
Inputs: map[string]scriptorium.ArtifactRef{
|
|
||||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
|
||||||
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
summary := struct {
|
|
||||||
PromptID string `json:"prompt_id"`
|
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
|
||||||
Model string `json:"model"`
|
|
||||||
MessageCount int `json:"message_count"`
|
|
||||||
InputHashes map[string]string `json:"input_hashes"`
|
|
||||||
}{
|
|
||||||
PromptID: prepared.PromptID,
|
|
||||||
SelectedProfileID: prepared.SelectedProfileID,
|
|
||||||
Model: prepared.EffectiveModelParams.Model,
|
|
||||||
MessageCount: len(prepared.Messages),
|
|
||||||
InputHashes: prepared.InputHashes,
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := json.NewEncoder(os.Stdout).Encode(summary); err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
4
examples/profiles/local-gpu.yml
Normal file
4
examples/profiles/local-gpu.yml
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
id: local-gpu
|
||||||
|
backend: local-gpu
|
||||||
|
model: local-model
|
||||||
|
reasoning_effort: low
|
||||||
25
examples/render-v0.9-features.sh
Executable file
25
examples/render-v0.9-features.sh
Executable file
@@ -0,0 +1,25 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
cd "$repo_root"
|
||||||
|
|
||||||
|
go run ./cmd/scriptorium render \
|
||||||
|
--config ./examples/config.full.yml \
|
||||||
|
--prompt generic.markdown_summary \
|
||||||
|
--prompt-version 1.0.0 \
|
||||||
|
--profile local-gpu \
|
||||||
|
--session-id example-session \
|
||||||
|
--reasoning-effort= \
|
||||||
|
--input transcript=./examples/fixtures/transcript.md \
|
||||||
|
--input glossary=./examples/fixtures/glossary.yml
|
||||||
|
|
||||||
|
go run ./cmd/scriptorium inspect prompt \
|
||||||
|
--config ./examples/config.full.yml \
|
||||||
|
--prompt generic.markdown_summary \
|
||||||
|
--format json
|
||||||
|
|
||||||
|
go run ./cmd/scriptorium inspect profile \
|
||||||
|
--config ./examples/config.full.yml \
|
||||||
|
--profile local-gpu \
|
||||||
|
--format json
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
package scriptorium
|
|
||||||
|
|
||||||
import "fmt"
|
|
||||||
|
|
||||||
// String returns a concise request summary without exposing direct API keys.
|
|
||||||
func (r RunRequest) String() string {
|
|
||||||
return r.redactedString()
|
|
||||||
}
|
|
||||||
|
|
||||||
// GoString returns a concise request summary without exposing direct API keys.
|
|
||||||
func (r RunRequest) GoString() string {
|
|
||||||
return r.redactedString()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r RunRequest) redactedString() string {
|
|
||||||
return fmt.Sprintf(
|
|
||||||
"scriptorium.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t Metadata:%d}",
|
|
||||||
r.PromptID,
|
|
||||||
r.PromptVersion,
|
|
||||||
r.ProfileID,
|
|
||||||
r.APIKey != "",
|
|
||||||
len(r.Inputs),
|
|
||||||
len(r.Vars),
|
|
||||||
r.Execution != nil,
|
|
||||||
r.Validation != nil,
|
|
||||||
len(r.Metadata),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// String returns a concise request summary without exposing direct API keys or
|
|
||||||
// rendered prompt content.
|
|
||||||
func (r GenerateRequest) String() string {
|
|
||||||
return r.redactedString()
|
|
||||||
}
|
|
||||||
|
|
||||||
// GoString returns a concise request summary without exposing direct API keys or
|
|
||||||
// rendered prompt content.
|
|
||||||
func (r GenerateRequest) GoString() string {
|
|
||||||
return r.redactedString()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r GenerateRequest) redactedString() string {
|
|
||||||
return fmt.Sprintf(
|
|
||||||
"scriptorium.GenerateRequest{Messages:%d Model:%q APIKeySet:%t StructuredOutputSet:%t ExtraParams:%d}",
|
|
||||||
len(r.Prompt.Messages),
|
|
||||||
r.Target.Model,
|
|
||||||
r.APIKey != "",
|
|
||||||
r.StructuredOutput != nil,
|
|
||||||
len(r.Target.ExtraParams),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
9
go.mod
9
go.mod
@@ -3,8 +3,13 @@ module gitea.maximumdirect.net/eric/scriptorium
|
|||||||
go 1.25.5
|
go 1.25.5
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
gitea.maximumdirect.net/eric/promptkit v0.9.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
|
|
||||||
require golang.org/x/text v0.14.0 // indirect
|
require (
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 // indirect
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 // indirect
|
||||||
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
||||||
|
golang.org/x/text v0.14.0 // indirect
|
||||||
|
)
|
||||||
|
|||||||
6
go.sum
6
go.sum
@@ -1,3 +1,9 @@
|
|||||||
|
gitea.maximumdirect.net/eric/promptkit v0.9.0 h1:IpvDRC8L6xRxQ9hpuyKOmMc5b6MeLTKYyx+h1YAjy08=
|
||||||
|
gitea.maximumdirect.net/eric/promptkit v0.9.0/go.mod h1:oMJ/WUJImUtwJ5e+6MAGECPYAErAkOaKel0G+3T/b4E=
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 h1:lc062euk2qseO//D762i3JaFyulDNML3eQQX7DkYTho=
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0/go.mod h1:AIa7kAu2mfrRQgcspe4L+DW51WqgnALQT60lqkEywJI=
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 h1:j9YY7wsTVjzke2kHH4YAzpU0oUpM+x+nXwl1IeS+2eg=
|
||||||
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0/go.mod h1:4RNS+LILDg4JbS4Ts9Lwy1C92wauXJIbeQaalps4Koo=
|
||||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||||
|
|||||||
200
internal/adapter/cli/inspect.go
Normal file
200
internal/adapter/cli/inspect.go
Normal file
@@ -0,0 +1,200 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/fs"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||||
|
appformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||||
|
)
|
||||||
|
|
||||||
|
type promptInspectionConfig struct {
|
||||||
|
configPath, promptDir, promptID, promptVersion, outputPath string
|
||||||
|
configExplicit bool
|
||||||
|
outputFormat appformat.OutputFormat
|
||||||
|
}
|
||||||
|
|
||||||
|
type profileInspectionConfig struct {
|
||||||
|
configPath, profileDir, profileID, outputPath string
|
||||||
|
configExplicit bool
|
||||||
|
outputFormat appformat.OutputFormat
|
||||||
|
}
|
||||||
|
|
||||||
|
// emptyPromptDefinitionFS satisfies Promptkit's engine-level prompt-source
|
||||||
|
// requirement without exposing the caller's working directory. Profile
|
||||||
|
// inspection never reads this source.
|
||||||
|
type emptyPromptDefinitionFS struct{}
|
||||||
|
|
||||||
|
func (emptyPromptDefinitionFS) Open(name string) (fs.File, error) {
|
||||||
|
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
|
||||||
|
}
|
||||||
|
|
||||||
|
func inspectCommand(args []string, stdout, stderr io.Writer) int {
|
||||||
|
if len(args) == 0 {
|
||||||
|
fmt.Fprintln(stderr, "inspect parse error: inspection mode is required")
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
if args[0] == "profile" {
|
||||||
|
return inspectProfileCommand(args[1:], stdout, stderr)
|
||||||
|
}
|
||||||
|
if args[0] != "prompt" {
|
||||||
|
fmt.Fprintln(stderr, "inspect parse error: unknown inspection mode")
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
cfg, err := parsePromptInspectionArgs(args[1:])
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "inspect parse error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
settings, err := resolveAppSettingsForPromptInspection(cfg)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
engine, err := newEngine(settings)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "engine error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectPrompt(context.Background(), cfg.promptID, cfg.promptVersion)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
data, err := appformat.FormatPromptInspection(inspection, cfg.outputFormat)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
if err := writeOutput(stdout, cfg.outputPath, data); err != nil {
|
||||||
|
fmt.Fprintf(stderr, "output write error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
return ExitOK
|
||||||
|
}
|
||||||
|
|
||||||
|
func inspectProfileCommand(args []string, stdout, stderr io.Writer) int {
|
||||||
|
cfg, err := parseProfileInspectionArgs(args)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "inspect parse error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
settings, err := resolveAppSettingsForProfileInspection(cfg)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
engine, err := newEngine(settings, promptkit.WithPromptFS(emptyPromptDefinitionFS{}, "."))
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "engine error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectProfile(context.Background(), cfg.profileID)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
data, err := appformat.FormatProfileInspection(inspection, cfg.outputFormat)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
if err := writeOutput(stdout, cfg.outputPath, data); err != nil {
|
||||||
|
fmt.Fprintf(stderr, "output write error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
return ExitOK
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseProfileInspectionArgs(args []string) (*profileInspectionConfig, error) {
|
||||||
|
cfg := &profileInspectionConfig{outputFormat: appformat.DefaultOutputFormat}
|
||||||
|
fs := flag.NewFlagSet("inspect profile", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
registerConfigPathFlag(fs, &cfg.configPath)
|
||||||
|
fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing execution profiles")
|
||||||
|
fs.StringVar(&cfg.profileID, "profile", "", "profile ID to inspect")
|
||||||
|
rawFormat := ""
|
||||||
|
fs.StringVar(&rawFormat, "format", "", "output format: text or json")
|
||||||
|
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
cfg.configExplicit = flagWasSet(fs, "config")
|
||||||
|
if fs.NArg() > 0 {
|
||||||
|
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(cfg.profileID) == "" {
|
||||||
|
return nil, errors.New("--profile is required")
|
||||||
|
}
|
||||||
|
format, err := appformat.ParseOutputFormat(rawFormat)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
cfg.outputFormat = format
|
||||||
|
if cfg.outputPath != "" {
|
||||||
|
cfg.outputPath = filepath.Clean(cfg.outputPath)
|
||||||
|
}
|
||||||
|
return cfg, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveAppSettingsForProfileInspection(cfg *profileInspectionConfig) (engineSettings, error) {
|
||||||
|
settings, err := resolveAppSettingsWithConfigPresence(cfg.configPath, cfg.configExplicit, appconfig.CLIOverrides{
|
||||||
|
ProfileDir: cfg.profileDir,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return engineSettings{}, err
|
||||||
|
}
|
||||||
|
return engineSettings{profileDir: settings.ProfileDir, backends: settings.Backends}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parsePromptInspectionArgs(args []string) (*promptInspectionConfig, error) {
|
||||||
|
cfg := &promptInspectionConfig{outputFormat: appformat.DefaultOutputFormat}
|
||||||
|
fs := flag.NewFlagSet("inspect prompt", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
registerConfigPathFlag(fs, &cfg.configPath)
|
||||||
|
fs.StringVar(&cfg.promptDir, "prompt-dir", "", "directory containing prompt definitions")
|
||||||
|
fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to inspect")
|
||||||
|
fs.StringVar(&cfg.promptVersion, "prompt-version", "", "optional prompt version")
|
||||||
|
rawFormat := ""
|
||||||
|
fs.StringVar(&rawFormat, "format", "", "output format: text or json")
|
||||||
|
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
cfg.configExplicit = flagWasSet(fs, "config")
|
||||||
|
if fs.NArg() > 0 {
|
||||||
|
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(cfg.promptID) == "" {
|
||||||
|
return nil, errors.New("--prompt is required")
|
||||||
|
}
|
||||||
|
format, err := appformat.ParseOutputFormat(rawFormat)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
cfg.outputFormat = format
|
||||||
|
if cfg.outputPath != "" {
|
||||||
|
cfg.outputPath = filepath.Clean(cfg.outputPath)
|
||||||
|
}
|
||||||
|
return cfg, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveAppSettingsForPromptInspection(cfg *promptInspectionConfig) (engineSettings, error) {
|
||||||
|
settings, err := resolveAppSettingsWithConfigPresence(cfg.configPath, cfg.configExplicit, appconfig.CLIOverrides{
|
||||||
|
PromptDir: cfg.promptDir,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return engineSettings{}, err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(settings.PromptDir) == "" {
|
||||||
|
return engineSettings{}, errors.New(errPromptDirRequired)
|
||||||
|
}
|
||||||
|
return engineSettings{promptDir: settings.PromptDir, backends: settings.Backends}, nil
|
||||||
|
}
|
||||||
@@ -12,7 +12,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
httpadapter "gitea.maximumdirect.net/eric/scriptorium/internal/adapter/http"
|
httpadapter "gitea.maximumdirect.net/eric/scriptorium/internal/adapter/http"
|
||||||
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
@@ -32,31 +32,36 @@ const (
|
|||||||
type runConfig struct {
|
type runConfig struct {
|
||||||
configPath string
|
configPath string
|
||||||
|
|
||||||
promptDir string
|
promptDir string
|
||||||
profileDir string
|
profileDir string
|
||||||
promptID string
|
promptID string
|
||||||
profileID string
|
promptVersion string
|
||||||
inputRaw listFlag
|
profileID string
|
||||||
varRaw listFlag
|
sessionID string
|
||||||
outputPath string
|
inputRaw listFlag
|
||||||
llmBaseURL string
|
varRaw listFlag
|
||||||
apiKeyEnv string
|
outputPath string
|
||||||
model string
|
llmBaseURL string
|
||||||
temperature float64
|
apiKeyEnv string
|
||||||
maxTokens int
|
model string
|
||||||
topP float64
|
temperature float64
|
||||||
schemaDir string
|
maxTokens int
|
||||||
timeout time.Duration
|
topP float64
|
||||||
|
reasoningEffort string
|
||||||
|
schemaDir string
|
||||||
|
backends []appconfig.BackendSettings
|
||||||
|
timeout time.Duration
|
||||||
|
|
||||||
defaultRenderFormat renderformat.PreparedRunOutputFormat
|
defaultRenderFormat renderformat.PreparedRunOutputFormat
|
||||||
|
|
||||||
llmBaseURLSet bool
|
llmBaseURLSet bool
|
||||||
apiKeyEnvSet bool
|
apiKeyEnvSet bool
|
||||||
modelSet bool
|
modelSet bool
|
||||||
temperatureSet bool
|
temperatureSet bool
|
||||||
maxTokensSet bool
|
maxTokensSet bool
|
||||||
topPSet bool
|
topPSet bool
|
||||||
timeoutSet bool
|
reasoningEffortSet bool
|
||||||
|
timeoutSet bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type renderConfig struct {
|
type renderConfig struct {
|
||||||
@@ -75,6 +80,7 @@ type serveConfig struct {
|
|||||||
maxRequestBytes int64
|
maxRequestBytes int64
|
||||||
maxArtifactBytes int64
|
maxArtifactBytes int64
|
||||||
maxResponseBytes int64
|
maxResponseBytes int64
|
||||||
|
backends []appconfig.BackendSettings
|
||||||
}
|
}
|
||||||
|
|
||||||
type commonCommandSettings struct {
|
type commonCommandSettings struct {
|
||||||
@@ -87,6 +93,7 @@ type commonCommandSettings struct {
|
|||||||
maxArtifactBytes int64
|
maxArtifactBytes int64
|
||||||
maxResponseBytes int64
|
maxResponseBytes int64
|
||||||
defaultRenderFormat renderformat.PreparedRunOutputFormat
|
defaultRenderFormat renderformat.PreparedRunOutputFormat
|
||||||
|
backends []appconfig.BackendSettings
|
||||||
}
|
}
|
||||||
|
|
||||||
type listFlag []string
|
type listFlag []string
|
||||||
@@ -113,6 +120,8 @@ func Run(args []string, stdout, stderr io.Writer) int {
|
|||||||
return renderCommand(args[1:], stdout, stderr)
|
return renderCommand(args[1:], stdout, stderr)
|
||||||
case "serve":
|
case "serve":
|
||||||
return serveCommand(args[1:], stderr)
|
return serveCommand(args[1:], stderr)
|
||||||
|
case "inspect":
|
||||||
|
return inspectCommand(args[1:], stdout, stderr)
|
||||||
default:
|
default:
|
||||||
fmt.Fprintf(stderr, "unknown command %q\n", args[0])
|
fmt.Fprintf(stderr, "unknown command %q\n", args[0])
|
||||||
printUsage(stderr)
|
printUsage(stderr)
|
||||||
@@ -133,7 +142,7 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
|
|||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
|
|
||||||
engine, err := newEngine(cfg)
|
engine, err := newEngine(cfg.engineSettings())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(stderr, "engine error: %v\n", err)
|
fmt.Fprintf(stderr, "engine error: %v\n", err)
|
||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
@@ -141,7 +150,7 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
|
|||||||
|
|
||||||
res, runErr := engine.Run(context.Background(), req)
|
res, runErr := engine.Run(context.Background(), req)
|
||||||
if runErr != nil {
|
if runErr != nil {
|
||||||
fmt.Fprintf(stderr, "run error: %v\n", runErr)
|
fmt.Fprintln(stderr, runErrorMessage(runErr))
|
||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -167,7 +176,7 @@ func renderCommand(args []string, stdout, stderr io.Writer) int {
|
|||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
|
|
||||||
engine, err := newEngine(&cfg.runConfig)
|
engine, err := newEngine(cfg.runConfig.engineSettings())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(stderr, "engine error: %v\n", err)
|
fmt.Fprintf(stderr, "engine error: %v\n", err)
|
||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
@@ -205,11 +214,7 @@ func serveCommand(args []string, stderr io.Writer) int {
|
|||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
|
|
||||||
engine, err := newEngine(&runConfig{
|
engine, err := newEngine(cfg.engineSettings(), promptkit.WithArtifactReader(artifactReader))
|
||||||
promptDir: cfg.promptDir,
|
|
||||||
profileDir: cfg.profileDir,
|
|
||||||
schemaDir: cfg.schemaDir,
|
|
||||||
}, scriptorium.WithArtifactReader(artifactReader))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(stderr, "engine error: %v\n", err)
|
fmt.Fprintf(stderr, "engine error: %v\n", err)
|
||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
@@ -329,6 +334,7 @@ func parseServeArgs(args []string) (*serveConfig, error) {
|
|||||||
cfg.maxRequestBytes = settings.maxRequestBytes
|
cfg.maxRequestBytes = settings.maxRequestBytes
|
||||||
cfg.maxArtifactBytes = settings.maxArtifactBytes
|
cfg.maxArtifactBytes = settings.maxArtifactBytes
|
||||||
cfg.maxResponseBytes = settings.maxResponseBytes
|
cfg.maxResponseBytes = settings.maxResponseBytes
|
||||||
|
cfg.backends = settings.backends
|
||||||
|
|
||||||
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
|
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -349,7 +355,9 @@ func registerExecutionRequestFlags(fs *flag.FlagSet, cfg *runConfig) {
|
|||||||
fs.StringVar(&cfg.promptDir, "prompt-dir", "", "directory containing prompt definition YAML files")
|
fs.StringVar(&cfg.promptDir, "prompt-dir", "", "directory containing prompt definition YAML files")
|
||||||
fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing execution profile YAML files")
|
fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing execution profile YAML files")
|
||||||
fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to run")
|
fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to run")
|
||||||
|
fs.StringVar(&cfg.promptVersion, "prompt-version", "", "optional prompt definition version")
|
||||||
fs.StringVar(&cfg.profileID, "profile", "", "optional execution profile ID; if omitted, prompt default_profile is used")
|
fs.StringVar(&cfg.profileID, "profile", "", "optional execution profile ID; if omitted, prompt default_profile is used")
|
||||||
|
fs.StringVar(&cfg.sessionID, "session-id", "", "optional session ID")
|
||||||
fs.Var(&cfg.inputRaw, "input", "input mapping(s): name=path (repeatable, comma-separated)")
|
fs.Var(&cfg.inputRaw, "input", "input mapping(s): name=path (repeatable, comma-separated)")
|
||||||
fs.Var(&cfg.varRaw, "var", "variable mapping(s): name=value (repeatable, comma-separated)")
|
fs.Var(&cfg.varRaw, "var", "variable mapping(s): name=value (repeatable, comma-separated)")
|
||||||
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")
|
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")
|
||||||
@@ -359,7 +367,8 @@ func registerExecutionRequestFlags(fs *flag.FlagSet, cfg *runConfig) {
|
|||||||
fs.Float64Var(&cfg.temperature, "temperature", 0, "optional temperature override")
|
fs.Float64Var(&cfg.temperature, "temperature", 0, "optional temperature override")
|
||||||
fs.IntVar(&cfg.maxTokens, "max-tokens", 0, "optional max tokens override")
|
fs.IntVar(&cfg.maxTokens, "max-tokens", 0, "optional max tokens override")
|
||||||
fs.Float64Var(&cfg.topP, "top-p", 0, "optional top_p override")
|
fs.Float64Var(&cfg.topP, "top-p", 0, "optional top_p override")
|
||||||
fs.DurationVar(&cfg.timeout, "timeout", defaults.LLMRequestTimeoutDefault, "LLM request timeout")
|
fs.StringVar(&cfg.reasoningEffort, "reasoning-effort", "", "optional reasoning effort override")
|
||||||
|
fs.DurationVar(&cfg.timeout, "timeout", 0, "LLM request timeout")
|
||||||
fs.StringVar(&cfg.promptID, "prompt-id", "", "deprecated alias for --prompt")
|
fs.StringVar(&cfg.promptID, "prompt-id", "", "deprecated alias for --prompt")
|
||||||
fs.StringVar(&cfg.profileID, "profile-id", "", "deprecated alias for --profile")
|
fs.StringVar(&cfg.profileID, "profile-id", "", "deprecated alias for --profile")
|
||||||
}
|
}
|
||||||
@@ -382,6 +391,7 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
|
|||||||
cfg.profileDir = settings.profileDir
|
cfg.profileDir = settings.profileDir
|
||||||
cfg.schemaDir = settings.schemaDir
|
cfg.schemaDir = settings.schemaDir
|
||||||
cfg.defaultRenderFormat = settings.defaultRenderFormat
|
cfg.defaultRenderFormat = settings.defaultRenderFormat
|
||||||
|
cfg.backends = settings.backends
|
||||||
|
|
||||||
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
|
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -389,9 +399,6 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
|
|||||||
if strings.TrimSpace(cfg.promptID) == "" {
|
if strings.TrimSpace(cfg.promptID) == "" {
|
||||||
return errors.New("--prompt is required")
|
return errors.New("--prompt is required")
|
||||||
}
|
}
|
||||||
if len(cfg.inputRaw) == 0 {
|
|
||||||
return errors.New("at least one --input is required")
|
|
||||||
}
|
|
||||||
cfg.promptDir = filepath.Clean(cfg.promptDir)
|
cfg.promptDir = filepath.Clean(cfg.promptDir)
|
||||||
if strings.TrimSpace(cfg.profileDir) != "" {
|
if strings.TrimSpace(cfg.profileDir) != "" {
|
||||||
cfg.profileDir = filepath.Clean(cfg.profileDir)
|
cfg.profileDir = filepath.Clean(cfg.profileDir)
|
||||||
@@ -405,6 +412,7 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
|
|||||||
cfg.temperatureSet = flagWasSet(fs, "temperature")
|
cfg.temperatureSet = flagWasSet(fs, "temperature")
|
||||||
cfg.maxTokensSet = flagWasSet(fs, "max-tokens")
|
cfg.maxTokensSet = flagWasSet(fs, "max-tokens")
|
||||||
cfg.topPSet = flagWasSet(fs, "top-p")
|
cfg.topPSet = flagWasSet(fs, "top-p")
|
||||||
|
cfg.reasoningEffortSet = flagWasSet(fs, "reasoning-effort")
|
||||||
cfg.timeoutSet = flagWasSet(fs, "timeout")
|
cfg.timeoutSet = flagWasSet(fs, "timeout")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -500,7 +508,11 @@ func registerConfigPathFlag(fs *flag.FlagSet, target *string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func resolveAppSettings(fs *flag.FlagSet, configPath string, overrides appconfig.CLIOverrides) (appconfig.AppSettings, error) {
|
func resolveAppSettings(fs *flag.FlagSet, configPath string, overrides appconfig.CLIOverrides) (appconfig.AppSettings, error) {
|
||||||
settings, err := appconfig.LoadConfig(configPath, flagWasSet(fs, "config"))
|
return resolveAppSettingsWithConfigPresence(configPath, flagWasSet(fs, "config"), overrides)
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveAppSettingsWithConfigPresence(configPath string, configExplicit bool, overrides appconfig.CLIOverrides) (appconfig.AppSettings, error) {
|
||||||
|
settings, err := appconfig.LoadConfig(configPath, configExplicit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return appconfig.AppSettings{}, fmt.Errorf("application config error: %w", err)
|
return appconfig.AppSettings{}, fmt.Errorf("application config error: %w", err)
|
||||||
}
|
}
|
||||||
@@ -528,6 +540,7 @@ func resolveCommonSettings(fs *flag.FlagSet, configPath string, overrides appcon
|
|||||||
maxArtifactBytes: settings.MaxArtifactBytes,
|
maxArtifactBytes: settings.MaxArtifactBytes,
|
||||||
maxResponseBytes: settings.MaxResponseBytes,
|
maxResponseBytes: settings.MaxResponseBytes,
|
||||||
defaultRenderFormat: settings.DefaultRenderFormat,
|
defaultRenderFormat: settings.DefaultRenderFormat,
|
||||||
|
backends: settings.Backends,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -538,36 +551,74 @@ func validateRequiredLibraryDirs(promptDir string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func newEngine(cfg *runConfig, options ...scriptorium.Option) (*scriptorium.Engine, error) {
|
type engineSettings struct {
|
||||||
return scriptorium.NewEngine(scriptorium.Config{
|
promptDir string
|
||||||
PromptDir: cfg.promptDir,
|
profileDir string
|
||||||
ProfileDir: cfg.profileDir,
|
schemaDir string
|
||||||
SchemaDir: cfg.schemaDir,
|
backends []appconfig.BackendSettings
|
||||||
}, options...)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildRunRequestFromConfig(cfg *runConfig) (scriptorium.RunRequest, error) {
|
func (c runConfig) engineSettings() engineSettings {
|
||||||
inputMappings, err := parseMappings(cfg.inputRaw, false)
|
return engineSettings{promptDir: c.promptDir, profileDir: c.profileDir, schemaDir: c.schemaDir, backends: c.backends}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c serveConfig) engineSettings() engineSettings {
|
||||||
|
return engineSettings{promptDir: c.promptDir, profileDir: c.profileDir, schemaDir: c.schemaDir, backends: c.backends}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newEngine(settings engineSettings, options ...promptkit.Option) (*promptkit.Engine, error) {
|
||||||
|
engineOptions := make([]promptkit.Option, 0, len(settings.backends)+len(options))
|
||||||
|
for _, configured := range settings.backends {
|
||||||
|
engineOptions = append(engineOptions, promptkit.WithBackend(promptkit.Backend{
|
||||||
|
ID: configured.ID,
|
||||||
|
Endpoint: configured.Endpoint,
|
||||||
|
APIKeyEnv: configured.APIKeyEnv,
|
||||||
|
ExtraParams: configured.ExtraParams,
|
||||||
|
ConcurrencyLimit: configured.ConcurrencyLimit,
|
||||||
|
QueueCapacity: configured.QueueCapacity,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
engineOptions = append(engineOptions, options...)
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: settings.promptDir,
|
||||||
|
ProfileDir: settings.profileDir,
|
||||||
|
SchemaDir: settings.schemaDir,
|
||||||
|
}, engineOptions...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return scriptorium.RunRequest{}, fmt.Errorf("input parse error: %w", err)
|
return nil, fmt.Errorf("engine initialization from application configuration: %w", err)
|
||||||
|
}
|
||||||
|
return engine, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildRunRequestFromConfig(cfg *runConfig) (promptkit.RunRequest, error) {
|
||||||
|
var inputMappings map[string]string
|
||||||
|
var err error
|
||||||
|
if len(cfg.inputRaw) > 0 {
|
||||||
|
inputMappings, err = parseMappings(cfg.inputRaw, false)
|
||||||
|
if err != nil {
|
||||||
|
return promptkit.RunRequest{}, fmt.Errorf("input parse error: %w", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
varMappings := map[string]string{}
|
varMappings := map[string]string{}
|
||||||
if len(cfg.varRaw) > 0 {
|
if len(cfg.varRaw) > 0 {
|
||||||
varMappings, err = parseMappings(cfg.varRaw, false)
|
varMappings, err = parseMappings(cfg.varRaw, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return scriptorium.RunRequest{}, fmt.Errorf("var parse error: %w", err)
|
return promptkit.RunRequest{}, fmt.Errorf("var parse error: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
inputs := make(map[string]scriptorium.ArtifactRef, len(inputMappings))
|
var inputs map[string]promptkit.ArtifactRef
|
||||||
for name, path := range inputMappings {
|
if len(inputMappings) > 0 {
|
||||||
inputs[name] = scriptorium.File(path)
|
inputs = make(map[string]promptkit.ArtifactRef, len(inputMappings))
|
||||||
|
for name, path := range inputMappings {
|
||||||
|
inputs[name] = promptkit.File(path)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var modelOverride *scriptorium.ExecutionTargetOverride
|
var modelOverride *promptkit.ExecutionTargetOverride
|
||||||
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.apiKeyEnvSet || cfg.timeoutSet {
|
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.reasoningEffortSet || cfg.apiKeyEnvSet || cfg.timeoutSet {
|
||||||
modelOverride = &scriptorium.ExecutionTargetOverride{
|
modelOverride = &promptkit.ExecutionTargetOverride{
|
||||||
Endpoint: cfg.llmBaseURL,
|
Endpoint: cfg.llmBaseURL,
|
||||||
Model: cfg.model,
|
Model: cfg.model,
|
||||||
APIKeyEnv: cfg.apiKeyEnv,
|
APIKeyEnv: cfg.apiKeyEnv,
|
||||||
@@ -581,18 +632,23 @@ func buildRunRequestFromConfig(cfg *runConfig) (scriptorium.RunRequest, error) {
|
|||||||
if cfg.topPSet {
|
if cfg.topPSet {
|
||||||
modelOverride.TopP = &cfg.topP
|
modelOverride.TopP = &cfg.topP
|
||||||
}
|
}
|
||||||
|
if cfg.reasoningEffortSet {
|
||||||
|
modelOverride.ReasoningEffort = &cfg.reasoningEffort
|
||||||
|
}
|
||||||
if cfg.timeoutSet {
|
if cfg.timeoutSet {
|
||||||
timeoutSeconds := int(cfg.timeout.Seconds())
|
timeoutSeconds := int(cfg.timeout.Seconds())
|
||||||
modelOverride.TimeoutSeconds = &timeoutSeconds
|
modelOverride.TimeoutSeconds = &timeoutSeconds
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return scriptorium.RunRequest{
|
return promptkit.RunRequest{
|
||||||
PromptID: cfg.promptID,
|
PromptID: cfg.promptID,
|
||||||
ProfileID: cfg.profileID,
|
PromptVersion: cfg.promptVersion,
|
||||||
Inputs: inputs,
|
ProfileID: cfg.profileID,
|
||||||
Vars: varMappings,
|
SessionID: cfg.sessionID,
|
||||||
Execution: modelOverride,
|
Inputs: inputs,
|
||||||
|
Vars: varMappings,
|
||||||
|
Execution: modelOverride,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -651,17 +707,17 @@ func writeOutput(stdout io.Writer, outputPath string, body []byte) error {
|
|||||||
return os.WriteFile(outputPath, body, 0644)
|
return os.WriteFile(outputPath, body, 0644)
|
||||||
}
|
}
|
||||||
|
|
||||||
func determineExitCode(runErr error, result *scriptorium.RunResult) int {
|
func determineExitCode(runErr error, result *promptkit.RunResult) int {
|
||||||
if runErr != nil {
|
if runErr != nil {
|
||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
if result != nil && result.Validation.Status == scriptorium.ValidationFailed {
|
if result != nil && result.Validation.Status == promptkit.ValidationFailed {
|
||||||
return ExitValidationFailed
|
return ExitValidationFailed
|
||||||
}
|
}
|
||||||
return ExitOK
|
return ExitOK
|
||||||
}
|
}
|
||||||
|
|
||||||
func printSummary(stderr io.Writer, res *scriptorium.RunResult) {
|
func printSummary(stderr io.Writer, res *promptkit.RunResult) {
|
||||||
if res == nil {
|
if res == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -682,12 +738,24 @@ func printSummary(stderr io.Writer, res *scriptorium.RunResult) {
|
|||||||
if res.Usage.CachedTokens != 0 || res.Usage.CacheWriteTokens != 0 {
|
if res.Usage.CachedTokens != 0 || res.Usage.CacheWriteTokens != 0 {
|
||||||
fmt.Fprintf(stderr, " cached_tokens=%d cache_write_tokens=%d", res.Usage.CachedTokens, res.Usage.CacheWriteTokens)
|
fmt.Fprintf(stderr, " cached_tokens=%d cache_write_tokens=%d", res.Usage.CachedTokens, res.Usage.CacheWriteTokens)
|
||||||
}
|
}
|
||||||
|
if res.SelectedBackendID != "" {
|
||||||
|
fmt.Fprintf(stderr, " backend=%s", res.SelectedBackendID)
|
||||||
|
}
|
||||||
fmt.Fprintln(stderr)
|
fmt.Fprintln(stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
func printUsage(w io.Writer) {
|
func runErrorMessage(err error) string {
|
||||||
fmt.Fprintln(w, "usage: scriptorium <run|render|serve> ...")
|
if errors.Is(err, promptkit.ErrCapacityExceeded) {
|
||||||
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID --input name=path [--input ...] [--profile ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--var k=v] [--out path] [--timeout 10m]")
|
return "run error: model backend capacity is exhausted"
|
||||||
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID --input name=path [--input ...] [--profile ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--var k=v] [--format text|json] [--out path] [--timeout 10m]")
|
}
|
||||||
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
|
return fmt.Sprintf("run error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func printUsage(w io.Writer) {
|
||||||
|
fmt.Fprintln(w, "usage: scriptorium <run|render|serve|inspect> ...")
|
||||||
|
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--timeout 10m]")
|
||||||
|
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m]")
|
||||||
|
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
|
||||||
|
fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--prompt-dir DIR] [--format text|json] [--out path]")
|
||||||
|
fmt.Fprintln(w, " inspect profile: scriptorium inspect profile --profile ID [--config PATH] [--profile-dir DIR] [--format text|json] [--out path]")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||||
@@ -87,9 +87,12 @@ func TestParseRunArgsRequiredFlags(t *testing.T) {
|
|||||||
t.Fatal("expected missing --prompt error")
|
t.Fatal("expected missing --prompt error")
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p"})
|
cfg, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p"})
|
||||||
if err == nil {
|
if err != nil {
|
||||||
t.Fatal("expected missing --input error")
|
t.Fatalf("expected omitted --input to be accepted, got %v", err)
|
||||||
|
}
|
||||||
|
if len(cfg.inputRaw) != 0 {
|
||||||
|
t.Fatalf("expected no input mappings, got %#v", cfg.inputRaw)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -98,13 +101,16 @@ func TestParseRunArgsFlagMapping(t *testing.T) {
|
|||||||
"--prompt-dir", "./prompts",
|
"--prompt-dir", "./prompts",
|
||||||
"--profile-dir", "./profiles",
|
"--profile-dir", "./profiles",
|
||||||
"--prompt", "prompt.a",
|
"--prompt", "prompt.a",
|
||||||
|
"--prompt-version", "2",
|
||||||
"--profile", "profile.a",
|
"--profile", "profile.a",
|
||||||
|
"--session-id", "session-1",
|
||||||
"--input", "a=b",
|
"--input", "a=b",
|
||||||
"--llm-base-url", "http://x/v1",
|
"--llm-base-url", "http://x/v1",
|
||||||
"--model", "m",
|
"--model", "m",
|
||||||
"--temperature", "0.7",
|
"--temperature", "0.7",
|
||||||
"--max-tokens", "111",
|
"--max-tokens", "111",
|
||||||
"--top-p", "0.8",
|
"--top-p", "0.8",
|
||||||
|
"--reasoning-effort", "medium",
|
||||||
"--timeout", "30s",
|
"--timeout", "30s",
|
||||||
"--api-key-env", "SCRIPTORIUM_API_KEY",
|
"--api-key-env", "SCRIPTORIUM_API_KEY",
|
||||||
})
|
})
|
||||||
@@ -114,8 +120,11 @@ func TestParseRunArgsFlagMapping(t *testing.T) {
|
|||||||
if cfg.promptDir != filepath.Clean("./prompts") || cfg.profileDir != filepath.Clean("./profiles") {
|
if cfg.promptDir != filepath.Clean("./prompts") || cfg.profileDir != filepath.Clean("./profiles") {
|
||||||
t.Fatalf("unexpected dirs: prompt=%q profile=%q", cfg.promptDir, cfg.profileDir)
|
t.Fatalf("unexpected dirs: prompt=%q profile=%q", cfg.promptDir, cfg.profileDir)
|
||||||
}
|
}
|
||||||
if cfg.promptID != "prompt.a" || cfg.profileID != "profile.a" {
|
if cfg.promptID != "prompt.a" || cfg.promptVersion != "2" || cfg.profileID != "profile.a" {
|
||||||
t.Fatalf("unexpected prompt/profile ids: %q %q", cfg.promptID, cfg.profileID)
|
t.Fatalf("unexpected prompt/version/profile ids: %q %q %q", cfg.promptID, cfg.promptVersion, cfg.profileID)
|
||||||
|
}
|
||||||
|
if cfg.sessionID != "session-1" || cfg.reasoningEffort != "medium" || !cfg.reasoningEffortSet {
|
||||||
|
t.Fatalf("unexpected session or reasoning configuration: %+v", cfg)
|
||||||
}
|
}
|
||||||
if !cfg.llmBaseURLSet || !cfg.modelSet || !cfg.temperatureSet || !cfg.maxTokensSet || !cfg.topPSet || !cfg.timeoutSet || !cfg.apiKeyEnvSet {
|
if !cfg.llmBaseURLSet || !cfg.modelSet || !cfg.temperatureSet || !cfg.maxTokensSet || !cfg.topPSet || !cfg.timeoutSet || !cfg.apiKeyEnvSet {
|
||||||
t.Fatalf("expected override flags set, got %+v", cfg)
|
t.Fatalf("expected override flags set, got %+v", cfg)
|
||||||
@@ -192,7 +201,7 @@ func TestParseServeArgsRejectsRuntimeOverrideFlags(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestUsageIncludesServeFileAndSizeLimitFlags(t *testing.T) {
|
func TestUsageIncludesExecutionAndServeFlags(t *testing.T) {
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Run(nil, io.Discard, &stderr)
|
code := Run(nil, io.Discard, &stderr)
|
||||||
if code != ExitRuntimeError {
|
if code != ExitRuntimeError {
|
||||||
@@ -201,10 +210,15 @@ func TestUsageIncludesServeFileAndSizeLimitFlags(t *testing.T) {
|
|||||||
|
|
||||||
usage := stderr.String()
|
usage := stderr.String()
|
||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
|
"--prompt-version VERSION",
|
||||||
|
"--session-id ID",
|
||||||
|
"--reasoning-effort VALUE",
|
||||||
"--artifact-root",
|
"--artifact-root",
|
||||||
"--max-request-bytes",
|
"--max-request-bytes",
|
||||||
"--max-artifact-bytes",
|
"--max-artifact-bytes",
|
||||||
"--max-response-bytes",
|
"--max-response-bytes",
|
||||||
|
"inspect prompt",
|
||||||
|
"inspect profile",
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(usage, want) {
|
if !strings.Contains(usage, want) {
|
||||||
t.Fatalf("expected usage to include %q, got %q", want, usage)
|
t.Fatalf("expected usage to include %q, got %q", want, usage)
|
||||||
@@ -222,8 +236,8 @@ func TestParseRunArgsTimeout(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected valid run args, got %v", err)
|
t.Fatalf("expected valid run args, got %v", err)
|
||||||
}
|
}
|
||||||
if cfg.timeout != defaults.LLMRequestTimeoutDefault {
|
if cfg.timeout != 0 {
|
||||||
t.Fatalf("expected default timeout %s, got %s", defaults.LLMRequestTimeoutDefault, cfg.timeout)
|
t.Fatalf("expected omitted timeout to remain unset, got %s", cfg.timeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg, err = parseRunArgs([]string{
|
cfg, err = parseRunArgs([]string{
|
||||||
@@ -689,6 +703,55 @@ func TestBuildRunRequestPreservesNumericOverridePresence(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBuildRunRequestPreservesReasoningEffortPresenceAndSessionID(t *testing.T) {
|
||||||
|
omitted, err := buildRunRequestFromConfig(&runConfig{promptID: "prompt-1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected omitted request to build, got %v", err)
|
||||||
|
}
|
||||||
|
if omitted.Execution != nil {
|
||||||
|
t.Fatalf("expected omitted reasoning flag to leave execution nil, got %#v", omitted.Execution)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
value string
|
||||||
|
}{
|
||||||
|
{name: "replacement", value: "high"},
|
||||||
|
{name: "clear", value: ""},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
req, err := buildRunRequestFromConfig(&runConfig{
|
||||||
|
promptID: "prompt-1",
|
||||||
|
sessionID: "session-1",
|
||||||
|
reasoningEffort: tc.value,
|
||||||
|
reasoningEffortSet: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected request to build, got %v", err)
|
||||||
|
}
|
||||||
|
if req.SessionID != "session-1" || req.Execution == nil || req.Execution.ReasoningEffort == nil || *req.Execution.ReasoningEffort != tc.value {
|
||||||
|
t.Fatalf("unexpected mapped request: %#v", req)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildRunRequestAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) {
|
||||||
|
req, err := buildRunRequestFromConfig(&runConfig{
|
||||||
|
promptID: "prompt-1",
|
||||||
|
promptVersion: "2",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected request without inputs to build, got %v", err)
|
||||||
|
}
|
||||||
|
if req.PromptVersion != "2" {
|
||||||
|
t.Fatalf("expected prompt version to be mapped, got %q", req.PromptVersion)
|
||||||
|
}
|
||||||
|
if req.Inputs != nil {
|
||||||
|
t.Fatalf("expected omitted inputs to remain nil, got %#v", req.Inputs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseRunArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) {
|
func TestParseRunArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) {
|
||||||
configPath := writeAppConfigFile(t, `
|
configPath := writeAppConfigFile(t, `
|
||||||
profile_dir: ./profiles
|
profile_dir: ./profiles
|
||||||
@@ -765,13 +828,13 @@ func TestDetermineExitCode(t *testing.T) {
|
|||||||
if got := determineExitCode(errors.New("boom"), nil); got != ExitRuntimeError {
|
if got := determineExitCode(errors.New("boom"), nil); got != ExitRuntimeError {
|
||||||
t.Fatalf("expected runtime exit code, got %d", got)
|
t.Fatalf("expected runtime exit code, got %d", got)
|
||||||
}
|
}
|
||||||
if got := determineExitCode(nil, &scriptorium.RunResult{Validation: scriptorium.ValidationResult{Status: scriptorium.ValidationFailed}}); got != ExitValidationFailed {
|
if got := determineExitCode(nil, &promptkit.RunResult{Validation: promptkit.ValidationResult{Status: promptkit.ValidationFailed}}); got != ExitValidationFailed {
|
||||||
t.Fatalf("expected validation exit code, got %d", got)
|
t.Fatalf("expected validation exit code, got %d", got)
|
||||||
}
|
}
|
||||||
if got := determineExitCode(nil, &scriptorium.RunResult{Validation: scriptorium.ValidationResult{Status: scriptorium.ValidationPassed}}); got != ExitOK {
|
if got := determineExitCode(nil, &promptkit.RunResult{Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed}}); got != ExitOK {
|
||||||
t.Fatalf("expected success exit code for passed validation, got %d", got)
|
t.Fatalf("expected success exit code for passed validation, got %d", got)
|
||||||
}
|
}
|
||||||
if got := determineExitCode(nil, &scriptorium.RunResult{Validation: scriptorium.ValidationResult{Status: scriptorium.ValidationSkipped}}); got != ExitOK {
|
if got := determineExitCode(nil, &promptkit.RunResult{Validation: promptkit.ValidationResult{Status: promptkit.ValidationSkipped}}); got != ExitOK {
|
||||||
t.Fatalf("expected success exit code for skipped validation, got %d", got)
|
t.Fatalf("expected success exit code for skipped validation, got %d", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -942,6 +1005,452 @@ profile_dir: %s
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRenderCommandUsesConfiguredCustomBackend(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom
|
||||||
|
version: "1"
|
||||||
|
default_profile: local-gpu
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "hello"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`id: local-gpu
|
||||||
|
backend: local-gpu
|
||||||
|
model: local-model
|
||||||
|
`), 0o644); err != nil {
|
||||||
|
t.Fatalf("write profile fixture: %v", err)
|
||||||
|
}
|
||||||
|
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
||||||
|
prompt_dir: %s
|
||||||
|
profile_dir: %s
|
||||||
|
backends:
|
||||||
|
local-gpu:
|
||||||
|
endpoint: http://localhost:11434/v1
|
||||||
|
extra_params:
|
||||||
|
provider_option: enabled
|
||||||
|
concurrency_limit: 2
|
||||||
|
queue_capacity: 0
|
||||||
|
`, lib.promptDir, lib.profileDir))
|
||||||
|
|
||||||
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--config", configPath,
|
||||||
|
"--prompt", "custom",
|
||||||
|
})
|
||||||
|
if code != ExitOK {
|
||||||
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout, "selected_backend_id: local-gpu") {
|
||||||
|
t.Fatalf("expected configured backend in prepared output, got:\n%s", stdout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderCommandMapsReasoningEffortAndSessionID(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
writePromptDefinition(t, lib.promptDir, "session.yaml", `id: session
|
||||||
|
version: "1"
|
||||||
|
default_profile: local
|
||||||
|
session_id: definition-session
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "hello"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
if err := os.WriteFile(filepath.Join(lib.profileDir, "local.yaml"), []byte(`id: local
|
||||||
|
endpoint: http://127.0.0.1:1/v1
|
||||||
|
model: local-model
|
||||||
|
reasoning_effort: low
|
||||||
|
`), 0o644); err != nil {
|
||||||
|
t.Fatalf("write profile fixture: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
wantReasoning string
|
||||||
|
wantSessionID string
|
||||||
|
absentReasoning bool
|
||||||
|
}{
|
||||||
|
{name: "omitted reasoning inherits profile", wantReasoning: "low", wantSessionID: "definition-session"},
|
||||||
|
{name: "nonblank reasoning replaces profile", args: []string{"--reasoning-effort", "high"}, wantReasoning: "high", wantSessionID: "definition-session"},
|
||||||
|
{name: "empty reasoning clears profile", args: []string{"--reasoning-effort="}, wantSessionID: "definition-session", absentReasoning: true},
|
||||||
|
{name: "direct session replaces definition", args: []string{"--session-id", "direct-session"}, wantReasoning: "low", wantSessionID: "direct-session"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
args := []string{"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "session"}
|
||||||
|
args = append(args, tc.args...)
|
||||||
|
code, stdout, stderr := runCLICommand(t, renderCommand, args)
|
||||||
|
if code != ExitOK {
|
||||||
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout, "session_id: "+tc.wantSessionID) {
|
||||||
|
t.Fatalf("expected session ID %q, got:\n%s", tc.wantSessionID, stdout)
|
||||||
|
}
|
||||||
|
hasReasoning := strings.Contains(stdout, "reasoning_effort:")
|
||||||
|
if tc.absentReasoning {
|
||||||
|
if hasReasoning {
|
||||||
|
t.Fatalf("expected cleared reasoning to be omitted, got:\n%s", stdout)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout, "reasoning_effort: "+tc.wantReasoning) {
|
||||||
|
t.Fatalf("expected reasoning effort %q, got:\n%s", tc.wantReasoning, stdout)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderCommandOmitsEmptyEffectiveSessionID(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
writePromptDefinition(t, lib.promptDir, "plain.yaml", `id: plain
|
||||||
|
version: "1"
|
||||||
|
default_profile: local
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "hello"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
|
||||||
|
|
||||||
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--prompt-dir", lib.promptDir,
|
||||||
|
"--profile-dir", lib.profileDir,
|
||||||
|
"--prompt", "plain",
|
||||||
|
})
|
||||||
|
if code != ExitOK {
|
||||||
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
if strings.Contains(stdout, "session_id:") {
|
||||||
|
t.Fatalf("expected no effective session ID, got:\n%s", stdout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderCommandRejectsOverlongSessionID(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
writePromptDefinition(t, lib.promptDir, "session.yaml", `id: session
|
||||||
|
version: "1"
|
||||||
|
default_profile: local
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "hello"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
|
||||||
|
|
||||||
|
code, _, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--prompt-dir", lib.promptDir,
|
||||||
|
"--profile-dir", lib.profileDir,
|
||||||
|
"--prompt", "session",
|
||||||
|
"--session-id", strings.Repeat("x", 257),
|
||||||
|
})
|
||||||
|
if code != ExitRuntimeError {
|
||||||
|
t.Fatalf("expected runtime error, got %d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr, "invalid run request") {
|
||||||
|
t.Fatalf("expected invalid-request context, got %q", stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectPromptCommandFormatsDefinitionWithoutProfileOrGeneration(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
writePromptDefinition(t, lib.promptDir, "inspect.yaml", `id: inspect
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "hello"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--prompt-dir", lib.promptDir, "--prompt", "inspect", "--format", "json"})
|
||||||
|
if code != ExitOK {
|
||||||
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout, `"prompt_id": "inspect"`) || !strings.Contains(stdout, `"inputs": []`) {
|
||||||
|
t.Fatalf("unexpected inspection output: %s", stdout)
|
||||||
|
}
|
||||||
|
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--prompt-dir", lib.promptDir, "--prompt", "missing"})
|
||||||
|
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
|
||||||
|
t.Fatalf("expected failed inspection without output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectionParsersPreserveExplicitEmptyConfigPath(t *testing.T) {
|
||||||
|
promptConfig, err := parsePromptInspectionArgs([]string{"--config=", "--prompt", "fixture"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse prompt inspection: %v", err)
|
||||||
|
}
|
||||||
|
if promptConfig.configPath != "" || !promptConfig.configExplicit {
|
||||||
|
t.Fatalf("expected explicit empty prompt config path, got %+v", promptConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
profileConfig, err := parseProfileInspectionArgs([]string{"--config=", "--profile", "fixture"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse profile inspection: %v", err)
|
||||||
|
}
|
||||||
|
if profileConfig.configPath != "" || !profileConfig.configExplicit {
|
||||||
|
t.Fatalf("expected explicit empty profile config path, got %+v", profileConfig)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptkitV09DefinitionsRenderThroughCLI(t *testing.T) {
|
||||||
|
fixtureRoot := promptkitV09FixtureRoot(t)
|
||||||
|
configPath := writePromptkitV09Config(t, fixtureRoot, true)
|
||||||
|
inputPath := filepath.Join(fixtureRoot, "inputs", "source.md")
|
||||||
|
|
||||||
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--config", configPath,
|
||||||
|
"--prompt", "compat.complete",
|
||||||
|
"--prompt-version", "1.0.0",
|
||||||
|
"--input", "source=" + inputPath,
|
||||||
|
"--var", "topic=testing",
|
||||||
|
"--format", "json",
|
||||||
|
})
|
||||||
|
if code != ExitOK {
|
||||||
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
|
||||||
|
var payload map[string]any
|
||||||
|
if err := json.Unmarshal([]byte(stdout), &payload); err != nil {
|
||||||
|
t.Fatalf("decode rendered fixture: %v\nbody=%s", err, stdout)
|
||||||
|
}
|
||||||
|
if payload["prompt_version"] != "1.0.0" || payload["selected_profile_id"] != "custom-derived" || payload["selected_backend_id"] != "fixture-custom" {
|
||||||
|
t.Fatalf("unexpected selected definition and target: %#v", payload)
|
||||||
|
}
|
||||||
|
if payload["session_id"] != "fixture-testing" {
|
||||||
|
t.Fatalf("expected rendered session template, got %#v", payload["session_id"])
|
||||||
|
}
|
||||||
|
|
||||||
|
outputContract := payload["output_contract"].(map[string]any)
|
||||||
|
if outputContract["format"] != "json" || outputContract["validation_mode"] != "json_schema" || outputContract["repair_attempts"] != float64(2) {
|
||||||
|
t.Fatalf("unexpected output contract: %#v", outputContract)
|
||||||
|
}
|
||||||
|
if _, ok := payload["structured_output"].(map[string]any)["json_schema"]; !ok {
|
||||||
|
t.Fatalf("expected loaded JSON schema metadata, got %#v", payload["structured_output"])
|
||||||
|
}
|
||||||
|
|
||||||
|
messages := payload["messages"].([]any)
|
||||||
|
wantRoles := []string{"developer", "system", "user", "assistant"}
|
||||||
|
for i, wantRole := range wantRoles {
|
||||||
|
message := messages[i].(map[string]any)
|
||||||
|
if message["role"] != wantRole {
|
||||||
|
t.Fatalf("message %d: expected role %q, got %#v", i, wantRole, message["role"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cacheControl := messages[0].(map[string]any)["cache_control"].(map[string]any)
|
||||||
|
if cacheControl["type"] != "ephemeral" || cacheControl["ttl"] != "1h" {
|
||||||
|
t.Fatalf("unexpected cache control: %#v", cacheControl)
|
||||||
|
}
|
||||||
|
if !strings.Contains(messages[2].(map[string]any)["content"].(string), "stable, synthetic material") {
|
||||||
|
t.Fatalf("file-backed input template was not rendered: %#v", messages[2])
|
||||||
|
}
|
||||||
|
inputHashes := payload["input_hashes"].(map[string]any)
|
||||||
|
if len(inputHashes) != 1 || inputHashes["source"] == "" {
|
||||||
|
t.Fatalf("required and omitted optional inputs were not preserved: %#v", inputHashes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptkitV09PromptInspectionSelectsVersionsAndContracts(t *testing.T) {
|
||||||
|
fixtureRoot := promptkitV09FixtureRoot(t)
|
||||||
|
configPath := writePromptkitV09Config(t, fixtureRoot, true)
|
||||||
|
tests := []struct {
|
||||||
|
promptID string
|
||||||
|
version string
|
||||||
|
format string
|
||||||
|
mode string
|
||||||
|
}{
|
||||||
|
{promptID: "compat.complete", version: "1.0.0", format: "json", mode: "json_schema"},
|
||||||
|
{promptID: "compat.complete", version: "2.0.0", format: "text", mode: "basic"},
|
||||||
|
{promptID: "compat.json", version: "1.0.0", format: "json", mode: "json"},
|
||||||
|
{promptID: "compat.none", version: "1.0.0", format: "text", mode: "none"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.promptID+"@"+tc.version, func(t *testing.T) {
|
||||||
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
||||||
|
"prompt",
|
||||||
|
"--config", configPath,
|
||||||
|
"--prompt", tc.promptID,
|
||||||
|
"--prompt-version", tc.version,
|
||||||
|
"--format", "json",
|
||||||
|
})
|
||||||
|
if code != ExitOK {
|
||||||
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
var inspection renderformat.PromptInspection
|
||||||
|
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
|
||||||
|
t.Fatalf("decode prompt inspection: %v\nbody=%s", err, stdout)
|
||||||
|
}
|
||||||
|
if inspection.PromptVersion != tc.version || inspection.OutputContract.Format != tc.format || inspection.OutputContract.ValidationMode != tc.mode {
|
||||||
|
t.Fatalf("unexpected prompt inspection: %+v", inspection)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptkitV09ProfileInspectionResolvesSupportedTargets(t *testing.T) {
|
||||||
|
const secret = "sentinel-profile-secret"
|
||||||
|
t.Setenv("FIXTURE_PROFILE_API_KEY", secret)
|
||||||
|
|
||||||
|
fixtureRoot := promptkitV09FixtureRoot(t)
|
||||||
|
configPath := writePromptkitV09Config(t, fixtureRoot, true)
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
profileID string
|
||||||
|
wantBackend string
|
||||||
|
wantModel string
|
||||||
|
wantAPIKeyEnv string
|
||||||
|
}{
|
||||||
|
{name: "inherited custom backend", profileID: "custom-derived", wantBackend: "fixture-custom", wantModel: "fixture-derived-model", wantAPIKeyEnv: "FIXTURE_PROFILE_API_KEY"},
|
||||||
|
{name: "endpoint only", profileID: "endpoint-only", wantModel: "fixture-endpoint-model"},
|
||||||
|
{name: "built in", profileID: "deepseek-4-flash", wantBackend: "openrouter", wantModel: "deepseek/deepseek-v4-flash", wantAPIKeyEnv: "OPENROUTER_API_KEY"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
args := []string{"profile", "--config", configPath, "--profile", tc.profileID, "--format", "json"}
|
||||||
|
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
|
||||||
|
if code != ExitOK {
|
||||||
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
if strings.Contains(stdout, secret) || strings.Contains(stderr, secret) {
|
||||||
|
t.Fatalf("inspection exposed environment secret: stdout=%q stderr=%q", stdout, stderr)
|
||||||
|
}
|
||||||
|
var inspection renderformat.ProfileInspection
|
||||||
|
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
|
||||||
|
t.Fatalf("decode profile inspection: %v\nbody=%s", err, stdout)
|
||||||
|
}
|
||||||
|
params := inspection.EffectiveModelParams
|
||||||
|
if inspection.ProfileID != tc.profileID || params.BackendID != tc.wantBackend || params.Model != tc.wantModel || params.APIKeyEnv != tc.wantAPIKeyEnv {
|
||||||
|
t.Fatalf("unexpected effective profile: %+v", inspection)
|
||||||
|
}
|
||||||
|
if tc.profileID == "custom-derived" {
|
||||||
|
if params.ServiceTier != "flex" || params.ReasoningEffort != "high" || params.TimeoutSeconds != 45 || len(params.ExtraParams) == 0 {
|
||||||
|
t.Fatalf("inherited profile controls were not resolved: %+v", params)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
configWithoutPromptDir := writePromptkitV09Config(t, fixtureRoot, false)
|
||||||
|
code, _, stderr := runCLICommand(t, inspectCommand, []string{
|
||||||
|
"profile",
|
||||||
|
"--config", configWithoutPromptDir,
|
||||||
|
"--profile-dir", filepath.Join(fixtureRoot, "profiles"),
|
||||||
|
"--profile", "custom-derived",
|
||||||
|
})
|
||||||
|
if code != ExitOK {
|
||||||
|
t.Fatalf("profile inspection unexpectedly required a prompt directory: %q", stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.T) {
|
||||||
|
fixtureRoot := promptkitV09FixtureRoot(t)
|
||||||
|
configPath := writePromptkitV09Config(t, fixtureRoot, true)
|
||||||
|
overrideDir := t.TempDir()
|
||||||
|
writeProfileFile(t, overrideDir, "custom-derived", "http://127.0.0.1:9000/v1", "override-model")
|
||||||
|
outPath := filepath.Join(t.TempDir(), "inspection.json")
|
||||||
|
|
||||||
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
||||||
|
"profile",
|
||||||
|
"--config", configPath,
|
||||||
|
"--profile-dir", overrideDir,
|
||||||
|
"--profile", "custom-derived",
|
||||||
|
"--format", "json",
|
||||||
|
"--out", outPath,
|
||||||
|
})
|
||||||
|
if code != ExitOK || stdout != "" {
|
||||||
|
t.Fatalf("expected file output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
output, err := os.ReadFile(outPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read profile inspection output: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(output), `"model": "override-model"`) || !strings.Contains(string(output), `"backend_id": ""`) {
|
||||||
|
t.Fatalf("profile directory override was not used: %s", output)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, profileID := range []string{"missing", "invalid"} {
|
||||||
|
t.Run(profileID, func(t *testing.T) {
|
||||||
|
profileDir := filepath.Join(fixtureRoot, "profiles")
|
||||||
|
if profileID == "invalid" {
|
||||||
|
profileDir = t.TempDir()
|
||||||
|
writePromptDefinition(t, profileDir, "invalid.yaml", "id: invalid\nendpoint: not-a-url\nmodel: fixture\n")
|
||||||
|
}
|
||||||
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
||||||
|
"profile",
|
||||||
|
"--config", configPath,
|
||||||
|
"--profile-dir", profileDir,
|
||||||
|
"--profile", profileID,
|
||||||
|
})
|
||||||
|
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
|
||||||
|
t.Fatalf("expected inspection failure without partial output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
backend string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "reserved ID",
|
||||||
|
backend: `openrouter:
|
||||||
|
endpoint: http://localhost:11434/v1`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid endpoint",
|
||||||
|
backend: `local-gpu:
|
||||||
|
endpoint: not-a-url`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid capacity relationship",
|
||||||
|
backend: `local-gpu:
|
||||||
|
endpoint: http://localhost:11434/v1
|
||||||
|
queue_capacity: 0`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "reserved extra parameter",
|
||||||
|
backend: `local-gpu:
|
||||||
|
endpoint: http://localhost:11434/v1
|
||||||
|
extra_params:
|
||||||
|
model: forbidden`,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
||||||
|
prompt_dir: %s
|
||||||
|
profile_dir: %s
|
||||||
|
backends:
|
||||||
|
%s
|
||||||
|
`, lib.promptDir, lib.profileDir, tc.backend))
|
||||||
|
cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected config decoding to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
_, err = newEngine(cfg.runConfig.engineSettings())
|
||||||
|
if !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("expected Promptkit ErrInvalidConfig, got %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "engine initialization from application configuration") {
|
||||||
|
t.Fatalf("expected application context, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRenderCommandExplicitTextFormatWorks(t *testing.T) {
|
func TestRenderCommandExplicitTextFormatWorks(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
||||||
@@ -1122,6 +1631,104 @@ func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRenderCommandUsesDefinitionInputRulesAndPromptVersions(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "model")
|
||||||
|
|
||||||
|
writePromptDefinition(t, lib.promptDir, "sole.yaml", `id: sole
|
||||||
|
version: "1"
|
||||||
|
default_profile: local
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "hello"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
writePromptDefinition(t, lib.promptDir, "versioned-one.yaml", `id: versioned
|
||||||
|
version: "1"
|
||||||
|
default_profile: local
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "one"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
writePromptDefinition(t, lib.promptDir, "versioned-two.yaml", `id: versioned
|
||||||
|
version: "2"
|
||||||
|
default_profile: local
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "two"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
writePromptDefinition(t, lib.promptDir, "optional.yaml", `id: optional
|
||||||
|
version: "1"
|
||||||
|
default_profile: local
|
||||||
|
inputs:
|
||||||
|
- name: note
|
||||||
|
required: false
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "hello"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
writePromptDefinition(t, lib.promptDir, "required.yaml", `id: required
|
||||||
|
version: "1"
|
||||||
|
default_profile: local
|
||||||
|
inputs:
|
||||||
|
- name: note
|
||||||
|
required: true
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "hello"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
writePromptDefinition(t, lib.promptDir, "template.yaml", `id: template
|
||||||
|
version: "1"
|
||||||
|
default_profile: local
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: '{{input "note"}}'
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
|
||||||
|
baseArgs := []string{"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
wantCode int
|
||||||
|
wantText string
|
||||||
|
}{
|
||||||
|
{name: "sole version selected when omitted", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt_version: 1"},
|
||||||
|
{name: "explicit version selected", args: []string{"--prompt", "versioned", "--prompt-version", "2"}, wantCode: ExitOK, wantText: "prompt_version: 2"},
|
||||||
|
{name: "multiple versions require selection", args: []string{"--prompt", "versioned"}, wantCode: ExitRuntimeError, wantText: "duplicate prompt definition id"},
|
||||||
|
{name: "no declared inputs", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt: sole"},
|
||||||
|
{name: "optional input omitted", args: []string{"--prompt", "optional"}, wantCode: ExitOK, wantText: "prompt: optional"},
|
||||||
|
{name: "required input omitted", args: []string{"--prompt", "required"}, wantCode: ExitRuntimeError, wantText: "required"},
|
||||||
|
{name: "template input omitted", args: []string{"--prompt", "template"}, wantCode: ExitRuntimeError, wantText: "note"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
code, stdout, stderr := runCLICommand(t, renderCommand, append(append([]string{}, baseArgs...), tc.args...))
|
||||||
|
if code != tc.wantCode {
|
||||||
|
t.Fatalf("expected exit %d, got %d stderr=%q", tc.wantCode, code, stderr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout+stderr, tc.wantText) {
|
||||||
|
t.Fatalf("expected output to contain %q, stdout=%q stderr=%q", tc.wantText, stdout, stderr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
|
func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||||
@@ -1242,12 +1849,12 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
|
|||||||
if err := writeOutput(&stdout, "", []byte("artifact-body")); err != nil {
|
if err := writeOutput(&stdout, "", []byte("artifact-body")); err != nil {
|
||||||
t.Fatalf("unexpected writeOutput error: %v", err)
|
t.Fatalf("unexpected writeOutput error: %v", err)
|
||||||
}
|
}
|
||||||
printSummary(&stderr, &scriptorium.RunResult{
|
printSummary(&stderr, &promptkit.RunResult{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
PromptVersion: "1",
|
PromptVersion: "1",
|
||||||
SelectedProfileID: "exec",
|
SelectedProfileID: "exec",
|
||||||
ModelName: "m",
|
ModelName: "m",
|
||||||
Validation: scriptorium.ValidationResult{Status: scriptorium.ValidationPassed, Mode: scriptorium.ValidationBasic},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic},
|
||||||
RenderedPromptHash: "h",
|
RenderedPromptHash: "h",
|
||||||
InputHashes: map[string]string{"in": "x"},
|
InputHashes: map[string]string{"in": "x"},
|
||||||
})
|
})
|
||||||
@@ -1261,20 +1868,23 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
|
|||||||
if strings.Contains(stderr.String(), "cached_tokens=") || strings.Contains(stderr.String(), "cache_write_tokens=") {
|
if strings.Contains(stderr.String(), "cached_tokens=") || strings.Contains(stderr.String(), "cache_write_tokens=") {
|
||||||
t.Fatalf("expected zero cache usage to be omitted from summary, got %q", stderr.String())
|
t.Fatalf("expected zero cache usage to be omitted from summary, got %q", stderr.String())
|
||||||
}
|
}
|
||||||
|
if strings.Contains(stderr.String(), "backend=") {
|
||||||
|
t.Fatalf("expected endpoint-only backend to be omitted from summary, got %q", stderr.String())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) {
|
func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) {
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
printSummary(&stderr, &scriptorium.RunResult{
|
printSummary(&stderr, &promptkit.RunResult{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
PromptVersion: "1",
|
PromptVersion: "1",
|
||||||
SelectedProfileID: "exec",
|
SelectedProfileID: "exec",
|
||||||
ModelName: "m",
|
ModelName: "m",
|
||||||
Validation: scriptorium.ValidationResult{Status: scriptorium.ValidationPassed, Mode: scriptorium.ValidationBasic},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic},
|
||||||
RenderedPromptHash: "h",
|
RenderedPromptHash: "h",
|
||||||
InputHashes: map[string]string{"in": "x"},
|
InputHashes: map[string]string{"in": "x"},
|
||||||
Usage: scriptorium.TokenUsage{
|
Usage: promptkit.TokenUsage{
|
||||||
PromptTokens: 10,
|
PromptTokens: 10,
|
||||||
CompletionTokens: 5,
|
CompletionTokens: 5,
|
||||||
TotalTokens: 15,
|
TotalTokens: 15,
|
||||||
@@ -1290,6 +1900,32 @@ func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) {
|
|||||||
if !strings.Contains(summary, "cached_tokens=0 cache_write_tokens=3") {
|
if !strings.Contains(summary, "cached_tokens=0 cache_write_tokens=3") {
|
||||||
t.Fatalf("expected cache usage in summary, got %q", summary)
|
t.Fatalf("expected cache usage in summary, got %q", summary)
|
||||||
}
|
}
|
||||||
|
if strings.Contains(summary, "backend=") {
|
||||||
|
t.Fatalf("expected endpoint-only backend to be omitted from summary, got %q", summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrintSummaryIncludesBackendWhenPresent(t *testing.T) {
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
printSummary(&stderr, &promptkit.RunResult{
|
||||||
|
PromptID: "p",
|
||||||
|
PromptVersion: "1",
|
||||||
|
SelectedProfileID: "exec",
|
||||||
|
SelectedBackendID: "local",
|
||||||
|
ModelName: "m",
|
||||||
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic},
|
||||||
|
RenderedPromptHash: "h",
|
||||||
|
})
|
||||||
|
if !strings.Contains(stderr.String(), "backend=local") {
|
||||||
|
t.Fatalf("expected backend in summary, got %q", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunErrorMessageDoesNotExposeCapacityDetails(t *testing.T) {
|
||||||
|
got := runErrorMessage(&promptkit.CapacityError{BackendID: "private-backend"})
|
||||||
|
if got != "run error: model backend capacity is exhausted" {
|
||||||
|
t.Fatalf("unexpected capacity diagnostic: %q", got)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type cliTestLibrary struct {
|
type cliTestLibrary struct {
|
||||||
@@ -1335,11 +1971,48 @@ func runCLICommand(t *testing.T, command func([]string, io.Writer, io.Writer) in
|
|||||||
return code, stdout.String(), stderr.String()
|
return code, stdout.String(), stderr.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func promptkitV09FixtureRoot(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
root, err := filepath.Abs(filepath.Join("..", "..", "..", "testdata", "promptkit-v0.9"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve Promptkit v0.9 fixture root: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(root); err != nil {
|
||||||
|
t.Fatalf("stat Promptkit v0.9 fixture root: %v", err)
|
||||||
|
}
|
||||||
|
return root
|
||||||
|
}
|
||||||
|
|
||||||
|
func writePromptkitV09Config(t *testing.T, fixtureRoot string, includeSources bool) string {
|
||||||
|
t.Helper()
|
||||||
|
sources := ""
|
||||||
|
if includeSources {
|
||||||
|
sources = fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\nschema_dir: %q\n", filepath.Join(fixtureRoot, "prompts"), filepath.Join(fixtureRoot, "profiles"), filepath.Join(fixtureRoot, "schemas"))
|
||||||
|
}
|
||||||
|
return writeAppConfigFile(t, sources+`backends:
|
||||||
|
fixture-custom:
|
||||||
|
endpoint: http://127.0.0.1:11434/v1
|
||||||
|
api_key_env: FIXTURE_BACKEND_API_KEY
|
||||||
|
extra_params:
|
||||||
|
backend_option:
|
||||||
|
enabled: true
|
||||||
|
concurrency_limit: 2
|
||||||
|
queue_capacity: 0
|
||||||
|
`)
|
||||||
|
}
|
||||||
|
|
||||||
func writePromptFile(t *testing.T, dir, id, defaultProfile string) {
|
func writePromptFile(t *testing.T, dir, id, defaultProfile string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
writePromptFileWithTemplate(t, dir, id, defaultProfile, "Summarize: {{input \"transcript\"}}")
|
writePromptFileWithTemplate(t, dir, id, defaultProfile, "Summarize: {{input \"transcript\"}}")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func writePromptDefinition(t *testing.T, dir, name, definition string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(definition), 0o644); err != nil {
|
||||||
|
t.Fatalf("write prompt definition: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func writePromptFileWithTemplate(t *testing.T, dir, id, defaultProfile, templateContent string) {
|
func writePromptFileWithTemplate(t *testing.T, dir, id, defaultProfile, templateContent string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
data := fmt.Sprintf(`id: %s
|
data := fmt.Sprintf(`id: %s
|
||||||
|
|||||||
@@ -13,120 +13,310 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
const scriptoriumModulePath = "gitea.maximumdirect.net/eric/scriptorium"
|
const (
|
||||||
|
scriptoriumModulePath = "gitea.maximumdirect.net/eric/scriptorium"
|
||||||
|
promptkitInternalPath = "gitea.maximumdirect.net/eric/promptkit/internal"
|
||||||
|
promptkitOpenRouterCatalogPath = "gitea.maximumdirect.net/eric/promptkit-backend-openrouter"
|
||||||
|
promptkitRakestrawhomeCatalogPath = "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome"
|
||||||
|
)
|
||||||
|
|
||||||
var forbiddenFrameworkPackageRoots = []string{
|
var (
|
||||||
scriptoriumModulePath + "/internal/domain",
|
removedFrameworkPackageRoots = []string{
|
||||||
scriptoriumModulePath + "/internal/usecase",
|
scriptoriumModulePath + "/internal/artifact",
|
||||||
scriptoriumModulePath + "/internal/promptdef",
|
scriptoriumModulePath + "/internal/domain",
|
||||||
scriptoriumModulePath + "/internal/prompt",
|
scriptoriumModulePath + "/internal/filecatalog",
|
||||||
scriptoriumModulePath + "/internal/profile",
|
scriptoriumModulePath + "/internal/llm",
|
||||||
scriptoriumModulePath + "/internal/validate",
|
scriptoriumModulePath + "/internal/profile",
|
||||||
scriptoriumModulePath + "/internal/llm",
|
scriptoriumModulePath + "/internal/prompt",
|
||||||
scriptoriumModulePath + "/internal/artifact",
|
scriptoriumModulePath + "/internal/promptdef",
|
||||||
}
|
scriptoriumModulePath + "/internal/usecase",
|
||||||
|
scriptoriumModulePath + "/internal/validate",
|
||||||
|
}
|
||||||
|
removedFrameworkDirectories = []string{
|
||||||
|
"internal/artifact",
|
||||||
|
"internal/domain",
|
||||||
|
"internal/filecatalog",
|
||||||
|
"internal/llm",
|
||||||
|
"internal/profile",
|
||||||
|
"internal/prompt",
|
||||||
|
"internal/promptdef",
|
||||||
|
"internal/usecase",
|
||||||
|
"internal/validate",
|
||||||
|
}
|
||||||
|
nonSourceDirectories = map[string]struct{}{
|
||||||
|
".cache": {},
|
||||||
|
".codebase-memory": {},
|
||||||
|
".git": {},
|
||||||
|
"build": {},
|
||||||
|
"coverage": {},
|
||||||
|
"dist": {},
|
||||||
|
"node_modules": {},
|
||||||
|
"out": {},
|
||||||
|
"testdata": {},
|
||||||
|
"vendor": {},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
type forbiddenFrameworkImport struct {
|
type forbiddenImport struct {
|
||||||
filePath string
|
filePath string
|
||||||
importPath string
|
importPath string
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestScriptoriumAdaptersUseOnlyPublicFrameworkBoundary(t *testing.T) {
|
func TestApplicationBoundary(t *testing.T) {
|
||||||
_, testFile, _, ok := runtime.Caller(0)
|
moduleRoot := moduleRootFromTestFile(t)
|
||||||
if !ok {
|
|
||||||
t.Fatal("locate dependency guard source")
|
|
||||||
}
|
|
||||||
|
|
||||||
adapterDir := filepath.Dir(testFile)
|
violations, err := findForbiddenProductionImports(moduleRoot)
|
||||||
violations, err := findForbiddenFrameworkImports([]string{
|
|
||||||
filepath.Join(adapterDir, "cli"),
|
|
||||||
filepath.Join(adapterDir, "http"),
|
|
||||||
filepath.Join(adapterDir, "..", "format"),
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("scan framework imports: %v", err)
|
t.Fatalf("scan production imports: %v", err)
|
||||||
}
|
}
|
||||||
for _, violation := range violations {
|
for _, violation := range violations {
|
||||||
t.Errorf("%s imports forbidden framework package %s", violation.filePath, violation.importPath)
|
t.Errorf("%s imports forbidden package %s", violation.filePath, violation.importPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertFrameworkImplementationAbsent(t, moduleRoot)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestForbiddenImportScannerDetectsFormerRoot(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
sourcePath := writeGoSource(t, root, "nested/consumer/root.go", scriptoriumModulePath)
|
||||||
|
|
||||||
|
violations, err := findForbiddenProductionImports(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("scan source fixture: %v", err)
|
||||||
|
}
|
||||||
|
assertSingleViolation(t, violations, sourcePath, scriptoriumModulePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestForbiddenImportScannerDetectsFormerFrameworkFamily(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
importPath := scriptoriumModulePath + "/internal/profile/builtin"
|
||||||
|
sourcePath := writeGoSource(t, root, "nested/consumer/profile.go", importPath)
|
||||||
|
|
||||||
|
violations, err := findForbiddenProductionImports(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("scan source fixture: %v", err)
|
||||||
|
}
|
||||||
|
assertSingleViolation(t, violations, sourcePath, importPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestForbiddenImportScannerDetectsPromptkitInternalPackages(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
importPath string
|
||||||
|
}{
|
||||||
|
{name: "exact internal root", importPath: promptkitInternalPath},
|
||||||
|
{name: "internal descendant", importPath: promptkitInternalPath + "/domain"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
sourcePath := writeGoSource(t, root, "nested/consumer/promptkit.go", tc.importPath)
|
||||||
|
|
||||||
|
violations, err := findForbiddenProductionImports(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("scan source fixture: %v", err)
|
||||||
|
}
|
||||||
|
assertSingleViolation(t, violations, sourcePath, tc.importPath)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestForbiddenFrameworkImportScannerDetectsNestedPackageFamilies(t *testing.T) {
|
func TestForbiddenImportScannerDetectsPromptkitCatalogPackages(t *testing.T) {
|
||||||
root := t.TempDir()
|
for _, importPath := range []string{
|
||||||
nestedDir := filepath.Join(root, "nested", "adapter")
|
promptkitOpenRouterCatalogPath,
|
||||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
promptkitRakestrawhomeCatalogPath,
|
||||||
t.Fatalf("create nested source directory: %v", err)
|
} {
|
||||||
}
|
t.Run(importPath, func(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
sourcePath := writeGoSource(t, root, "nested/consumer/catalog.go", importPath)
|
||||||
|
|
||||||
sourcePath := filepath.Join(nestedDir, "imports.go")
|
violations, err := findForbiddenProductionImports(root)
|
||||||
source := `package nested
|
if err != nil {
|
||||||
|
t.Fatalf("scan source fixture: %v", err)
|
||||||
|
}
|
||||||
|
assertSingleViolation(t, violations, sourcePath, importPath)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestForbiddenImportScannerAllowsRetainedApplicationPackages(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
sourcePath := filepath.Join(root, "nested/consumer/application.go")
|
||||||
|
if err := os.MkdirAll(filepath.Dir(sourcePath), 0o755); err != nil {
|
||||||
|
t.Fatalf("create source fixture directory: %v", err)
|
||||||
|
}
|
||||||
|
source := `package consumer
|
||||||
|
|
||||||
import (
|
import (
|
||||||
_ "gitea.maximumdirect.net/eric/scriptorium"
|
_ "gitea.maximumdirect.net/eric/promptkit"
|
||||||
_ "gitea.maximumdirect.net/eric/scriptorium/internal/profile/builtin"
|
_ "gitea.maximumdirect.net/eric/scriptorium/internal/adapter/http"
|
||||||
|
_ "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||||
|
_ "gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
|
_ "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||||
)
|
)
|
||||||
`
|
`
|
||||||
if err := os.WriteFile(sourcePath, []byte(source), 0o644); err != nil {
|
if err := os.WriteFile(sourcePath, []byte(source), 0o644); err != nil {
|
||||||
t.Fatalf("write nested source fixture: %v", err)
|
t.Fatalf("write source fixture: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
violations, err := findForbiddenFrameworkImports([]string{root})
|
violations, err := findForbiddenProductionImports(root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("scan nested source fixture: %v", err)
|
t.Fatalf("scan source fixture: %v", err)
|
||||||
}
|
}
|
||||||
if len(violations) != 1 {
|
if len(violations) != 0 {
|
||||||
t.Fatalf("expected one forbidden import, got %#v", violations)
|
t.Fatalf("expected retained application imports to be allowed, got %#v", violations)
|
||||||
}
|
|
||||||
if violations[0].filePath != sourcePath {
|
|
||||||
t.Fatalf("unexpected importing file: %q", violations[0].filePath)
|
|
||||||
}
|
|
||||||
wantImport := scriptoriumModulePath + "/internal/profile/builtin"
|
|
||||||
if violations[0].importPath != wantImport {
|
|
||||||
t.Fatalf("unexpected forbidden import: %q", violations[0].importPath)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func findForbiddenFrameworkImports(roots []string) ([]forbiddenFrameworkImport, error) {
|
func findForbiddenProductionImports(root string) ([]forbiddenImport, error) {
|
||||||
var violations []forbiddenFrameworkImport
|
var violations []forbiddenImport
|
||||||
for _, root := range roots {
|
err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
|
||||||
err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
|
if err != nil {
|
||||||
if err != nil {
|
return err
|
||||||
return err
|
}
|
||||||
}
|
if entry.IsDir() {
|
||||||
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".go") || strings.HasSuffix(entry.Name(), "_test.go") {
|
if path != root && shouldSkipSourceDirectory(entry.Name()) {
|
||||||
return nil
|
return filepath.SkipDir
|
||||||
}
|
|
||||||
|
|
||||||
file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.ImportsOnly)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("parse imports in %s: %w", path, err)
|
|
||||||
}
|
|
||||||
for _, imported := range file.Imports {
|
|
||||||
importPath, err := strconv.Unquote(imported.Path.Value)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("parse import path in %s: %w", path, err)
|
|
||||||
}
|
|
||||||
if isForbiddenFrameworkImport(importPath) {
|
|
||||||
violations = append(violations, forbiddenFrameworkImport{
|
|
||||||
filePath: path,
|
|
||||||
importPath: importPath,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("walk source root %s: %w", root, err)
|
|
||||||
}
|
}
|
||||||
|
if !strings.HasSuffix(entry.Name(), ".go") || strings.HasSuffix(entry.Name(), "_test.go") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.ImportsOnly)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("parse imports in %s: %w", path, err)
|
||||||
|
}
|
||||||
|
for _, imported := range file.Imports {
|
||||||
|
importPath, err := strconv.Unquote(imported.Path.Value)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("parse import path in %s: %w", path, err)
|
||||||
|
}
|
||||||
|
if isForbiddenProductionImport(importPath) {
|
||||||
|
violations = append(violations, forbiddenImport{
|
||||||
|
filePath: path,
|
||||||
|
importPath: importPath,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("walk repository root %s: %w", root, err)
|
||||||
}
|
}
|
||||||
return violations, nil
|
return violations, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func isForbiddenFrameworkImport(importPath string) bool {
|
func shouldSkipSourceDirectory(name string) bool {
|
||||||
for _, root := range forbiddenFrameworkPackageRoots {
|
_, skip := nonSourceDirectories[name]
|
||||||
|
return skip
|
||||||
|
}
|
||||||
|
|
||||||
|
func isForbiddenProductionImport(importPath string) bool {
|
||||||
|
if importPath == scriptoriumModulePath {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if importPath == promptkitInternalPath || strings.HasPrefix(importPath, promptkitInternalPath+"/") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if importPath == promptkitOpenRouterCatalogPath || strings.HasPrefix(importPath, promptkitOpenRouterCatalogPath+"/") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if importPath == promptkitRakestrawhomeCatalogPath || strings.HasPrefix(importPath, promptkitRakestrawhomeCatalogPath+"/") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, root := range removedFrameworkPackageRoots {
|
||||||
if importPath == root || strings.HasPrefix(importPath, root+"/") {
|
if importPath == root || strings.HasPrefix(importPath, root+"/") {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func moduleRootFromTestFile(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
_, testFile, _, ok := runtime.Caller(0)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("locate dependency guard source")
|
||||||
|
}
|
||||||
|
root, err := findModuleRoot(filepath.Dir(testFile))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return root
|
||||||
|
}
|
||||||
|
|
||||||
|
func findModuleRoot(start string) (string, error) {
|
||||||
|
dir, err := filepath.Abs(start)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("resolve module search path: %w", err)
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
goMod := filepath.Join(dir, "go.mod")
|
||||||
|
if info, err := os.Stat(goMod); err == nil && !info.IsDir() {
|
||||||
|
return dir, nil
|
||||||
|
} else if err != nil && !os.IsNotExist(err) {
|
||||||
|
return "", fmt.Errorf("inspect %s: %w", goMod, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
parent := filepath.Dir(dir)
|
||||||
|
if parent == dir {
|
||||||
|
return "", fmt.Errorf("locate go.mod from %s", start)
|
||||||
|
}
|
||||||
|
dir = parent
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertFrameworkImplementationAbsent(t *testing.T, moduleRoot string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
entries, err := os.ReadDir(moduleRoot)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read module root: %v", err)
|
||||||
|
}
|
||||||
|
for _, entry := range entries {
|
||||||
|
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".go") && !strings.HasSuffix(entry.Name(), "_test.go") {
|
||||||
|
t.Errorf("module root contains production Go file %s", entry.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, relativePath := range removedFrameworkDirectories {
|
||||||
|
path := filepath.Join(moduleRoot, filepath.FromSlash(relativePath))
|
||||||
|
if _, err := os.Stat(path); err == nil {
|
||||||
|
t.Errorf("removed framework directory still exists: %s", relativePath)
|
||||||
|
} else if !os.IsNotExist(err) {
|
||||||
|
t.Errorf("inspect removed framework directory %s: %v", relativePath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeGoSource(t *testing.T, root, relativePath, importPath string) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
sourcePath := filepath.Join(root, filepath.FromSlash(relativePath))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(sourcePath), 0o755); err != nil {
|
||||||
|
t.Fatalf("create source fixture directory: %v", err)
|
||||||
|
}
|
||||||
|
source := fmt.Sprintf("package consumer\n\nimport _ %q\n", importPath)
|
||||||
|
if err := os.WriteFile(sourcePath, []byte(source), 0o644); err != nil {
|
||||||
|
t.Fatalf("write source fixture: %v", err)
|
||||||
|
}
|
||||||
|
return sourcePath
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertSingleViolation(t *testing.T, violations []forbiddenImport, sourcePath, importPath string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
if len(violations) != 1 {
|
||||||
|
t.Fatalf("expected one forbidden import, got %#v", violations)
|
||||||
|
}
|
||||||
|
if violations[0].filePath != sourcePath {
|
||||||
|
t.Fatalf("unexpected importing file: %q", violations[0].filePath)
|
||||||
|
}
|
||||||
|
if violations[0].importPath != importPath {
|
||||||
|
t.Fatalf("unexpected forbidden import: %q", violations[0].importPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -25,7 +25,7 @@ const fallbackArtifactContentType = "text/plain"
|
|||||||
// NewRestrictedArtifactReader creates the HTTP artifact reader for a rooted
|
// NewRestrictedArtifactReader creates the HTTP artifact reader for a rooted
|
||||||
// filesystem and optional byte limit. An empty root permits inline artifacts
|
// filesystem and optional byte limit. An empty root permits inline artifacts
|
||||||
// but denies file references; a zero limit permits artifacts of any size.
|
// but denies file references; a zero limit permits artifacts of any size.
|
||||||
func NewRestrictedArtifactReader(root string, maxBytes int64) (scriptorium.ArtifactReader, error) {
|
func NewRestrictedArtifactReader(root string, maxBytes int64) (promptkit.ArtifactReader, error) {
|
||||||
if maxBytes < 0 {
|
if maxBytes < 0 {
|
||||||
return nil, fmt.Errorf("artifact size limit must be greater than or equal to 0")
|
return nil, fmt.Errorf("artifact size limit must be greater than or equal to 0")
|
||||||
}
|
}
|
||||||
@@ -46,9 +46,9 @@ type restrictedArtifactReader struct {
|
|||||||
maxBytes int64
|
maxBytes int64
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ scriptorium.ArtifactReader = (*restrictedArtifactReader)(nil)
|
var _ promptkit.ArtifactReader = (*restrictedArtifactReader)(nil)
|
||||||
|
|
||||||
func (r *restrictedArtifactReader) Read(ctx context.Context, ref scriptorium.ArtifactRef) (*scriptorium.Artifact, error) {
|
func (r *restrictedArtifactReader) Read(ctx context.Context, ref promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return nil, ctx.Err()
|
return nil, ctx.Err()
|
||||||
@@ -56,22 +56,22 @@ func (r *restrictedArtifactReader) Read(ctx context.Context, ref scriptorium.Art
|
|||||||
}
|
}
|
||||||
|
|
||||||
switch ref.Type {
|
switch ref.Type {
|
||||||
case scriptorium.ArtifactRefInline:
|
case promptkit.ArtifactRefInline:
|
||||||
return readInlineArtifact(ref)
|
return readInlineArtifact(ref)
|
||||||
case scriptorium.ArtifactRefFile:
|
case promptkit.ArtifactRefFile:
|
||||||
return r.readFileArtifact(ref)
|
return r.readFileArtifact(ref)
|
||||||
default:
|
default:
|
||||||
return nil, fmt.Errorf("unsupported artifact reference type %q", ref.Type)
|
return nil, fmt.Errorf("unsupported artifact reference type %q", ref.Type)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func readInlineArtifact(ref scriptorium.ArtifactRef) (*scriptorium.Artifact, error) {
|
func readInlineArtifact(ref promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||||
if ref.Body == "" {
|
if ref.Body == "" {
|
||||||
return nil, errors.New("inline artifact body is required")
|
return nil, errors.New("inline artifact body is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
body := []byte(ref.Body)
|
body := []byte(ref.Body)
|
||||||
return &scriptorium.Artifact{
|
return &promptkit.Artifact{
|
||||||
ContentType: fallbackArtifactContentType,
|
ContentType: fallbackArtifactContentType,
|
||||||
Body: body,
|
Body: body,
|
||||||
Size: int64(len(body)),
|
Size: int64(len(body)),
|
||||||
@@ -80,7 +80,7 @@ func readInlineArtifact(ref scriptorium.ArtifactRef) (*scriptorium.Artifact, err
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *restrictedArtifactReader) readFileArtifact(ref scriptorium.ArtifactRef) (*scriptorium.Artifact, error) {
|
func (r *restrictedArtifactReader) readFileArtifact(ref promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||||
if ref.URI == "" {
|
if ref.URI == "" {
|
||||||
return nil, errors.New("file artifact path is required")
|
return nil, errors.New("file artifact path is required")
|
||||||
}
|
}
|
||||||
@@ -119,7 +119,7 @@ func (r *restrictedArtifactReader) resolveLexicalPath(rawPath string) (string, e
|
|||||||
return absCandidate, nil
|
return absCandidate, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func readArtifactFile(path string, maxBytes int64) (*scriptorium.Artifact, error) {
|
func readArtifactFile(path string, maxBytes int64) (*promptkit.Artifact, error) {
|
||||||
file, err := os.Open(path)
|
file, err := os.Open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
||||||
@@ -150,7 +150,7 @@ func readArtifactFile(path string, maxBytes int64) (*scriptorium.Artifact, error
|
|||||||
if contentType == "" {
|
if contentType == "" {
|
||||||
contentType = fallbackArtifactContentType
|
contentType = fallbackArtifactContentType
|
||||||
}
|
}
|
||||||
return &scriptorium.Artifact{
|
return &promptkit.Artifact{
|
||||||
Name: filepath.Base(path),
|
Name: filepath.Base(path),
|
||||||
ContentType: contentType,
|
ContentType: contentType,
|
||||||
Body: body,
|
Body: body,
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRestrictedArtifactReaderReadsContainedFiles(t *testing.T) {
|
func TestRestrictedArtifactReaderReadsContainedFiles(t *testing.T) {
|
||||||
@@ -37,9 +37,9 @@ func TestRestrictedArtifactReaderReadsContainedFiles(t *testing.T) {
|
|||||||
t.Fatalf("construct restricted reader: %v", err)
|
t.Fatalf("construct restricted reader: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, ref := range []scriptorium.ArtifactRef{
|
for _, ref := range []promptkit.ArtifactRef{
|
||||||
{Type: scriptorium.ArtifactRefFile, URI: "nested/../input.html"},
|
{Type: promptkit.ArtifactRefFile, URI: "nested/../input.html"},
|
||||||
{Type: scriptorium.ArtifactRefFile, URI: inputPath},
|
{Type: promptkit.ArtifactRefFile, URI: inputPath},
|
||||||
} {
|
} {
|
||||||
artifact, err := reader.Read(context.Background(), ref)
|
artifact, err := reader.Read(context.Background(), ref)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -56,7 +56,7 @@ func TestRestrictedArtifactReaderReadsContainedFiles(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
artifact, err := reader.Read(context.Background(), scriptorium.File("input.unknown"))
|
artifact, err := reader.Read(context.Background(), promptkit.File("input.unknown"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("read unknown-extension path: %v", err)
|
t.Fatalf("read unknown-extension path: %v", err)
|
||||||
}
|
}
|
||||||
@@ -64,9 +64,9 @@ func TestRestrictedArtifactReaderReadsContainedFiles(t *testing.T) {
|
|||||||
t.Fatalf("unexpected fallback content type: %q", artifact.ContentType)
|
t.Fatalf("unexpected fallback content type: %q", artifact.ContentType)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, ref := range []scriptorium.ArtifactRef{
|
for _, ref := range []promptkit.ArtifactRef{
|
||||||
{Type: scriptorium.ArtifactRefFile, URI: filepath.Join("..", filepath.Base(outside), "secret.txt")},
|
{Type: promptkit.ArtifactRefFile, URI: filepath.Join("..", filepath.Base(outside), "secret.txt")},
|
||||||
{Type: scriptorium.ArtifactRefFile, URI: filepath.Join(outside, "secret.txt")},
|
{Type: promptkit.ArtifactRefFile, URI: filepath.Join(outside, "secret.txt")},
|
||||||
} {
|
} {
|
||||||
_, err := reader.Read(context.Background(), ref)
|
_, err := reader.Read(context.Background(), ref)
|
||||||
if !errors.Is(err, ErrFileOutsideRoot) {
|
if !errors.Is(err, ErrFileOutsideRoot) {
|
||||||
@@ -90,7 +90,7 @@ func TestRestrictedArtifactReaderFollowsSymlinkAfterLexicalCheck(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct restricted reader: %v", err)
|
t.Fatalf("construct restricted reader: %v", err)
|
||||||
}
|
}
|
||||||
artifact, err := reader.Read(context.Background(), scriptorium.File("linked.txt"))
|
artifact, err := reader.Read(context.Background(), promptkit.File("linked.txt"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("read symlink inside root: %v", err)
|
t.Fatalf("read symlink inside root: %v", err)
|
||||||
}
|
}
|
||||||
@@ -105,7 +105,7 @@ func TestRestrictedArtifactReaderWithoutRootDeniesFiles(t *testing.T) {
|
|||||||
t.Fatalf("construct rootless reader: %v", err)
|
t.Fatalf("construct rootless reader: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
artifact, err := reader.Read(context.Background(), scriptorium.Inline("inline"))
|
artifact, err := reader.Read(context.Background(), promptkit.Inline("inline"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("read inline artifact: %v", err)
|
t.Fatalf("read inline artifact: %v", err)
|
||||||
}
|
}
|
||||||
@@ -113,7 +113,7 @@ func TestRestrictedArtifactReaderWithoutRootDeniesFiles(t *testing.T) {
|
|||||||
t.Fatalf("unexpected inline artifact: %#v", artifact)
|
t.Fatalf("unexpected inline artifact: %#v", artifact)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = reader.Read(context.Background(), scriptorium.File("input.txt"))
|
_, err = reader.Read(context.Background(), promptkit.File("input.txt"))
|
||||||
if !errors.Is(err, ErrFileNotAllowed) {
|
if !errors.Is(err, ErrFileNotAllowed) {
|
||||||
t.Fatalf("expected ErrFileNotAllowed, got %v", err)
|
t.Fatalf("expected ErrFileNotAllowed, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -132,11 +132,11 @@ func TestRestrictedArtifactReaderEnforcesLimits(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct limited reader: %v", err)
|
t.Fatalf("construct limited reader: %v", err)
|
||||||
}
|
}
|
||||||
artifact, err := reader.Read(context.Background(), scriptorium.File("exact.txt"))
|
artifact, err := reader.Read(context.Background(), promptkit.File("exact.txt"))
|
||||||
if err != nil || string(artifact.Body) != "12345" {
|
if err != nil || string(artifact.Body) != "12345" {
|
||||||
t.Fatalf("expected exact-limit artifact, got %#v and %v", artifact, err)
|
t.Fatalf("expected exact-limit artifact, got %#v and %v", artifact, err)
|
||||||
}
|
}
|
||||||
_, err = reader.Read(context.Background(), scriptorium.File("large.txt"))
|
_, err = reader.Read(context.Background(), promptkit.File("large.txt"))
|
||||||
if !errors.Is(err, ErrFileTooLarge) {
|
if !errors.Is(err, ErrFileTooLarge) {
|
||||||
t.Fatalf("expected ErrFileTooLarge, got %v", err)
|
t.Fatalf("expected ErrFileTooLarge, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -145,7 +145,7 @@ func TestRestrictedArtifactReaderEnforcesLimits(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct unlimited reader: %v", err)
|
t.Fatalf("construct unlimited reader: %v", err)
|
||||||
}
|
}
|
||||||
artifact, err = unlimited.Read(context.Background(), scriptorium.File("large.txt"))
|
artifact, err = unlimited.Read(context.Background(), promptkit.File("large.txt"))
|
||||||
if err != nil || string(artifact.Body) != "123456" {
|
if err != nil || string(artifact.Body) != "123456" {
|
||||||
t.Fatalf("expected unlimited artifact, got %#v and %v", artifact, err)
|
t.Fatalf("expected unlimited artifact, got %#v and %v", artifact, err)
|
||||||
}
|
}
|
||||||
@@ -163,9 +163,9 @@ func TestRestrictedArtifactReaderRejectsCanceledAndMalformedReferences(t *testin
|
|||||||
|
|
||||||
canceledCtx, cancel := context.WithCancel(context.Background())
|
canceledCtx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
for _, ref := range []scriptorium.ArtifactRef{
|
for _, ref := range []promptkit.ArtifactRef{
|
||||||
scriptorium.Inline("input"),
|
promptkit.Inline("input"),
|
||||||
scriptorium.File("input.txt"),
|
promptkit.File("input.txt"),
|
||||||
} {
|
} {
|
||||||
_, err := reader.Read(canceledCtx, ref)
|
_, err := reader.Read(canceledCtx, ref)
|
||||||
if !errors.Is(err, context.Canceled) {
|
if !errors.Is(err, context.Canceled) {
|
||||||
@@ -173,10 +173,10 @@ func TestRestrictedArtifactReaderRejectsCanceledAndMalformedReferences(t *testin
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, ref := range []scriptorium.ArtifactRef{
|
for _, ref := range []promptkit.ArtifactRef{
|
||||||
{Type: scriptorium.ArtifactRefType("unsupported")},
|
{Type: promptkit.ArtifactRefType("unsupported")},
|
||||||
{Type: scriptorium.ArtifactRefInline},
|
{Type: promptkit.ArtifactRefInline},
|
||||||
{Type: scriptorium.ArtifactRefFile},
|
{Type: promptkit.ArtifactRefFile},
|
||||||
} {
|
} {
|
||||||
if _, err := reader.Read(context.Background(), ref); err == nil {
|
if _, err := reader.Read(context.Background(), ref); err == nil {
|
||||||
t.Fatalf("expected malformed reference %#v to fail", ref)
|
t.Fatalf("expected malformed reference %#v to fail", ref)
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ type runRequestDTO struct {
|
|||||||
PromptID string `json:"prompt_id"`
|
PromptID string `json:"prompt_id"`
|
||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
PromptVersion string `json:"prompt_version,omitempty"`
|
||||||
ProfileID string `json:"profile_id,omitempty"`
|
ProfileID string `json:"profile_id,omitempty"`
|
||||||
|
SessionID string `json:"session_id,omitempty"`
|
||||||
Inputs map[string]inputRefDTO `json:"inputs"`
|
Inputs map[string]inputRefDTO `json:"inputs"`
|
||||||
Vars map[string]string `json:"vars,omitempty"`
|
Vars map[string]string `json:"vars,omitempty"`
|
||||||
Model *modelOverrideRequestDTO `json:"model,omitempty"`
|
Model *modelOverrideRequestDTO `json:"model,omitempty"`
|
||||||
@@ -28,7 +29,7 @@ type modelOverrideRequestDTO struct {
|
|||||||
TopP *float64 `json:"top_p,omitempty"`
|
TopP *float64 `json:"top_p,omitempty"`
|
||||||
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||||
ServiceTier string `json:"service_tier,omitempty"`
|
ServiceTier string `json:"service_tier,omitempty"`
|
||||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
ReasoningEffort *string `json:"reasoning_effort,omitempty"`
|
||||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||||
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||||
}
|
}
|
||||||
@@ -56,6 +57,8 @@ type metadataDTO struct {
|
|||||||
PromptHash string `json:"prompt_hash"`
|
PromptHash string `json:"prompt_hash"`
|
||||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
|
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||||
|
SessionID string `json:"session_id,omitempty"`
|
||||||
ModelName string `json:"model_name"`
|
ModelName string `json:"model_name"`
|
||||||
Endpoint string `json:"endpoint"`
|
Endpoint string `json:"endpoint"`
|
||||||
ModelParams modelParamsDTO `json:"model_params"`
|
ModelParams modelParamsDTO `json:"model_params"`
|
||||||
@@ -71,6 +74,7 @@ type metadataDTO struct {
|
|||||||
|
|
||||||
type modelParamsDTO struct {
|
type modelParamsDTO struct {
|
||||||
Endpoint string `json:"endpoint"`
|
Endpoint string `json:"endpoint"`
|
||||||
|
BackendID string `json:"backend_id,omitempty"`
|
||||||
Model string `json:"model"`
|
Model string `json:"model"`
|
||||||
Temperature float64 `json:"temperature"`
|
Temperature float64 `json:"temperature"`
|
||||||
MaxTokens int `json:"max_tokens"`
|
MaxTokens int `json:"max_tokens"`
|
||||||
|
|||||||
@@ -8,12 +8,12 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Runner interface {
|
type Runner interface {
|
||||||
Run(ctx context.Context, req scriptorium.RunRequest) (*scriptorium.RunResult, error)
|
Run(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type Handler struct {
|
type Handler struct {
|
||||||
@@ -76,29 +76,28 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeError(w, http.StatusBadRequest, "invalid_request", "prompt_id is required")
|
writeError(w, http.StatusBadRequest, "invalid_request", "prompt_id is required")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if len(req.Inputs) == 0 {
|
var mappedInputs map[string]promptkit.ArtifactRef
|
||||||
writeError(w, http.StatusBadRequest, "invalid_request", "inputs is required")
|
if len(req.Inputs) > 0 {
|
||||||
return
|
mappedInputs = make(map[string]promptkit.ArtifactRef, len(req.Inputs))
|
||||||
}
|
for name, in := range req.Inputs {
|
||||||
|
mappedInputs[name] = promptkit.ArtifactRef{
|
||||||
mappedInputs := make(map[string]scriptorium.ArtifactRef, len(req.Inputs))
|
Type: promptkit.ArtifactRefType(in.Type),
|
||||||
for name, in := range req.Inputs {
|
URI: in.URI,
|
||||||
mappedInputs[name] = scriptorium.ArtifactRef{
|
Body: in.Body,
|
||||||
Type: scriptorium.ArtifactRefType(in.Type),
|
}
|
||||||
URI: in.URI,
|
|
||||||
Body: in.Body,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var model *scriptorium.ExecutionTargetOverride
|
var model *promptkit.ExecutionTargetOverride
|
||||||
if req.Model != nil {
|
if req.Model != nil {
|
||||||
model = executionTargetOverrideFromModelOverrideDTO(req.Model)
|
model = executionTargetOverrideFromModelOverrideDTO(req.Model)
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := h.runner.Run(r.Context(), scriptorium.RunRequest{
|
res, err := h.runner.Run(r.Context(), promptkit.RunRequest{
|
||||||
PromptID: req.PromptID,
|
PromptID: req.PromptID,
|
||||||
PromptVersion: req.PromptVersion,
|
PromptVersion: req.PromptVersion,
|
||||||
ProfileID: req.ProfileID,
|
ProfileID: req.ProfileID,
|
||||||
|
SessionID: req.SessionID,
|
||||||
Inputs: mappedInputs,
|
Inputs: mappedInputs,
|
||||||
Vars: req.Vars,
|
Vars: req.Vars,
|
||||||
Execution: model,
|
Execution: model,
|
||||||
@@ -126,6 +125,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
PromptHash: res.PromptHash,
|
PromptHash: res.PromptHash,
|
||||||
RenderedPromptHash: res.RenderedPromptHash,
|
RenderedPromptHash: res.RenderedPromptHash,
|
||||||
SelectedProfileID: res.SelectedProfileID,
|
SelectedProfileID: res.SelectedProfileID,
|
||||||
|
SelectedBackendID: res.SelectedBackendID,
|
||||||
|
SessionID: res.SessionID,
|
||||||
ModelName: res.ModelName,
|
ModelName: res.ModelName,
|
||||||
Endpoint: res.Endpoint,
|
Endpoint: res.Endpoint,
|
||||||
ModelParams: modelParamsDTOFromExecutionTarget(res.EffectiveModelParams),
|
ModelParams: modelParamsDTOFromExecutionTarget(res.EffectiveModelParams),
|
||||||
@@ -152,11 +153,11 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeLimitedJSON(w, http.StatusOK, resp, h.options.MaxResponseBytes)
|
writeLimitedJSON(w, http.StatusOK, resp, h.options.MaxResponseBytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
func executionTargetOverrideFromModelOverrideDTO(dto *modelOverrideRequestDTO) *scriptorium.ExecutionTargetOverride {
|
func executionTargetOverrideFromModelOverrideDTO(dto *modelOverrideRequestDTO) *promptkit.ExecutionTargetOverride {
|
||||||
if dto == nil {
|
if dto == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return &scriptorium.ExecutionTargetOverride{
|
return &promptkit.ExecutionTargetOverride{
|
||||||
Endpoint: dto.Endpoint,
|
Endpoint: dto.Endpoint,
|
||||||
Model: dto.Model,
|
Model: dto.Model,
|
||||||
Temperature: dto.Temperature,
|
Temperature: dto.Temperature,
|
||||||
@@ -170,9 +171,10 @@ func executionTargetOverrideFromModelOverrideDTO(dto *modelOverrideRequestDTO) *
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func modelParamsDTOFromExecutionTarget(target scriptorium.ExecutionTarget) modelParamsDTO {
|
func modelParamsDTOFromExecutionTarget(target promptkit.ExecutionTarget) modelParamsDTO {
|
||||||
return modelParamsDTO{
|
return modelParamsDTO{
|
||||||
Endpoint: target.Endpoint,
|
Endpoint: target.Endpoint,
|
||||||
|
BackendID: target.BackendID,
|
||||||
Model: target.Model,
|
Model: target.Model,
|
||||||
Temperature: target.Temperature,
|
Temperature: target.Temperature,
|
||||||
MaxTokens: target.MaxTokens,
|
MaxTokens: target.MaxTokens,
|
||||||
@@ -185,7 +187,7 @@ func modelParamsDTOFromExecutionTarget(target scriptorium.ExecutionTarget) model
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func mapValidation(v scriptorium.ValidationResult) validationDTO {
|
func mapValidation(v promptkit.ValidationResult) validationDTO {
|
||||||
return validationDTO{
|
return validationDTO{
|
||||||
Status: string(v.Status),
|
Status: string(v.Status),
|
||||||
Mode: string(v.Mode),
|
Mode: string(v.Mode),
|
||||||
@@ -198,31 +200,33 @@ func mapValidation(v scriptorium.ValidationResult) validationDTO {
|
|||||||
|
|
||||||
func mapRunError(err error) (int, string, string) {
|
func mapRunError(err error) (int, string, string) {
|
||||||
switch {
|
switch {
|
||||||
case errors.Is(err, scriptorium.ErrPromptNotFound):
|
case errors.Is(err, promptkit.ErrPromptNotFound):
|
||||||
return http.StatusNotFound, "prompt_not_found", "prompt definition not found"
|
return http.StatusNotFound, "prompt_not_found", "prompt definition not found"
|
||||||
case errors.Is(err, scriptorium.ErrProfileNotFound):
|
case errors.Is(err, promptkit.ErrProfileNotFound):
|
||||||
return http.StatusNotFound, "profile_not_found", "execution profile not found"
|
return http.StatusNotFound, "profile_not_found", "execution profile not found"
|
||||||
case errors.Is(err, scriptorium.ErrProfileRequired):
|
case errors.Is(err, promptkit.ErrProfileRequired):
|
||||||
return http.StatusBadRequest, "profile_required", "profile_id is required when prompt default_profile is not set"
|
return http.StatusBadRequest, "profile_required", "profile_id is required when prompt default_profile is not set"
|
||||||
case errors.Is(err, scriptorium.ErrAPIKeyEnvMissing):
|
case errors.Is(err, promptkit.ErrAPIKeyEnvMissing):
|
||||||
return http.StatusBadRequest, "api_key_env_missing", "api_key_env is set but the environment variable is missing"
|
return http.StatusBadRequest, "api_key_env_missing", "api_key_env is set but the environment variable is missing"
|
||||||
case errors.Is(err, scriptorium.ErrPromptLoad):
|
case errors.Is(err, promptkit.ErrPromptLoad):
|
||||||
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
|
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
|
||||||
case errors.Is(err, scriptorium.ErrProfileLoad):
|
case errors.Is(err, promptkit.ErrProfileLoad):
|
||||||
return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile"
|
return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile"
|
||||||
case errors.Is(err, scriptorium.ErrInvalidRequest):
|
case errors.Is(err, promptkit.ErrInvalidRequest):
|
||||||
return http.StatusBadRequest, "invalid_request", "invalid run request"
|
return http.StatusBadRequest, "invalid_request", "invalid run request"
|
||||||
case errors.Is(err, ErrFileNotAllowed), errors.Is(err, ErrFileOutsideRoot):
|
case errors.Is(err, ErrFileNotAllowed), errors.Is(err, ErrFileOutsideRoot):
|
||||||
return http.StatusBadRequest, "artifact_not_allowed", "file input artifact is not allowed"
|
return http.StatusBadRequest, "artifact_not_allowed", "file input artifact is not allowed"
|
||||||
case errors.Is(err, ErrFileTooLarge):
|
case errors.Is(err, ErrFileTooLarge):
|
||||||
return http.StatusRequestEntityTooLarge, "artifact_too_large", "file input artifact is too large"
|
return http.StatusRequestEntityTooLarge, "artifact_too_large", "file input artifact is too large"
|
||||||
case errors.Is(err, scriptorium.ErrArtifactLoad):
|
case errors.Is(err, promptkit.ErrArtifactLoad):
|
||||||
return http.StatusBadRequest, "artifact_read_failed", "failed to read input artifact"
|
return http.StatusBadRequest, "artifact_read_failed", "failed to read input artifact"
|
||||||
case errors.Is(err, scriptorium.ErrPromptRender):
|
case errors.Is(err, promptkit.ErrPromptRender):
|
||||||
return http.StatusBadRequest, "prompt_render_failed", "failed to render prompt"
|
return http.StatusBadRequest, "prompt_render_failed", "failed to render prompt"
|
||||||
case errors.Is(err, scriptorium.ErrLLMGenerate):
|
case errors.Is(err, promptkit.ErrCapacityExceeded):
|
||||||
|
return http.StatusServiceUnavailable, "capacity_exceeded", "model backend capacity is exhausted"
|
||||||
|
case errors.Is(err, promptkit.ErrLLMGenerate):
|
||||||
return http.StatusBadGateway, "llm_failed", "model generation request failed"
|
return http.StatusBadGateway, "llm_failed", "model generation request failed"
|
||||||
case errors.Is(err, scriptorium.ErrValidation):
|
case errors.Is(err, promptkit.ErrValidation):
|
||||||
return http.StatusInternalServerError, "validation_runtime_failed", "validation runtime failed"
|
return http.StatusInternalServerError, "validation_runtime_failed", "validation runtime failed"
|
||||||
default:
|
default:
|
||||||
return http.StatusInternalServerError, "internal_error", "internal server error"
|
return http.StatusInternalServerError, "internal_error", "internal server error"
|
||||||
|
|||||||
@@ -11,19 +11,20 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
type fakeRunner struct {
|
type fakeRunner struct {
|
||||||
result *scriptorium.RunResult
|
result *promptkit.RunResult
|
||||||
err error
|
err error
|
||||||
last scriptorium.RunRequest
|
last promptkit.RunRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeRunner) Run(ctx context.Context, req scriptorium.RunRequest) (*scriptorium.RunResult, error) {
|
func (f *fakeRunner) Run(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
|
||||||
f.last = req
|
f.last = req
|
||||||
if f.err != nil {
|
if f.err != nil {
|
||||||
return nil, f.err
|
return nil, f.err
|
||||||
@@ -37,7 +38,7 @@ func TestMaintainedHTTPRunExampleMatchesRequestContract(t *testing.T) {
|
|||||||
t.Fatalf("read maintained HTTP request example: %v", err)
|
t.Fatalf("read maintained HTTP request example: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
runner := &fakeRunner{result: &scriptorium.RunResult{}}
|
runner := &fakeRunner{result: &promptkit.RunResult{}}
|
||||||
h := NewHandler(runner)
|
h := NewHandler(runner)
|
||||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewReader(body))
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewReader(body))
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
@@ -65,8 +66,36 @@ func TestMaintainedHTTPRunExampleMatchesRequestContract(t *testing.T) {
|
|||||||
|
|
||||||
type handlerLLMClient struct{}
|
type handlerLLMClient struct{}
|
||||||
|
|
||||||
func (handlerLLMClient) Generate(ctx context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
|
func (handlerLLMClient) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
return &scriptorium.GenerateResponse{Content: "ok"}, nil
|
return &promptkit.GenerateResponse{Content: "ok"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type blockingLLMClient struct {
|
||||||
|
started chan struct{}
|
||||||
|
release chan struct{}
|
||||||
|
current int32
|
||||||
|
peak int32
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *blockingLLMClient) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
current := atomic.AddInt32(&c.current, 1)
|
||||||
|
defer atomic.AddInt32(&c.current, -1)
|
||||||
|
for {
|
||||||
|
peak := atomic.LoadInt32(&c.peak)
|
||||||
|
if current <= peak || atomic.CompareAndSwapInt32(&c.peak, peak, current) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case c.started <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-c.release:
|
||||||
|
return &promptkit.GenerateResponse{Content: "ok"}, nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
||||||
@@ -75,24 +104,26 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
const envName = "SCRIPTORIUM_API_KEY"
|
const envName = "SCRIPTORIUM_API_KEY"
|
||||||
const secret = "never-include-me"
|
const secret = "never-include-me"
|
||||||
|
|
||||||
r := &fakeRunner{result: &scriptorium.RunResult{
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
RunID: "11111111-1111-4111-8111-111111111111",
|
RunID: "11111111-1111-4111-8111-111111111111",
|
||||||
Artifact: scriptorium.Artifact{
|
Artifact: promptkit.Artifact{
|
||||||
Name: "output",
|
Name: "output",
|
||||||
ContentType: "text/plain",
|
ContentType: "text/plain",
|
||||||
Body: []byte("hello"),
|
Body: []byte("hello"),
|
||||||
Size: 5,
|
Size: 5,
|
||||||
Hash: "abc",
|
Hash: "abc",
|
||||||
},
|
},
|
||||||
Validation: scriptorium.ValidationResult{Status: scriptorium.ValidationPassed, Mode: scriptorium.ValidationBasic, IsValid: true},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
PromptID: "prompt-1",
|
PromptID: "prompt-1",
|
||||||
PromptVersion: "1.0.0",
|
PromptVersion: "1.0.0",
|
||||||
PromptHash: "phash",
|
PromptHash: "phash",
|
||||||
RenderedPromptHash: "rhash",
|
RenderedPromptHash: "rhash",
|
||||||
SelectedProfileID: "exec-default",
|
SelectedProfileID: "exec-default",
|
||||||
|
SelectedBackendID: "local",
|
||||||
ModelName: "m1",
|
ModelName: "m1",
|
||||||
Endpoint: "http://llm/v1",
|
Endpoint: "http://llm/v1",
|
||||||
EffectiveModelParams: scriptorium.ExecutionTarget{
|
EffectiveModelParams: promptkit.ExecutionTarget{
|
||||||
|
BackendID: "local",
|
||||||
Endpoint: "http://llm/v1",
|
Endpoint: "http://llm/v1",
|
||||||
Model: "m1",
|
Model: "m1",
|
||||||
Temperature: 0.2,
|
Temperature: 0.2,
|
||||||
@@ -103,7 +134,7 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
APIKeyEnv: envName,
|
APIKeyEnv: envName,
|
||||||
},
|
},
|
||||||
InputHashes: map[string]string{"transcript": "h1"},
|
InputHashes: map[string]string{"transcript": "h1"},
|
||||||
Usage: scriptorium.TokenUsage{
|
Usage: promptkit.TokenUsage{
|
||||||
PromptTokens: 1,
|
PromptTokens: 1,
|
||||||
CompletionTokens: 2,
|
CompletionTokens: 2,
|
||||||
TotalTokens: 3,
|
TotalTokens: 3,
|
||||||
@@ -151,6 +182,9 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
if metadata["selected_profile_id"] != "exec-default" {
|
if metadata["selected_profile_id"] != "exec-default" {
|
||||||
t.Fatalf("unexpected metadata.selected_profile_id: %#v", metadata["selected_profile_id"])
|
t.Fatalf("unexpected metadata.selected_profile_id: %#v", metadata["selected_profile_id"])
|
||||||
}
|
}
|
||||||
|
if metadata["selected_backend_id"] != "local" {
|
||||||
|
t.Fatalf("unexpected metadata.selected_backend_id: %#v", metadata["selected_backend_id"])
|
||||||
|
}
|
||||||
if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" {
|
if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" {
|
||||||
t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"])
|
t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"])
|
||||||
}
|
}
|
||||||
@@ -162,6 +196,9 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
t.Fatalf("unexpected cache usage metadata: %#v", usage)
|
t.Fatalf("unexpected cache usage metadata: %#v", usage)
|
||||||
}
|
}
|
||||||
modelParams := metadata["model_params"].(map[string]any)
|
modelParams := metadata["model_params"].(map[string]any)
|
||||||
|
if modelParams["backend_id"] != "local" {
|
||||||
|
t.Fatalf("expected model_params.backend_id=local, got %#v", modelParams["backend_id"])
|
||||||
|
}
|
||||||
if modelParams["api_key_env"] != envName {
|
if modelParams["api_key_env"] != envName {
|
||||||
t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"])
|
t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"])
|
||||||
}
|
}
|
||||||
@@ -192,6 +229,124 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandlerAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) {
|
||||||
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationNone, IsValid: true},
|
||||||
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
|
}}
|
||||||
|
h := NewHandler(r)
|
||||||
|
|
||||||
|
for _, body := range []string{
|
||||||
|
`{"prompt_id":"prompt-1","prompt_version":"2"}`,
|
||||||
|
`{"prompt_id":"prompt-1","prompt_version":"2","inputs":{}}`,
|
||||||
|
} {
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(body))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if r.last.PromptVersion != "2" {
|
||||||
|
t.Fatalf("expected prompt version to be mapped, got %q", r.last.PromptVersion)
|
||||||
|
}
|
||||||
|
if r.last.Inputs != nil {
|
||||||
|
t.Fatalf("expected omitted or empty inputs to remain nil, got %#v", r.last.Inputs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerDelegatesDefinitionInputRequirements(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
definition string
|
||||||
|
wantStatus int
|
||||||
|
wantCode string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "no declared inputs",
|
||||||
|
definition: `id: p
|
||||||
|
version: "1"
|
||||||
|
default_profile: exec
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "hello"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`,
|
||||||
|
wantStatus: http.StatusOK,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "optional input omitted",
|
||||||
|
definition: `id: p
|
||||||
|
version: "1"
|
||||||
|
default_profile: exec
|
||||||
|
inputs:
|
||||||
|
- name: note
|
||||||
|
required: false
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "hello"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`,
|
||||||
|
wantStatus: http.StatusOK,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "required input omitted",
|
||||||
|
definition: `id: p
|
||||||
|
version: "1"
|
||||||
|
default_profile: exec
|
||||||
|
inputs:
|
||||||
|
- name: note
|
||||||
|
required: true
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "hello"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`,
|
||||||
|
wantStatus: http.StatusBadRequest,
|
||||||
|
wantCode: "prompt_render_failed",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "template input omitted",
|
||||||
|
definition: `id: p
|
||||||
|
version: "1"
|
||||||
|
default_profile: exec
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: '{{input "note"}}'
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`,
|
||||||
|
wantStatus: http.StatusBadRequest,
|
||||||
|
wantCode: "prompt_render_failed",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
h := newDefinitionHandler(t, tc.definition)
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p"}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != tc.wantStatus {
|
||||||
|
t.Fatalf("expected %d, got %d body=%s", tc.wantStatus, w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if tc.wantCode != "" {
|
||||||
|
assertHTTPErrorCode(t, w, tc.wantStatus, tc.wantCode)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandlerInlineRefsWorkWithoutArtifactRoot(t *testing.T) {
|
func TestHandlerInlineRefsWorkWithoutArtifactRoot(t *testing.T) {
|
||||||
h := newArtifactRootHandler(t, "")
|
h := newArtifactRootHandler(t, "")
|
||||||
|
|
||||||
@@ -292,13 +447,13 @@ func TestHandlerFileRefsOutsideArtifactRootAreRejected(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
|
func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
|
||||||
r := &fakeRunner{result: &scriptorium.RunResult{
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
Artifact: scriptorium.Artifact{Body: []byte("ok")},
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
PromptID: "prompt-1",
|
PromptID: "prompt-1",
|
||||||
PromptVersion: "1.0.0",
|
PromptVersion: "1.0.0",
|
||||||
SelectedProfileID: "prompt-default",
|
SelectedProfileID: "prompt-default",
|
||||||
Validation: scriptorium.ValidationResult{Status: scriptorium.ValidationPassed, Mode: scriptorium.ValidationBasic, IsValid: true},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
EffectiveModelParams: scriptorium.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
}}
|
}}
|
||||||
h := NewHandler(r)
|
h := NewHandler(r)
|
||||||
|
|
||||||
@@ -327,10 +482,10 @@ func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
|
func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
|
||||||
r := &fakeRunner{result: &scriptorium.RunResult{
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
Artifact: scriptorium.Artifact{Body: []byte("ok")},
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
Validation: scriptorium.ValidationResult{Status: scriptorium.ValidationPassed, Mode: scriptorium.ValidationBasic, IsValid: true},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
EffectiveModelParams: scriptorium.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
}}
|
}}
|
||||||
h := NewHandler(r)
|
h := NewHandler(r)
|
||||||
|
|
||||||
@@ -365,10 +520,12 @@ func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
|
|||||||
if got.Endpoint != "http://override/v1" ||
|
if got.Endpoint != "http://override/v1" ||
|
||||||
got.Model != "override-model" ||
|
got.Model != "override-model" ||
|
||||||
got.ServiceTier != "flex" ||
|
got.ServiceTier != "flex" ||
|
||||||
got.ReasoningEffort != "medium" ||
|
|
||||||
got.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
|
got.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
|
||||||
t.Fatalf("unexpected mapped execution target: %+v", got)
|
t.Fatalf("unexpected mapped execution target: %+v", got)
|
||||||
}
|
}
|
||||||
|
if got.ReasoningEffort == nil || *got.ReasoningEffort != "medium" {
|
||||||
|
t.Fatalf("unexpected mapped reasoning_effort: %#v", got.ReasoningEffort)
|
||||||
|
}
|
||||||
if got.Temperature == nil || *got.Temperature != 0.6 {
|
if got.Temperature == nil || *got.Temperature != 0.6 {
|
||||||
t.Fatalf("unexpected mapped temperature: %#v", got.Temperature)
|
t.Fatalf("unexpected mapped temperature: %#v", got.Temperature)
|
||||||
}
|
}
|
||||||
@@ -386,11 +543,256 @@ func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandlerModelOverridePreservesReasoningEffortPresence(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
model string
|
||||||
|
wantPresent bool
|
||||||
|
wantValue string
|
||||||
|
}{
|
||||||
|
{name: "omitted", model: `{}`},
|
||||||
|
{name: "nonblank", model: `{"reasoning_effort":"high"}`, wantPresent: true, wantValue: "high"},
|
||||||
|
{name: "explicit empty", model: `{"reasoning_effort":""}`, wantPresent: true},
|
||||||
|
{name: "null", model: `{"reasoning_effort":null}`},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
|
}}
|
||||||
|
h := NewHandler(r)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"prompt-1","inputs":{"transcript":{"type":"file","uri":"./t.md"}},"model":`+tc.model+`}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if r.last.Execution == nil {
|
||||||
|
t.Fatal("expected execution override")
|
||||||
|
}
|
||||||
|
if (r.last.Execution.ReasoningEffort != nil) != tc.wantPresent {
|
||||||
|
t.Fatalf("unexpected reasoning_effort presence: %#v", r.last.Execution.ReasoningEffort)
|
||||||
|
}
|
||||||
|
if tc.wantPresent && *r.last.Execution.ReasoningEffort != tc.wantValue {
|
||||||
|
t.Fatalf("unexpected reasoning_effort: got %q want %q", *r.last.Execution.ReasoningEffort, tc.wantValue)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerMapsSessionIDAndReportsEffectiveResultSessionID(t *testing.T) {
|
||||||
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
|
SessionID: "effective-session",
|
||||||
|
}}
|
||||||
|
h := NewHandler(r)
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"prompt-1","session_id":"request-session"}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if r.last.SessionID != "request-session" {
|
||||||
|
t.Fatalf("expected session ID in run request, got %q", r.last.SessionID)
|
||||||
|
}
|
||||||
|
var resp map[string]any
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||||
|
t.Fatalf("invalid JSON response: %v", err)
|
||||||
|
}
|
||||||
|
metadata := resp["metadata"].(map[string]any)
|
||||||
|
if metadata["session_id"] != "effective-session" {
|
||||||
|
t.Fatalf("expected effective session ID in response, got %#v", metadata["session_id"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerOmitsEmptyEffectiveSessionID(t *testing.T) {
|
||||||
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
|
}}
|
||||||
|
h := NewHandler(r)
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"prompt-1"}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
var resp map[string]any
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||||
|
t.Fatalf("invalid JSON response: %v", err)
|
||||||
|
}
|
||||||
|
metadata := resp["metadata"].(map[string]any)
|
||||||
|
if _, ok := metadata["session_id"]; ok {
|
||||||
|
t.Fatalf("expected empty effective session ID to be omitted, got %#v", metadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerSessionIDUsesPromptkitResolution(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
definitionSession string
|
||||||
|
requestSession string
|
||||||
|
wantSession string
|
||||||
|
}{
|
||||||
|
{name: "definition session", definitionSession: "definition-session", wantSession: "definition-session"},
|
||||||
|
{name: "direct session", definitionSession: "definition-session", requestSession: "direct-session", wantSession: "direct-session"},
|
||||||
|
{name: "no effective session"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
promptDir := t.TempDir()
|
||||||
|
profileDir := t.TempDir()
|
||||||
|
sessionLine := ""
|
||||||
|
if tc.definitionSession != "" {
|
||||||
|
sessionLine = "session_id: " + tc.definitionSession + "\n"
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte("id: p\nversion: \"1\"\ndefault_profile: exec\n"+sessionLine+`messages:
|
||||||
|
- role: user
|
||||||
|
content: "hi"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`), 0o644); err != nil {
|
||||||
|
t.Fatalf("write prompt fixture: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("id: exec\nendpoint: http://127.0.0.1:1/v1\nmodel: test\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write profile fixture: %v", err)
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{PromptDir: promptDir, ProfileDir: profileDir}, promptkit.WithLLMClient(handlerLLMClient{}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("new engine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
body := `{"prompt_id":"p"}`
|
||||||
|
if tc.requestSession != "" {
|
||||||
|
body = `{"prompt_id":"p","session_id":"` + tc.requestSession + `"}`
|
||||||
|
}
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
NewHandler(engine).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(body)))
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
var resp map[string]any
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||||
|
t.Fatalf("invalid JSON response: %v", err)
|
||||||
|
}
|
||||||
|
metadata := resp["metadata"].(map[string]any)
|
||||||
|
if tc.wantSession == "" {
|
||||||
|
if _, ok := metadata["session_id"]; ok {
|
||||||
|
t.Fatalf("expected session_id to be omitted, got %#v", metadata)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if metadata["session_id"] != tc.wantSession {
|
||||||
|
t.Fatalf("expected effective session ID %q, got %#v", tc.wantSession, metadata["session_id"])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerSharesBackendCapacityAcrossConcurrentRequests(t *testing.T) {
|
||||||
|
promptDir := t.TempDir()
|
||||||
|
profileDir := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`id: p
|
||||||
|
version: "1"
|
||||||
|
default_profile: limited
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "hi"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`), 0o644); err != nil {
|
||||||
|
t.Fatalf("write prompt fixture: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`id: limited
|
||||||
|
backend: limited
|
||||||
|
model: test
|
||||||
|
`), 0o644); err != nil {
|
||||||
|
t.Fatalf("write profile fixture: %v", err)
|
||||||
|
}
|
||||||
|
queueCapacity := 0
|
||||||
|
client := &blockingLLMClient{started: make(chan struct{}, 1), release: make(chan struct{})}
|
||||||
|
engine, err := promptkit.NewEngine(
|
||||||
|
promptkit.Config{PromptDir: promptDir, ProfileDir: profileDir},
|
||||||
|
promptkit.WithBackend(promptkit.Backend{
|
||||||
|
ID: "limited",
|
||||||
|
Endpoint: "http://127.0.0.1:1/v1",
|
||||||
|
ConcurrencyLimit: 1,
|
||||||
|
QueueCapacity: &queueCapacity,
|
||||||
|
}),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("new engine: %v", err)
|
||||||
|
}
|
||||||
|
h := NewHandler(engine)
|
||||||
|
first := httptest.NewRecorder()
|
||||||
|
firstDone := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(firstDone)
|
||||||
|
h.ServeHTTP(first, httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p"}`)))
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-client.started:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("first request did not reach generation")
|
||||||
|
}
|
||||||
|
|
||||||
|
second := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(second, httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p"}`)))
|
||||||
|
assertHTTPErrorCode(t, second, http.StatusServiceUnavailable, "capacity_exceeded")
|
||||||
|
if second.Header().Get("Retry-After") != "" {
|
||||||
|
t.Fatalf("expected no Retry-After header, got %q", second.Header().Get("Retry-After"))
|
||||||
|
}
|
||||||
|
if strings.Contains(second.Body.String(), "limited") {
|
||||||
|
t.Fatalf("capacity response leaked backend details: %s", second.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
close(client.release)
|
||||||
|
select {
|
||||||
|
case <-firstDone:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("first request did not complete")
|
||||||
|
}
|
||||||
|
if first.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected first request to succeed, got %d body=%s", first.Code, first.Body.String())
|
||||||
|
}
|
||||||
|
if atomic.LoadInt32(&client.peak) != 1 {
|
||||||
|
t.Fatalf("expected peak generation concurrency of one, got %d", atomic.LoadInt32(&client.peak))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerRejectsOverlongSessionIDAndNonStringReasoningEffort(t *testing.T) {
|
||||||
|
engine := newHandlerEngine(t)
|
||||||
|
for _, body := range []string{
|
||||||
|
`{"prompt_id":"p","session_id":"` + strings.Repeat("x", 257) + `"}`,
|
||||||
|
`{"prompt_id":"p","model":{"reasoning_effort":1}}`,
|
||||||
|
} {
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
NewHandler(engine).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(body)))
|
||||||
|
if w.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("expected 400, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandlerModelOverrideAcceptsJSONCompatibleExtraParams(t *testing.T) {
|
func TestHandlerModelOverrideAcceptsJSONCompatibleExtraParams(t *testing.T) {
|
||||||
r := &fakeRunner{result: &scriptorium.RunResult{
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
Artifact: scriptorium.Artifact{Body: []byte("ok")},
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
Validation: scriptorium.ValidationResult{Status: scriptorium.ValidationPassed, Mode: scriptorium.ValidationBasic, IsValid: true},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
EffectiveModelParams: scriptorium.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
}}
|
}}
|
||||||
h := NewHandler(r)
|
h := NewHandler(r)
|
||||||
|
|
||||||
@@ -431,10 +833,10 @@ func TestHandlerModelOverrideAcceptsJSONCompatibleExtraParams(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerModelOverrideExplicitZeroTemperatureMapsAsPresent(t *testing.T) {
|
func TestHandlerModelOverrideExplicitZeroTemperatureMapsAsPresent(t *testing.T) {
|
||||||
r := &fakeRunner{result: &scriptorium.RunResult{
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
Artifact: scriptorium.Artifact{Body: []byte("ok")},
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
Validation: scriptorium.ValidationResult{Status: scriptorium.ValidationPassed, Mode: scriptorium.ValidationBasic, IsValid: true},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
EffectiveModelParams: scriptorium.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0},
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0},
|
||||||
}}
|
}}
|
||||||
h := NewHandler(r)
|
h := NewHandler(r)
|
||||||
|
|
||||||
@@ -458,10 +860,10 @@ func TestHandlerModelOverrideExplicitZeroTemperatureMapsAsPresent(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerModelOverrideOmittedTemperatureMapsAsAbsent(t *testing.T) {
|
func TestHandlerModelOverrideOmittedTemperatureMapsAsAbsent(t *testing.T) {
|
||||||
r := &fakeRunner{result: &scriptorium.RunResult{
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
Artifact: scriptorium.Artifact{Body: []byte("ok")},
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
Validation: scriptorium.ValidationResult{Status: scriptorium.ValidationPassed, Mode: scriptorium.ValidationBasic, IsValid: true},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
EffectiveModelParams: scriptorium.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0.7},
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0.7},
|
||||||
}}
|
}}
|
||||||
h := NewHandler(r)
|
h := NewHandler(r)
|
||||||
|
|
||||||
@@ -495,16 +897,16 @@ func TestHandlerModelOverrideOmittedTemperatureMapsAsAbsent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing.T) {
|
func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing.T) {
|
||||||
r := &fakeRunner{result: &scriptorium.RunResult{
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
Artifact: scriptorium.Artifact{
|
Artifact: promptkit.Artifact{
|
||||||
Name: "output",
|
Name: "output",
|
||||||
ContentType: "text/plain",
|
ContentType: "text/plain",
|
||||||
Body: []byte("ok"),
|
Body: []byte("ok"),
|
||||||
Size: 2,
|
Size: 2,
|
||||||
Hash: "abc",
|
Hash: "abc",
|
||||||
},
|
},
|
||||||
Validation: scriptorium.ValidationResult{Status: scriptorium.ValidationPassed, Mode: scriptorium.ValidationBasic, IsValid: true},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
EffectiveModelParams: scriptorium.ExecutionTarget{
|
EffectiveModelParams: promptkit.ExecutionTarget{
|
||||||
Endpoint: "http://llm/v1",
|
Endpoint: "http://llm/v1",
|
||||||
Model: "gpt-test",
|
Model: "gpt-test",
|
||||||
Temperature: 0.4,
|
Temperature: 0.4,
|
||||||
@@ -624,10 +1026,10 @@ func TestHandlerMalformedJSONBelowLimitStillBadRequest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerResponseTooLarge(t *testing.T) {
|
func TestHandlerResponseTooLarge(t *testing.T) {
|
||||||
h := NewHandlerWithOptions(&fakeRunner{result: &scriptorium.RunResult{
|
h := NewHandlerWithOptions(&fakeRunner{result: &promptkit.RunResult{
|
||||||
Artifact: scriptorium.Artifact{Body: []byte(strings.Repeat("x", 128))},
|
Artifact: promptkit.Artifact{Body: []byte(strings.Repeat("x", 128))},
|
||||||
Validation: scriptorium.ValidationResult{Status: scriptorium.ValidationPassed, Mode: scriptorium.ValidationBasic, IsValid: true},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
EffectiveModelParams: scriptorium.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
}}, HandlerOptions{MaxRequestBytes: 1024, MaxResponseBytes: 64})
|
}}, HandlerOptions{MaxRequestBytes: 1024, MaxResponseBytes: 64})
|
||||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
@@ -638,11 +1040,11 @@ func TestHandlerResponseTooLarge(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerRawOutputDoesNotBypassResponseLimit(t *testing.T) {
|
func TestHandlerRawOutputDoesNotBypassResponseLimit(t *testing.T) {
|
||||||
h := NewHandlerWithOptions(&fakeRunner{result: &scriptorium.RunResult{
|
h := NewHandlerWithOptions(&fakeRunner{result: &promptkit.RunResult{
|
||||||
Artifact: scriptorium.Artifact{Body: []byte("ok")},
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
RawOutput: strings.Repeat("raw", 80),
|
RawOutput: strings.Repeat("raw", 80),
|
||||||
Validation: scriptorium.ValidationResult{Status: scriptorium.ValidationPassed, Mode: scriptorium.ValidationBasic, IsValid: true},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
EffectiveModelParams: scriptorium.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
}}, HandlerOptions{MaxRequestBytes: 1024, MaxResponseBytes: 128})
|
}}, HandlerOptions{MaxRequestBytes: 1024, MaxResponseBytes: 128})
|
||||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
"prompt_id":"p",
|
"prompt_id":"p",
|
||||||
@@ -710,20 +1112,21 @@ func TestHandlerPublicErrorMapping(t *testing.T) {
|
|||||||
message string
|
message string
|
||||||
avoidCause string
|
avoidCause string
|
||||||
}{
|
}{
|
||||||
{name: "prompt not found", err: scriptorium.ErrPromptNotFound, status: http.StatusNotFound, code: "prompt_not_found", message: "prompt definition not found"},
|
{name: "prompt not found", err: promptkit.ErrPromptNotFound, status: http.StatusNotFound, code: "prompt_not_found", message: "prompt definition not found"},
|
||||||
{name: "prompt load", err: wrap(scriptorium.ErrPromptLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition", avoidCause: "read failed"},
|
{name: "prompt load", err: wrap(promptkit.ErrPromptLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition", avoidCause: "read failed"},
|
||||||
{name: "missing profile/default", err: wrap(scriptorium.ErrProfileRequired, scriptorium.ErrInvalidRequest), status: http.StatusBadRequest, code: "profile_required", message: "profile_id is required when prompt default_profile is not set"},
|
{name: "missing profile/default", err: wrap(promptkit.ErrProfileRequired, promptkit.ErrInvalidRequest), status: http.StatusBadRequest, code: "profile_required", message: "profile_id is required when prompt default_profile is not set"},
|
||||||
{name: "profile not found", err: scriptorium.ErrProfileNotFound, status: http.StatusNotFound, code: "profile_not_found", message: "execution profile not found"},
|
{name: "profile not found", err: promptkit.ErrProfileNotFound, status: http.StatusNotFound, code: "profile_not_found", message: "execution profile not found"},
|
||||||
{name: "profile load", err: wrap(scriptorium.ErrProfileLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile", avoidCause: "read failed"},
|
{name: "profile load", err: wrap(promptkit.ErrProfileLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile", avoidCause: "read failed"},
|
||||||
{name: "api key env missing", err: wrap(scriptorium.ErrAPIKeyEnvMissing, scriptorium.ErrInvalidRequest), status: http.StatusBadRequest, code: "api_key_env_missing", message: "api_key_env is set but the environment variable is missing"},
|
{name: "api key env missing", err: wrap(promptkit.ErrAPIKeyEnvMissing, promptkit.ErrInvalidRequest), status: http.StatusBadRequest, code: "api_key_env_missing", message: "api_key_env is set but the environment variable is missing"},
|
||||||
{name: "invalid request", err: scriptorium.ErrInvalidRequest, status: http.StatusBadRequest, code: "invalid_request", message: "invalid run request"},
|
{name: "invalid request", err: promptkit.ErrInvalidRequest, status: http.StatusBadRequest, code: "invalid_request", message: "invalid run request"},
|
||||||
{name: "file denied", err: ErrFileNotAllowed, status: http.StatusBadRequest, code: "artifact_not_allowed", message: "file input artifact is not allowed"},
|
{name: "file denied", err: ErrFileNotAllowed, status: http.StatusBadRequest, code: "artifact_not_allowed", message: "file input artifact is not allowed"},
|
||||||
{name: "file outside root", err: ErrFileOutsideRoot, status: http.StatusBadRequest, code: "artifact_not_allowed", message: "file input artifact is not allowed"},
|
{name: "file outside root", err: ErrFileOutsideRoot, status: http.StatusBadRequest, code: "artifact_not_allowed", message: "file input artifact is not allowed"},
|
||||||
{name: "file too large", err: ErrFileTooLarge, status: http.StatusRequestEntityTooLarge, code: "artifact_too_large", message: "file input artifact is too large"},
|
{name: "file too large", err: ErrFileTooLarge, status: http.StatusRequestEntityTooLarge, code: "artifact_too_large", message: "file input artifact is too large"},
|
||||||
{name: "artifact", err: wrap(scriptorium.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"},
|
{name: "artifact", err: wrap(promptkit.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"},
|
||||||
{name: "prompt render", err: wrap(scriptorium.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"},
|
{name: "prompt render", err: wrap(promptkit.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"},
|
||||||
{name: "llm", err: wrap(scriptorium.ErrLLMGenerate, fmt.Errorf("llm failed")), status: http.StatusBadGateway, code: "llm_failed", message: "model generation request failed", avoidCause: "llm failed"},
|
{name: "capacity", err: &promptkit.CapacityError{BackendID: "private-backend"}, status: http.StatusServiceUnavailable, code: "capacity_exceeded", message: "model backend capacity is exhausted", avoidCause: "private-backend"},
|
||||||
{name: "validation runtime", err: wrap(scriptorium.ErrValidation, fmt.Errorf("validator broke")), status: http.StatusInternalServerError, code: "validation_runtime_failed", message: "validation runtime failed", avoidCause: "validator broke"},
|
{name: "llm", err: wrap(promptkit.ErrLLMGenerate, fmt.Errorf("llm failed")), status: http.StatusBadGateway, code: "llm_failed", message: "model generation request failed", avoidCause: "llm failed"},
|
||||||
|
{name: "validation runtime", err: wrap(promptkit.ErrValidation, fmt.Errorf("validator broke")), status: http.StatusInternalServerError, code: "validation_runtime_failed", message: "validation runtime failed", avoidCause: "validator broke"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
@@ -776,12 +1179,12 @@ func TestHandlerRawAPIKeyRejectedByStrictJSON(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerValidationFailureStillSuccessAndRawOutputOptIn(t *testing.T) {
|
func TestHandlerValidationFailureStillSuccessAndRawOutputOptIn(t *testing.T) {
|
||||||
h := NewHandler(&fakeRunner{result: &scriptorium.RunResult{
|
h := NewHandler(&fakeRunner{result: &promptkit.RunResult{
|
||||||
Artifact: scriptorium.Artifact{Body: []byte("bad json")},
|
Artifact: promptkit.Artifact{Body: []byte("bad json")},
|
||||||
RawOutput: "bad json",
|
RawOutput: "bad json",
|
||||||
Validation: scriptorium.ValidationResult{
|
Validation: promptkit.ValidationResult{
|
||||||
Status: scriptorium.ValidationFailed,
|
Status: promptkit.ValidationFailed,
|
||||||
Mode: scriptorium.ValidationJSON,
|
Mode: promptkit.ValidationJSON,
|
||||||
Errors: []string{"invalid JSON"},
|
Errors: []string{"invalid JSON"},
|
||||||
},
|
},
|
||||||
}})
|
}})
|
||||||
@@ -839,22 +1242,22 @@ func newArtifactRootHandlerWithLimit(t *testing.T, root string, maxArtifactBytes
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected restricted artifact reader: %v", err)
|
t.Fatalf("expected restricted artifact reader: %v", err)
|
||||||
}
|
}
|
||||||
return NewHandler(newHandlerEngine(t, scriptorium.WithArtifactReader(reader)))
|
return NewHandler(newHandlerEngine(t, promptkit.WithArtifactReader(reader)))
|
||||||
}
|
}
|
||||||
|
|
||||||
func newHandlerEngine(t *testing.T, options ...scriptorium.Option) *scriptorium.Engine {
|
func newHandlerEngine(t *testing.T, options ...promptkit.Option) *promptkit.Engine {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
return newHandlerEngineWithOptions(t, append(options, scriptorium.WithLLMClient(handlerLLMClient{}))...)
|
return newHandlerEngineWithOptions(t, append(options, promptkit.WithLLMClient(handlerLLMClient{}))...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func newHandlerEngineWithDefaultClient(t *testing.T, options ...scriptorium.Option) *scriptorium.Engine {
|
func newHandlerEngineWithDefaultClient(t *testing.T, options ...promptkit.Option) *promptkit.Engine {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
return newHandlerEngineWithOptions(t, options...)
|
return newHandlerEngineWithOptions(t, options...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func newHandlerEngineWithOptions(t *testing.T, options ...scriptorium.Option) *scriptorium.Engine {
|
func newHandlerEngineWithOptions(t *testing.T, options ...promptkit.Option) *promptkit.Engine {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
promptDir := t.TempDir()
|
promptDir := t.TempDir()
|
||||||
@@ -879,7 +1282,7 @@ model: model
|
|||||||
t.Fatalf("write profile fixture: %v", err)
|
t.Fatalf("write profile fixture: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
PromptDir: promptDir,
|
PromptDir: promptDir,
|
||||||
ProfileDir: profileDir,
|
ProfileDir: profileDir,
|
||||||
}, options...)
|
}, options...)
|
||||||
@@ -889,6 +1292,31 @@ model: model
|
|||||||
return engine
|
return engine
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newDefinitionHandler(t *testing.T, definition string) *Handler {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
promptDir := t.TempDir()
|
||||||
|
profileDir := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(definition), 0o644); err != nil {
|
||||||
|
t.Fatalf("write prompt fixture: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`id: exec
|
||||||
|
endpoint: http://example.invalid/v1
|
||||||
|
model: model
|
||||||
|
`), 0o644); err != nil {
|
||||||
|
t.Fatalf("write profile fixture: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: promptDir,
|
||||||
|
ProfileDir: profileDir,
|
||||||
|
}, promptkit.WithLLMClient(handlerLLMClient{}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct public engine: %v", err)
|
||||||
|
}
|
||||||
|
return NewHandler(engine)
|
||||||
|
}
|
||||||
|
|
||||||
func assertHTTPErrorCode(t *testing.T, w *httptest.ResponseRecorder, status int, code string) {
|
func assertHTTPErrorCode(t *testing.T, w *httptest.ResponseRecorder, status int, code string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
|||||||
@@ -1,122 +0,0 @@
|
|||||||
package artifact
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"mime"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrUnsupportedRefType = errors.New("unsupported artifact reference type")
|
|
||||||
ErrMissingInlineBody = errors.New("missing body for inline artifact")
|
|
||||||
ErrMissingFilePath = errors.New("missing file path for file artifact")
|
|
||||||
)
|
|
||||||
|
|
||||||
// Reader resolves artifact references into actual artifacts.
|
|
||||||
type Reader interface {
|
|
||||||
Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// CompositeReader routes artifact resolution based on the reference type.
|
|
||||||
type CompositeReader struct {
|
|
||||||
inlineReader *inlineReader
|
|
||||||
fileReader Reader
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewCompositeReader() Reader {
|
|
||||||
return &CompositeReader{
|
|
||||||
inlineReader: &inlineReader{},
|
|
||||||
fileReader: &fileReader{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *CompositeReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
switch ref.Type {
|
|
||||||
case domain.ArtifactRefInline:
|
|
||||||
return c.inlineReader.Read(ctx, ref)
|
|
||||||
case domain.ArtifactRefFile:
|
|
||||||
return c.fileReader.Read(ctx, ref)
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedRefType, ref.Type)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type inlineReader struct{}
|
|
||||||
|
|
||||||
func (r *inlineReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
if ref.Body == "" {
|
|
||||||
return nil, ErrMissingInlineBody
|
|
||||||
}
|
|
||||||
|
|
||||||
body := []byte(ref.Body)
|
|
||||||
return &domain.Artifact{
|
|
||||||
ContentType: defaults.ContentTypeTextPlain,
|
|
||||||
Body: body,
|
|
||||||
Size: int64(len(body)),
|
|
||||||
Hash: fmt.Sprintf("%x", sha256.Sum256(body)),
|
|
||||||
URI: ref.URI,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type fileReader struct{}
|
|
||||||
|
|
||||||
func (r *fileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
if ref.URI == "" {
|
|
||||||
return nil, ErrMissingFilePath
|
|
||||||
}
|
|
||||||
|
|
||||||
return readFileArtifact(ref.URI)
|
|
||||||
}
|
|
||||||
|
|
||||||
func readFileArtifact(path string) (*domain.Artifact, error) {
|
|
||||||
file, err := os.Open(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
data, err := io.ReadAll(file)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
contentType := mime.TypeByExtension(filepath.Ext(path))
|
|
||||||
if contentType == "" {
|
|
||||||
contentType = defaults.ContentTypeTextPlain
|
|
||||||
}
|
|
||||||
|
|
||||||
return &domain.Artifact{
|
|
||||||
Name: filepath.Base(path),
|
|
||||||
ContentType: contentType,
|
|
||||||
Body: data,
|
|
||||||
URI: path,
|
|
||||||
Size: int64(len(data)),
|
|
||||||
Hash: fmt.Sprintf("%x", sha256.Sum256(data)),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
@@ -1,105 +0,0 @@
|
|||||||
package artifact
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCompositeReader_Read(t *testing.T) {
|
|
||||||
reader := NewCompositeReader()
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
t.Run("inline artifact", func(t *testing.T) {
|
|
||||||
ref := domain.ArtifactRef{
|
|
||||||
Type: domain.ArtifactRefInline,
|
|
||||||
Body: "hello world",
|
|
||||||
}
|
|
||||||
art, err := reader.Read(ctx, ref)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if string(art.Body) != "hello world" {
|
|
||||||
t.Errorf("expected 'hello world', got %s", string(art.Body))
|
|
||||||
}
|
|
||||||
if art.ContentType != "text/plain" {
|
|
||||||
t.Errorf("expected text/plain content type, got %q", art.ContentType)
|
|
||||||
}
|
|
||||||
if art.Hash != "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" {
|
|
||||||
t.Errorf("unexpected hash: %s", art.Hash)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("inline artifact missing body", func(t *testing.T) {
|
|
||||||
ref := domain.ArtifactRef{
|
|
||||||
Type: domain.ArtifactRefInline,
|
|
||||||
Body: "",
|
|
||||||
}
|
|
||||||
_, err := reader.Read(ctx, ref)
|
|
||||||
if !errors.Is(err, ErrMissingInlineBody) {
|
|
||||||
t.Errorf("expected ErrMissingInlineBody, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("unsupported ref type", func(t *testing.T) {
|
|
||||||
ref := domain.ArtifactRef{
|
|
||||||
Type: domain.ArtifactRefType("unsupported"),
|
|
||||||
URI: "unsupported://bucket/key",
|
|
||||||
}
|
|
||||||
_, err := reader.Read(ctx, ref)
|
|
||||||
if !errors.Is(err, ErrUnsupportedRefType) {
|
|
||||||
t.Error("expected error for unsupported type")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFileReader_Read(t *testing.T) {
|
|
||||||
content := []byte("test file content")
|
|
||||||
tmpFile, err := os.CreateTemp("", "artifact_test_*.txt")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer os.Remove(tmpFile.Name())
|
|
||||||
|
|
||||||
if _, err := tmpFile.Write(content); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
tmpFile.Close()
|
|
||||||
|
|
||||||
reader := NewCompositeReader()
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
t.Run("file artifact loading", func(t *testing.T) {
|
|
||||||
ref := domain.ArtifactRef{
|
|
||||||
Type: domain.ArtifactRefFile,
|
|
||||||
URI: tmpFile.Name(),
|
|
||||||
}
|
|
||||||
art, err := reader.Read(ctx, ref)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if string(art.Body) != string(content) {
|
|
||||||
t.Errorf("expected %s, got %s", string(content), string(art.Body))
|
|
||||||
}
|
|
||||||
if art.Name == "" {
|
|
||||||
t.Error("expected name to be inferred from filename")
|
|
||||||
}
|
|
||||||
if art.Hash != "60f5237ed4049f0382661ef009d2bc42e48c3ceb3edb6600f7024e7ab3b838f3" {
|
|
||||||
t.Errorf("unexpected hash: %s", art.Hash)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("missing file path", func(t *testing.T) {
|
|
||||||
ref := domain.ArtifactRef{
|
|
||||||
Type: domain.ArtifactRefFile,
|
|
||||||
URI: "",
|
|
||||||
}
|
|
||||||
_, err := reader.Read(ctx, ref)
|
|
||||||
if !errors.Is(err, ErrMissingFilePath) {
|
|
||||||
t.Errorf("expected ErrMissingFilePath, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
@@ -32,11 +33,29 @@ var (
|
|||||||
|
|
||||||
// Config is the on-disk YAML shape for application-level settings.
|
// Config is the on-disk YAML shape for application-level settings.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
PromptDir string `yaml:"prompt_dir"`
|
PromptDir string `yaml:"prompt_dir"`
|
||||||
ProfileDir string `yaml:"profile_dir"`
|
ProfileDir string `yaml:"profile_dir"`
|
||||||
SchemaDir string `yaml:"schema_dir"`
|
SchemaDir string `yaml:"schema_dir"`
|
||||||
Server ServerConfig `yaml:"server"`
|
Server ServerConfig `yaml:"server"`
|
||||||
Defaults DefaultsConfig `yaml:"defaults"`
|
Defaults DefaultsConfig `yaml:"defaults"`
|
||||||
|
Backends map[string]BackendConfig `yaml:"backends"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type BackendConfig struct {
|
||||||
|
Endpoint string `yaml:"endpoint"`
|
||||||
|
APIKeyEnv string `yaml:"api_key_env"`
|
||||||
|
ExtraParams map[string]any `yaml:"extra_params"`
|
||||||
|
ConcurrencyLimit int `yaml:"concurrency_limit"`
|
||||||
|
QueueCapacity *int `yaml:"queue_capacity"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type BackendSettings struct {
|
||||||
|
ID string
|
||||||
|
Endpoint string
|
||||||
|
APIKeyEnv string
|
||||||
|
ExtraParams map[string]any
|
||||||
|
ConcurrencyLimit int
|
||||||
|
QueueCapacity *int
|
||||||
}
|
}
|
||||||
|
|
||||||
type ServerConfig struct {
|
type ServerConfig struct {
|
||||||
@@ -62,6 +81,7 @@ type AppSettings struct {
|
|||||||
MaxArtifactBytes int64
|
MaxArtifactBytes int64
|
||||||
MaxResponseBytes int64
|
MaxResponseBytes int64
|
||||||
DefaultRenderFormat renderformat.PreparedRunOutputFormat
|
DefaultRenderFormat renderformat.PreparedRunOutputFormat
|
||||||
|
Backends []BackendSettings
|
||||||
}
|
}
|
||||||
|
|
||||||
// CLIOverrides can be applied after config load to enforce precedence.
|
// CLIOverrides can be applied after config load to enforce precedence.
|
||||||
@@ -245,6 +265,25 @@ func applyConfig(base AppSettings, cfg Config) (AppSettings, error) {
|
|||||||
}
|
}
|
||||||
out.DefaultRenderFormat = parsed
|
out.DefaultRenderFormat = parsed
|
||||||
}
|
}
|
||||||
|
if len(cfg.Backends) > 0 {
|
||||||
|
ids := make([]string, 0, len(cfg.Backends))
|
||||||
|
for id := range cfg.Backends {
|
||||||
|
ids = append(ids, id)
|
||||||
|
}
|
||||||
|
sort.Strings(ids)
|
||||||
|
out.Backends = make([]BackendSettings, 0, len(ids))
|
||||||
|
for _, id := range ids {
|
||||||
|
backend := cfg.Backends[id]
|
||||||
|
out.Backends = append(out.Backends, BackendSettings{
|
||||||
|
ID: id,
|
||||||
|
Endpoint: backend.Endpoint,
|
||||||
|
APIKeyEnv: backend.APIKeyEnv,
|
||||||
|
ExtraParams: backend.ExtraParams,
|
||||||
|
ConcurrencyLimit: backend.ConcurrencyLimit,
|
||||||
|
QueueCapacity: backend.QueueCapacity,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
@@ -20,7 +21,7 @@ func TestLoadConfigMissingImplicitPathUsesBuiltInDefaults(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
want := BuiltInDefaults()
|
want := BuiltInDefaults()
|
||||||
if got != want {
|
if !reflect.DeepEqual(got, want) {
|
||||||
t.Fatalf("unexpected settings: got=%+v want=%+v", got, want)
|
t.Fatalf("unexpected settings: got=%+v want=%+v", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -100,6 +101,78 @@ func TestLoadConfigAPIKeyFieldIsRejectedAsUnknown(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadConfigBackendsRetainsResolvedSettingsInSortedOrder(t *testing.T) {
|
||||||
|
path := writeConfigFile(t, "config.yml", `
|
||||||
|
backends:
|
||||||
|
zebra:
|
||||||
|
endpoint: https://zebra.example/v1
|
||||||
|
api_key_env: ZEBRA_API_KEY
|
||||||
|
extra_params:
|
||||||
|
provider_option: enabled
|
||||||
|
nested:
|
||||||
|
enabled: true
|
||||||
|
attempts: 2
|
||||||
|
concurrency_limit: 2
|
||||||
|
alpha:
|
||||||
|
endpoint: http://alpha.example/v1
|
||||||
|
queue_capacity: 0
|
||||||
|
`)
|
||||||
|
|
||||||
|
got, err := LoadConfig(path, true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if len(got.Backends) != 2 {
|
||||||
|
t.Fatalf("expected two backends, got %#v", got.Backends)
|
||||||
|
}
|
||||||
|
if got.Backends[0].ID != "alpha" || got.Backends[1].ID != "zebra" {
|
||||||
|
t.Fatalf("expected sorted backend IDs, got %#v", got.Backends)
|
||||||
|
}
|
||||||
|
if got.Backends[0].QueueCapacity == nil || *got.Backends[0].QueueCapacity != 0 {
|
||||||
|
t.Fatalf("expected explicit zero queue capacity, got %#v", got.Backends[0].QueueCapacity)
|
||||||
|
}
|
||||||
|
if got.Backends[1].QueueCapacity != nil {
|
||||||
|
t.Fatalf("expected omitted queue capacity to remain nil, got %#v", got.Backends[1].QueueCapacity)
|
||||||
|
}
|
||||||
|
wantParams := map[string]any{
|
||||||
|
"provider_option": "enabled",
|
||||||
|
"nested": map[string]any{
|
||||||
|
"enabled": true,
|
||||||
|
"attempts": 2,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got.Backends[1].ExtraParams, wantParams) {
|
||||||
|
t.Fatalf("unexpected extra params: got=%#v want=%#v", got.Backends[1].ExtraParams, wantParams)
|
||||||
|
}
|
||||||
|
if got.Backends[1].Endpoint != "https://zebra.example/v1" || got.Backends[1].APIKeyEnv != "ZEBRA_API_KEY" || got.Backends[1].ConcurrencyLimit != 2 {
|
||||||
|
t.Fatalf("unexpected zebra backend: %#v", got.Backends[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadConfigRejectsUnknownOrSecretBackendFields(t *testing.T) {
|
||||||
|
for name, body := range map[string]string{
|
||||||
|
"unknown": "backends:\n local:\n endpoint: http://localhost:11434/v1\n unexpected: value\n",
|
||||||
|
"secret": "backends:\n local:\n endpoint: http://localhost:11434/v1\n api_key: secret\n",
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
path := writeConfigFile(t, "config.yml", body)
|
||||||
|
_, err := LoadConfig(path, true)
|
||||||
|
if !errors.Is(err, ErrInvalidConfigYAML) {
|
||||||
|
t.Fatalf("expected ErrInvalidConfigYAML, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadConfigRejectsInvalidBackendFieldTypes(t *testing.T) {
|
||||||
|
path := writeConfigFile(t, "config.yml", "backends:\n local:\n endpoint: http://localhost:11434/v1\n queue_capacity: not-a-number\n")
|
||||||
|
|
||||||
|
_, err := LoadConfig(path, true)
|
||||||
|
if !errors.Is(err, ErrInvalidConfigYAML) {
|
||||||
|
t.Fatalf("expected ErrInvalidConfigYAML, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadConfigValidConfigSetsDirectoriesAndServerAddr(t *testing.T) {
|
func TestLoadConfigValidConfigSetsDirectoriesAndServerAddr(t *testing.T) {
|
||||||
path := writeConfigFile(t, "config.yml", `
|
path := writeConfigFile(t, "config.yml", `
|
||||||
prompt_dir: ./prompts
|
prompt_dir: ./prompts
|
||||||
@@ -196,7 +269,7 @@ func TestLoadConfigEmptyFileResolvesToBuiltInDefaults(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
want := BuiltInDefaults()
|
want := BuiltInDefaults()
|
||||||
if got != want {
|
if !reflect.DeepEqual(got, want) {
|
||||||
t.Fatalf("unexpected settings: got=%+v want=%+v", got, want)
|
t.Fatalf("unexpected settings: got=%+v want=%+v", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,39 +1,13 @@
|
|||||||
package defaults
|
package defaults
|
||||||
|
|
||||||
import (
|
import "time"
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
HTTPAddrDefault = ":8080"
|
HTTPAddrDefault = ":8080"
|
||||||
SchemaDirDefault = "."
|
SchemaDirDefault = "."
|
||||||
OutputArtifactName = "output"
|
|
||||||
ContentTypeTextPlain = "text/plain"
|
|
||||||
ContentTypeTextMarkdown = "text/markdown"
|
|
||||||
ContentTypeApplicationJSON = "application/json"
|
|
||||||
OpenAIChatCompletionsPath = "/chat/completions"
|
|
||||||
HTTPMaxRequestBytesDefault = 16 * 1024 * 1024
|
HTTPMaxRequestBytesDefault = 16 * 1024 * 1024
|
||||||
HTTPMaxArtifactBytesDefault = 16 * 1024 * 1024
|
HTTPMaxArtifactBytesDefault = 16 * 1024 * 1024
|
||||||
HTTPMaxResponseBytesDefault = 16 * 1024 * 1024
|
HTTPMaxResponseBytesDefault = 16 * 1024 * 1024
|
||||||
|
|
||||||
ExecutionDefaultTemperature = 0.0
|
|
||||||
ExecutionDefaultMaxTokens = 0
|
|
||||||
ExecutionDefaultTopP = 1.0
|
|
||||||
ExecutionDefaultTimeoutSeconds = 600
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var HTTPReadHeaderTimeoutDefault = 10 * time.Second
|
||||||
LLMRequestTimeoutDefault = 10 * time.Minute
|
|
||||||
HTTPReadHeaderTimeoutDefault = 10 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
func ExecutionTargetDefault() domain.ExecutionTarget {
|
|
||||||
return domain.ExecutionTarget{
|
|
||||||
Temperature: ExecutionDefaultTemperature,
|
|
||||||
MaxTokens: ExecutionDefaultMaxTokens,
|
|
||||||
TopP: ExecutionDefaultTopP,
|
|
||||||
TimeoutSeconds: ExecutionDefaultTimeoutSeconds,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,288 +0,0 @@
|
|||||||
package domain
|
|
||||||
|
|
||||||
import (
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ArtifactRefType defines how an artifact is referenced.
|
|
||||||
type ArtifactRefType string
|
|
||||||
|
|
||||||
const (
|
|
||||||
ArtifactRefInline ArtifactRefType = "inline"
|
|
||||||
ArtifactRefFile ArtifactRefType = "file"
|
|
||||||
)
|
|
||||||
|
|
||||||
// OutputFormat defines the desired format of the generated artifact.
|
|
||||||
type OutputFormat string
|
|
||||||
|
|
||||||
const (
|
|
||||||
FormatText OutputFormat = "text"
|
|
||||||
FormatMarkdown OutputFormat = "markdown"
|
|
||||||
FormatJSON OutputFormat = "json"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ValidationMode defines how the output should be validated.
|
|
||||||
type ValidationMode string
|
|
||||||
|
|
||||||
const (
|
|
||||||
ValidationNone ValidationMode = "none"
|
|
||||||
ValidationBasic ValidationMode = "basic"
|
|
||||||
ValidationJSON ValidationMode = "json"
|
|
||||||
ValidationJSONSchema ValidationMode = "json_schema"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ValidationStatus defines the result of a validation check.
|
|
||||||
type ValidationStatus string
|
|
||||||
|
|
||||||
const (
|
|
||||||
ValidationPassed ValidationStatus = "passed"
|
|
||||||
ValidationFailed ValidationStatus = "failed"
|
|
||||||
ValidationSkipped ValidationStatus = "skipped"
|
|
||||||
)
|
|
||||||
|
|
||||||
// CacheControlType defines provider cache behavior for prompt content.
|
|
||||||
type CacheControlType string
|
|
||||||
|
|
||||||
const (
|
|
||||||
CacheControlEphemeral CacheControlType = "ephemeral"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// SessionIDMaxLength is OpenRouter's documented maximum session_id length.
|
|
||||||
SessionIDMaxLength = 256
|
|
||||||
)
|
|
||||||
|
|
||||||
// CacheControl describes provider cache metadata attached to prompt content.
|
|
||||||
type CacheControl struct {
|
|
||||||
Type CacheControlType `yaml:"type" json:"type"`
|
|
||||||
TTL string `yaml:"ttl,omitempty" json:"ttl,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// RunRequest represents a request to generate a single artifact.
|
|
||||||
type RunRequest struct {
|
|
||||||
PromptID string
|
|
||||||
PromptVersion string
|
|
||||||
ProfileID string
|
|
||||||
APIKey string `json:"-" yaml:"-"`
|
|
||||||
Inputs map[string]ArtifactRef
|
|
||||||
Vars map[string]string
|
|
||||||
Execution *ExecutionTargetOverride
|
|
||||||
Validation *OutputContract
|
|
||||||
Metadata map[string]string
|
|
||||||
}
|
|
||||||
|
|
||||||
// RunResult represents the complete result of a prompt execution run.
|
|
||||||
type RunResult struct {
|
|
||||||
RunID string
|
|
||||||
Artifact Artifact
|
|
||||||
RawOutput string
|
|
||||||
Validation ValidationResult
|
|
||||||
PromptID string
|
|
||||||
PromptVersion string
|
|
||||||
PromptHash string
|
|
||||||
RenderedPromptHash string
|
|
||||||
SelectedProfileID string
|
|
||||||
ModelName string
|
|
||||||
Endpoint string
|
|
||||||
EffectiveModelParams ExecutionTarget
|
|
||||||
InputHashes map[string]string
|
|
||||||
Usage TokenUsage
|
|
||||||
StartTime time.Time
|
|
||||||
EndTime time.Time
|
|
||||||
Duration time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
|
|
||||||
// It must never include resolved API key values, model output, or validation data.
|
|
||||||
type PreparedRun struct {
|
|
||||||
PromptID string `json:"prompt_id"`
|
|
||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
|
||||||
PromptHash string `json:"prompt_hash,omitempty"`
|
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
|
||||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
|
||||||
TargetPresence ExecutionTargetPresence `json:"-"`
|
|
||||||
OutputContract OutputContract `json:"output_contract"`
|
|
||||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
|
||||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
|
||||||
SessionID string `json:"session_id,omitempty"`
|
|
||||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
|
||||||
Messages []RenderedMessage `json:"messages"`
|
|
||||||
StartTime time.Time `json:"start_time,omitempty"`
|
|
||||||
EndTime time.Time `json:"end_time,omitempty"`
|
|
||||||
DurationMS int64 `json:"duration_ms,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ArtifactRef represents a reference to an input artifact.
|
|
||||||
type ArtifactRef struct {
|
|
||||||
Type ArtifactRefType
|
|
||||||
URI string
|
|
||||||
Body string // Used for inline
|
|
||||||
}
|
|
||||||
|
|
||||||
// Artifact represents the actual loaded content of a reference.
|
|
||||||
type Artifact struct {
|
|
||||||
Name string
|
|
||||||
ContentType string
|
|
||||||
Body []byte
|
|
||||||
URI string
|
|
||||||
Size int64
|
|
||||||
Hash string
|
|
||||||
}
|
|
||||||
|
|
||||||
// PromptDefinition represents a configured prompt execution definition.
|
|
||||||
type PromptDefinition struct {
|
|
||||||
ID string `yaml:"id"`
|
|
||||||
Version string `yaml:"version"`
|
|
||||||
DefaultProfile string `yaml:"default_profile"`
|
|
||||||
Description string `yaml:"description"`
|
|
||||||
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
|
|
||||||
Inputs []PromptInput `yaml:"inputs"`
|
|
||||||
Templates []PromptMessageTemplate `yaml:"templates"`
|
|
||||||
OutputFormat OutputFormat `yaml:"output_format"`
|
|
||||||
Validation OutputContract `yaml:"validation"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// PromptInput describes one named input expected by a prompt definition.
|
|
||||||
type PromptInput struct {
|
|
||||||
Name string `yaml:"name"`
|
|
||||||
Required bool `yaml:"required"`
|
|
||||||
ContentType string `yaml:"content_type"`
|
|
||||||
Description string `yaml:"description"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// PromptMessageTemplate defines a template for a chat message.
|
|
||||||
type PromptMessageTemplate struct {
|
|
||||||
Role string `yaml:"role"`
|
|
||||||
Content string `yaml:"content"`
|
|
||||||
ContentFile string `yaml:"content_file"`
|
|
||||||
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExecutionProfile describes how and where to execute a model.
|
|
||||||
type ExecutionProfile struct {
|
|
||||||
ID string `yaml:"id"`
|
|
||||||
Endpoint string `yaml:"endpoint"`
|
|
||||||
Model string `yaml:"model"`
|
|
||||||
Temperature float64 `yaml:"temperature"`
|
|
||||||
MaxTokens int `yaml:"max_tokens"`
|
|
||||||
TopP float64 `yaml:"top_p"`
|
|
||||||
TimeoutSeconds int `yaml:"timeout_seconds"`
|
|
||||||
ServiceTier string `yaml:"service_tier"`
|
|
||||||
ReasoningEffort string `yaml:"reasoning_effort"`
|
|
||||||
APIKeyEnv string `yaml:"api_key_env"`
|
|
||||||
APIKeyRequired bool `yaml:"-" json:"-"`
|
|
||||||
ExtraParams map[string]any `yaml:"extra_params"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExecutionTargetOverride represents per-request runtime setting overrides.
|
|
||||||
type ExecutionTargetOverride struct {
|
|
||||||
Endpoint string `json:"endpoint,omitempty"`
|
|
||||||
Model string `json:"model,omitempty"`
|
|
||||||
Temperature *float64 `json:"temperature,omitempty"`
|
|
||||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
|
||||||
TopP *float64 `json:"top_p,omitempty"`
|
|
||||||
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
|
||||||
ServiceTier string `json:"service_tier,omitempty"`
|
|
||||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
|
||||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
|
||||||
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExecutionTargetPresence tracks which effective runtime fields came from an
|
|
||||||
// explicit request override even when the resolved value is a zero value.
|
|
||||||
type ExecutionTargetPresence struct {
|
|
||||||
Temperature bool
|
|
||||||
MaxTokens bool
|
|
||||||
TopP bool
|
|
||||||
TimeoutSeconds bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExecutionTarget represents effective model runtime settings for a run.
|
|
||||||
type ExecutionTarget struct {
|
|
||||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
|
||||||
Model string `yaml:"model" json:"model"`
|
|
||||||
Temperature float64 `yaml:"temperature" json:"temperature"`
|
|
||||||
MaxTokens int `yaml:"max_tokens" json:"max_tokens"`
|
|
||||||
TopP float64 `yaml:"top_p" json:"top_p"`
|
|
||||||
TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"`
|
|
||||||
ServiceTier string `yaml:"service_tier" json:"service_tier"`
|
|
||||||
ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"`
|
|
||||||
APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"`
|
|
||||||
APIKey string `yaml:"-" json:"-"`
|
|
||||||
APIKeyRequired bool `yaml:"-" json:"-"`
|
|
||||||
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// OutputContract defines the requirements for the output artifact.
|
|
||||||
type OutputContract struct {
|
|
||||||
Format OutputFormat `yaml:"format"`
|
|
||||||
ValidationMode ValidationMode `yaml:"validation_mode"`
|
|
||||||
SchemaPath string `yaml:"schema_path"`
|
|
||||||
RepairAttempts int `yaml:"repair_attempts"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// RenderedPrompt represents the prompt after template application.
|
|
||||||
type RenderedPrompt struct {
|
|
||||||
SessionID string `json:"session_id,omitempty"`
|
|
||||||
Messages []RenderedMessage `json:"messages"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// RenderedMessage is a single message in a rendered prompt.
|
|
||||||
type RenderedMessage struct {
|
|
||||||
Role string `json:"role"`
|
|
||||||
Content string `json:"content"`
|
|
||||||
CacheControl *CacheControl `json:"cache_control,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// GenerateRequest is the internal request passed to the LLM client.
|
|
||||||
type GenerateRequest struct {
|
|
||||||
Prompt RenderedPrompt
|
|
||||||
Target ExecutionTarget
|
|
||||||
TargetPresence ExecutionTargetPresence
|
|
||||||
StructuredOutput *StructuredOutputSpec
|
|
||||||
}
|
|
||||||
|
|
||||||
// StructuredOutputType indicates which provider-level output mode is requested.
|
|
||||||
type StructuredOutputType string
|
|
||||||
|
|
||||||
const (
|
|
||||||
StructuredOutputJSONSchema StructuredOutputType = "json_schema"
|
|
||||||
)
|
|
||||||
|
|
||||||
// StructuredOutputSpec describes provider-level structured output requirements.
|
|
||||||
type StructuredOutputSpec struct {
|
|
||||||
Type StructuredOutputType `json:"type"`
|
|
||||||
JSONSchema *StructuredOutputJSONSpec `json:"json_schema,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// StructuredOutputJSONSpec contains json_schema output constraints.
|
|
||||||
type StructuredOutputJSONSpec struct {
|
|
||||||
Name string `json:"name"`
|
|
||||||
Strict bool `json:"strict"`
|
|
||||||
Schema any `json:"schema"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// GenerateResponse is the response received from the LLM client.
|
|
||||||
type GenerateResponse struct {
|
|
||||||
Content string
|
|
||||||
Usage TokenUsage
|
|
||||||
}
|
|
||||||
|
|
||||||
// TokenUsage tracks token consumption.
|
|
||||||
type TokenUsage struct {
|
|
||||||
PromptTokens int
|
|
||||||
CompletionTokens int
|
|
||||||
TotalTokens int
|
|
||||||
CachedTokens int
|
|
||||||
CacheWriteTokens int
|
|
||||||
}
|
|
||||||
|
|
||||||
// ValidationResult represents the outcome of an output validation.
|
|
||||||
type ValidationResult struct {
|
|
||||||
Status ValidationStatus
|
|
||||||
Mode ValidationMode
|
|
||||||
Errors []string
|
|
||||||
SchemaPath string
|
|
||||||
RepairAttempts int
|
|
||||||
IsValid bool
|
|
||||||
}
|
|
||||||
@@ -1,141 +0,0 @@
|
|||||||
package domain
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
|
|
||||||
const envName = "SCRIPTORIUM_TEST_API_KEY"
|
|
||||||
const secret = "super-secret-value"
|
|
||||||
t.Setenv(envName, secret)
|
|
||||||
|
|
||||||
prepared := PreparedRun{
|
|
||||||
PromptID: "prompt.id",
|
|
||||||
PromptVersion: "v1",
|
|
||||||
PromptHash: "prompt-hash",
|
|
||||||
SelectedProfileID: "local-fast",
|
|
||||||
EffectiveModelParams: ExecutionTarget{
|
|
||||||
Endpoint: "http://llm/v1",
|
|
||||||
Model: "gpt-test",
|
|
||||||
APIKeyEnv: envName,
|
|
||||||
APIKey: secret,
|
|
||||||
},
|
|
||||||
InputHashes: map[string]string{"transcript": "hash-1"},
|
|
||||||
RenderedPromptHash: "rendered-hash",
|
|
||||||
Messages: []RenderedMessage{
|
|
||||||
{Role: "system", Content: "You are helpful."},
|
|
||||||
{Role: "user", Content: "Summarize this."},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
b, err := json.Marshal(prepared)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
out := string(b)
|
|
||||||
if strings.Contains(out, secret) {
|
|
||||||
t.Fatalf("prepared run JSON unexpectedly contains secret value: %s", out)
|
|
||||||
}
|
|
||||||
if !strings.Contains(out, `"api_key_env":"`+envName+`"`) {
|
|
||||||
t.Fatalf("prepared run JSON should include api_key_env name: %s", out)
|
|
||||||
}
|
|
||||||
|
|
||||||
var top map[string]any
|
|
||||||
if err := json.Unmarshal(b, &top); err != nil {
|
|
||||||
t.Fatalf("unmarshal failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, forbidden := range []string{"raw_output", "validation", "artifact"} {
|
|
||||||
if _, ok := top[forbidden]; ok {
|
|
||||||
t.Fatalf("prepared run JSON should not include %q", forbidden)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPreparedRunJSONIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
|
||||||
prepared := PreparedRun{
|
|
||||||
PromptID: "prompt.id",
|
|
||||||
SelectedProfileID: "local-fast",
|
|
||||||
EffectiveModelParams: ExecutionTarget{
|
|
||||||
Endpoint: "http://llm/v1",
|
|
||||||
Model: "gpt-test",
|
|
||||||
},
|
|
||||||
RenderedPromptHash: "rendered-hash",
|
|
||||||
Messages: []RenderedMessage{
|
|
||||||
{
|
|
||||||
Role: "system",
|
|
||||||
Content: "You are helpful.",
|
|
||||||
CacheControl: &CacheControl{
|
|
||||||
Type: CacheControlEphemeral,
|
|
||||||
TTL: "1h",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{Role: "user", Content: "Summarize this."},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
b, err := json.Marshal(prepared)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var decoded struct {
|
|
||||||
Messages []map[string]any `json:"messages"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(b, &decoded); err != nil {
|
|
||||||
t.Fatalf("unmarshal failed: %v", err)
|
|
||||||
}
|
|
||||||
if len(decoded.Messages) != 2 {
|
|
||||||
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
|
|
||||||
}
|
|
||||||
|
|
||||||
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("expected cache_control on first message, got %#v", decoded.Messages[0])
|
|
||||||
}
|
|
||||||
if cacheControl["type"] != string(CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
|
||||||
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
|
||||||
}
|
|
||||||
if _, ok := decoded.Messages[1]["cache_control"]; ok {
|
|
||||||
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPreparedRunJSONIncludesSessionIDOnlyWhenPresent(t *testing.T) {
|
|
||||||
prepared := PreparedRun{
|
|
||||||
PromptID: "prompt.id",
|
|
||||||
SelectedProfileID: "local-fast",
|
|
||||||
EffectiveModelParams: ExecutionTarget{
|
|
||||||
Endpoint: "http://llm/v1",
|
|
||||||
Model: "gpt-test",
|
|
||||||
},
|
|
||||||
SessionID: "session-123",
|
|
||||||
RenderedPromptHash: "rendered-hash",
|
|
||||||
Messages: []RenderedMessage{{Role: "user", Content: "Summarize this."}},
|
|
||||||
}
|
|
||||||
|
|
||||||
b, err := json.Marshal(prepared)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var decoded map[string]any
|
|
||||||
if err := json.Unmarshal(b, &decoded); err != nil {
|
|
||||||
t.Fatalf("unmarshal failed: %v", err)
|
|
||||||
}
|
|
||||||
if decoded["session_id"] != "session-123" {
|
|
||||||
t.Fatalf("expected session_id in prepared run JSON, got %#v", decoded["session_id"])
|
|
||||||
}
|
|
||||||
|
|
||||||
prepared.SessionID = ""
|
|
||||||
b, err = json.Marshal(prepared)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal failed: %v", err)
|
|
||||||
}
|
|
||||||
if strings.Contains(string(b), "session_id") {
|
|
||||||
t.Fatalf("expected empty session_id to be omitted, got %s", b)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,142 +0,0 @@
|
|||||||
package filecatalog
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io/fs"
|
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"path/filepath"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// FindYAMLFiles returns sorted full paths for .yaml and .yml files under root.
|
|
||||||
func FindYAMLFiles(ctx context.Context, root string) ([]string, error) {
|
|
||||||
var files []string
|
|
||||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
if d.IsDir() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if !IsYAMLFile(d.Name()) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
files = append(files, path)
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
sort.Strings(files)
|
|
||||||
return files, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// FindFSYAMLFiles returns sorted paths for .yaml and .yml files under root in fsys.
|
|
||||||
func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, error) {
|
|
||||||
cleanRoot := CleanFSRoot(root)
|
|
||||||
var files []string
|
|
||||||
err := fs.WalkDir(fsys, cleanRoot, func(name string, d fs.DirEntry, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
if d.IsDir() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if !IsYAMLFile(d.Name()) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
files = append(files, name)
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
sort.Strings(files)
|
|
||||||
return files, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// RelativePath computes a clean relative path from root to path.
|
|
||||||
func RelativePath(root string, filePath string) string {
|
|
||||||
rel, err := filepath.Rel(root, filePath)
|
|
||||||
if err != nil {
|
|
||||||
return filepath.Clean(filePath)
|
|
||||||
}
|
|
||||||
return filepath.Clean(rel)
|
|
||||||
}
|
|
||||||
|
|
||||||
// CleanFSRoot normalizes a root path for use with fs.FS.
|
|
||||||
func CleanFSRoot(root string) string {
|
|
||||||
root = strings.TrimSpace(root)
|
|
||||||
if root == "" || root == "." {
|
|
||||||
return "."
|
|
||||||
}
|
|
||||||
return path.Clean(root)
|
|
||||||
}
|
|
||||||
|
|
||||||
// DisplayPath returns name relative to root for messages about fs.FS paths.
|
|
||||||
func DisplayPath(root string, name string) string {
|
|
||||||
cleanRoot := CleanFSRoot(root)
|
|
||||||
cleanName := path.Clean(name)
|
|
||||||
if cleanRoot == "." {
|
|
||||||
return cleanName
|
|
||||||
}
|
|
||||||
prefix := strings.TrimSuffix(cleanRoot, "/") + "/"
|
|
||||||
if strings.HasPrefix(cleanName, prefix) {
|
|
||||||
return strings.TrimPrefix(cleanName, prefix)
|
|
||||||
}
|
|
||||||
return cleanName
|
|
||||||
}
|
|
||||||
|
|
||||||
// ResolveFSPath resolves userPath from baseDir and keeps it inside root.
|
|
||||||
func ResolveFSPath(root string, baseDir string, userPath string) (string, string, error) {
|
|
||||||
cleanRoot := CleanFSRoot(root)
|
|
||||||
cleanBase := path.Clean(strings.TrimSpace(baseDir))
|
|
||||||
if cleanBase == "" {
|
|
||||||
cleanBase = cleanRoot
|
|
||||||
}
|
|
||||||
if !containsFSPath(cleanRoot, cleanBase) {
|
|
||||||
return "", "", fmt.Errorf("base path %q is outside source root %q", cleanBase, cleanRoot)
|
|
||||||
}
|
|
||||||
|
|
||||||
cleanUserPath := strings.TrimSpace(userPath)
|
|
||||||
if cleanUserPath == "" {
|
|
||||||
return "", "", fmt.Errorf("path is required")
|
|
||||||
}
|
|
||||||
cleanUserPath = path.Clean(cleanUserPath)
|
|
||||||
if path.IsAbs(cleanUserPath) {
|
|
||||||
return "", "", fmt.Errorf("path %q must be relative", userPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
resolved := path.Clean(path.Join(cleanBase, cleanUserPath))
|
|
||||||
if !containsFSPath(cleanRoot, resolved) {
|
|
||||||
return "", "", fmt.Errorf("path %q escapes source root %q", userPath, cleanRoot)
|
|
||||||
}
|
|
||||||
return resolved, DisplayPath(cleanRoot, resolved), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func containsFSPath(root string, name string) bool {
|
|
||||||
root = CleanFSRoot(root)
|
|
||||||
name = path.Clean(name)
|
|
||||||
if root == "." {
|
|
||||||
return name == "." || (name != ".." && !strings.HasPrefix(name, "../"))
|
|
||||||
}
|
|
||||||
return name == root || strings.HasPrefix(name, strings.TrimSuffix(root, "/")+"/")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stem strips .yaml or .yml from a file name.
|
|
||||||
func Stem(name string) string {
|
|
||||||
name = strings.TrimSuffix(name, ".yaml")
|
|
||||||
name = strings.TrimSuffix(name, ".yml")
|
|
||||||
return name
|
|
||||||
}
|
|
||||||
|
|
||||||
func IsYAMLFile(name string) bool {
|
|
||||||
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
|
|
||||||
}
|
|
||||||
@@ -1,270 +0,0 @@
|
|||||||
package filecatalog
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"reflect"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"testing/fstest"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestFindYAMLFilesNestedSortedAndFiltered(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
mustWriteFile(t, filepath.Join(root, "z", "prompt.yml"), "id: z")
|
|
||||||
mustWriteFile(t, filepath.Join(root, "a", "profile.yaml"), "id: a")
|
|
||||||
mustWriteFile(t, filepath.Join(root, "a", "ignore.txt"), "not yaml")
|
|
||||||
mustWriteFile(t, filepath.Join(root, "b", "ignore.yaml.bak"), "not yaml")
|
|
||||||
|
|
||||||
got, err := FindYAMLFiles(context.Background(), root)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []string{
|
|
||||||
filepath.Join(root, "a", "profile.yaml"),
|
|
||||||
filepath.Join(root, "z", "prompt.yml"),
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(got, want) {
|
|
||||||
t.Fatalf("expected sorted YAML files %v, got %v", want, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFindYAMLFilesHonorsContextCancellation(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
mustWriteFile(t, filepath.Join(root, "one.yaml"), "id: one")
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
|
|
||||||
_, err := FindYAMLFiles(ctx, root)
|
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("expected context.Canceled, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFindFSYAMLFilesNestedSortedAndFiltered(t *testing.T) {
|
|
||||||
fsys := fstest.MapFS{
|
|
||||||
"prompts/z/prompt.yml": &fstest.MapFile{Data: []byte("id: z")},
|
|
||||||
"prompts/a/profile.yaml": &fstest.MapFile{Data: []byte("id: a")},
|
|
||||||
"prompts/a/ignore.txt": &fstest.MapFile{Data: []byte("not yaml")},
|
|
||||||
"prompts/b/ignore.yaml.bak": &fstest.MapFile{Data: []byte("not yaml")},
|
|
||||||
"other/ignored.yaml": &fstest.MapFile{Data: []byte("id: ignored")},
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := FindFSYAMLFiles(context.Background(), fsys, " prompts ")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []string{
|
|
||||||
"prompts/a/profile.yaml",
|
|
||||||
"prompts/z/prompt.yml",
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(got, want) {
|
|
||||||
t.Fatalf("expected sorted YAML files %v, got %v", want, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFindFSYAMLFilesHonorsContextCancellation(t *testing.T) {
|
|
||||||
fsys := fstest.MapFS{
|
|
||||||
"one.yaml": &fstest.MapFile{Data: []byte("id: one")},
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
|
|
||||||
_, err := FindFSYAMLFiles(ctx, fsys, ".")
|
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("expected context.Canceled, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRelativePathNested(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
path := filepath.Join(root, "nested", "profiles", "local.yaml")
|
|
||||||
got := RelativePath(root, path)
|
|
||||||
want := filepath.Join("nested", "profiles", "local.yaml")
|
|
||||||
if got != want {
|
|
||||||
t.Fatalf("expected relative path %q, got %q", want, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCleanFSRoot(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
root string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{name: "empty", root: "", want: "."},
|
|
||||||
{name: "dot", root: ".", want: "."},
|
|
||||||
{name: "trimmed", root: " prompts/../profiles ", want: "profiles"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
if got := CleanFSRoot(tc.root); got != tc.want {
|
|
||||||
t.Fatalf("expected %q, got %q", tc.want, got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDisplayPath(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
root string
|
|
||||||
path string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{name: "root dot", root: ".", path: "profiles/local.yaml", want: "profiles/local.yaml"},
|
|
||||||
{name: "nested root", root: "profiles", path: "profiles/local.yaml", want: "local.yaml"},
|
|
||||||
{name: "outside root", root: "profiles", path: "other/local.yaml", want: "other/local.yaml"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
if got := DisplayPath(tc.root, tc.path); got != tc.want {
|
|
||||||
t.Fatalf("expected %q, got %q", tc.want, got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveFSPath(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
root string
|
|
||||||
baseDir string
|
|
||||||
userPath string
|
|
||||||
wantPath string
|
|
||||||
wantDisplay string
|
|
||||||
wantErr string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "sibling inside root",
|
|
||||||
root: "prompts",
|
|
||||||
baseDir: "prompts/nested",
|
|
||||||
userPath: "./messages/user.tmpl",
|
|
||||||
wantPath: "prompts/nested/messages/user.tmpl",
|
|
||||||
wantDisplay: "nested/messages/user.tmpl",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "parent inside root",
|
|
||||||
root: "prompts",
|
|
||||||
baseDir: "prompts/nested",
|
|
||||||
userPath: "../shared/user.tmpl",
|
|
||||||
wantPath: "prompts/shared/user.tmpl",
|
|
||||||
wantDisplay: "shared/user.tmpl",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "escape rejected",
|
|
||||||
root: "prompts",
|
|
||||||
baseDir: "prompts/nested",
|
|
||||||
userPath: "../../outside.tmpl",
|
|
||||||
wantErr: "escapes source root",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "absolute path rejected",
|
|
||||||
root: "prompts",
|
|
||||||
baseDir: "prompts/nested",
|
|
||||||
userPath: "/outside.tmpl",
|
|
||||||
wantErr: "must be relative",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "empty path rejected",
|
|
||||||
root: "prompts",
|
|
||||||
baseDir: "prompts/nested",
|
|
||||||
userPath: " ",
|
|
||||||
wantErr: "path is required",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "dot root allows normal relative path",
|
|
||||||
root: ".",
|
|
||||||
baseDir: ".",
|
|
||||||
userPath: "schemas/events.schema.json",
|
|
||||||
wantPath: "schemas/events.schema.json",
|
|
||||||
wantDisplay: "schemas/events.schema.json",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "dot root rejects parent escape",
|
|
||||||
root: ".",
|
|
||||||
baseDir: ".",
|
|
||||||
userPath: "../outside.tmpl",
|
|
||||||
wantErr: "escapes source root",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
gotPath, gotDisplay, err := ResolveFSPath(tc.root, tc.baseDir, tc.userPath)
|
|
||||||
if tc.wantErr != "" {
|
|
||||||
if err == nil {
|
|
||||||
t.Fatalf("expected error containing %q", tc.wantErr)
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), tc.wantErr) {
|
|
||||||
t.Fatalf("expected error to contain %q, got %v", tc.wantErr, err)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if gotPath != tc.wantPath || gotDisplay != tc.wantDisplay {
|
|
||||||
t.Fatalf("expected path/display %q/%q, got %q/%q", tc.wantPath, tc.wantDisplay, gotPath, gotDisplay)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStemStripsYAMLExtensions(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
in string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{name: "yaml", in: "prompt.yaml", want: "prompt"},
|
|
||||||
{name: "yml", in: "profile.yml", want: "profile"},
|
|
||||||
{name: "other", in: "file.txt", want: "file.txt"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
if got := Stem(tc.in); got != tc.want {
|
|
||||||
t.Fatalf("expected %q, got %q", tc.want, got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIsYAMLFile(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
in string
|
|
||||||
want bool
|
|
||||||
}{
|
|
||||||
{name: "yaml", in: "prompt.yaml", want: true},
|
|
||||||
{name: "yml", in: "profile.yml", want: true},
|
|
||||||
{name: "backup", in: "profile.yaml.bak", want: false},
|
|
||||||
{name: "uppercase", in: "profile.YAML", want: false},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
if got := IsYAMLFile(tc.in); got != tc.want {
|
|
||||||
t.Fatalf("expected %v, got %v", tc.want, got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func mustWriteFile(t *testing.T, path string, content string) {
|
|
||||||
t.Helper()
|
|
||||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
||||||
t.Fatalf("failed to create directory: %v", err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
|
||||||
t.Fatalf("failed to write file %q: %v", path, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
122
internal/format/inspection.go
Normal file
122
internal/format/inspection.go
Normal file
@@ -0,0 +1,122 @@
|
|||||||
|
package format
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type PromptInspection struct {
|
||||||
|
PromptID string `json:"prompt_id"`
|
||||||
|
PromptVersion string `json:"prompt_version"`
|
||||||
|
PromptHash string `json:"prompt_hash"`
|
||||||
|
DefaultProfileID string `json:"default_profile_id"`
|
||||||
|
Inputs []PromptInspectionInput `json:"inputs"`
|
||||||
|
OutputContract OutputContract `json:"output_contract"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type PromptInspectionInput struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Required bool `json:"required"`
|
||||||
|
ContentType string `json:"content_type"`
|
||||||
|
Description string `json:"description"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type OutputContract struct {
|
||||||
|
Format string `json:"format"`
|
||||||
|
ValidationMode string `json:"validation_mode"`
|
||||||
|
SchemaPath string `json:"schema_path"`
|
||||||
|
RepairAttempts int `json:"repair_attempts"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ProfileInspection struct {
|
||||||
|
ProfileID string `json:"profile_id"`
|
||||||
|
EffectiveModelParams ProfileModelParams `json:"effective_model_params"`
|
||||||
|
APIKeyRequired bool `json:"api_key_required"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ProfileModelParams struct {
|
||||||
|
BackendID string `json:"backend_id"`
|
||||||
|
Endpoint string `json:"endpoint"`
|
||||||
|
Model string `json:"model"`
|
||||||
|
Temperature float64 `json:"temperature"`
|
||||||
|
MaxTokens int `json:"max_tokens"`
|
||||||
|
TopP float64 `json:"top_p"`
|
||||||
|
TimeoutSeconds int `json:"timeout_seconds"`
|
||||||
|
ServiceTier string `json:"service_tier"`
|
||||||
|
ReasoningEffort string `json:"reasoning_effort"`
|
||||||
|
APIKeyEnv string `json:"api_key_env"`
|
||||||
|
ExtraParams map[string]any `json:"extra_params"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func FormatPromptInspection(value *promptkit.PromptInspection, outputFormat OutputFormat) ([]byte, error) {
|
||||||
|
if value == nil {
|
||||||
|
return nil, errors.New("prompt inspection is nil")
|
||||||
|
}
|
||||||
|
dto := PromptInspection{
|
||||||
|
PromptID: value.PromptID, PromptVersion: value.PromptVersion, PromptHash: value.PromptHash, DefaultProfileID: value.DefaultProfileID,
|
||||||
|
Inputs: make([]PromptInspectionInput, len(value.Inputs)),
|
||||||
|
OutputContract: OutputContract{Format: string(value.OutputContract.Format), ValidationMode: string(value.OutputContract.ValidationMode), SchemaPath: value.OutputContract.SchemaPath, RepairAttempts: value.OutputContract.RepairAttempts},
|
||||||
|
}
|
||||||
|
for i, input := range value.Inputs {
|
||||||
|
dto.Inputs[i] = PromptInspectionInput{Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description}
|
||||||
|
}
|
||||||
|
switch outputFormat {
|
||||||
|
case OutputFormatJSON:
|
||||||
|
data, err := json.MarshalIndent(dto, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return append(data, '\n'), nil
|
||||||
|
case OutputFormatText:
|
||||||
|
var b bytes.Buffer
|
||||||
|
fmt.Fprintf(&b, "prompt_id: %s\nprompt_version: %s\nprompt_hash: %s\ndefault_profile_id: %s\ninputs:", dto.PromptID, dto.PromptVersion, dto.PromptHash, dto.DefaultProfileID)
|
||||||
|
if len(dto.Inputs) == 0 {
|
||||||
|
fmt.Fprintln(&b, " []")
|
||||||
|
} else {
|
||||||
|
fmt.Fprintln(&b)
|
||||||
|
for _, input := range dto.Inputs {
|
||||||
|
fmt.Fprintf(&b, " - name: %s\n required: %t\n content_type: %s\n description: %s\n", input.Name, input.Required, input.ContentType, strconv.Quote(input.Description))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, "output_contract:\n format: %s\n validation_mode: %s\n schema_path: %s\n repair_attempts: %d\n", dto.OutputContract.Format, dto.OutputContract.ValidationMode, dto.OutputContract.SchemaPath, dto.OutputContract.RepairAttempts)
|
||||||
|
return b.Bytes(), nil
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func FormatProfileInspection(value *promptkit.ProfileInspection, outputFormat OutputFormat) ([]byte, error) {
|
||||||
|
if value == nil {
|
||||||
|
return nil, errors.New("profile inspection is nil")
|
||||||
|
}
|
||||||
|
target := value.EffectiveModelParams
|
||||||
|
params := map[string]any{}
|
||||||
|
for key, item := range target.ExtraParams {
|
||||||
|
params[key] = item
|
||||||
|
}
|
||||||
|
dto := ProfileInspection{ProfileID: value.ProfileID, APIKeyRequired: value.APIKeyRequired, EffectiveModelParams: ProfileModelParams{BackendID: target.BackendID, Endpoint: target.Endpoint, Model: target.Model, Temperature: target.Temperature, MaxTokens: target.MaxTokens, TopP: target.TopP, TimeoutSeconds: target.TimeoutSeconds, ServiceTier: target.ServiceTier, ReasoningEffort: target.ReasoningEffort, APIKeyEnv: target.APIKeyEnv, ExtraParams: params}}
|
||||||
|
switch outputFormat {
|
||||||
|
case OutputFormatJSON:
|
||||||
|
data, err := json.MarshalIndent(dto, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return append(data, '\n'), nil
|
||||||
|
case OutputFormatText:
|
||||||
|
var b bytes.Buffer
|
||||||
|
fmt.Fprintf(&b, "profile_id: %s\neffective_model_params:\n backend_id: %s\n endpoint: %s\n model: %s\n temperature: %g\n max_tokens: %d\n top_p: %g\n timeout_seconds: %d\n service_tier: %s\n reasoning_effort: %s\n api_key_env: %s\n extra_params: ", dto.ProfileID, dto.EffectiveModelParams.BackendID, dto.EffectiveModelParams.Endpoint, dto.EffectiveModelParams.Model, dto.EffectiveModelParams.Temperature, dto.EffectiveModelParams.MaxTokens, dto.EffectiveModelParams.TopP, dto.EffectiveModelParams.TimeoutSeconds, dto.EffectiveModelParams.ServiceTier, dto.EffectiveModelParams.ReasoningEffort, dto.EffectiveModelParams.APIKeyEnv)
|
||||||
|
paramsJSON, err := json.Marshal(dto.EffectiveModelParams.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, "%s\napi_key_required: %t\n", paramsJSON, dto.APIKeyRequired)
|
||||||
|
return b.Bytes(), nil
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
|
||||||
|
}
|
||||||
|
}
|
||||||
124
internal/format/inspection_test.go
Normal file
124
internal/format/inspection_test.go
Normal file
@@ -0,0 +1,124 @@
|
|||||||
|
package format
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFormatPromptInspectionPreservesDeclaredOrderAndEmptyValues(t *testing.T) {
|
||||||
|
value := &promptkit.PromptInspection{PromptID: "p", PromptVersion: "1", PromptHash: "hash", Inputs: []promptkit.PromptInputDefinition{{Name: "first", Required: true, ContentType: "text/plain", Description: "first input"}, {Name: "second"}}, OutputContract: promptkit.OutputContract{Format: "text", ValidationMode: promptkit.ValidationNone}}
|
||||||
|
text, err := FormatPromptInspection(value, OutputFormatText)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(text), "default_profile_id: \ninputs:\n - name: first") || strings.Index(string(text), "name: first") > strings.Index(string(text), "name: second") {
|
||||||
|
t.Fatalf("unexpected text inspection: %s", text)
|
||||||
|
}
|
||||||
|
jsonOutput, err := FormatPromptInspection(value, OutputFormatJSON)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(string(jsonOutput), "\n") || !strings.Contains(string(jsonOutput), `"default_profile_id": ""`) {
|
||||||
|
t.Fatalf("unexpected json inspection: %s", jsonOutput)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFormatPromptInspectionRejectsNil(t *testing.T) {
|
||||||
|
if _, err := FormatPromptInspection(nil, OutputFormatText); err == nil {
|
||||||
|
t.Fatal("expected nil inspection error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFormatProfileInspectionPreservesSafeEffectiveValues(t *testing.T) {
|
||||||
|
const secret = "sentinel-secret-must-not-appear"
|
||||||
|
t.Setenv("FIXTURE_PROFILE_API_KEY", secret)
|
||||||
|
|
||||||
|
value := &promptkit.ProfileInspection{
|
||||||
|
ProfileID: "custom-derived",
|
||||||
|
EffectiveModelParams: promptkit.ExecutionTarget{
|
||||||
|
BackendID: "fixture-custom",
|
||||||
|
Endpoint: "http://127.0.0.1:11434/v1",
|
||||||
|
Model: "fixture-model",
|
||||||
|
Temperature: 0.25,
|
||||||
|
MaxTokens: 640,
|
||||||
|
TopP: 0.9,
|
||||||
|
TimeoutSeconds: 45,
|
||||||
|
ServiceTier: "flex",
|
||||||
|
ReasoningEffort: "high",
|
||||||
|
APIKeyEnv: "FIXTURE_PROFILE_API_KEY",
|
||||||
|
ExtraParams: map[string]any{
|
||||||
|
"zeta": true,
|
||||||
|
"alpha": map[string]any{"nested": []any{"first", 2.0}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
APIKeyRequired: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
jsonOutput, err := FormatProfileInspection(value, OutputFormatJSON)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
secondJSONOutput, err := FormatProfileInspection(value, OutputFormatJSON)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(jsonOutput) != string(secondJSONOutput) || !strings.HasSuffix(string(jsonOutput), "\n") {
|
||||||
|
t.Fatalf("expected deterministic newline-terminated JSON, got %q", jsonOutput)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(jsonOutput), secret) {
|
||||||
|
t.Fatalf("inspection exposed an environment secret: %s", jsonOutput)
|
||||||
|
}
|
||||||
|
|
||||||
|
var decoded ProfileInspection
|
||||||
|
if err := json.Unmarshal(jsonOutput, &decoded); err != nil {
|
||||||
|
t.Fatalf("decode profile inspection: %v", err)
|
||||||
|
}
|
||||||
|
if decoded.ProfileID != "custom-derived" || decoded.EffectiveModelParams.BackendID != "fixture-custom" {
|
||||||
|
t.Fatalf("unexpected profile identity: %+v", decoded)
|
||||||
|
}
|
||||||
|
if decoded.EffectiveModelParams.APIKeyEnv != "FIXTURE_PROFILE_API_KEY" || !decoded.APIKeyRequired {
|
||||||
|
t.Fatalf("unexpected credential metadata: %+v", decoded)
|
||||||
|
}
|
||||||
|
alpha, ok := decoded.EffectiveModelParams.ExtraParams["alpha"].(map[string]any)
|
||||||
|
if !ok || len(alpha["nested"].([]any)) != 2 {
|
||||||
|
t.Fatalf("nested extra parameters were not preserved: %#v", decoded.EffectiveModelParams.ExtraParams)
|
||||||
|
}
|
||||||
|
|
||||||
|
textOutput, err := FormatProfileInspection(value, OutputFormatText)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
textValue := string(textOutput)
|
||||||
|
if !strings.Contains(textValue, "backend_id: fixture-custom") ||
|
||||||
|
!strings.Contains(textValue, "api_key_env: FIXTURE_PROFILE_API_KEY") ||
|
||||||
|
!strings.Contains(textValue, `extra_params: {"alpha":{"nested":["first",2]},"zeta":true}`) ||
|
||||||
|
!strings.Contains(textValue, "api_key_required: true") {
|
||||||
|
t.Fatalf("unexpected text inspection: %s", textValue)
|
||||||
|
}
|
||||||
|
if strings.Contains(textValue, secret) {
|
||||||
|
t.Fatalf("text inspection exposed an environment secret: %s", textValue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFormatProfileInspectionPreservesEmptyBackendAndRejectsNil(t *testing.T) {
|
||||||
|
output, err := FormatProfileInspection(&promptkit.ProfileInspection{
|
||||||
|
ProfileID: "endpoint-only",
|
||||||
|
EffectiveModelParams: promptkit.ExecutionTarget{
|
||||||
|
Endpoint: "http://127.0.0.1:8000/v1",
|
||||||
|
Model: "fixture-model",
|
||||||
|
},
|
||||||
|
}, OutputFormatJSON)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(output), `"backend_id": ""`) ||
|
||||||
|
!strings.Contains(string(output), `"extra_params": {}`) {
|
||||||
|
t.Fatalf("expected explicit empty backend and object extra params, got %s", output)
|
||||||
|
}
|
||||||
|
if _, err := FormatProfileInspection(nil, OutputFormatText); err == nil {
|
||||||
|
t.Fatal("expected nil profile inspection error")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -9,37 +9,52 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
var ErrUnknownPreparedRunFormat = errors.New("unknown prepared run format")
|
var ErrUnknownPreparedRunFormat = errors.New("unknown prepared run format")
|
||||||
|
|
||||||
// PreparedRunOutputFormat is the output format for prepared render data.
|
// OutputFormat selects an application-owned textual or JSON representation.
|
||||||
type PreparedRunOutputFormat string
|
type OutputFormat string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
PreparedRunFormatText PreparedRunOutputFormat = "text"
|
OutputFormatText OutputFormat = "text"
|
||||||
PreparedRunFormatJSON PreparedRunOutputFormat = "json"
|
OutputFormatJSON OutputFormat = "json"
|
||||||
|
|
||||||
DefaultPreparedRunOutputFormat PreparedRunOutputFormat = PreparedRunFormatText
|
DefaultOutputFormat OutputFormat = OutputFormatText
|
||||||
|
)
|
||||||
|
|
||||||
|
// PreparedRunOutputFormat is the output format for prepared render data.
|
||||||
|
type PreparedRunOutputFormat = OutputFormat
|
||||||
|
|
||||||
|
const (
|
||||||
|
PreparedRunFormatText = OutputFormatText
|
||||||
|
PreparedRunFormatJSON = OutputFormatJSON
|
||||||
|
|
||||||
|
DefaultPreparedRunOutputFormat = DefaultOutputFormat
|
||||||
)
|
)
|
||||||
|
|
||||||
// PreparedRunFormatter serializes a prepared run without performing use case work.
|
// PreparedRunFormatter serializes a prepared run without performing use case work.
|
||||||
type PreparedRunFormatter interface {
|
type PreparedRunFormatter interface {
|
||||||
Format(prepared *scriptorium.PreparedRun) ([]byte, error)
|
Format(prepared *promptkit.PreparedRun) ([]byte, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParsePreparedRunOutputFormat parses a format name.
|
// ParsePreparedRunOutputFormat parses a format name.
|
||||||
func ParsePreparedRunOutputFormat(raw string) (PreparedRunOutputFormat, error) {
|
func ParsePreparedRunOutputFormat(raw string) (PreparedRunOutputFormat, error) {
|
||||||
switch PreparedRunOutputFormat(strings.ToLower(strings.TrimSpace(raw))) {
|
return ParseOutputFormat(raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseOutputFormat parses the shared inspection and prepared-run format names.
|
||||||
|
func ParseOutputFormat(raw string) (OutputFormat, error) {
|
||||||
|
switch OutputFormat(strings.ToLower(strings.TrimSpace(raw))) {
|
||||||
case "":
|
case "":
|
||||||
return DefaultPreparedRunOutputFormat, nil
|
return DefaultOutputFormat, nil
|
||||||
case PreparedRunFormatText:
|
case OutputFormatText:
|
||||||
return PreparedRunFormatText, nil
|
return OutputFormatText, nil
|
||||||
case PreparedRunFormatJSON:
|
case OutputFormatJSON:
|
||||||
return PreparedRunFormatJSON, nil
|
return OutputFormatJSON, nil
|
||||||
default:
|
default:
|
||||||
return "", fmt.Errorf("%w: %q (supported: %s, %s)", ErrUnknownPreparedRunFormat, raw, PreparedRunFormatText, PreparedRunFormatJSON)
|
return "", fmt.Errorf("%w: %q (supported: %s, %s)", ErrUnknownPreparedRunFormat, raw, OutputFormatText, OutputFormatJSON)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -56,7 +71,7 @@ func FormatterForPreparedRun(outputFormat PreparedRunOutputFormat) (PreparedRunF
|
|||||||
}
|
}
|
||||||
|
|
||||||
// FormatPreparedRun formats a prepared run using the selected format.
|
// FormatPreparedRun formats a prepared run using the selected format.
|
||||||
func FormatPreparedRun(prepared *scriptorium.PreparedRun, outputFormat PreparedRunOutputFormat) ([]byte, error) {
|
func FormatPreparedRun(prepared *promptkit.PreparedRun, outputFormat PreparedRunOutputFormat) ([]byte, error) {
|
||||||
formatter, err := FormatterForPreparedRun(outputFormat)
|
formatter, err := FormatterForPreparedRun(outputFormat)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -65,7 +80,7 @@ func FormatPreparedRun(prepared *scriptorium.PreparedRun, outputFormat PreparedR
|
|||||||
}
|
}
|
||||||
|
|
||||||
// FormatPreparedRunByName parses a format name and formats a prepared run.
|
// FormatPreparedRunByName parses a format name and formats a prepared run.
|
||||||
func FormatPreparedRunByName(prepared *scriptorium.PreparedRun, rawFormat string) ([]byte, error) {
|
func FormatPreparedRunByName(prepared *promptkit.PreparedRun, rawFormat string) ([]byte, error) {
|
||||||
outputFormat, err := ParsePreparedRunOutputFormat(rawFormat)
|
outputFormat, err := ParsePreparedRunOutputFormat(rawFormat)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -75,7 +90,7 @@ func FormatPreparedRunByName(prepared *scriptorium.PreparedRun, rawFormat string
|
|||||||
|
|
||||||
type jsonPreparedRunFormatter struct{}
|
type jsonPreparedRunFormatter struct{}
|
||||||
|
|
||||||
func (jsonPreparedRunFormatter) Format(prepared *scriptorium.PreparedRun) ([]byte, error) {
|
func (jsonPreparedRunFormatter) Format(prepared *promptkit.PreparedRun) ([]byte, error) {
|
||||||
if prepared == nil {
|
if prepared == nil {
|
||||||
return nil, errors.New("prepared run is nil")
|
return nil, errors.New("prepared run is nil")
|
||||||
}
|
}
|
||||||
@@ -84,7 +99,7 @@ func (jsonPreparedRunFormatter) Format(prepared *scriptorium.PreparedRun) ([]byt
|
|||||||
|
|
||||||
type textPreparedRunFormatter struct{}
|
type textPreparedRunFormatter struct{}
|
||||||
|
|
||||||
func (textPreparedRunFormatter) Format(prepared *scriptorium.PreparedRun) ([]byte, error) {
|
func (textPreparedRunFormatter) Format(prepared *promptkit.PreparedRun) ([]byte, error) {
|
||||||
if prepared == nil {
|
if prepared == nil {
|
||||||
return nil, errors.New("prepared run is nil")
|
return nil, errors.New("prepared run is nil")
|
||||||
}
|
}
|
||||||
@@ -93,6 +108,9 @@ func (textPreparedRunFormatter) Format(prepared *scriptorium.PreparedRun) ([]byt
|
|||||||
fmt.Fprintf(&b, "prompt: %s\n", prepared.PromptID)
|
fmt.Fprintf(&b, "prompt: %s\n", prepared.PromptID)
|
||||||
fmt.Fprintf(&b, "prompt_version: %s\n", prepared.PromptVersion)
|
fmt.Fprintf(&b, "prompt_version: %s\n", prepared.PromptVersion)
|
||||||
fmt.Fprintf(&b, "selected_profile_id: %s\n", prepared.SelectedProfileID)
|
fmt.Fprintf(&b, "selected_profile_id: %s\n", prepared.SelectedProfileID)
|
||||||
|
if prepared.SelectedBackendID != "" {
|
||||||
|
fmt.Fprintf(&b, "selected_backend_id: %s\n", prepared.SelectedBackendID)
|
||||||
|
}
|
||||||
if prepared.PromptHash != "" {
|
if prepared.PromptHash != "" {
|
||||||
fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash)
|
fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash)
|
||||||
}
|
}
|
||||||
@@ -146,7 +164,7 @@ func (textPreparedRunFormatter) Format(prepared *scriptorium.PreparedRun) ([]byt
|
|||||||
|
|
||||||
fmt.Fprintln(&b, "messages:")
|
fmt.Fprintln(&b, "messages:")
|
||||||
roleOrder := make([]string, 0)
|
roleOrder := make([]string, 0)
|
||||||
byRole := make(map[string][]scriptorium.RenderedMessage)
|
byRole := make(map[string][]promptkit.RenderedMessage)
|
||||||
for _, msg := range prepared.Messages {
|
for _, msg := range prepared.Messages {
|
||||||
if _, exists := byRole[msg.Role]; !exists {
|
if _, exists := byRole[msg.Role]; !exists {
|
||||||
roleOrder = append(roleOrder, msg.Role)
|
roleOrder = append(roleOrder, msg.Role)
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) {
|
func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) {
|
||||||
@@ -22,6 +22,7 @@ func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) {
|
|||||||
"prompt: prompt.id",
|
"prompt: prompt.id",
|
||||||
"prompt_version: v1",
|
"prompt_version: v1",
|
||||||
"selected_profile_id: local-fast",
|
"selected_profile_id: local-fast",
|
||||||
|
"selected_backend_id: local",
|
||||||
"endpoint: http://llm/v1",
|
"endpoint: http://llm/v1",
|
||||||
"model: gpt-test",
|
"model: gpt-test",
|
||||||
"temperature: 0.4",
|
"temperature: 0.4",
|
||||||
@@ -107,12 +108,12 @@ func TestTextFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) {
|
|||||||
|
|
||||||
func TestTextFormatterIncludesMessageCacheControlBeforeContent(t *testing.T) {
|
func TestTextFormatterIncludesMessageCacheControlBeforeContent(t *testing.T) {
|
||||||
prepared := samplePreparedRun()
|
prepared := samplePreparedRun()
|
||||||
prepared.Messages = []scriptorium.RenderedMessage{
|
prepared.Messages = []promptkit.RenderedMessage{
|
||||||
{
|
{
|
||||||
Role: "system",
|
Role: "system",
|
||||||
Content: "System guidance.",
|
Content: "System guidance.",
|
||||||
CacheControl: &scriptorium.CacheControl{
|
CacheControl: &promptkit.CacheControl{
|
||||||
Type: scriptorium.CacheControlEphemeral,
|
Type: promptkit.CacheControlEphemeral,
|
||||||
TTL: "1h",
|
TTL: "1h",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -147,12 +148,12 @@ func TestTextFormatterIncludesSessionIDWhenPresent(t *testing.T) {
|
|||||||
|
|
||||||
func TestTextFormatterOmitsEmptyCacheControlTTL(t *testing.T) {
|
func TestTextFormatterOmitsEmptyCacheControlTTL(t *testing.T) {
|
||||||
prepared := samplePreparedRun()
|
prepared := samplePreparedRun()
|
||||||
prepared.Messages = []scriptorium.RenderedMessage{
|
prepared.Messages = []promptkit.RenderedMessage{
|
||||||
{
|
{
|
||||||
Role: "system",
|
Role: "system",
|
||||||
Content: "System guidance.",
|
Content: "System guidance.",
|
||||||
CacheControl: &scriptorium.CacheControl{
|
CacheControl: &promptkit.CacheControl{
|
||||||
Type: scriptorium.CacheControlEphemeral,
|
Type: promptkit.CacheControlEphemeral,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -230,12 +231,12 @@ func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
|||||||
|
|
||||||
func TestJSONFormatterIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
func TestJSONFormatterIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
||||||
prepared := samplePreparedRun()
|
prepared := samplePreparedRun()
|
||||||
prepared.Messages = []scriptorium.RenderedMessage{
|
prepared.Messages = []promptkit.RenderedMessage{
|
||||||
{
|
{
|
||||||
Role: "system",
|
Role: "system",
|
||||||
Content: "System guidance.",
|
Content: "System guidance.",
|
||||||
CacheControl: &scriptorium.CacheControl{
|
CacheControl: &promptkit.CacheControl{
|
||||||
Type: scriptorium.CacheControlEphemeral,
|
Type: promptkit.CacheControlEphemeral,
|
||||||
TTL: "1h",
|
TTL: "1h",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -261,7 +262,7 @@ func TestJSONFormatterIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
|||||||
if !ok {
|
if !ok {
|
||||||
t.Fatalf("expected first message cache_control, got %#v", decoded.Messages[0])
|
t.Fatalf("expected first message cache_control, got %#v", decoded.Messages[0])
|
||||||
}
|
}
|
||||||
if cacheControl["type"] != string(scriptorium.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
if cacheControl["type"] != string(promptkit.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
||||||
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
||||||
}
|
}
|
||||||
if _, ok := decoded.Messages[1]["cache_control"]; ok {
|
if _, ok := decoded.Messages[1]["cache_control"]; ok {
|
||||||
@@ -340,13 +341,15 @@ func TestFormatPreparedRunByNameUnknownFailsClearly(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func samplePreparedRun() *scriptorium.PreparedRun {
|
func samplePreparedRun() *promptkit.PreparedRun {
|
||||||
return &scriptorium.PreparedRun{
|
return &promptkit.PreparedRun{
|
||||||
PromptID: "prompt.id",
|
PromptID: "prompt.id",
|
||||||
PromptVersion: "v1",
|
PromptVersion: "v1",
|
||||||
PromptHash: "prompt-hash",
|
PromptHash: "prompt-hash",
|
||||||
SelectedProfileID: "local-fast",
|
SelectedProfileID: "local-fast",
|
||||||
EffectiveModelParams: scriptorium.ExecutionTarget{
|
SelectedBackendID: "local",
|
||||||
|
EffectiveModelParams: promptkit.ExecutionTarget{
|
||||||
|
BackendID: "local",
|
||||||
Endpoint: "http://llm/v1",
|
Endpoint: "http://llm/v1",
|
||||||
Model: "gpt-test",
|
Model: "gpt-test",
|
||||||
Temperature: 0.4,
|
Temperature: 0.4,
|
||||||
@@ -362,7 +365,7 @@ func samplePreparedRun() *scriptorium.PreparedRun {
|
|||||||
"glossary": "hash-glossary",
|
"glossary": "hash-glossary",
|
||||||
},
|
},
|
||||||
RenderedPromptHash: "rendered-hash",
|
RenderedPromptHash: "rendered-hash",
|
||||||
Messages: []scriptorium.RenderedMessage{
|
Messages: []promptkit.RenderedMessage{
|
||||||
{Role: "system", Content: "System guidance."},
|
{Role: "system", Content: "System guidance."},
|
||||||
{Role: "user", Content: "Summarize the transcript.\nInclude key entities."},
|
{Role: "user", Content: "Summarize the transcript.\nInclude key entities."},
|
||||||
{Role: "user", Content: "Second user message."},
|
{Role: "user", Content: "Second user message."},
|
||||||
|
|||||||
@@ -1,11 +0,0 @@
|
|||||||
package llm
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Client executes a rendered prompt against an LLM endpoint.
|
|
||||||
type Client interface {
|
|
||||||
Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error)
|
|
||||||
}
|
|
||||||
@@ -1,385 +0,0 @@
|
|||||||
package llm
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
"unicode/utf8"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrInvalidConfig = errors.New("invalid llm client configuration")
|
|
||||||
ErrInvalidRequest = errors.New("invalid generate request")
|
|
||||||
ErrRequestFailed = errors.New("llm request failed")
|
|
||||||
ErrUnexpectedStatus = errors.New("llm returned non-success status")
|
|
||||||
ErrMalformedResponse = errors.New("malformed llm response")
|
|
||||||
)
|
|
||||||
|
|
||||||
type OpenAICompatibleConfig struct {
|
|
||||||
BaseURL string
|
|
||||||
Model string
|
|
||||||
Timeout time.Duration
|
|
||||||
HTTPClient *http.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
type OpenAICompatibleClient struct {
|
|
||||||
baseURL string
|
|
||||||
defaultModel string
|
|
||||||
httpClient *http.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) {
|
|
||||||
baseURL := strings.TrimSpace(cfg.BaseURL)
|
|
||||||
if baseURL != "" {
|
|
||||||
if _, err := url.ParseRequestURI(baseURL); err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
timeout := cfg.Timeout
|
|
||||||
if timeout <= 0 {
|
|
||||||
timeout = defaults.LLMRequestTimeoutDefault
|
|
||||||
}
|
|
||||||
|
|
||||||
var client *http.Client
|
|
||||||
if cfg.HTTPClient != nil {
|
|
||||||
cloned := *cfg.HTTPClient
|
|
||||||
if cloned.Timeout <= 0 {
|
|
||||||
cloned.Timeout = timeout
|
|
||||||
}
|
|
||||||
client = &cloned
|
|
||||||
} else {
|
|
||||||
client = &http.Client{Timeout: timeout}
|
|
||||||
}
|
|
||||||
|
|
||||||
return &OpenAICompatibleClient{
|
|
||||||
baseURL: strings.TrimRight(baseURL, "/"),
|
|
||||||
defaultModel: cfg.Model,
|
|
||||||
httpClient: client,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
|
||||||
if req.Target.TimeoutSeconds < 0 {
|
|
||||||
return nil, fmt.Errorf("%w: timeout_seconds must be greater than or equal to 0", ErrInvalidRequest)
|
|
||||||
}
|
|
||||||
|
|
||||||
endpoint := strings.TrimSpace(req.Target.Endpoint)
|
|
||||||
if endpoint == "" {
|
|
||||||
endpoint = c.baseURL
|
|
||||||
}
|
|
||||||
if endpoint == "" {
|
|
||||||
return nil, fmt.Errorf("%w: endpoint is required", ErrInvalidRequest)
|
|
||||||
}
|
|
||||||
endpoint = strings.TrimRight(endpoint, "/") + defaults.OpenAIChatCompletionsPath
|
|
||||||
|
|
||||||
wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
wirePayload, err := openAIChatRequestPayload(wireReq)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
payload, err := json.Marshal(wirePayload)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
requestContext := ctx
|
|
||||||
if req.Target.TimeoutSeconds > 0 {
|
|
||||||
var cancel context.CancelFunc
|
|
||||||
requestContext, cancel = context.WithTimeout(
|
|
||||||
ctx,
|
|
||||||
time.Duration(req.Target.TimeoutSeconds)*time.Second,
|
|
||||||
)
|
|
||||||
defer cancel()
|
|
||||||
}
|
|
||||||
|
|
||||||
httpReq, err := http.NewRequestWithContext(requestContext, http.MethodPost, endpoint, bytes.NewReader(payload))
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
|
|
||||||
}
|
|
||||||
httpReq.Header.Set("Content-Type", "application/json")
|
|
||||||
if apiKey := strings.TrimSpace(req.Target.APIKey); apiKey != "" {
|
|
||||||
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
|
||||||
} else if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" {
|
|
||||||
apiKey := strings.TrimSpace(os.Getenv(envName))
|
|
||||||
if apiKey == "" {
|
|
||||||
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
|
|
||||||
}
|
|
||||||
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
httpClient := c.httpClient
|
|
||||||
if httpClient == nil {
|
|
||||||
httpClient = &http.Client{Timeout: defaults.LLMRequestTimeoutDefault}
|
|
||||||
}
|
|
||||||
|
|
||||||
httpResp, err := httpClient.Do(httpReq)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %v", ErrRequestFailed, err)
|
|
||||||
}
|
|
||||||
defer httpResp.Body.Close()
|
|
||||||
|
|
||||||
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
|
|
||||||
_, _ = io.Copy(io.Discard, io.LimitReader(httpResp.Body, 4096))
|
|
||||||
return nil, fmt.Errorf("%w: status=%d", ErrUnexpectedStatus, httpResp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
var wireResp openAIChatResponse
|
|
||||||
if err := json.NewDecoder(httpResp.Body).Decode(&wireResp); err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to decode response: %v", ErrMalformedResponse, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(wireResp.Choices) == 0 {
|
|
||||||
return nil, fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
|
|
||||||
}
|
|
||||||
content := wireResp.Choices[0].Message.Content
|
|
||||||
if content == "" {
|
|
||||||
return nil, fmt.Errorf("%w: first choice has empty message content", ErrMalformedResponse)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &domain.GenerateResponse{
|
|
||||||
Content: content,
|
|
||||||
Usage: domain.TokenUsage{
|
|
||||||
PromptTokens: wireResp.Usage.PromptTokens,
|
|
||||||
CompletionTokens: wireResp.Usage.CompletionTokens,
|
|
||||||
TotalTokens: wireResp.Usage.TotalTokens,
|
|
||||||
CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens,
|
|
||||||
CacheWriteTokens: wireResp.Usage.CacheWriteTokens,
|
|
||||||
},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) {
|
|
||||||
model := strings.TrimSpace(req.Target.Model)
|
|
||||||
if model == "" {
|
|
||||||
model = strings.TrimSpace(defaultModel)
|
|
||||||
}
|
|
||||||
if model == "" {
|
|
||||||
return openAIChatRequest{}, errors.New("model is required")
|
|
||||||
}
|
|
||||||
|
|
||||||
wireReq := openAIChatRequest{
|
|
||||||
Model: model,
|
|
||||||
}
|
|
||||||
if sessionID := strings.TrimSpace(req.Prompt.SessionID); sessionID != "" {
|
|
||||||
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
|
|
||||||
return openAIChatRequest{}, fmt.Errorf("session_id length %d exceeds maximum %d", n, domain.SessionIDMaxLength)
|
|
||||||
}
|
|
||||||
wireReq.SessionID = sessionID
|
|
||||||
}
|
|
||||||
|
|
||||||
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
|
|
||||||
for _, msg := range req.Prompt.Messages {
|
|
||||||
wireReq.Messages = append(wireReq.Messages, openAIChatRequestMessageFromRenderedMessage(msg))
|
|
||||||
}
|
|
||||||
|
|
||||||
if req.Target.Temperature != 0 || req.TargetPresence.Temperature {
|
|
||||||
wireReq.Temperature = &req.Target.Temperature
|
|
||||||
}
|
|
||||||
if req.Target.MaxTokens != 0 || req.TargetPresence.MaxTokens {
|
|
||||||
wireReq.MaxTokens = &req.Target.MaxTokens
|
|
||||||
}
|
|
||||||
if req.Target.TopP != 0 || req.TargetPresence.TopP {
|
|
||||||
wireReq.TopP = &req.Target.TopP
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(req.Target.ServiceTier) != "" {
|
|
||||||
wireReq.ServiceTier = req.Target.ServiceTier
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(req.Target.ReasoningEffort) != "" {
|
|
||||||
wireReq.ReasoningEffort = req.Target.ReasoningEffort
|
|
||||||
}
|
|
||||||
if len(req.Target.ExtraParams) > 0 {
|
|
||||||
wireReq.ExtraParams = req.Target.ExtraParams
|
|
||||||
}
|
|
||||||
if req.StructuredOutput != nil {
|
|
||||||
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
|
|
||||||
if err != nil {
|
|
||||||
return openAIChatRequest{}, err
|
|
||||||
}
|
|
||||||
wireReq.ResponseFormat = responseFormat
|
|
||||||
}
|
|
||||||
|
|
||||||
return wireReq, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type openAIChatRequest struct {
|
|
||||||
Model string `json:"model"`
|
|
||||||
SessionID string `json:"session_id,omitempty"`
|
|
||||||
Messages []openAIChatRequestMessage `json:"messages"`
|
|
||||||
Temperature *float64 `json:"temperature,omitempty"`
|
|
||||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
|
||||||
TopP *float64 `json:"top_p,omitempty"`
|
|
||||||
ServiceTier string `json:"service_tier,omitempty"`
|
|
||||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
|
||||||
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"`
|
|
||||||
ExtraParams map[string]any `json:"-"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
|
||||||
out := map[string]any{
|
|
||||||
"model": req.Model,
|
|
||||||
"messages": req.Messages,
|
|
||||||
}
|
|
||||||
if req.SessionID != "" {
|
|
||||||
out["session_id"] = req.SessionID
|
|
||||||
}
|
|
||||||
if req.Temperature != nil {
|
|
||||||
out["temperature"] = *req.Temperature
|
|
||||||
}
|
|
||||||
if req.MaxTokens != nil {
|
|
||||||
out["max_tokens"] = *req.MaxTokens
|
|
||||||
}
|
|
||||||
if req.TopP != nil {
|
|
||||||
out["top_p"] = *req.TopP
|
|
||||||
}
|
|
||||||
if req.ServiceTier != "" {
|
|
||||||
out["service_tier"] = req.ServiceTier
|
|
||||||
}
|
|
||||||
if req.ReasoningEffort != "" {
|
|
||||||
out["reasoning_effort"] = req.ReasoningEffort
|
|
||||||
}
|
|
||||||
if req.ResponseFormat != nil {
|
|
||||||
out["response_format"] = req.ResponseFormat
|
|
||||||
}
|
|
||||||
|
|
||||||
for key, value := range req.ExtraParams {
|
|
||||||
if key == "" {
|
|
||||||
return nil, errors.New("extra_params key must not be empty")
|
|
||||||
}
|
|
||||||
if _, reserved := reservedOpenAIChatRequestFields[key]; reserved {
|
|
||||||
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
|
|
||||||
}
|
|
||||||
if _, err := json.Marshal(value); err != nil {
|
|
||||||
return nil, fmt.Errorf("extra_params.%s must be JSON-serializable: %w", key, err)
|
|
||||||
}
|
|
||||||
out[key] = value
|
|
||||||
}
|
|
||||||
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var reservedOpenAIChatRequestFields = map[string]struct{}{
|
|
||||||
"model": {},
|
|
||||||
"session_id": {},
|
|
||||||
"messages": {},
|
|
||||||
"temperature": {},
|
|
||||||
"max_tokens": {},
|
|
||||||
"top_p": {},
|
|
||||||
"service_tier": {},
|
|
||||||
"reasoning_effort": {},
|
|
||||||
"response_format": {},
|
|
||||||
}
|
|
||||||
|
|
||||||
type openAIChatRequestMessage struct {
|
|
||||||
Role string `json:"role"`
|
|
||||||
Content any `json:"content"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type openAIChatTextContentBlock struct {
|
|
||||||
Type string `json:"type"`
|
|
||||||
Text string `json:"text"`
|
|
||||||
CacheControl *openAICacheControl `json:"cache_control,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type openAICacheControl struct {
|
|
||||||
Type string `json:"type"`
|
|
||||||
TTL string `json:"ttl,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type openAIChatResponseMessage struct {
|
|
||||||
Role string `json:"role"`
|
|
||||||
Content string `json:"content"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type openAIChatResponse struct {
|
|
||||||
Choices []struct {
|
|
||||||
Message openAIChatResponseMessage `json:"message"`
|
|
||||||
} `json:"choices"`
|
|
||||||
Usage struct {
|
|
||||||
PromptTokens int `json:"prompt_tokens"`
|
|
||||||
CompletionTokens int `json:"completion_tokens"`
|
|
||||||
TotalTokens int `json:"total_tokens"`
|
|
||||||
PromptTokensDetails struct {
|
|
||||||
CachedTokens int `json:"cached_tokens"`
|
|
||||||
} `json:"prompt_tokens_details"`
|
|
||||||
CacheWriteTokens int `json:"cache_write_tokens"`
|
|
||||||
} `json:"usage"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type openAIResponseFormat struct {
|
|
||||||
Type string `json:"type"`
|
|
||||||
JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type openAIJSONSchemaEnvelope struct {
|
|
||||||
Name string `json:"name"`
|
|
||||||
Strict bool `json:"strict"`
|
|
||||||
Schema any `json:"schema"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func openAIChatRequestMessageFromRenderedMessage(msg domain.RenderedMessage) openAIChatRequestMessage {
|
|
||||||
wireMsg := openAIChatRequestMessage{
|
|
||||||
Role: msg.Role,
|
|
||||||
Content: msg.Content,
|
|
||||||
}
|
|
||||||
if msg.CacheControl == nil {
|
|
||||||
return wireMsg
|
|
||||||
}
|
|
||||||
|
|
||||||
wireMsg.Content = []openAIChatTextContentBlock{
|
|
||||||
{
|
|
||||||
Type: "text",
|
|
||||||
Text: msg.Content,
|
|
||||||
CacheControl: &openAICacheControl{
|
|
||||||
Type: string(msg.CacheControl.Type),
|
|
||||||
TTL: msg.CacheControl.TTL,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
return wireMsg
|
|
||||||
}
|
|
||||||
|
|
||||||
func toOpenAIResponseFormat(spec *domain.StructuredOutputSpec) (*openAIResponseFormat, error) {
|
|
||||||
if spec == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
switch spec.Type {
|
|
||||||
case domain.StructuredOutputJSONSchema:
|
|
||||||
if spec.JSONSchema == nil {
|
|
||||||
return nil, errors.New("json_schema structured output requires schema payload")
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(spec.JSONSchema.Name) == "" {
|
|
||||||
return nil, errors.New("json_schema structured output requires non-empty schema name")
|
|
||||||
}
|
|
||||||
if spec.JSONSchema.Schema == nil {
|
|
||||||
return nil, errors.New("json_schema structured output requires schema document")
|
|
||||||
}
|
|
||||||
return &openAIResponseFormat{
|
|
||||||
Type: "json_schema",
|
|
||||||
JSONSchema: &openAIJSONSchemaEnvelope{
|
|
||||||
Name: spec.JSONSchema.Name,
|
|
||||||
Strict: spec.JSONSchema.Strict,
|
|
||||||
Schema: spec.JSONSchema.Schema,
|
|
||||||
},
|
|
||||||
}, nil
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("unsupported structured output type %q", spec.Type)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,9 +0,0 @@
|
|||||||
id: aion-2
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: aion-labs/aion-2.0
|
|
||||||
temperature: 0.72
|
|
||||||
reasoning_effort: high
|
|
||||||
top_p: 0.95
|
|
||||||
timeout_seconds: 180
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: claude-fable-latest
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: "~anthropic/claude-fable-latest"
|
|
||||||
reasoning_effort: high
|
|
||||||
timeout_seconds: 600
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: claude-haiku-latest
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: "~anthropic/claude-haiku-latest"
|
|
||||||
reasoning_effort: medium
|
|
||||||
timeout_seconds: 240
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: claude-opus-latest
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: "~anthropic/claude-opus-latest"
|
|
||||||
reasoning_effort: high
|
|
||||||
timeout_seconds: 240
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: claude-sonnet-latest
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: "~anthropic/claude-sonnet-latest"
|
|
||||||
reasoning_effort: high
|
|
||||||
timeout_seconds: 240
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: deepseek-3-2
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: deepseek/deepseek-v3.2
|
|
||||||
reasoning_effort: high
|
|
||||||
timeout_seconds: 180
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: deepseek-4-flash
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: deepseek/deepseek-v4-flash
|
|
||||||
#reasoning_effort: medium
|
|
||||||
timeout_seconds: 180
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: deepseek-4-pro
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: deepseek/deepseek-v4-pro
|
|
||||||
reasoning_effort: high
|
|
||||||
timeout_seconds: 180
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
id: gemini-2-flash-lite
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: "google/gemini-2.5-flash-lite"
|
|
||||||
#temperature: 0.15
|
|
||||||
reasoning_effort: high
|
|
||||||
#top_p: 0.98
|
|
||||||
timeout_seconds: 240
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
id: gemini-2-flash
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: "google/gemini-2.5-flash"
|
|
||||||
#temperature: 0.15
|
|
||||||
reasoning_effort: high
|
|
||||||
#top_p: 0.98
|
|
||||||
timeout_seconds: 240
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
id: gemini-2-pro
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: "google/gemini-2.5-pro"
|
|
||||||
#temperature: 0.15
|
|
||||||
reasoning_effort: high
|
|
||||||
#top_p: 0.98
|
|
||||||
timeout_seconds: 240
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
id: gemini-3-flash-lite
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: "google/gemini-3.1-flash-lite"
|
|
||||||
#temperature: 0.15
|
|
||||||
reasoning_effort: high
|
|
||||||
#top_p: 0.98
|
|
||||||
timeout_seconds: 240
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
id: gemini-flash-latest
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: "~google/gemini-flash-latest"
|
|
||||||
#temperature: 0.15
|
|
||||||
reasoning_effort: high
|
|
||||||
#top_p: 0.98
|
|
||||||
timeout_seconds: 240
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
id: gemini-pro-latest
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: "~google/gemini-pro-latest"
|
|
||||||
#temperature: 0.15
|
|
||||||
reasoning_effort: high
|
|
||||||
#top_p: 0.98
|
|
||||||
timeout_seconds: 240
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
id: gemma-4-31b
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: google/gemma-4-31b-it:exacto
|
|
||||||
temperature: 0.15
|
|
||||||
reasoning_effort: high
|
|
||||||
top_p: 0.98
|
|
||||||
timeout_seconds: 240
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
id: minimax-m2
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: minimax/minimax-m2.5
|
|
||||||
temperature: 0.5
|
|
||||||
reasoning_effort: high
|
|
||||||
top_p: 0.95
|
|
||||||
timeout_seconds: 180
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
id: minimax-m3
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: minimax/minimax-m3
|
|
||||||
#temperature: 0.5
|
|
||||||
reasoning_effort: high
|
|
||||||
#top_p: 0.95
|
|
||||||
timeout_seconds: 180
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: mistral-large-2512
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: mistralai/mistral-large-2512
|
|
||||||
temperature: 0.15
|
|
||||||
top_p: 0.98
|
|
||||||
timeout_seconds: 180
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
id: mistral-medium-3-5
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: mistralai/mistral-medium-3-5
|
|
||||||
temperature: 0.15
|
|
||||||
reasoning_effort: high
|
|
||||||
top_p: 0.98
|
|
||||||
timeout_seconds: 180
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: mistral-small-3
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: mistralai/mistral-small-3.2-24b-instruct
|
|
||||||
temperature: 0.05
|
|
||||||
top_p: 1.0
|
|
||||||
timeout_seconds: 180
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
id: mistral-small-4
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: mistralai/mistral-small-2603
|
|
||||||
temperature: 0.1
|
|
||||||
reasoning_effort: high
|
|
||||||
top_p: 0.98
|
|
||||||
timeout_seconds: 180
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: nemotron-3-ultra
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: nvidia/nemotron-3-ultra-550b-a55b
|
|
||||||
reasoning_effort: high
|
|
||||||
timeout_seconds: 180
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: gpt-5-mini
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: "openai/gpt-5.4-mini"
|
|
||||||
reasoning_effort: high
|
|
||||||
timeout_seconds: 240
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: gpt-5-nano
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
model: "openai/gpt-5.4-nano"
|
|
||||||
reasoning_effort: high
|
|
||||||
timeout_seconds: 240
|
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
package builtin
|
|
||||||
|
|
||||||
import (
|
|
||||||
"embed"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
|
||||||
)
|
|
||||||
|
|
||||||
const assetRoot = "assets"
|
|
||||||
|
|
||||||
//go:embed assets/**/*.yml
|
|
||||||
var assets embed.FS
|
|
||||||
|
|
||||||
func NewRepository() profile.Repository {
|
|
||||||
return profile.NewFSRepository(assets, assetRoot)
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewRepositoryWithPrimary(primary profile.Repository) profile.Repository {
|
|
||||||
if primary == nil {
|
|
||||||
return NewRepository()
|
|
||||||
}
|
|
||||||
return profile.NewOverlayRepository(primary, NewRepository())
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewRepositoryWithDirectory(dir string) profile.Repository {
|
|
||||||
if strings.TrimSpace(dir) == "" {
|
|
||||||
return NewRepository()
|
|
||||||
}
|
|
||||||
return NewRepositoryWithPrimary(profile.NewFilesystemRepository(dir))
|
|
||||||
}
|
|
||||||
@@ -1,127 +0,0 @@
|
|||||||
package builtin
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"io/fs"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
|
||||||
"gopkg.in/yaml.v3"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
|
|
||||||
repo := NewRepository()
|
|
||||||
ids := loadBuiltInProfileIDs(t)
|
|
||||||
if len(ids) == 0 {
|
|
||||||
t.Fatal("expected built-in profiles")
|
|
||||||
}
|
|
||||||
|
|
||||||
for id := range ids {
|
|
||||||
t.Run(id, func(t *testing.T) {
|
|
||||||
p, err := repo.GetProfile(context.Background(), id)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected built-in profile %q to load, got %v", id, err)
|
|
||||||
}
|
|
||||||
if p.ID != id {
|
|
||||||
t.Fatalf("expected profile id %q, got %q", id, p.ID)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuiltInProfilesDoNotContainDuplicateIDsOrRawAPIKeys(t *testing.T) {
|
|
||||||
loadBuiltInProfileIDs(t)
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadBuiltInProfileIDs(t *testing.T) map[string]string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
ids := map[string]string{}
|
|
||||||
err := fs.WalkDir(assets, assetRoot, func(name string, d fs.DirEntry, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if d.IsDir() || !strings.HasSuffix(name, ".yml") {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := assets.ReadFile(name)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read built-in profile %s: %v", name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var raw map[string]any
|
|
||||||
if err := yaml.Unmarshal(data, &raw); err != nil {
|
|
||||||
t.Fatalf("failed to decode built-in profile %s: %v", name, err)
|
|
||||||
}
|
|
||||||
if _, ok := raw["api_key"]; ok {
|
|
||||||
t.Fatalf("built-in profile %s contains raw api_key", name)
|
|
||||||
}
|
|
||||||
id, ok := raw["id"].(string)
|
|
||||||
if !ok || strings.TrimSpace(id) == "" {
|
|
||||||
t.Fatalf("built-in profile %s has missing id", name)
|
|
||||||
}
|
|
||||||
if previous, ok := ids[id]; ok {
|
|
||||||
t.Fatalf("duplicate built-in profile id %q in %s and %s", id, previous, name)
|
|
||||||
}
|
|
||||||
ids[id] = name
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to walk built-in profiles: %v", err)
|
|
||||||
}
|
|
||||||
return ids
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRepositoryWithPrimaryUsesPrimaryBeforeBuiltIns(t *testing.T) {
|
|
||||||
repo := NewRepositoryWithPrimary(staticProfileRepo{
|
|
||||||
profiles: map[string]string{"mistral-small-3": "custom-model"},
|
|
||||||
})
|
|
||||||
|
|
||||||
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected profile to load, got %v", err)
|
|
||||||
}
|
|
||||||
if p.Model != "custom-model" {
|
|
||||||
t.Fatalf("expected primary profile to override built-in, got %+v", p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRepositoryWithPrimaryFallsBackToBuiltIns(t *testing.T) {
|
|
||||||
repo := NewRepositoryWithPrimary(staticProfileRepo{})
|
|
||||||
|
|
||||||
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected built-in profile to load, got %v", err)
|
|
||||||
}
|
|
||||||
if p.ID != "mistral-small-3" {
|
|
||||||
t.Fatalf("unexpected profile: %+v", p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRepositoryWithPrimaryDoesNotFallBackAfterPrimaryError(t *testing.T) {
|
|
||||||
repo := NewRepositoryWithPrimary(staticProfileRepo{err: profile.ErrInvalidProfile})
|
|
||||||
|
|
||||||
_, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
|
||||||
if !errors.Is(err, profile.ErrInvalidProfile) {
|
|
||||||
t.Fatalf("expected primary error, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type staticProfileRepo struct {
|
|
||||||
profiles map[string]string
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
|
||||||
if r.err != nil {
|
|
||||||
return nil, r.err
|
|
||||||
}
|
|
||||||
if model, ok := r.profiles[id]; ok {
|
|
||||||
return &domain.ExecutionProfile{ID: id, Endpoint: "http://primary/v1", Model: model}, nil
|
|
||||||
}
|
|
||||||
return nil, profile.ErrProfileNotFound
|
|
||||||
}
|
|
||||||
@@ -1,213 +0,0 @@
|
|||||||
package profile
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io/fs"
|
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/filecatalog"
|
|
||||||
"gopkg.in/yaml.v3"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrProfileNotFound = errors.New("execution profile not found")
|
|
||||||
ErrInvalidYAML = errors.New("invalid YAML format")
|
|
||||||
ErrInvalidProfile = errors.New("invalid execution profile configuration")
|
|
||||||
ErrRawAPIKeyNotAllowed = errors.New("raw api_key is not allowed; use api_key_env")
|
|
||||||
)
|
|
||||||
|
|
||||||
type filesystemRepository struct {
|
|
||||||
dir string
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewFilesystemRepository(dir string) Repository {
|
|
||||||
return &filesystemRepository{dir: dir}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
|
||||||
return loadProfile(ctx, os.DirFS(r.dir), ".", id)
|
|
||||||
}
|
|
||||||
|
|
||||||
type fsRepository struct {
|
|
||||||
fsys fs.FS
|
|
||||||
root string
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewFSRepository(fsys fs.FS, root string) Repository {
|
|
||||||
return &fsRepository{fsys: fsys, root: root}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fsRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
|
||||||
return loadProfile(ctx, r.fsys, r.root, id)
|
|
||||||
}
|
|
||||||
|
|
||||||
type overlayRepository struct {
|
|
||||||
primary Repository
|
|
||||||
fallback Repository
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewOverlayRepository(primary, fallback Repository) Repository {
|
|
||||||
return &overlayRepository{primary: primary, fallback: fallback}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *overlayRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
|
||||||
if r.primary != nil {
|
|
||||||
prof, err := r.primary.GetProfile(ctx, id)
|
|
||||||
if err == nil {
|
|
||||||
return prof, nil
|
|
||||||
}
|
|
||||||
if !errors.Is(err, ErrProfileNotFound) {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if r.fallback == nil {
|
|
||||||
return nil, ErrProfileNotFound
|
|
||||||
}
|
|
||||||
return r.fallback.GetProfile(ctx, id)
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*domain.ExecutionProfile, error) {
|
|
||||||
if strings.TrimSpace(id) == "" {
|
|
||||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
|
|
||||||
}
|
|
||||||
if fsys == nil {
|
|
||||||
return nil, fmt.Errorf("failed to read profile directory: filesystem is nil")
|
|
||||||
}
|
|
||||||
|
|
||||||
files, err := filecatalog.FindFSYAMLFiles(ctx, fsys, root)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read profile directory: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var matches []profileMatch
|
|
||||||
for _, fullPath := range files {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
relPath := filecatalog.DisplayPath(root, fullPath)
|
|
||||||
fileMatch := filecatalog.Stem(path.Base(fullPath)) == id
|
|
||||||
data, err := fs.ReadFile(fsys, fullPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
|
|
||||||
}
|
|
||||||
metadata := readProfileFileMetadata(data)
|
|
||||||
idMatch := fileMatch || metadata.id == id
|
|
||||||
if metadata.hasRawAPIKey {
|
|
||||||
if idMatch {
|
|
||||||
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
var prof domain.ExecutionProfile
|
|
||||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
|
||||||
decoder.KnownFields(true)
|
|
||||||
if err := decoder.Decode(&prof); err != nil {
|
|
||||||
if idMatch {
|
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if prof.ID != id {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := validateProfile(&prof); err != nil {
|
|
||||||
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
|
||||||
return nil, fmt.Errorf("%w: %s", err, relPath)
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
|
|
||||||
}
|
|
||||||
matches = append(matches, profileMatch{
|
|
||||||
profile: &prof,
|
|
||||||
path: relPath,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(matches) > 1 {
|
|
||||||
paths := make([]string, 0, len(matches))
|
|
||||||
for _, match := range matches {
|
|
||||||
paths = append(paths, match.path)
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("%w: duplicate execution profile id %q found in: %s", ErrInvalidProfile, id, strings.Join(paths, ", "))
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(matches) == 1 {
|
|
||||||
return matches[0].profile, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil, ErrProfileNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
type profileMatch struct {
|
|
||||||
profile *domain.ExecutionProfile
|
|
||||||
path string
|
|
||||||
}
|
|
||||||
|
|
||||||
type profileFileMetadata struct {
|
|
||||||
id string
|
|
||||||
hasRawAPIKey bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func readProfileFileMetadata(data []byte) profileFileMetadata {
|
|
||||||
var node yaml.Node
|
|
||||||
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&node); err != nil {
|
|
||||||
return profileFileMetadata{}
|
|
||||||
}
|
|
||||||
if node.Kind != yaml.DocumentNode || len(node.Content) == 0 {
|
|
||||||
return profileFileMetadata{}
|
|
||||||
}
|
|
||||||
mapping := node.Content[0]
|
|
||||||
if mapping.Kind != yaml.MappingNode {
|
|
||||||
return profileFileMetadata{}
|
|
||||||
}
|
|
||||||
|
|
||||||
var metadata profileFileMetadata
|
|
||||||
for i := 0; i+1 < len(mapping.Content); i += 2 {
|
|
||||||
key := mapping.Content[i]
|
|
||||||
value := mapping.Content[i+1]
|
|
||||||
switch key.Value {
|
|
||||||
case "id":
|
|
||||||
metadata.id = strings.TrimSpace(value.Value)
|
|
||||||
case "api_key":
|
|
||||||
metadata.hasRawAPIKey = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return metadata
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateProfile(p *domain.ExecutionProfile) error {
|
|
||||||
if strings.TrimSpace(p.ID) == "" {
|
|
||||||
return errors.New("id is required")
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(p.Endpoint) == "" {
|
|
||||||
return errors.New("endpoint is required")
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(p.Model) == "" {
|
|
||||||
return errors.New("model is required")
|
|
||||||
}
|
|
||||||
|
|
||||||
if p.Temperature < 0 || p.Temperature > 2 {
|
|
||||||
return errors.New("temperature must be between 0 and 2")
|
|
||||||
}
|
|
||||||
if p.MaxTokens < 0 {
|
|
||||||
return errors.New("max_tokens must be greater than or equal to 0")
|
|
||||||
}
|
|
||||||
if p.TopP < 0 || p.TopP > 1 {
|
|
||||||
return errors.New("top_p must be between 0 and 1")
|
|
||||||
}
|
|
||||||
if p.TimeoutSeconds < 0 {
|
|
||||||
return errors.New("timeout_seconds must be greater than or equal to 0")
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
package profile
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Repository loads execution profiles.
|
|
||||||
type Repository interface {
|
|
||||||
GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error)
|
|
||||||
}
|
|
||||||
@@ -1,479 +0,0 @@
|
|||||||
package profile
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"testing/fstest"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestFilesystemRepository_GetProfile(t *testing.T) {
|
|
||||||
tmpDir, err := os.MkdirTemp("", "execution_profile_test")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer os.RemoveAll(tmpDir)
|
|
||||||
|
|
||||||
files, err := os.ReadDir("testdata")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read testdata: %v", err)
|
|
||||||
}
|
|
||||||
for _, f := range files {
|
|
||||||
src := filepath.Join("testdata", f.Name())
|
|
||||||
dst := filepath.Join(tmpDir, f.Name())
|
|
||||||
data, err := os.ReadFile(src)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(dst, data, 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
repo := NewFilesystemRepository(tmpDir)
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
t.Run("valid local profile", func(t *testing.T) {
|
|
||||||
p, err := repo.GetProfile(ctx, "local-default")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if p.ID != "local-default" {
|
|
||||||
t.Fatalf("unexpected id: %q", p.ID)
|
|
||||||
}
|
|
||||||
if p.Endpoint == "" || p.Model == "" {
|
|
||||||
t.Fatalf("expected endpoint/model to be set: %+v", p)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("valid profile with api_key_env", func(t *testing.T) {
|
|
||||||
p, err := repo.GetProfile(ctx, "local-secure")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if p.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
|
|
||||||
t.Fatalf("unexpected api_key_env: %q", p.APIKeyEnv)
|
|
||||||
}
|
|
||||||
if p.ReasoningEffort != "medium" {
|
|
||||||
t.Fatalf("unexpected reasoning_effort: %q", p.ReasoningEffort)
|
|
||||||
}
|
|
||||||
if p.ServiceTier != "priority" {
|
|
||||||
t.Fatalf("unexpected service_tier: %q", p.ServiceTier)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("valid nested profile", func(t *testing.T) {
|
|
||||||
nestedDir := filepath.Join(tmpDir, "local")
|
|
||||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
writeProfileTestFile(t, filepath.Join(nestedDir, "nested-local.yaml"), `
|
|
||||||
id: nested-local
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: nested-model
|
|
||||||
temperature: 0.1
|
|
||||||
`)
|
|
||||||
|
|
||||||
p, err := repo.GetProfile(ctx, "nested-local")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if p.Model != "nested-model" {
|
|
||||||
t.Fatalf("unexpected model: %q", p.Model)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("valid profile with JSON-compatible extra params", func(t *testing.T) {
|
|
||||||
writeProfileTestFile(t, filepath.Join(tmpDir, "json-extra-params.yaml"), `
|
|
||||||
id: json-extra-params
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: nested-model
|
|
||||||
extra_params:
|
|
||||||
string_value: enabled
|
|
||||||
number_value: 42
|
|
||||||
boolean_value: true
|
|
||||||
object_value:
|
|
||||||
nested: value
|
|
||||||
count: 2
|
|
||||||
array_value:
|
|
||||||
- first
|
|
||||||
- 3
|
|
||||||
- false
|
|
||||||
`)
|
|
||||||
|
|
||||||
p, err := repo.GetProfile(ctx, "json-extra-params")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var got map[string]any
|
|
||||||
encoded, err := json.Marshal(p.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected extra_params to marshal as JSON, got %v", err)
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(encoded, &got); err != nil {
|
|
||||||
t.Fatalf("expected extra_params JSON to decode, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got["string_value"] != "enabled" {
|
|
||||||
t.Fatalf("unexpected string extra param: %#v", got["string_value"])
|
|
||||||
}
|
|
||||||
if got["number_value"] != float64(42) {
|
|
||||||
t.Fatalf("unexpected number extra param: %#v", got["number_value"])
|
|
||||||
}
|
|
||||||
if got["boolean_value"] != true {
|
|
||||||
t.Fatalf("unexpected boolean extra param: %#v", got["boolean_value"])
|
|
||||||
}
|
|
||||||
objectValue, ok := got["object_value"].(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("expected object extra param, got %#v", got["object_value"])
|
|
||||||
}
|
|
||||||
if objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
|
|
||||||
t.Fatalf("unexpected object extra param: %#v", objectValue)
|
|
||||||
}
|
|
||||||
arrayValue, ok := got["array_value"].([]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("expected array extra param, got %#v", got["array_value"])
|
|
||||||
}
|
|
||||||
if len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
|
|
||||||
t.Fatalf("unexpected array extra param: %#v", arrayValue)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
|
|
||||||
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
|
|
||||||
id: duplicate-profile
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: first-model
|
|
||||||
`)
|
|
||||||
nestedDir := filepath.Join(tmpDir, "duplicates")
|
|
||||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
writeProfileTestFile(t, filepath.Join(nestedDir, "duplicate-profile-b.yaml"), `
|
|
||||||
id: duplicate-profile
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: second-model
|
|
||||||
`)
|
|
||||||
|
|
||||||
_, err := repo.GetProfile(ctx, "duplicate-profile")
|
|
||||||
if !errors.Is(err, ErrInvalidProfile) {
|
|
||||||
t.Fatalf("expected duplicate profile to return ErrInvalidProfile, got %v", err)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"duplicate execution profile id", "duplicate-profile-a.yaml", filepath.Join("duplicates", "duplicate-profile-b.yaml")} {
|
|
||||||
if !strings.Contains(err.Error(), want) {
|
|
||||||
t.Fatalf("expected error to contain %q, got %v", want, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("nested raw api_key rejected for likely target file", func(t *testing.T) {
|
|
||||||
nestedDir := filepath.Join(tmpDir, "secure")
|
|
||||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
writeProfileTestFile(t, filepath.Join(nestedDir, "not_named_like_id.yaml"), `
|
|
||||||
id: nested_raw_api_key
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: m
|
|
||||||
api_key: secret
|
|
||||||
`)
|
|
||||||
|
|
||||||
_, err := repo.GetProfile(ctx, "nested_raw_api_key")
|
|
||||||
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
|
||||||
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), filepath.Join("secure", "not_named_like_id.yaml")) {
|
|
||||||
t.Fatalf("expected nested path in error, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("raw api_key in non-target profile is ignored", func(t *testing.T) {
|
|
||||||
writeProfileTestFile(t, filepath.Join(tmpDir, "raw-api-key-non-target.yaml"), `
|
|
||||||
id: raw-api-key-non-target
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: m
|
|
||||||
api_key: secret
|
|
||||||
`)
|
|
||||||
|
|
||||||
_, err := repo.GetProfile(ctx, "does-not-exist-with-raw-key-nearby")
|
|
||||||
if !errors.Is(err, ErrProfileNotFound) {
|
|
||||||
t.Fatalf("expected ErrProfileNotFound for non-target raw api_key file, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("invalid yaml", func(t *testing.T) {
|
|
||||||
_, err := repo.GetProfile(ctx, "invalid_yaml")
|
|
||||||
if !errors.Is(err, ErrInvalidYAML) {
|
|
||||||
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("missing id", func(t *testing.T) {
|
|
||||||
_, err := repo.GetProfile(ctx, "missing_id")
|
|
||||||
if !errors.Is(err, ErrProfileNotFound) {
|
|
||||||
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("missing endpoint", func(t *testing.T) {
|
|
||||||
_, err := repo.GetProfile(ctx, "missing-endpoint")
|
|
||||||
if !errors.Is(err, ErrInvalidProfile) {
|
|
||||||
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("missing model", func(t *testing.T) {
|
|
||||||
_, err := repo.GetProfile(ctx, "missing-model")
|
|
||||||
if !errors.Is(err, ErrInvalidProfile) {
|
|
||||||
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("unknown field", func(t *testing.T) {
|
|
||||||
_, err := repo.GetProfile(ctx, "unknown_field")
|
|
||||||
if !errors.Is(err, ErrInvalidYAML) {
|
|
||||||
t.Fatalf("expected ErrInvalidYAML for strict decode unknown field, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("raw api_key rejected", func(t *testing.T) {
|
|
||||||
_, err := repo.GetProfile(ctx, "raw_api_key")
|
|
||||||
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
|
||||||
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("profile not found", func(t *testing.T) {
|
|
||||||
_, err := repo.GetProfile(ctx, "does-not-exist")
|
|
||||||
if !errors.Is(err, ErrProfileNotFound) {
|
|
||||||
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeProfileTestFile(t *testing.T, path string, content string) {
|
|
||||||
t.Helper()
|
|
||||||
if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil {
|
|
||||||
t.Fatalf("failed to write profile test file %q: %v", path, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFSRepository(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
t.Run("loads valid profiles from nested directories", func(t *testing.T) {
|
|
||||||
repo := NewFSRepository(fstest.MapFS{
|
|
||||||
"profiles/provider/nested.yaml": profileMapFile(`
|
|
||||||
id: nested-profile
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: nested-model
|
|
||||||
temperature: 0.1
|
|
||||||
`),
|
|
||||||
}, "profiles")
|
|
||||||
|
|
||||||
p, err := repo.GetProfile(ctx, "nested-profile")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if p.ID != "nested-profile" || p.Model != "nested-model" {
|
|
||||||
t.Fatalf("unexpected profile: %+v", p)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("rejects unknown YAML fields", func(t *testing.T) {
|
|
||||||
repo := NewFSRepository(fstest.MapFS{
|
|
||||||
"profiles/unknown.yaml": profileMapFile(`
|
|
||||||
id: unknown-profile
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: model
|
|
||||||
unknown: value
|
|
||||||
`),
|
|
||||||
}, "profiles")
|
|
||||||
|
|
||||||
_, err := repo.GetProfile(ctx, "unknown-profile")
|
|
||||||
if !errors.Is(err, ErrInvalidYAML) {
|
|
||||||
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("rejects raw api_key in selected profile", func(t *testing.T) {
|
|
||||||
repo := NewFSRepository(fstest.MapFS{
|
|
||||||
"profiles/raw.yaml": profileMapFile(`
|
|
||||||
id: raw-profile
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: model
|
|
||||||
api_key: secret
|
|
||||||
`),
|
|
||||||
}, "profiles")
|
|
||||||
|
|
||||||
_, err := repo.GetProfile(ctx, "raw-profile")
|
|
||||||
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
|
||||||
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("ignores raw api_key in non-selected profiles", func(t *testing.T) {
|
|
||||||
repo := NewFSRepository(fstest.MapFS{
|
|
||||||
"profiles/raw.yaml": profileMapFile(`
|
|
||||||
id: raw-profile
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: model
|
|
||||||
api_key: secret
|
|
||||||
`),
|
|
||||||
"profiles/valid.yaml": profileMapFile(`
|
|
||||||
id: valid-profile
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: model
|
|
||||||
`),
|
|
||||||
}, "profiles")
|
|
||||||
|
|
||||||
p, err := repo.GetProfile(ctx, "valid-profile")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if p.ID != "valid-profile" {
|
|
||||||
t.Fatalf("unexpected profile: %+v", p)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("rejects duplicate IDs within one source", func(t *testing.T) {
|
|
||||||
repo := NewFSRepository(fstest.MapFS{
|
|
||||||
"profiles/a.yaml": profileMapFile(`
|
|
||||||
id: duplicate-profile
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: first
|
|
||||||
`),
|
|
||||||
"profiles/nested/b.yaml": profileMapFile(`
|
|
||||||
id: duplicate-profile
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: second
|
|
||||||
`),
|
|
||||||
}, "profiles")
|
|
||||||
|
|
||||||
_, err := repo.GetProfile(ctx, "duplicate-profile")
|
|
||||||
if !errors.Is(err, ErrInvalidProfile) {
|
|
||||||
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"duplicate execution profile id", "a.yaml", "nested/b.yaml"} {
|
|
||||||
if !strings.Contains(err.Error(), want) {
|
|
||||||
t.Fatalf("expected error to contain %q, got %v", want, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOverlayRepository(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"}
|
|
||||||
fallbackProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://fallback", Model: "fallback"}
|
|
||||||
|
|
||||||
t.Run("returns primary matches before fallback matches", func(t *testing.T) {
|
|
||||||
repo := NewOverlayRepository(
|
|
||||||
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": primaryProfile}},
|
|
||||||
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
|
|
||||||
)
|
|
||||||
|
|
||||||
p, err := repo.GetProfile(ctx, "shared")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if p.Model != "primary" {
|
|
||||||
t.Fatalf("expected primary profile, got %+v", p)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("falls back on primary not found", func(t *testing.T) {
|
|
||||||
repo := NewOverlayRepository(
|
|
||||||
staticProfileRepo{},
|
|
||||||
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
|
|
||||||
)
|
|
||||||
|
|
||||||
p, err := repo.GetProfile(ctx, "shared")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if p.Model != "fallback" {
|
|
||||||
t.Fatalf("expected fallback profile, got %+v", p)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("does not fall back after primary load errors", func(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
err error
|
|
||||||
}{
|
|
||||||
{name: "invalid yaml", err: ErrInvalidYAML},
|
|
||||||
{name: "invalid profile", err: ErrInvalidProfile},
|
|
||||||
{name: "raw api key", err: ErrRawAPIKeyNotAllowed},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
repo := NewOverlayRepository(
|
|
||||||
staticProfileRepo{err: tc.err},
|
|
||||||
staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}},
|
|
||||||
)
|
|
||||||
|
|
||||||
_, err := repo.GetProfile(ctx, "shared")
|
|
||||||
if !errors.Is(err, tc.err) {
|
|
||||||
t.Fatalf("expected %v, got %v", tc.err, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("returns not found when both sources miss", func(t *testing.T) {
|
|
||||||
repo := NewOverlayRepository(staticProfileRepo{}, staticProfileRepo{})
|
|
||||||
|
|
||||||
_, err := repo.GetProfile(ctx, "missing")
|
|
||||||
if !errors.Is(err, ErrProfileNotFound) {
|
|
||||||
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("nil primary uses fallback", func(t *testing.T) {
|
|
||||||
repo := NewOverlayRepository(nil, staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}})
|
|
||||||
|
|
||||||
p, err := repo.GetProfile(ctx, "shared")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if p.Model != "fallback" {
|
|
||||||
t.Fatalf("expected fallback profile, got %+v", p)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("nil fallback returns not found after primary miss", func(t *testing.T) {
|
|
||||||
repo := NewOverlayRepository(staticProfileRepo{}, nil)
|
|
||||||
|
|
||||||
_, err := repo.GetProfile(ctx, "missing")
|
|
||||||
if !errors.Is(err, ErrProfileNotFound) {
|
|
||||||
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func profileMapFile(content string) *fstest.MapFile {
|
|
||||||
return &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))}
|
|
||||||
}
|
|
||||||
|
|
||||||
type staticProfileRepo struct {
|
|
||||||
profiles map[string]*domain.ExecutionProfile
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
|
||||||
if r.err != nil {
|
|
||||||
return nil, r.err
|
|
||||||
}
|
|
||||||
if p, ok := r.profiles[id]; ok {
|
|
||||||
cp := *p
|
|
||||||
return &cp, nil
|
|
||||||
}
|
|
||||||
return nil, ErrProfileNotFound
|
|
||||||
}
|
|
||||||
3
internal/profile/testdata/invalid_yaml.yaml
vendored
3
internal/profile/testdata/invalid_yaml.yaml
vendored
@@ -1,3 +0,0 @@
|
|||||||
id: invalid_yaml
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: [broken
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
id: missing-endpoint
|
|
||||||
model: gpt-4o-mini
|
|
||||||
2
internal/profile/testdata/missing_id.yaml
vendored
2
internal/profile/testdata/missing_id.yaml
vendored
@@ -1,2 +0,0 @@
|
|||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: gpt-4o-mini
|
|
||||||
2
internal/profile/testdata/missing_model.yaml
vendored
2
internal/profile/testdata/missing_model.yaml
vendored
@@ -1,2 +0,0 @@
|
|||||||
id: missing-model
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
4
internal/profile/testdata/raw_api_key.yaml
vendored
4
internal/profile/testdata/raw_api_key.yaml
vendored
@@ -1,4 +0,0 @@
|
|||||||
id: raw-api-key
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: gpt-4o-mini
|
|
||||||
api_key: super-secret-should-not-be-here
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user