eric/scriptorium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add Promptkit definition conformance coverage

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-29 15:12:27 +00:00
parent 3e5f5c5198
commit 8936ca7c18
15 changed files with 434 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
examples/render-v0.9-features.sh
View File

@@ -7,6 +7,7 @@ cd "$repo_root"
go run ./cmd/scriptorium render \
--config ./examples/config.full.yml \
--prompt generic.markdown_summary \
--prompt-version 1.0.0 \
--profile local-gpu \
--session-id example-session \
--reasoning-effort= \

221
internal/adapter/cli/run_test.go
View File

@@ -1181,6 +1181,197 @@ output:
}
}
func TestPromptkitV09DefinitionsRenderThroughCLI(t *testing.T) {
fixtureRoot := promptkitV09FixtureRoot(t)
configPath := writePromptkitV09Config(t, fixtureRoot, true)
inputPath := filepath.Join(fixtureRoot, "inputs", "source.md")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", configPath,
"--prompt", "compat.complete",
"--prompt-version", "1.0.0",
"--input", "source=" + inputPath,
"--var", "topic=testing",
"--format", "json",
})
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
var payload map[string]any
if err := json.Unmarshal([]byte(stdout), &payload); err != nil {
t.Fatalf("decode rendered fixture: %v\nbody=%s", err, stdout)
}
if payload["prompt_version"] != "1.0.0" || payload["selected_profile_id"] != "custom-derived" || payload["selected_backend_id"] != "fixture-custom" {
t.Fatalf("unexpected selected definition and target: %#v", payload)
}
if payload["session_id"] != "fixture-testing" {
t.Fatalf("expected rendered session template, got %#v", payload["session_id"])
}
outputContract := payload["output_contract"].(map[string]any)
if outputContract["format"] != "json" || outputContract["validation_mode"] != "json_schema" || outputContract["repair_attempts"] != float64(2) {
t.Fatalf("unexpected output contract: %#v", outputContract)
}
if _, ok := payload["structured_output"].(map[string]any)["json_schema"]; !ok {
t.Fatalf("expected loaded JSON schema metadata, got %#v", payload["structured_output"])
}
messages := payload["messages"].([]any)
wantRoles := []string{"developer", "system", "user", "assistant"}
for i, wantRole := range wantRoles {
message := messages[i].(map[string]any)
if message["role"] != wantRole {
t.Fatalf("message %d: expected role %q, got %#v", i, wantRole, message["role"])
}
}
cacheControl := messages[0].(map[string]any)["cache_control"].(map[string]any)
if cacheControl["type"] != "ephemeral" || cacheControl["ttl"] != "1h" {
t.Fatalf("unexpected cache control: %#v", cacheControl)
}
if !strings.Contains(messages[2].(map[string]any)["content"].(string), "stable, synthetic material") {
t.Fatalf("file-backed input template was not rendered: %#v", messages[2])
}
inputHashes := payload["input_hashes"].(map[string]any)
if len(inputHashes) != 1 || inputHashes["source"] == "" {
t.Fatalf("required and omitted optional inputs were not preserved: %#v", inputHashes)
}
}
func TestPromptkitV09PromptInspectionSelectsVersionsAndContracts(t *testing.T) {
fixtureRoot := promptkitV09FixtureRoot(t)
configPath := writePromptkitV09Config(t, fixtureRoot, true)
tests := []struct {
promptID string
version string
format string
mode string
}{
{promptID: "compat.complete", version: "1.0.0", format: "json", mode: "json_schema"},
{promptID: "compat.complete", version: "2.0.0", format: "text", mode: "basic"},
{promptID: "compat.json", version: "1.0.0", format: "json", mode: "json"},
{promptID: "compat.none", version: "1.0.0", format: "text", mode: "none"},
}
for _, tc := range tests {
t.Run(tc.promptID+"@"+tc.version, func(t *testing.T) {
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
"prompt",
"--config", configPath,
"--prompt", tc.promptID,
"--prompt-version", tc.version,
"--format", "json",
})
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
var inspection renderformat.PromptInspection
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
t.Fatalf("decode prompt inspection: %v\nbody=%s", err, stdout)
}
if inspection.PromptVersion != tc.version || inspection.OutputContract.Format != tc.format || inspection.OutputContract.ValidationMode != tc.mode {
t.Fatalf("unexpected prompt inspection: %+v", inspection)
}
})
}
}
func TestPromptkitV09ProfileInspectionResolvesSupportedTargets(t *testing.T) {
const secret = "sentinel-profile-secret"
t.Setenv("FIXTURE_PROFILE_API_KEY", secret)
fixtureRoot := promptkitV09FixtureRoot(t)
configPath := writePromptkitV09Config(t, fixtureRoot, false)
profileDir := filepath.Join(fixtureRoot, "profiles")
tests := []struct {
name string
profileID string
profileDir string
wantBackend string
wantModel string
wantAPIKeyEnv string
}{
{name: "inherited custom backend", profileID: "custom-derived", profileDir: profileDir, wantBackend: "fixture-custom", wantModel: "fixture-derived-model", wantAPIKeyEnv: "FIXTURE_PROFILE_API_KEY"},
{name: "endpoint only", profileID: "endpoint-only", profileDir: profileDir, wantModel: "fixture-endpoint-model"},
{name: "built in", profileID: "deepseek-4-flash", wantBackend: "openrouter", wantModel: "deepseek/deepseek-v4-flash", wantAPIKeyEnv: "OPENROUTER_API_KEY"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
args := []string{"profile", "--config", configPath, "--profile", tc.profileID, "--format", "json"}
if tc.profileDir != "" {
args = append(args, "--profile-dir", tc.profileDir)
}
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
if strings.Contains(stdout, secret) || strings.Contains(stderr, secret) {
t.Fatalf("inspection exposed environment secret: stdout=%q stderr=%q", stdout, stderr)
}
var inspection renderformat.ProfileInspection
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
t.Fatalf("decode profile inspection: %v\nbody=%s", err, stdout)
}
params := inspection.EffectiveModelParams
if inspection.ProfileID != tc.profileID || params.BackendID != tc.wantBackend || params.Model != tc.wantModel || params.APIKeyEnv != tc.wantAPIKeyEnv {
t.Fatalf("unexpected effective profile: %+v", inspection)
}
if tc.profileID == "custom-derived" {
if params.ServiceTier != "flex" || params.ReasoningEffort != "high" || params.TimeoutSeconds != 45 || len(params.ExtraParams) == 0 {
t.Fatalf("inherited profile controls were not resolved: %+v", params)
}
}
})
}
}
func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.T) {
fixtureRoot := promptkitV09FixtureRoot(t)
configPath := writePromptkitV09Config(t, fixtureRoot, false)
overrideDir := t.TempDir()
writeProfileFile(t, overrideDir, "custom-derived", "http://127.0.0.1:9000/v1", "override-model")
outPath := filepath.Join(t.TempDir(), "inspection.json")
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
"profile",
"--config", configPath,
"--profile-dir", overrideDir,
"--profile", "custom-derived",
"--format", "json",
"--out", outPath,
})
if code != ExitOK || stdout != "" {
t.Fatalf("expected file output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output, err := os.ReadFile(outPath)
if err != nil {
t.Fatalf("read profile inspection output: %v", err)
}
if !strings.Contains(string(output), `"model": "override-model"`) || !strings.Contains(string(output), `"backend_id": ""`) {
t.Fatalf("profile directory override was not used: %s", output)
}
for _, profileID := range []string{"missing", "invalid"} {
t.Run(profileID, func(t *testing.T) {
profileDir := filepath.Join(fixtureRoot, "profiles")
if profileID == "invalid" {
profileDir = t.TempDir()
writePromptDefinition(t, profileDir, "invalid.yaml", "id: invalid\nendpoint: not-a-url\nmodel: fixture\n")
}
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
"profile",
"--config", configPath,
"--profile-dir", profileDir,
"--profile", profileID,
})
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
t.Fatalf("expected inspection failure without partial output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
})
}
}
func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
lib := newCLITestLibrary(t)
@@ -1754,6 +1945,36 @@ func runCLICommand(t *testing.T, command func([]string, io.Writer, io.Writer) in
return code, stdout.String(), stderr.String()
}
func promptkitV09FixtureRoot(t *testing.T) string {
t.Helper()
root, err := filepath.Abs(filepath.Join("..", "..", "..", "testdata", "promptkit-v0.9"))
if err != nil {
t.Fatalf("resolve Promptkit v0.9 fixture root: %v", err)
}
if _, err := os.Stat(root); err != nil {
t.Fatalf("stat Promptkit v0.9 fixture root: %v", err)
}
return root
}
func writePromptkitV09Config(t *testing.T, fixtureRoot string, includeSources bool) string {
t.Helper()
sources := ""
if includeSources {
sources = fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\nschema_dir: %q\n", filepath.Join(fixtureRoot, "prompts"), filepath.Join(fixtureRoot, "profiles"), filepath.Join(fixtureRoot, "schemas"))
}
return writeAppConfigFile(t, sources+`backends:
fixture-custom:
endpoint: http://127.0.0.1:11434/v1
api_key_env: FIXTURE_BACKEND_API_KEY
extra_params:
backend_option:
enabled: true
concurrency_limit: 2
queue_capacity: 0
`)
}
func writePromptFile(t *testing.T, dir, id, defaultProfile string) {
t.Helper()
writePromptFileWithTemplate(t, dir, id, defaultProfile, "Summarize: {{input \"transcript\"}}")

92
internal/format/inspection_test.go
View File

@@ -1,6 +1,7 @@
package format
import (
"encoding/json"
"strings"
"testing"
@@ -30,3 +31,94 @@ func TestFormatPromptInspectionRejectsNil(t *testing.T) {
t.Fatal("expected nil inspection error")
}
}
func TestFormatProfileInspectionPreservesSafeEffectiveValues(t *testing.T) {
const secret = "sentinel-secret-must-not-appear"
t.Setenv("FIXTURE_PROFILE_API_KEY", secret)
value := &promptkit.ProfileInspection{
ProfileID: "custom-derived",
EffectiveModelParams: promptkit.ExecutionTarget{
BackendID: "fixture-custom",
Endpoint: "http://127.0.0.1:11434/v1",
Model: "fixture-model",
Temperature: 0.25,
MaxTokens: 640,
TopP: 0.9,
TimeoutSeconds: 45,
ServiceTier: "flex",
ReasoningEffort: "high",
APIKeyEnv: "FIXTURE_PROFILE_API_KEY",
ExtraParams: map[string]any{
"zeta": true,
"alpha": map[string]any{"nested": []any{"first", 2.0}},
},
},
APIKeyRequired: true,
}
jsonOutput, err := FormatProfileInspection(value, OutputFormatJSON)
if err != nil {
t.Fatal(err)
}
secondJSONOutput, err := FormatProfileInspection(value, OutputFormatJSON)
if err != nil {
t.Fatal(err)
}
if string(jsonOutput) != string(secondJSONOutput) || !strings.HasSuffix(string(jsonOutput), "\n") {
t.Fatalf("expected deterministic newline-terminated JSON, got %q", jsonOutput)
}
if strings.Contains(string(jsonOutput), secret) {
t.Fatalf("inspection exposed an environment secret: %s", jsonOutput)
}
var decoded ProfileInspection
if err := json.Unmarshal(jsonOutput, &decoded); err != nil {
t.Fatalf("decode profile inspection: %v", err)
}
if decoded.ProfileID != "custom-derived" || decoded.EffectiveModelParams.BackendID != "fixture-custom" {
t.Fatalf("unexpected profile identity: %+v", decoded)
}
if decoded.EffectiveModelParams.APIKeyEnv != "FIXTURE_PROFILE_API_KEY" || !decoded.APIKeyRequired {
t.Fatalf("unexpected credential metadata: %+v", decoded)
}
alpha, ok := decoded.EffectiveModelParams.ExtraParams["alpha"].(map[string]any)
if !ok || len(alpha["nested"].([]any)) != 2 {
t.Fatalf("nested extra parameters were not preserved: %#v", decoded.EffectiveModelParams.ExtraParams)
}
textOutput, err := FormatProfileInspection(value, OutputFormatText)
if err != nil {
t.Fatal(err)
}
textValue := string(textOutput)
if !strings.Contains(textValue, "backend_id: fixture-custom") ||
!strings.Contains(textValue, "api_key_env: FIXTURE_PROFILE_API_KEY") ||
!strings.Contains(textValue, `extra_params: {"alpha":{"nested":["first",2]},"zeta":true}`) ||
!strings.Contains(textValue, "api_key_required: true") {
t.Fatalf("unexpected text inspection: %s", textValue)
}
if strings.Contains(textValue, secret) {
t.Fatalf("text inspection exposed an environment secret: %s", textValue)
}
}
func TestFormatProfileInspectionPreservesEmptyBackendAndRejectsNil(t *testing.T) {
output, err := FormatProfileInspection(&promptkit.ProfileInspection{
ProfileID: "endpoint-only",
EffectiveModelParams: promptkit.ExecutionTarget{
Endpoint: "http://127.0.0.1:8000/v1",
Model: "fixture-model",
},
}, OutputFormatJSON)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(output), `"backend_id": ""`) ||
!strings.Contains(string(output), `"extra_params": {}`) {
t.Fatalf("expected explicit empty backend and object extra params, got %s", output)
}
if _, err := FormatProfileInspection(nil, OutputFormatText); err == nil {
t.Fatal("expected nil profile inspection error")
}
}

12
testdata/promptkit-v0.9/config.yml vendored Normal file
View File

@@ -0,0 +1,12 @@
prompt_dir: ./testdata/promptkit-v0.9/prompts
profile_dir: ./testdata/promptkit-v0.9/profiles
schema_dir: ./testdata/promptkit-v0.9/schemas
backends:
fixture-custom:
endpoint: http://127.0.0.1:11434/v1
api_key_env: FIXTURE_BACKEND_API_KEY
extra_params:
backend_option:
enabled: true
concurrency_limit: 2
queue_capacity: 0

1
testdata/promptkit-v0.9/inputs/source.md vendored Normal file
View File

@@ -0,0 +1 @@
The fixture source contains stable, synthetic material.

16
testdata/promptkit-v0.9/profiles/custom-base.yaml vendored Normal file
View File

@@ -0,0 +1,16 @@
id: custom-base
backend: fixture-custom
model: fixture-base-model
temperature: 0.25
max_tokens: 640
top_p: 0.9
timeout_seconds: 45
service_tier: flex
reasoning_effort: medium
api_key_env: FIXTURE_PROFILE_API_KEY
extra_params:
routing:
order:
- primary
- fallback
allow_fallback: true

4
testdata/promptkit-v0.9/profiles/custom-derived.yml vendored Normal file
View File

@@ -0,0 +1,4 @@
id: custom-derived
base_profile: custom-base
model: fixture-derived-model
reasoning_effort: high

10
testdata/promptkit-v0.9/profiles/endpoint-only.yaml vendored Normal file
View File

@@ -0,0 +1,10 @@
id: endpoint-only
endpoint: http://127.0.0.1:8000/v1
model: fixture-endpoint-model
temperature: 0.1
max_tokens: 128
top_p: 0.75
timeout_seconds: 30
extra_params:
endpoint_option:
nested: true

31
testdata/promptkit-v0.9/prompts/complete-v1.yaml vendored Normal file
View File

@@ -0,0 +1,31 @@
id: compat.complete
version: "1.0.0"
description: Exercise the complete Promptkit v0.9 prompt-definition surface.
default_profile: custom-derived
session_id: 'fixture-{{.topic}}'
inputs:
- name: source
required: true
content_type: text/markdown
description: Source material rendered by a file-backed message.
- name: context
required: false
content_type: text/plain
description: Optional caller context.
messages:
- role: developer
content: 'Use the topic {{.topic}}.'
cache_control:
type: ephemeral
ttl: 1h
- role: system
content_file: ./messages/complete.system.md
- role: user
content_file: ./messages/complete.user.md
- role: assistant
content: Return one JSON object.
output:
format: json
validation_mode: json_schema
schema_path: complete.schema.json
repair_attempts: 2

10
testdata/promptkit-v0.9/prompts/complete-v2.yml vendored Normal file
View File

@@ -0,0 +1,10 @@
id: compat.complete
version: "2.0.0"
default_profile: endpoint-only
messages:
- role: user
content: Version two has no declared inputs.
output:
format: text
validation_mode: basic
repair_attempts: 1

10
testdata/promptkit-v0.9/prompts/json.yaml vendored Normal file
View File

@@ -0,0 +1,10 @@
id: compat.json
version: "1.0.0"
default_profile: endpoint-only
messages:
- role: user
content: Return JSON.
output:
format: json
validation_mode: json
repair_attempts: 0

1
testdata/promptkit-v0.9/prompts/messages/complete.system.md vendored Normal file
View File

@@ -0,0 +1 @@
Use the supplied source without inventing details.

4
testdata/promptkit-v0.9/prompts/messages/complete.user.md vendored Normal file
View File

@@ -0,0 +1,4 @@
Topic: {{.topic}}
Source:
{{input "source"}}

10
testdata/promptkit-v0.9/prompts/none.yaml vendored Normal file
View File

@@ -0,0 +1,10 @@
id: compat.none
version: "1.0.0"
default_profile: endpoint-only
messages:
- role: user
content: Return text.
output:
format: text
validation_mode: none
repair_attempts: 0

11
testdata/promptkit-v0.9/schemas/complete.schema.json vendored Normal file
View File

@@ -0,0 +1,11 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"required": ["summary"],
"properties": {
"summary": {
"type": "string"
}
},
"additionalProperties": false
}