Add Promptkit definition conformance coverage

examples/render-v0.9-features.sh

@@ -7,6 +7,7 @@ cd "$repo_root"
 go run ./cmd/scriptorium render \
   --config ./examples/config.full.yml \
   --prompt generic.markdown_summary \
+  --prompt-version 1.0.0 \
   --profile local-gpu \
   --session-id example-session \
   --reasoning-effort= \

internal/adapter/cli/run_test.go

@@ -1181,6 +1181,197 @@ output:
 	}
 }
 
+func TestPromptkitV09DefinitionsRenderThroughCLI(t *testing.T) {
+	fixtureRoot := promptkitV09FixtureRoot(t)
+	configPath := writePromptkitV09Config(t, fixtureRoot, true)
+	inputPath := filepath.Join(fixtureRoot, "inputs", "source.md")
+
+	code, stdout, stderr := runCLICommand(t, renderCommand, []string{
+		"--config", configPath,
+		"--prompt", "compat.complete",
+		"--prompt-version", "1.0.0",
+		"--input", "source=" + inputPath,
+		"--var", "topic=testing",
+		"--format", "json",
+	})
+	if code != ExitOK {
+		t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
+	}
+
+	var payload map[string]any
+	if err := json.Unmarshal([]byte(stdout), &payload); err != nil {
+		t.Fatalf("decode rendered fixture: %v\nbody=%s", err, stdout)
+	}
+	if payload["prompt_version"] != "1.0.0" || payload["selected_profile_id"] != "custom-derived" || payload["selected_backend_id"] != "fixture-custom" {
+		t.Fatalf("unexpected selected definition and target: %#v", payload)
+	}
+	if payload["session_id"] != "fixture-testing" {
+		t.Fatalf("expected rendered session template, got %#v", payload["session_id"])
+	}
+
+	outputContract := payload["output_contract"].(map[string]any)
+	if outputContract["format"] != "json" || outputContract["validation_mode"] != "json_schema" || outputContract["repair_attempts"] != float64(2) {
+		t.Fatalf("unexpected output contract: %#v", outputContract)
+	}
+	if _, ok := payload["structured_output"].(map[string]any)["json_schema"]; !ok {
+		t.Fatalf("expected loaded JSON schema metadata, got %#v", payload["structured_output"])
+	}
+
+	messages := payload["messages"].([]any)
+	wantRoles := []string{"developer", "system", "user", "assistant"}
+	for i, wantRole := range wantRoles {
+		message := messages[i].(map[string]any)
+		if message["role"] != wantRole {
+			t.Fatalf("message %d: expected role %q, got %#v", i, wantRole, message["role"])
+		}
+	}
+	cacheControl := messages[0].(map[string]any)["cache_control"].(map[string]any)
+	if cacheControl["type"] != "ephemeral" || cacheControl["ttl"] != "1h" {
+		t.Fatalf("unexpected cache control: %#v", cacheControl)
+	}
+	if !strings.Contains(messages[2].(map[string]any)["content"].(string), "stable, synthetic material") {
+		t.Fatalf("file-backed input template was not rendered: %#v", messages[2])
+	}
+	inputHashes := payload["input_hashes"].(map[string]any)
+	if len(inputHashes) != 1 || inputHashes["source"] == "" {
+		t.Fatalf("required and omitted optional inputs were not preserved: %#v", inputHashes)
+	}
+}
+
+func TestPromptkitV09PromptInspectionSelectsVersionsAndContracts(t *testing.T) {
+	fixtureRoot := promptkitV09FixtureRoot(t)
+	configPath := writePromptkitV09Config(t, fixtureRoot, true)
+	tests := []struct {
+		promptID string
+		version  string
+		format   string
+		mode     string
+	}{
+		{promptID: "compat.complete", version: "1.0.0", format: "json", mode: "json_schema"},
+		{promptID: "compat.complete", version: "2.0.0", format: "text", mode: "basic"},
+		{promptID: "compat.json", version: "1.0.0", format: "json", mode: "json"},
+		{promptID: "compat.none", version: "1.0.0", format: "text", mode: "none"},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.promptID+"@"+tc.version, func(t *testing.T) {
+			code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
+				"prompt",
+				"--config", configPath,
+				"--prompt", tc.promptID,
+				"--prompt-version", tc.version,
+				"--format", "json",
+			})
+			if code != ExitOK {
+				t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
+			}
+			var inspection renderformat.PromptInspection
+			if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
+				t.Fatalf("decode prompt inspection: %v\nbody=%s", err, stdout)
+			}
+			if inspection.PromptVersion != tc.version || inspection.OutputContract.Format != tc.format || inspection.OutputContract.ValidationMode != tc.mode {
+				t.Fatalf("unexpected prompt inspection: %+v", inspection)
+			}
+		})
+	}
+}
+
+func TestPromptkitV09ProfileInspectionResolvesSupportedTargets(t *testing.T) {
+	const secret = "sentinel-profile-secret"
+	t.Setenv("FIXTURE_PROFILE_API_KEY", secret)
+
+	fixtureRoot := promptkitV09FixtureRoot(t)
+	configPath := writePromptkitV09Config(t, fixtureRoot, false)
+	profileDir := filepath.Join(fixtureRoot, "profiles")
+	tests := []struct {
+		name          string
+		profileID     string
+		profileDir    string
+		wantBackend   string
+		wantModel     string
+		wantAPIKeyEnv string
+	}{
+		{name: "inherited custom backend", profileID: "custom-derived", profileDir: profileDir, wantBackend: "fixture-custom", wantModel: "fixture-derived-model", wantAPIKeyEnv: "FIXTURE_PROFILE_API_KEY"},
+		{name: "endpoint only", profileID: "endpoint-only", profileDir: profileDir, wantModel: "fixture-endpoint-model"},
+		{name: "built in", profileID: "deepseek-4-flash", wantBackend: "openrouter", wantModel: "deepseek/deepseek-v4-flash", wantAPIKeyEnv: "OPENROUTER_API_KEY"},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			args := []string{"profile", "--config", configPath, "--profile", tc.profileID, "--format", "json"}
+			if tc.profileDir != "" {
+				args = append(args, "--profile-dir", tc.profileDir)
+			}
+			code, stdout, stderr := runCLICommand(t, inspectCommand, args)
+			if code != ExitOK {
+				t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
+			}
+			if strings.Contains(stdout, secret) || strings.Contains(stderr, secret) {
+				t.Fatalf("inspection exposed environment secret: stdout=%q stderr=%q", stdout, stderr)
+			}
+			var inspection renderformat.ProfileInspection
+			if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
+				t.Fatalf("decode profile inspection: %v\nbody=%s", err, stdout)
+			}
+			params := inspection.EffectiveModelParams
+			if inspection.ProfileID != tc.profileID || params.BackendID != tc.wantBackend || params.Model != tc.wantModel || params.APIKeyEnv != tc.wantAPIKeyEnv {
+				t.Fatalf("unexpected effective profile: %+v", inspection)
+			}
+			if tc.profileID == "custom-derived" {
+				if params.ServiceTier != "flex" || params.ReasoningEffort != "high" || params.TimeoutSeconds != 45 || len(params.ExtraParams) == 0 {
+					t.Fatalf("inherited profile controls were not resolved: %+v", params)
+				}
+			}
+		})
+	}
+}
+
+func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.T) {
+	fixtureRoot := promptkitV09FixtureRoot(t)
+	configPath := writePromptkitV09Config(t, fixtureRoot, false)
+	overrideDir := t.TempDir()
+	writeProfileFile(t, overrideDir, "custom-derived", "http://127.0.0.1:9000/v1", "override-model")
+	outPath := filepath.Join(t.TempDir(), "inspection.json")
+
+	code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
+		"profile",
+		"--config", configPath,
+		"--profile-dir", overrideDir,
+		"--profile", "custom-derived",
+		"--format", "json",
+		"--out", outPath,
+	})
+	if code != ExitOK || stdout != "" {
+		t.Fatalf("expected file output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
+	}
+	output, err := os.ReadFile(outPath)
+	if err != nil {
+		t.Fatalf("read profile inspection output: %v", err)
+	}
+	if !strings.Contains(string(output), `"model": "override-model"`) || !strings.Contains(string(output), `"backend_id": ""`) {
+		t.Fatalf("profile directory override was not used: %s", output)
+	}
+
+	for _, profileID := range []string{"missing", "invalid"} {
+		t.Run(profileID, func(t *testing.T) {
+			profileDir := filepath.Join(fixtureRoot, "profiles")
+			if profileID == "invalid" {
+				profileDir = t.TempDir()
+				writePromptDefinition(t, profileDir, "invalid.yaml", "id: invalid\nendpoint: not-a-url\nmodel: fixture\n")
+			}
+			code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
+				"profile",
+				"--config", configPath,
+				"--profile-dir", profileDir,
+				"--profile", profileID,
+			})
+			if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
+				t.Fatalf("expected inspection failure without partial output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
+			}
+		})
+	}
+}
+
 func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
 	lib := newCLITestLibrary(t)
 
@@ -1754,6 +1945,36 @@ func runCLICommand(t *testing.T, command func([]string, io.Writer, io.Writer) in
 	return code, stdout.String(), stderr.String()
 }
 
+func promptkitV09FixtureRoot(t *testing.T) string {
+	t.Helper()
+	root, err := filepath.Abs(filepath.Join("..", "..", "..", "testdata", "promptkit-v0.9"))
+	if err != nil {
+		t.Fatalf("resolve Promptkit v0.9 fixture root: %v", err)
+	}
+	if _, err := os.Stat(root); err != nil {
+		t.Fatalf("stat Promptkit v0.9 fixture root: %v", err)
+	}
+	return root
+}
+
+func writePromptkitV09Config(t *testing.T, fixtureRoot string, includeSources bool) string {
+	t.Helper()
+	sources := ""
+	if includeSources {
+		sources = fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\nschema_dir: %q\n", filepath.Join(fixtureRoot, "prompts"), filepath.Join(fixtureRoot, "profiles"), filepath.Join(fixtureRoot, "schemas"))
+	}
+	return writeAppConfigFile(t, sources+`backends:
+  fixture-custom:
+    endpoint: http://127.0.0.1:11434/v1
+    api_key_env: FIXTURE_BACKEND_API_KEY
+    extra_params:
+      backend_option:
+        enabled: true
+    concurrency_limit: 2
+    queue_capacity: 0
+`)
+}
+
 func writePromptFile(t *testing.T, dir, id, defaultProfile string) {
 	t.Helper()
 	writePromptFileWithTemplate(t, dir, id, defaultProfile, "Summarize: {{input \"transcript\"}}")

internal/format/inspection_test.go

@@ -1,6 +1,7 @@
 package format
 
 import (
+	"encoding/json"
 	"strings"
 	"testing"
 
@@ -30,3 +31,94 @@ func TestFormatPromptInspectionRejectsNil(t *testing.T) {
 		t.Fatal("expected nil inspection error")
 	}
 }
+
+func TestFormatProfileInspectionPreservesSafeEffectiveValues(t *testing.T) {
+	const secret = "sentinel-secret-must-not-appear"
+	t.Setenv("FIXTURE_PROFILE_API_KEY", secret)
+
+	value := &promptkit.ProfileInspection{
+		ProfileID: "custom-derived",
+		EffectiveModelParams: promptkit.ExecutionTarget{
+			BackendID:       "fixture-custom",
+			Endpoint:        "http://127.0.0.1:11434/v1",
+			Model:           "fixture-model",
+			Temperature:     0.25,
+			MaxTokens:       640,
+			TopP:            0.9,
+			TimeoutSeconds:  45,
+			ServiceTier:     "flex",
+			ReasoningEffort: "high",
+			APIKeyEnv:       "FIXTURE_PROFILE_API_KEY",
+			ExtraParams: map[string]any{
+				"zeta":  true,
+				"alpha": map[string]any{"nested": []any{"first", 2.0}},
+			},
+		},
+		APIKeyRequired: true,
+	}
+
+	jsonOutput, err := FormatProfileInspection(value, OutputFormatJSON)
+	if err != nil {
+		t.Fatal(err)
+	}
+	secondJSONOutput, err := FormatProfileInspection(value, OutputFormatJSON)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(jsonOutput) != string(secondJSONOutput) || !strings.HasSuffix(string(jsonOutput), "\n") {
+		t.Fatalf("expected deterministic newline-terminated JSON, got %q", jsonOutput)
+	}
+	if strings.Contains(string(jsonOutput), secret) {
+		t.Fatalf("inspection exposed an environment secret: %s", jsonOutput)
+	}
+
+	var decoded ProfileInspection
+	if err := json.Unmarshal(jsonOutput, &decoded); err != nil {
+		t.Fatalf("decode profile inspection: %v", err)
+	}
+	if decoded.ProfileID != "custom-derived" || decoded.EffectiveModelParams.BackendID != "fixture-custom" {
+		t.Fatalf("unexpected profile identity: %+v", decoded)
+	}
+	if decoded.EffectiveModelParams.APIKeyEnv != "FIXTURE_PROFILE_API_KEY" || !decoded.APIKeyRequired {
+		t.Fatalf("unexpected credential metadata: %+v", decoded)
+	}
+	alpha, ok := decoded.EffectiveModelParams.ExtraParams["alpha"].(map[string]any)
+	if !ok || len(alpha["nested"].([]any)) != 2 {
+		t.Fatalf("nested extra parameters were not preserved: %#v", decoded.EffectiveModelParams.ExtraParams)
+	}
+
+	textOutput, err := FormatProfileInspection(value, OutputFormatText)
+	if err != nil {
+		t.Fatal(err)
+	}
+	textValue := string(textOutput)
+	if !strings.Contains(textValue, "backend_id: fixture-custom") ||
+		!strings.Contains(textValue, "api_key_env: FIXTURE_PROFILE_API_KEY") ||
+		!strings.Contains(textValue, `extra_params: {"alpha":{"nested":["first",2]},"zeta":true}`) ||
+		!strings.Contains(textValue, "api_key_required: true") {
+		t.Fatalf("unexpected text inspection: %s", textValue)
+	}
+	if strings.Contains(textValue, secret) {
+		t.Fatalf("text inspection exposed an environment secret: %s", textValue)
+	}
+}
+
+func TestFormatProfileInspectionPreservesEmptyBackendAndRejectsNil(t *testing.T) {
+	output, err := FormatProfileInspection(&promptkit.ProfileInspection{
+		ProfileID: "endpoint-only",
+		EffectiveModelParams: promptkit.ExecutionTarget{
+			Endpoint: "http://127.0.0.1:8000/v1",
+			Model:    "fixture-model",
+		},
+	}, OutputFormatJSON)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(output), `"backend_id": ""`) ||
+		!strings.Contains(string(output), `"extra_params": {}`) {
+		t.Fatalf("expected explicit empty backend and object extra params, got %s", output)
+	}
+	if _, err := FormatProfileInspection(nil, OutputFormatText); err == nil {
+		t.Fatal("expected nil profile inspection error")
+	}
+}

testdata/promptkit-v0.9/config.yml

@@ -0,0 +1,12 @@
+prompt_dir: ./testdata/promptkit-v0.9/prompts
+profile_dir: ./testdata/promptkit-v0.9/profiles
+schema_dir: ./testdata/promptkit-v0.9/schemas
+backends:
+  fixture-custom:
+    endpoint: http://127.0.0.1:11434/v1
+    api_key_env: FIXTURE_BACKEND_API_KEY
+    extra_params:
+      backend_option:
+        enabled: true
+    concurrency_limit: 2
+    queue_capacity: 0

testdata/promptkit-v0.9/inputs/source.md

@@ -0,0 +1 @@
+The fixture source contains stable, synthetic material.

testdata/promptkit-v0.9/profiles/custom-base.yaml

@@ -0,0 +1,16 @@
+id: custom-base
+backend: fixture-custom
+model: fixture-base-model
+temperature: 0.25
+max_tokens: 640
+top_p: 0.9
+timeout_seconds: 45
+service_tier: flex
+reasoning_effort: medium
+api_key_env: FIXTURE_PROFILE_API_KEY
+extra_params:
+  routing:
+    order:
+      - primary
+      - fallback
+    allow_fallback: true

testdata/promptkit-v0.9/profiles/custom-derived.yml

@@ -0,0 +1,4 @@
+id: custom-derived
+base_profile: custom-base
+model: fixture-derived-model
+reasoning_effort: high

testdata/promptkit-v0.9/profiles/endpoint-only.yaml

@@ -0,0 +1,10 @@
+id: endpoint-only
+endpoint: http://127.0.0.1:8000/v1
+model: fixture-endpoint-model
+temperature: 0.1
+max_tokens: 128
+top_p: 0.75
+timeout_seconds: 30
+extra_params:
+  endpoint_option:
+    nested: true

testdata/promptkit-v0.9/prompts/complete-v1.yaml

@@ -0,0 +1,31 @@
+id: compat.complete
+version: "1.0.0"
+description: Exercise the complete Promptkit v0.9 prompt-definition surface.
+default_profile: custom-derived
+session_id: 'fixture-{{.topic}}'
+inputs:
+  - name: source
+    required: true
+    content_type: text/markdown
+    description: Source material rendered by a file-backed message.
+  - name: context
+    required: false
+    content_type: text/plain
+    description: Optional caller context.
+messages:
+  - role: developer
+    content: 'Use the topic {{.topic}}.'
+    cache_control:
+      type: ephemeral
+      ttl: 1h
+  - role: system
+    content_file: ./messages/complete.system.md
+  - role: user
+    content_file: ./messages/complete.user.md
+  - role: assistant
+    content: Return one JSON object.
+output:
+  format: json
+  validation_mode: json_schema
+  schema_path: complete.schema.json
+  repair_attempts: 2

testdata/promptkit-v0.9/prompts/complete-v2.yml

@@ -0,0 +1,10 @@
+id: compat.complete
+version: "2.0.0"
+default_profile: endpoint-only
+messages:
+  - role: user
+    content: Version two has no declared inputs.
+output:
+  format: text
+  validation_mode: basic
+  repair_attempts: 1

testdata/promptkit-v0.9/prompts/json.yaml

@@ -0,0 +1,10 @@
+id: compat.json
+version: "1.0.0"
+default_profile: endpoint-only
+messages:
+  - role: user
+    content: Return JSON.
+output:
+  format: json
+  validation_mode: json
+  repair_attempts: 0

testdata/promptkit-v0.9/prompts/messages/complete.system.md

@@ -0,0 +1 @@
+Use the supplied source without inventing details.

testdata/promptkit-v0.9/prompts/messages/complete.user.md

@@ -0,0 +1,4 @@
+Topic: {{.topic}}
+
+Source:
+{{input "source"}}

testdata/promptkit-v0.9/prompts/none.yaml

@@ -0,0 +1,10 @@
+id: compat.none
+version: "1.0.0"
+default_profile: endpoint-only
+messages:
+  - role: user
+    content: Return text.
+output:
+  format: text
+  validation_mode: none
+  repair_attempts: 0

testdata/promptkit-v0.9/schemas/complete.schema.json

@@ -0,0 +1,11 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "required": ["summary"],
+  "properties": {
+    "summary": {
+      "type": "string"
+    }
+  },
+  "additionalProperties": false
+}