diff --git a/examples/render-v0.9-features.sh b/examples/render-v0.9-features.sh index 6038629..eddbcb9 100755 --- a/examples/render-v0.9-features.sh +++ b/examples/render-v0.9-features.sh @@ -7,6 +7,7 @@ cd "$repo_root" go run ./cmd/scriptorium render \ --config ./examples/config.full.yml \ --prompt generic.markdown_summary \ + --prompt-version 1.0.0 \ --profile local-gpu \ --session-id example-session \ --reasoning-effort= \ diff --git a/internal/adapter/cli/run_test.go b/internal/adapter/cli/run_test.go index b76733c..1bbde0f 100644 --- a/internal/adapter/cli/run_test.go +++ b/internal/adapter/cli/run_test.go @@ -1181,6 +1181,197 @@ output: } } +func TestPromptkitV09DefinitionsRenderThroughCLI(t *testing.T) { + fixtureRoot := promptkitV09FixtureRoot(t) + configPath := writePromptkitV09Config(t, fixtureRoot, true) + inputPath := filepath.Join(fixtureRoot, "inputs", "source.md") + + code, stdout, stderr := runCLICommand(t, renderCommand, []string{ + "--config", configPath, + "--prompt", "compat.complete", + "--prompt-version", "1.0.0", + "--input", "source=" + inputPath, + "--var", "topic=testing", + "--format", "json", + }) + if code != ExitOK { + t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) + } + + var payload map[string]any + if err := json.Unmarshal([]byte(stdout), &payload); err != nil { + t.Fatalf("decode rendered fixture: %v\nbody=%s", err, stdout) + } + if payload["prompt_version"] != "1.0.0" || payload["selected_profile_id"] != "custom-derived" || payload["selected_backend_id"] != "fixture-custom" { + t.Fatalf("unexpected selected definition and target: %#v", payload) + } + if payload["session_id"] != "fixture-testing" { + t.Fatalf("expected rendered session template, got %#v", payload["session_id"]) + } + + outputContract := payload["output_contract"].(map[string]any) + if outputContract["format"] != "json" || outputContract["validation_mode"] != "json_schema" || outputContract["repair_attempts"] != float64(2) { + t.Fatalf("unexpected output contract: %#v", outputContract) + } + if _, ok := payload["structured_output"].(map[string]any)["json_schema"]; !ok { + t.Fatalf("expected loaded JSON schema metadata, got %#v", payload["structured_output"]) + } + + messages := payload["messages"].([]any) + wantRoles := []string{"developer", "system", "user", "assistant"} + for i, wantRole := range wantRoles { + message := messages[i].(map[string]any) + if message["role"] != wantRole { + t.Fatalf("message %d: expected role %q, got %#v", i, wantRole, message["role"]) + } + } + cacheControl := messages[0].(map[string]any)["cache_control"].(map[string]any) + if cacheControl["type"] != "ephemeral" || cacheControl["ttl"] != "1h" { + t.Fatalf("unexpected cache control: %#v", cacheControl) + } + if !strings.Contains(messages[2].(map[string]any)["content"].(string), "stable, synthetic material") { + t.Fatalf("file-backed input template was not rendered: %#v", messages[2]) + } + inputHashes := payload["input_hashes"].(map[string]any) + if len(inputHashes) != 1 || inputHashes["source"] == "" { + t.Fatalf("required and omitted optional inputs were not preserved: %#v", inputHashes) + } +} + +func TestPromptkitV09PromptInspectionSelectsVersionsAndContracts(t *testing.T) { + fixtureRoot := promptkitV09FixtureRoot(t) + configPath := writePromptkitV09Config(t, fixtureRoot, true) + tests := []struct { + promptID string + version string + format string + mode string + }{ + {promptID: "compat.complete", version: "1.0.0", format: "json", mode: "json_schema"}, + {promptID: "compat.complete", version: "2.0.0", format: "text", mode: "basic"}, + {promptID: "compat.json", version: "1.0.0", format: "json", mode: "json"}, + {promptID: "compat.none", version: "1.0.0", format: "text", mode: "none"}, + } + + for _, tc := range tests { + t.Run(tc.promptID+"@"+tc.version, func(t *testing.T) { + code, stdout, stderr := runCLICommand(t, inspectCommand, []string{ + "prompt", + "--config", configPath, + "--prompt", tc.promptID, + "--prompt-version", tc.version, + "--format", "json", + }) + if code != ExitOK { + t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) + } + var inspection renderformat.PromptInspection + if err := json.Unmarshal([]byte(stdout), &inspection); err != nil { + t.Fatalf("decode prompt inspection: %v\nbody=%s", err, stdout) + } + if inspection.PromptVersion != tc.version || inspection.OutputContract.Format != tc.format || inspection.OutputContract.ValidationMode != tc.mode { + t.Fatalf("unexpected prompt inspection: %+v", inspection) + } + }) + } +} + +func TestPromptkitV09ProfileInspectionResolvesSupportedTargets(t *testing.T) { + const secret = "sentinel-profile-secret" + t.Setenv("FIXTURE_PROFILE_API_KEY", secret) + + fixtureRoot := promptkitV09FixtureRoot(t) + configPath := writePromptkitV09Config(t, fixtureRoot, false) + profileDir := filepath.Join(fixtureRoot, "profiles") + tests := []struct { + name string + profileID string + profileDir string + wantBackend string + wantModel string + wantAPIKeyEnv string + }{ + {name: "inherited custom backend", profileID: "custom-derived", profileDir: profileDir, wantBackend: "fixture-custom", wantModel: "fixture-derived-model", wantAPIKeyEnv: "FIXTURE_PROFILE_API_KEY"}, + {name: "endpoint only", profileID: "endpoint-only", profileDir: profileDir, wantModel: "fixture-endpoint-model"}, + {name: "built in", profileID: "deepseek-4-flash", wantBackend: "openrouter", wantModel: "deepseek/deepseek-v4-flash", wantAPIKeyEnv: "OPENROUTER_API_KEY"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + args := []string{"profile", "--config", configPath, "--profile", tc.profileID, "--format", "json"} + if tc.profileDir != "" { + args = append(args, "--profile-dir", tc.profileDir) + } + code, stdout, stderr := runCLICommand(t, inspectCommand, args) + if code != ExitOK { + t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) + } + if strings.Contains(stdout, secret) || strings.Contains(stderr, secret) { + t.Fatalf("inspection exposed environment secret: stdout=%q stderr=%q", stdout, stderr) + } + var inspection renderformat.ProfileInspection + if err := json.Unmarshal([]byte(stdout), &inspection); err != nil { + t.Fatalf("decode profile inspection: %v\nbody=%s", err, stdout) + } + params := inspection.EffectiveModelParams + if inspection.ProfileID != tc.profileID || params.BackendID != tc.wantBackend || params.Model != tc.wantModel || params.APIKeyEnv != tc.wantAPIKeyEnv { + t.Fatalf("unexpected effective profile: %+v", inspection) + } + if tc.profileID == "custom-derived" { + if params.ServiceTier != "flex" || params.ReasoningEffort != "high" || params.TimeoutSeconds != 45 || len(params.ExtraParams) == 0 { + t.Fatalf("inherited profile controls were not resolved: %+v", params) + } + } + }) + } +} + +func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.T) { + fixtureRoot := promptkitV09FixtureRoot(t) + configPath := writePromptkitV09Config(t, fixtureRoot, false) + overrideDir := t.TempDir() + writeProfileFile(t, overrideDir, "custom-derived", "http://127.0.0.1:9000/v1", "override-model") + outPath := filepath.Join(t.TempDir(), "inspection.json") + + code, stdout, stderr := runCLICommand(t, inspectCommand, []string{ + "profile", + "--config", configPath, + "--profile-dir", overrideDir, + "--profile", "custom-derived", + "--format", "json", + "--out", outPath, + }) + if code != ExitOK || stdout != "" { + t.Fatalf("expected file output, got code=%d stdout=%q stderr=%q", code, stdout, stderr) + } + output, err := os.ReadFile(outPath) + if err != nil { + t.Fatalf("read profile inspection output: %v", err) + } + if !strings.Contains(string(output), `"model": "override-model"`) || !strings.Contains(string(output), `"backend_id": ""`) { + t.Fatalf("profile directory override was not used: %s", output) + } + + for _, profileID := range []string{"missing", "invalid"} { + t.Run(profileID, func(t *testing.T) { + profileDir := filepath.Join(fixtureRoot, "profiles") + if profileID == "invalid" { + profileDir = t.TempDir() + writePromptDefinition(t, profileDir, "invalid.yaml", "id: invalid\nendpoint: not-a-url\nmodel: fixture\n") + } + code, stdout, stderr := runCLICommand(t, inspectCommand, []string{ + "profile", + "--config", configPath, + "--profile-dir", profileDir, + "--profile", profileID, + }) + if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") { + t.Fatalf("expected inspection failure without partial output, got code=%d stdout=%q stderr=%q", code, stdout, stderr) + } + }) + } +} + func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) { lib := newCLITestLibrary(t) @@ -1754,6 +1945,36 @@ func runCLICommand(t *testing.T, command func([]string, io.Writer, io.Writer) in return code, stdout.String(), stderr.String() } +func promptkitV09FixtureRoot(t *testing.T) string { + t.Helper() + root, err := filepath.Abs(filepath.Join("..", "..", "..", "testdata", "promptkit-v0.9")) + if err != nil { + t.Fatalf("resolve Promptkit v0.9 fixture root: %v", err) + } + if _, err := os.Stat(root); err != nil { + t.Fatalf("stat Promptkit v0.9 fixture root: %v", err) + } + return root +} + +func writePromptkitV09Config(t *testing.T, fixtureRoot string, includeSources bool) string { + t.Helper() + sources := "" + if includeSources { + sources = fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\nschema_dir: %q\n", filepath.Join(fixtureRoot, "prompts"), filepath.Join(fixtureRoot, "profiles"), filepath.Join(fixtureRoot, "schemas")) + } + return writeAppConfigFile(t, sources+`backends: + fixture-custom: + endpoint: http://127.0.0.1:11434/v1 + api_key_env: FIXTURE_BACKEND_API_KEY + extra_params: + backend_option: + enabled: true + concurrency_limit: 2 + queue_capacity: 0 +`) +} + func writePromptFile(t *testing.T, dir, id, defaultProfile string) { t.Helper() writePromptFileWithTemplate(t, dir, id, defaultProfile, "Summarize: {{input \"transcript\"}}") diff --git a/internal/format/inspection_test.go b/internal/format/inspection_test.go index 9280cbf..a8067ec 100644 --- a/internal/format/inspection_test.go +++ b/internal/format/inspection_test.go @@ -1,6 +1,7 @@ package format import ( + "encoding/json" "strings" "testing" @@ -30,3 +31,94 @@ func TestFormatPromptInspectionRejectsNil(t *testing.T) { t.Fatal("expected nil inspection error") } } + +func TestFormatProfileInspectionPreservesSafeEffectiveValues(t *testing.T) { + const secret = "sentinel-secret-must-not-appear" + t.Setenv("FIXTURE_PROFILE_API_KEY", secret) + + value := &promptkit.ProfileInspection{ + ProfileID: "custom-derived", + EffectiveModelParams: promptkit.ExecutionTarget{ + BackendID: "fixture-custom", + Endpoint: "http://127.0.0.1:11434/v1", + Model: "fixture-model", + Temperature: 0.25, + MaxTokens: 640, + TopP: 0.9, + TimeoutSeconds: 45, + ServiceTier: "flex", + ReasoningEffort: "high", + APIKeyEnv: "FIXTURE_PROFILE_API_KEY", + ExtraParams: map[string]any{ + "zeta": true, + "alpha": map[string]any{"nested": []any{"first", 2.0}}, + }, + }, + APIKeyRequired: true, + } + + jsonOutput, err := FormatProfileInspection(value, OutputFormatJSON) + if err != nil { + t.Fatal(err) + } + secondJSONOutput, err := FormatProfileInspection(value, OutputFormatJSON) + if err != nil { + t.Fatal(err) + } + if string(jsonOutput) != string(secondJSONOutput) || !strings.HasSuffix(string(jsonOutput), "\n") { + t.Fatalf("expected deterministic newline-terminated JSON, got %q", jsonOutput) + } + if strings.Contains(string(jsonOutput), secret) { + t.Fatalf("inspection exposed an environment secret: %s", jsonOutput) + } + + var decoded ProfileInspection + if err := json.Unmarshal(jsonOutput, &decoded); err != nil { + t.Fatalf("decode profile inspection: %v", err) + } + if decoded.ProfileID != "custom-derived" || decoded.EffectiveModelParams.BackendID != "fixture-custom" { + t.Fatalf("unexpected profile identity: %+v", decoded) + } + if decoded.EffectiveModelParams.APIKeyEnv != "FIXTURE_PROFILE_API_KEY" || !decoded.APIKeyRequired { + t.Fatalf("unexpected credential metadata: %+v", decoded) + } + alpha, ok := decoded.EffectiveModelParams.ExtraParams["alpha"].(map[string]any) + if !ok || len(alpha["nested"].([]any)) != 2 { + t.Fatalf("nested extra parameters were not preserved: %#v", decoded.EffectiveModelParams.ExtraParams) + } + + textOutput, err := FormatProfileInspection(value, OutputFormatText) + if err != nil { + t.Fatal(err) + } + textValue := string(textOutput) + if !strings.Contains(textValue, "backend_id: fixture-custom") || + !strings.Contains(textValue, "api_key_env: FIXTURE_PROFILE_API_KEY") || + !strings.Contains(textValue, `extra_params: {"alpha":{"nested":["first",2]},"zeta":true}`) || + !strings.Contains(textValue, "api_key_required: true") { + t.Fatalf("unexpected text inspection: %s", textValue) + } + if strings.Contains(textValue, secret) { + t.Fatalf("text inspection exposed an environment secret: %s", textValue) + } +} + +func TestFormatProfileInspectionPreservesEmptyBackendAndRejectsNil(t *testing.T) { + output, err := FormatProfileInspection(&promptkit.ProfileInspection{ + ProfileID: "endpoint-only", + EffectiveModelParams: promptkit.ExecutionTarget{ + Endpoint: "http://127.0.0.1:8000/v1", + Model: "fixture-model", + }, + }, OutputFormatJSON) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(output), `"backend_id": ""`) || + !strings.Contains(string(output), `"extra_params": {}`) { + t.Fatalf("expected explicit empty backend and object extra params, got %s", output) + } + if _, err := FormatProfileInspection(nil, OutputFormatText); err == nil { + t.Fatal("expected nil profile inspection error") + } +} diff --git a/testdata/promptkit-v0.9/config.yml b/testdata/promptkit-v0.9/config.yml new file mode 100644 index 0000000..9c0fbf6 --- /dev/null +++ b/testdata/promptkit-v0.9/config.yml @@ -0,0 +1,12 @@ +prompt_dir: ./testdata/promptkit-v0.9/prompts +profile_dir: ./testdata/promptkit-v0.9/profiles +schema_dir: ./testdata/promptkit-v0.9/schemas +backends: + fixture-custom: + endpoint: http://127.0.0.1:11434/v1 + api_key_env: FIXTURE_BACKEND_API_KEY + extra_params: + backend_option: + enabled: true + concurrency_limit: 2 + queue_capacity: 0 diff --git a/testdata/promptkit-v0.9/inputs/source.md b/testdata/promptkit-v0.9/inputs/source.md new file mode 100644 index 0000000..7a0f54a --- /dev/null +++ b/testdata/promptkit-v0.9/inputs/source.md @@ -0,0 +1 @@ +The fixture source contains stable, synthetic material. diff --git a/testdata/promptkit-v0.9/profiles/custom-base.yaml b/testdata/promptkit-v0.9/profiles/custom-base.yaml new file mode 100644 index 0000000..66c219c --- /dev/null +++ b/testdata/promptkit-v0.9/profiles/custom-base.yaml @@ -0,0 +1,16 @@ +id: custom-base +backend: fixture-custom +model: fixture-base-model +temperature: 0.25 +max_tokens: 640 +top_p: 0.9 +timeout_seconds: 45 +service_tier: flex +reasoning_effort: medium +api_key_env: FIXTURE_PROFILE_API_KEY +extra_params: + routing: + order: + - primary + - fallback + allow_fallback: true diff --git a/testdata/promptkit-v0.9/profiles/custom-derived.yml b/testdata/promptkit-v0.9/profiles/custom-derived.yml new file mode 100644 index 0000000..7d60463 --- /dev/null +++ b/testdata/promptkit-v0.9/profiles/custom-derived.yml @@ -0,0 +1,4 @@ +id: custom-derived +base_profile: custom-base +model: fixture-derived-model +reasoning_effort: high diff --git a/testdata/promptkit-v0.9/profiles/endpoint-only.yaml b/testdata/promptkit-v0.9/profiles/endpoint-only.yaml new file mode 100644 index 0000000..0586652 --- /dev/null +++ b/testdata/promptkit-v0.9/profiles/endpoint-only.yaml @@ -0,0 +1,10 @@ +id: endpoint-only +endpoint: http://127.0.0.1:8000/v1 +model: fixture-endpoint-model +temperature: 0.1 +max_tokens: 128 +top_p: 0.75 +timeout_seconds: 30 +extra_params: + endpoint_option: + nested: true diff --git a/testdata/promptkit-v0.9/prompts/complete-v1.yaml b/testdata/promptkit-v0.9/prompts/complete-v1.yaml new file mode 100644 index 0000000..d27c9ae --- /dev/null +++ b/testdata/promptkit-v0.9/prompts/complete-v1.yaml @@ -0,0 +1,31 @@ +id: compat.complete +version: "1.0.0" +description: Exercise the complete Promptkit v0.9 prompt-definition surface. +default_profile: custom-derived +session_id: 'fixture-{{.topic}}' +inputs: + - name: source + required: true + content_type: text/markdown + description: Source material rendered by a file-backed message. + - name: context + required: false + content_type: text/plain + description: Optional caller context. +messages: + - role: developer + content: 'Use the topic {{.topic}}.' + cache_control: + type: ephemeral + ttl: 1h + - role: system + content_file: ./messages/complete.system.md + - role: user + content_file: ./messages/complete.user.md + - role: assistant + content: Return one JSON object. +output: + format: json + validation_mode: json_schema + schema_path: complete.schema.json + repair_attempts: 2 diff --git a/testdata/promptkit-v0.9/prompts/complete-v2.yml b/testdata/promptkit-v0.9/prompts/complete-v2.yml new file mode 100644 index 0000000..6af9c16 --- /dev/null +++ b/testdata/promptkit-v0.9/prompts/complete-v2.yml @@ -0,0 +1,10 @@ +id: compat.complete +version: "2.0.0" +default_profile: endpoint-only +messages: + - role: user + content: Version two has no declared inputs. +output: + format: text + validation_mode: basic + repair_attempts: 1 diff --git a/testdata/promptkit-v0.9/prompts/json.yaml b/testdata/promptkit-v0.9/prompts/json.yaml new file mode 100644 index 0000000..1fbbc1f --- /dev/null +++ b/testdata/promptkit-v0.9/prompts/json.yaml @@ -0,0 +1,10 @@ +id: compat.json +version: "1.0.0" +default_profile: endpoint-only +messages: + - role: user + content: Return JSON. +output: + format: json + validation_mode: json + repair_attempts: 0 diff --git a/testdata/promptkit-v0.9/prompts/messages/complete.system.md b/testdata/promptkit-v0.9/prompts/messages/complete.system.md new file mode 100644 index 0000000..cc73b8e --- /dev/null +++ b/testdata/promptkit-v0.9/prompts/messages/complete.system.md @@ -0,0 +1 @@ +Use the supplied source without inventing details. diff --git a/testdata/promptkit-v0.9/prompts/messages/complete.user.md b/testdata/promptkit-v0.9/prompts/messages/complete.user.md new file mode 100644 index 0000000..6452389 --- /dev/null +++ b/testdata/promptkit-v0.9/prompts/messages/complete.user.md @@ -0,0 +1,4 @@ +Topic: {{.topic}} + +Source: +{{input "source"}} diff --git a/testdata/promptkit-v0.9/prompts/none.yaml b/testdata/promptkit-v0.9/prompts/none.yaml new file mode 100644 index 0000000..79aae45 --- /dev/null +++ b/testdata/promptkit-v0.9/prompts/none.yaml @@ -0,0 +1,10 @@ +id: compat.none +version: "1.0.0" +default_profile: endpoint-only +messages: + - role: user + content: Return text. +output: + format: text + validation_mode: none + repair_attempts: 0 diff --git a/testdata/promptkit-v0.9/schemas/complete.schema.json b/testdata/promptkit-v0.9/schemas/complete.schema.json new file mode 100644 index 0000000..772e16f --- /dev/null +++ b/testdata/promptkit-v0.9/schemas/complete.schema.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["summary"], + "properties": { + "summary": { + "type": "string" + } + }, + "additionalProperties": false +}