eric/scriptorium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add Promptkit definition conformance coverage

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-29 15:12:27 +00:00
parent 3e5f5c5198
commit 8936ca7c18
15 changed files with 434 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

221
internal/adapter/cli/run_test.go
View File

@@ -1181,6 +1181,197 @@ output:
}
}
func TestPromptkitV09DefinitionsRenderThroughCLI(t *testing.T) {
fixtureRoot := promptkitV09FixtureRoot(t)
configPath := writePromptkitV09Config(t, fixtureRoot, true)
inputPath := filepath.Join(fixtureRoot, "inputs", "source.md")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", configPath,
"--prompt", "compat.complete",
"--prompt-version", "1.0.0",
"--input", "source=" + inputPath,
"--var", "topic=testing",
"--format", "json",
})
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
var payload map[string]any
if err := json.Unmarshal([]byte(stdout), &payload); err != nil {
t.Fatalf("decode rendered fixture: %v\nbody=%s", err, stdout)
}
if payload["prompt_version"] != "1.0.0" || payload["selected_profile_id"] != "custom-derived" || payload["selected_backend_id"] != "fixture-custom" {
t.Fatalf("unexpected selected definition and target: %#v", payload)
}
if payload["session_id"] != "fixture-testing" {
t.Fatalf("expected rendered session template, got %#v", payload["session_id"])
}
outputContract := payload["output_contract"].(map[string]any)
if outputContract["format"] != "json" || outputContract["validation_mode"] != "json_schema" || outputContract["repair_attempts"] != float64(2) {
t.Fatalf("unexpected output contract: %#v", outputContract)
}
if _, ok := payload["structured_output"].(map[string]any)["json_schema"]; !ok {
t.Fatalf("expected loaded JSON schema metadata, got %#v", payload["structured_output"])
}
messages := payload["messages"].([]any)
wantRoles := []string{"developer", "system", "user", "assistant"}
for i, wantRole := range wantRoles {
message := messages[i].(map[string]any)
if message["role"] != wantRole {
t.Fatalf("message %d: expected role %q, got %#v", i, wantRole, message["role"])
}
}
cacheControl := messages[0].(map[string]any)["cache_control"].(map[string]any)
if cacheControl["type"] != "ephemeral" || cacheControl["ttl"] != "1h" {
t.Fatalf("unexpected cache control: %#v", cacheControl)
}
if !strings.Contains(messages[2].(map[string]any)["content"].(string), "stable, synthetic material") {
t.Fatalf("file-backed input template was not rendered: %#v", messages[2])
}
inputHashes := payload["input_hashes"].(map[string]any)
if len(inputHashes) != 1 || inputHashes["source"] == "" {
t.Fatalf("required and omitted optional inputs were not preserved: %#v", inputHashes)
}
}
func TestPromptkitV09PromptInspectionSelectsVersionsAndContracts(t *testing.T) {
fixtureRoot := promptkitV09FixtureRoot(t)
configPath := writePromptkitV09Config(t, fixtureRoot, true)
tests := []struct {
promptID string
version string
format string
mode string
}{
{promptID: "compat.complete", version: "1.0.0", format: "json", mode: "json_schema"},
{promptID: "compat.complete", version: "2.0.0", format: "text", mode: "basic"},
{promptID: "compat.json", version: "1.0.0", format: "json", mode: "json"},
{promptID: "compat.none", version: "1.0.0", format: "text", mode: "none"},
}
for _, tc := range tests {
t.Run(tc.promptID+"@"+tc.version, func(t *testing.T) {
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
"prompt",
"--config", configPath,
"--prompt", tc.promptID,
"--prompt-version", tc.version,
"--format", "json",
})
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
var inspection renderformat.PromptInspection
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
t.Fatalf("decode prompt inspection: %v\nbody=%s", err, stdout)
}
if inspection.PromptVersion != tc.version || inspection.OutputContract.Format != tc.format || inspection.OutputContract.ValidationMode != tc.mode {
t.Fatalf("unexpected prompt inspection: %+v", inspection)
}
})
}
}
func TestPromptkitV09ProfileInspectionResolvesSupportedTargets(t *testing.T) {
const secret = "sentinel-profile-secret"
t.Setenv("FIXTURE_PROFILE_API_KEY", secret)
fixtureRoot := promptkitV09FixtureRoot(t)
configPath := writePromptkitV09Config(t, fixtureRoot, false)
profileDir := filepath.Join(fixtureRoot, "profiles")
tests := []struct {
name string
profileID string
profileDir string
wantBackend string
wantModel string
wantAPIKeyEnv string
}{
{name: "inherited custom backend", profileID: "custom-derived", profileDir: profileDir, wantBackend: "fixture-custom", wantModel: "fixture-derived-model", wantAPIKeyEnv: "FIXTURE_PROFILE_API_KEY"},
{name: "endpoint only", profileID: "endpoint-only", profileDir: profileDir, wantModel: "fixture-endpoint-model"},
{name: "built in", profileID: "deepseek-4-flash", wantBackend: "openrouter", wantModel: "deepseek/deepseek-v4-flash", wantAPIKeyEnv: "OPENROUTER_API_KEY"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
args := []string{"profile", "--config", configPath, "--profile", tc.profileID, "--format", "json"}
if tc.profileDir != "" {
args = append(args, "--profile-dir", tc.profileDir)
}
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
if strings.Contains(stdout, secret) || strings.Contains(stderr, secret) {
t.Fatalf("inspection exposed environment secret: stdout=%q stderr=%q", stdout, stderr)
}
var inspection renderformat.ProfileInspection
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
t.Fatalf("decode profile inspection: %v\nbody=%s", err, stdout)
}
params := inspection.EffectiveModelParams
if inspection.ProfileID != tc.profileID || params.BackendID != tc.wantBackend || params.Model != tc.wantModel || params.APIKeyEnv != tc.wantAPIKeyEnv {
t.Fatalf("unexpected effective profile: %+v", inspection)
}
if tc.profileID == "custom-derived" {
if params.ServiceTier != "flex" || params.ReasoningEffort != "high" || params.TimeoutSeconds != 45 || len(params.ExtraParams) == 0 {
t.Fatalf("inherited profile controls were not resolved: %+v", params)
}
}
})
}
}
func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.T) {
fixtureRoot := promptkitV09FixtureRoot(t)
configPath := writePromptkitV09Config(t, fixtureRoot, false)
overrideDir := t.TempDir()
writeProfileFile(t, overrideDir, "custom-derived", "http://127.0.0.1:9000/v1", "override-model")
outPath := filepath.Join(t.TempDir(), "inspection.json")
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
"profile",
"--config", configPath,
"--profile-dir", overrideDir,
"--profile", "custom-derived",
"--format", "json",
"--out", outPath,
})
if code != ExitOK || stdout != "" {
t.Fatalf("expected file output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output, err := os.ReadFile(outPath)
if err != nil {
t.Fatalf("read profile inspection output: %v", err)
}
if !strings.Contains(string(output), `"model": "override-model"`) || !strings.Contains(string(output), `"backend_id": ""`) {
t.Fatalf("profile directory override was not used: %s", output)
}
for _, profileID := range []string{"missing", "invalid"} {
t.Run(profileID, func(t *testing.T) {
profileDir := filepath.Join(fixtureRoot, "profiles")
if profileID == "invalid" {
profileDir = t.TempDir()
writePromptDefinition(t, profileDir, "invalid.yaml", "id: invalid\nendpoint: not-a-url\nmodel: fixture\n")
}
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
"profile",
"--config", configPath,
"--profile-dir", profileDir,
"--profile", profileID,
})
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
t.Fatalf("expected inspection failure without partial output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
})
}
}
func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
lib := newCLITestLibrary(t)
@@ -1754,6 +1945,36 @@ func runCLICommand(t *testing.T, command func([]string, io.Writer, io.Writer) in
return code, stdout.String(), stderr.String()
}
func promptkitV09FixtureRoot(t *testing.T) string {
t.Helper()
root, err := filepath.Abs(filepath.Join("..", "..", "..", "testdata", "promptkit-v0.9"))
if err != nil {
t.Fatalf("resolve Promptkit v0.9 fixture root: %v", err)
}
if _, err := os.Stat(root); err != nil {
t.Fatalf("stat Promptkit v0.9 fixture root: %v", err)
}
return root
}
func writePromptkitV09Config(t *testing.T, fixtureRoot string, includeSources bool) string {
t.Helper()
sources := ""
if includeSources {
sources = fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\nschema_dir: %q\n", filepath.Join(fixtureRoot, "prompts"), filepath.Join(fixtureRoot, "profiles"), filepath.Join(fixtureRoot, "schemas"))
}
return writeAppConfigFile(t, sources+`backends:
fixture-custom:
endpoint: http://127.0.0.1:11434/v1
api_key_env: FIXTURE_BACKEND_API_KEY
extra_params:
backend_option:
enabled: true
concurrency_limit: 2
queue_capacity: 0
`)
}
func writePromptFile(t *testing.T, dir, id, defaultProfile string) {
t.Helper()
writePromptFileWithTemplate(t, dir, id, defaultProfile, "Summarize: {{input \"transcript\"}}")