eric/scriptorium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 9 Wiki Activity

Adopt Promptkit v0.11 and maintained catalog profiles

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-26 15:24:55 +00:00
parent 5c3b0cf4f6
commit 2a06a911bc
16 changed files with 180 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
internal/adapter/cli/run_test.go
View File

@@ -1201,9 +1201,9 @@ func TestInspectionParsersPreserveExplicitEmptyConfigPath(t *testing.T) {
}
}
func TestPromptkitV09DefinitionsRenderThroughCLI(t *testing.T) {
fixtureRoot := promptkitV09FixtureRoot(t)
configPath := writePromptkitV09Config(t, fixtureRoot, true)
func TestPromptkitDefinitionsRenderThroughCLI(t *testing.T) {
fixtureRoot := definitionFixtureRoot(t)
configPath := writeDefinitionFixtureConfig(t, fixtureRoot, true)
inputPath := filepath.Join(fixtureRoot, "inputs", "source.md")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
@@ -1258,9 +1258,9 @@ func TestPromptkitV09DefinitionsRenderThroughCLI(t *testing.T) {
}
}
func TestPromptkitV09PromptInspectionSelectsVersionsAndContracts(t *testing.T) {
fixtureRoot := promptkitV09FixtureRoot(t)
configPath := writePromptkitV09Config(t, fixtureRoot, true)
func TestPromptkitPromptInspectionSelectsVersionsAndContracts(t *testing.T) {
fixtureRoot := definitionFixtureRoot(t)
configPath := writeDefinitionFixtureConfig(t, fixtureRoot, true)
tests := []struct {
promptID string
version string
@@ -1296,12 +1296,12 @@ func TestPromptkitV09PromptInspectionSelectsVersionsAndContracts(t *testing.T) {
}
}
func TestPromptkitV09ProfileInspectionResolvesSupportedTargets(t *testing.T) {
func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) {
const secret = "sentinel-profile-secret"
t.Setenv("FIXTURE_PROFILE_API_KEY", secret)
fixtureRoot := promptkitV09FixtureRoot(t)
configPath := writePromptkitV09Config(t, fixtureRoot, true)
fixtureRoot := definitionFixtureRoot(t)
configPath := writeDefinitionFixtureConfig(t, fixtureRoot, true)
tests := []struct {
name string
profileID string
@@ -1311,7 +1311,7 @@ func TestPromptkitV09ProfileInspectionResolvesSupportedTargets(t *testing.T) {
}{
{name: "inherited custom backend", profileID: "custom-derived", wantBackend: "fixture-custom", wantModel: "fixture-derived-model", wantAPIKeyEnv: "FIXTURE_PROFILE_API_KEY"},
{name: "endpoint only", profileID: "endpoint-only", wantModel: "fixture-endpoint-model"},
{name: "built in", profileID: "deepseek-4-flash", wantBackend: "openrouter", wantModel: "deepseek/deepseek-v4-flash", wantAPIKeyEnv: "OPENROUTER_API_KEY"},
{name: "built in", profileID: "promptkit-openrouter-auto", wantBackend: "openrouter", wantModel: "openrouter/auto", wantAPIKeyEnv: "OPENROUTER_API_KEY"},
}
for _, tc := range tests {
@@ -1340,7 +1340,7 @@ func TestPromptkitV09ProfileInspectionResolvesSupportedTargets(t *testing.T) {
})
}
configWithoutPromptDir := writePromptkitV09Config(t, fixtureRoot, false)
configWithoutPromptDir := writeDefinitionFixtureConfig(t, fixtureRoot, false)
code, _, stderr := runCLICommand(t, inspectCommand, []string{
"profile",
"--config", configWithoutPromptDir,
@@ -1352,9 +1352,92 @@ func TestPromptkitV09ProfileInspectionResolvesSupportedTargets(t *testing.T) {
}
}
func TestMaintainedProfilesResolveAcrossSelectionAndSourcePrecedence(t *testing.T) {
promptDir := t.TempDir()
profileDir := t.TempDir()
writePromptDefinition(t, promptDir, "maintained.yaml", `id: maintained
version: "1.0.0"
default_profile: promptkit-openrouter-standard
messages:
- role: user
content: Hello
output:
format: text
validation_mode: none
`)
configPath := writeAppConfigFile(t, fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\n", promptDir, profileDir))
for _, profileID := range []string{"promptkit-openrouter-standard", "promptkit-openrouter-auto"} {
t.Run(profileID+" inspection", func(t *testing.T) {
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"profile", "--config", configPath, "--profile", profileID, "--format", "json"})
if code != ExitOK {
t.Fatalf("inspect %s: code=%d stderr=%q", profileID, code, stderr)
}
var inspection renderformat.ProfileInspection
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
t.Fatal(err)
}
if inspection.ProfileID != profileID || inspection.EffectiveModelParams.BackendID != "openrouter" || inspection.EffectiveModelParams.Model == "" {
t.Fatalf("maintained profile did not resolve: %+v", inspection)
}
if profileID == "promptkit-openrouter-auto" && inspection.EffectiveModelParams.Model != "openrouter/auto" {
t.Fatalf("unexpected auto target: %+v", inspection.EffectiveModelParams)
}
})
}
for _, tc := range []struct {
name string
selection []string
wantID string
}{
{name: "prompt default", wantID: "promptkit-openrouter-standard"},
{name: "explicit selection", selection: []string{"--profile", "promptkit-openrouter-auto"}, wantID: "promptkit-openrouter-auto"},
} {
t.Run(tc.name, func(t *testing.T) {
args := append([]string{"--config", configPath, "--prompt", "maintained", "--format", "json"}, tc.selection...)
code, stdout, stderr := runCLICommand(t, renderCommand, args)
if code != ExitOK {
t.Fatalf("render: code=%d stderr=%q", code, stderr)
}
var rendered struct {
SelectedProfileID string `json:"selected_profile_id"`
SelectedBackendID string `json:"selected_backend_id"`
}
if err := json.Unmarshal([]byte(stdout), &rendered); err != nil {
t.Fatal(err)
}
if rendered.SelectedProfileID != tc.wantID || rendered.SelectedBackendID != "openrouter" {
t.Fatalf("unexpected selection: %+v", rendered)
}
})
}
writePromptDefinition(t, profileDir, "derived.yaml", "id: derived-maintained\nbase_profile: promptkit-openrouter-standard\nmodel: fixture-inherited-model\n")
writePromptDefinition(t, profileDir, "replacement.yaml", "id: promptkit-openrouter-auto\nbackend: openrouter\nmodel: fixture-override-model\n")
for _, tc := range []struct{ id, model string }{
{id: "derived-maintained", model: "fixture-inherited-model"},
{id: "promptkit-openrouter-auto", model: "fixture-override-model"},
} {
t.Run(tc.id+" source", func(t *testing.T) {
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"profile", "--config", configPath, "--profile", tc.id, "--format", "json"})
if code != ExitOK {
t.Fatalf("inspect: code=%d stderr=%q", code, stderr)
}
var inspection renderformat.ProfileInspection
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
t.Fatal(err)
}
if inspection.EffectiveModelParams.BackendID != "openrouter" || inspection.EffectiveModelParams.Model != tc.model {
t.Fatalf("unexpected effective target: %+v", inspection)
}
})
}
}
func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.T) {
fixtureRoot := promptkitV09FixtureRoot(t)
configPath := writePromptkitV09Config(t, fixtureRoot, true)
fixtureRoot := definitionFixtureRoot(t)
configPath := writeDefinitionFixtureConfig(t, fixtureRoot, true)
overrideDir := t.TempDir()
writeProfileFile(t, overrideDir, "custom-derived", "http://127.0.0.1:9000/v1", "override-model")
outPath := filepath.Join(t.TempDir(), "inspection.json")
@@ -1971,7 +2054,7 @@ func runCLICommand(t *testing.T, command func([]string, io.Writer, io.Writer) in
return code, stdout.String(), stderr.String()
}
func promptkitV09FixtureRoot(t *testing.T) string {
func definitionFixtureRoot(t *testing.T) string {
t.Helper()
root, err := filepath.Abs(filepath.Join("..", "..", "..", "testdata", "promptkit-v0.9"))
if err != nil {
@@ -1983,7 +2066,7 @@ func promptkitV09FixtureRoot(t *testing.T) string {
return root
}
func writePromptkitV09Config(t *testing.T, fixtureRoot string, includeSources bool) string {
func writeDefinitionFixtureConfig(t *testing.T, fixtureRoot string, includeSources bool) string {
t.Helper()
sources := ""
if includeSources {