eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adopt published version two catalogs

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-22 12:26:22 +00:00
parent ba1e85a805
commit 2356f7c73d
6 changed files with 52 additions and 515 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/roadmap/implementation.md
View File

@@ -186,6 +186,8 @@ effective, and no public exports or provider routing logic have been added.
## Stage 5: Adopt Published V2 Catalogs And Replace Snapshot Tests
**Completed:** 2026-09-22
1. Update catalog imports in production and tests to `/v2`. Select the exact
Stage 2 versions through normal Go tooling, tidy module metadata, and review
`go.mod`/`go.sum`. No `latest`, unpublished revisions, unrelated upgrades,

4
go.mod
View File

@@ -3,8 +3,8 @@ module gitea.maximumdirect.net/eric/promptkit
go 1.25.5
require (
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260922.0
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260922.0
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
gopkg.in/yaml.v3 v3.0.1
)

8
go.sum
View File

@@ -1,7 +1,7 @@
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 h1:lc062euk2qseO//D762i3JaFyulDNML3eQQX7DkYTho=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0/go.mod h1:AIa7kAu2mfrRQgcspe4L+DW51WqgnALQT60lqkEywJI=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 h1:j9YY7wsTVjzke2kHH4YAzpU0oUpM+x+nXwl1IeS+2eg=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0/go.mod h1:4RNS+LILDg4JbS4Ts9Lwy1C92wauXJIbeQaalps4Koo=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260922.0 h1:uxvVFOylxXrzcg0jJLXcRBIj73BGNjvnsRuMoOeoYKw=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260922.0/go.mod h1:a3VBbh9oaGXD6ruxhtwHj+AWW6DY/SSyby1QtMmYSUo=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260922.0 h1:Y0EXc3eh12eKW2NrOviVqdk3oBo6UUjopt1YAa1B3Oc=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260922.0/go.mod h1:mtgD1E0SGm0e/I8Ffhz+AYX7GN7RKUWJXehNFIKHs4Y=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=

125
internal/catalog/catalog_test.go
View File

@@ -1,24 +1,19 @@
package catalog
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
"testing/fstest"
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter"
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome"
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2"
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2"
)
func TestLoadPublishedCatalogsMatchCompatibilityFixture(t *testing.T) {
func TestLoadPublishedCatalogs(t *testing.T) {
loaded, err := Load(
Source{Name: "OpenRouter", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
Source{Name: "Rakestrawhome", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
@@ -26,22 +21,48 @@ func TestLoadPublishedCatalogsMatchCompatibilityFixture(t *testing.T) {
if err != nil {
t.Fatalf("load published catalogs: %v", err)
}
expected := loadCompatibilityFixture(t)
expectedIDs := fixtureProfileIDs(t, expected)
if !reflect.DeepEqual(loaded.profileIDs, expectedIDs) {
t.Fatalf("published profile IDs differ from compatibility fixture: got %q, want %q", loaded.profileIDs, expectedIDs)
if loaded.Profiles == nil || len(loaded.profileIDs) == 0 {
t.Fatalf("published catalogs have no usable profiles: %#v", loaded)
}
actual := catalogValue(t, loaded)
actualJSON, err := json.Marshal(actual)
if err != nil {
t.Fatalf("encode loaded catalogs: %v", err)
if _, err := loaded.Profiles.GetProfile(context.Background(), "gemini-flash-latest"); err != nil {
t.Fatalf("load maintained standard-profile base: %v", err)
}
expectedJSON, err := json.Marshal(expected)
if err != nil {
t.Fatalf("encode compatibility fixture: %v", err)
expectedBackends := map[string]struct {
endpoint string
apiKeyEnv string
concurrencyLimit int
queueCapacity int
}{
"openrouter": {
endpoint: "https://openrouter.ai/api/v1",
apiKeyEnv: "OPENROUTER_API_KEY",
concurrencyLimit: 16,
queueCapacity: 1024,
},
"rakestrawhome": {
endpoint: "https://inference.ai.rakestrawhome.com/v1",
apiKeyEnv: "RAKESTRAWHOME_INFERENCE_API_KEY",
concurrencyLimit: 4,
queueCapacity: 1024,
},
}
if !bytes.Equal(actualJSON, expectedJSON) {
t.Fatalf("published catalogs differ from compatibility fixture: got %#v, want %#v", actual, expected)
if len(loaded.Backends) != len(expectedBackends) {
t.Fatalf("published backend count = %d, want %d", len(loaded.Backends), len(expectedBackends))
}
for _, definition := range loaded.Backends {
expected, found := expectedBackends[definition.ID]
if !found {
t.Fatalf("unexpected published backend %q", definition.ID)
}
if definition.Endpoint != expected.endpoint ||
definition.APIKeyEnv != expected.apiKeyEnv ||
definition.ConcurrencyLimit != expected.concurrencyLimit ||
definition.QueueCapacity != expected.queueCapacity ||
!definition.QueueCapacitySet ||
definition.ExtraParams != nil {
t.Fatalf("unexpected published backend definition: %#v", definition)
}
}
}
@@ -298,68 +319,6 @@ func TestLoadReturnsDefensiveCatalogValues(t *testing.T) {
}
}
func loadCompatibilityFixture(t *testing.T) map[string]any {
t.Helper()
data, err := os.ReadFile(filepath.Join("..", "..", "testdata", "builtin-catalog-v1.json"))
if err != nil {
t.Fatalf("read fixture: %v", err)
}
var value map[string]any
if err := json.Unmarshal(data, &value); err != nil {
t.Fatalf("decode fixture: %v", err)
}
return value
}
func fixtureProfileIDs(t *testing.T, fixture map[string]any) []string {
t.Helper()
profiles, ok := fixture["profiles"].([]any)
if !ok {
t.Fatal("compatibility fixture profiles are malformed")
}
ids := make([]string, 0, len(profiles))
for _, entry := range profiles {
profileValue, ok := entry.(map[string]any)
if !ok {
t.Fatal("compatibility fixture profile is malformed")
}
id, ok := profileValue["id"].(string)
if !ok {
t.Fatal("compatibility fixture profile ID is malformed")
}
ids = append(ids, id)
}
sort.Strings(ids)
return ids
}
func catalogValue(t *testing.T, loaded Set) map[string]any {
t.Helper()
backends := make([]any, 0, len(loaded.Backends))
for _, backend := range loaded.Backends {
backends = append(backends, map[string]any{"id": backend.ID, "endpoint": backend.Endpoint, "api_key_env": backend.APIKeyEnv, "extra_params": interfaceValue(backend.ExtraParams), "concurrency_limit": backend.ConcurrencyLimit, "queue_capacity": backend.QueueCapacity, "queue_capacity_set": backend.QueueCapacitySet})
}
sort.Slice(backends, func(left, right int) bool {
return backends[left].(map[string]any)["id"].(string) < backends[right].(map[string]any)["id"].(string)
})
actualProfiles := make([]any, 0, len(loaded.profileIDs))
for _, id := range loaded.profileIDs {
profile, err := loaded.Profiles.GetProfile(context.Background(), id)
if err != nil {
t.Fatalf("load profile %q: %v", id, err)
}
actualProfiles = append(actualProfiles, map[string]any{"id": profile.ID, "base_profile": profile.BaseProfileID, "backend": profile.BackendID, "endpoint": profile.Endpoint, "model": profile.Model, "temperature": profile.Temperature, "max_tokens": profile.MaxTokens, "top_p": profile.TopP, "timeout_seconds": profile.TimeoutSeconds, "service_tier": profile.ServiceTier, "reasoning_effort": profile.ReasoningEffort, "api_key_env": profile.APIKeyEnv, "api_key_required": profile.APIKeyRequired, "extra_params": interfaceValue(profile.ExtraParams)})
}
return map[string]any{"backends": backends, "profiles": actualProfiles}
}
func interfaceValue(value map[string]any) any {
if value == nil {
return nil
}
return value
}
func testSource(id string, fsys fs.FS) Source {
return Source{Name: id, ExpectedBackendID: id, FS: fsys, Root: "catalog"}
}

4
internal/engine/engine.go
View File

@@ -11,8 +11,8 @@ import (
"strings"
"time"
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter"
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome"
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2"
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2"
"gitea.maximumdirect.net/eric/promptkit/internal/artifact"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"

424
testdata/builtin-catalog-v1.json vendored
View File

@@ -1,424 +0,0 @@
{
"backends": [
{
"id": "openrouter",
"endpoint": "https://openrouter.ai/api/v1",
"api_key_env": "OPENROUTER_API_KEY",
"extra_params": null,
"concurrency_limit": 16,
"queue_capacity": 1024,
"queue_capacity_set": true
},
{
"id": "rakestrawhome",
"endpoint": "https://inference.ai.rakestrawhome.com/v1",
"api_key_env": "RAKESTRAWHOME_INFERENCE_API_KEY",
"extra_params": null,
"concurrency_limit": 4,
"queue_capacity": 1024,
"queue_capacity_set": true
}
],
"profiles": [
{
"id": "aion-2",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "aion-labs/aion-2.0",
"temperature": 0.72,
"max_tokens": 0,
"top_p": 0.95,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "claude-fable-latest",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "~anthropic/claude-fable-latest",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 600,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "claude-haiku-latest",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "~anthropic/claude-haiku-latest",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "medium",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "claude-opus-latest",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "~anthropic/claude-opus-latest",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "claude-sonnet-latest",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "~anthropic/claude-sonnet-latest",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "deepseek-3-2",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "deepseek/deepseek-v3.2",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "deepseek-4-flash",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "deepseek/deepseek-v4-flash",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "deepseek-4-pro",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "deepseek/deepseek-v4-pro",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemini-2-flash",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "google/gemini-2.5-flash",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemini-2-flash-lite",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "google/gemini-2.5-flash-lite",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemini-2-pro",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "google/gemini-2.5-pro",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemini-3-flash-lite",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "google/gemini-3.1-flash-lite",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemini-flash-latest",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "~google/gemini-flash-latest",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemini-pro-latest",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "~google/gemini-pro-latest",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemma-4-31b",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "google/gemma-4-31b-it:exacto",
"temperature": 0.15,
"max_tokens": 0,
"top_p": 0.98,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gpt-5-mini",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "openai/gpt-5.4-mini",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gpt-5-nano",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "openai/gpt-5.4-nano",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "minimax-m2",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "minimax/minimax-m2.5",
"temperature": 0.5,
"max_tokens": 0,
"top_p": 0.95,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "minimax-m3",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "minimax/minimax-m3",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "mistral-large-2512",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "mistralai/mistral-large-2512",
"temperature": 0.15,
"max_tokens": 0,
"top_p": 0.98,
"timeout_seconds": 180,
"service_tier": "",
"reasoning_effort": "",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "mistral-medium-3-5",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "mistralai/mistral-medium-3-5",
"temperature": 0.15,
"max_tokens": 0,
"top_p": 0.98,
"timeout_seconds": 180,
"service_tier": "",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "mistral-small-3",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "mistralai/mistral-small-3.2-24b-instruct",
"temperature": 0.05,
"max_tokens": 0,
"top_p": 1,
"timeout_seconds": 180,
"service_tier": "",
"reasoning_effort": "",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "mistral-small-4",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "mistralai/mistral-small-2603",
"temperature": 0.1,
"max_tokens": 0,
"top_p": 0.98,
"timeout_seconds": 180,
"service_tier": "",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "nemotron-3-ultra",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "nvidia/nemotron-3-ultra-550b-a55b",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "rakestrawhome-gemma-4-31b",
"base_profile": "",
"backend": "rakestrawhome",
"endpoint": "",
"model": "google/gemma-4-31b-it",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 0,
"service_tier": "",
"reasoning_effort": "",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
}
]
}