eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Freeze the built-in catalog compatibility baseline

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-26 14:49:25 +00:00
parent b45cea4084
commit 59be507d2f
3 changed files with 586 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/roadmap/implementation.md
View File

@@ -140,7 +140,7 @@ Promptkit's private adapter:
## Stage 1: Freeze The Pre-Extraction Compatibility Baseline
**Status:** Pending
**Status:** Complete
### Repository

161
internal/profile/builtin/repository_test.go
View File

@@ -2,14 +2,175 @@ package builtin
import (
"context"
"encoding/json"
"errors"
"io"
"io/fs"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gopkg.in/yaml.v3"
)
type catalogFixture struct {
Backends []backendFixture `json:"backends"`
Profiles []profileFixture `json:"profiles"`
}
type backendFixture struct {
ID string `json:"id"`
Endpoint string `json:"endpoint"`
APIKeyEnv string `json:"api_key_env"`
ExtraParams map[string]any `json:"extra_params"`
ConcurrencyLimit int `json:"concurrency_limit"`
QueueCapacity int `json:"queue_capacity"`
QueueCapacitySet bool `json:"queue_capacity_set"`
}
type profileFixture struct {
ID string `json:"id"`
BaseProfileID string `json:"base_profile"`
BackendID string `json:"backend"`
Endpoint string `json:"endpoint"`
Model string `json:"model"`
Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"`
TopP float64 `json:"top_p"`
TimeoutSeconds int `json:"timeout_seconds"`
ServiceTier string `json:"service_tier"`
ReasoningEffort string `json:"reasoning_effort"`
APIKeyEnv string `json:"api_key_env"`
APIKeyRequired bool `json:"api_key_required"`
ExtraParams map[string]any `json:"extra_params"`
}
func TestBuiltInCatalogMatchesCompatibilityFixture(t *testing.T) {
fixture := loadCatalogFixture(t)
assertCatalogFixtureOrdering(t, fixture)
registry, err := backend.NewRegistry(nil)
if err != nil {
t.Fatalf("construct built-in backend registry: %v", err)
}
actualBackends := make(map[string]backendFixture)
for id := range registry.CapacityPolicies() {
definition, err := registry.GetBackend(id)
if err != nil {
t.Fatalf("load built-in backend %q: %v", id, err)
}
actualBackends[id] = backendFixtureFromDefinition(definition)
}
expectedBackends := make(map[string]backendFixture, len(fixture.Backends))
for _, expected := range fixture.Backends {
expectedBackends[expected.ID] = expected
}
if !mapsEqual(actualBackends, expectedBackends) {
t.Fatalf("built-in backends differ from compatibility fixture: got %#v, want %#v", actualBackends, expectedBackends)
}
profilePaths := loadBuiltInProfileIDs(t)
actualProfiles := make(map[string]profileFixture, len(profilePaths))
for id := range profilePaths {
definition, err := NewRepository().GetProfile(context.Background(), id)
if err != nil {
t.Fatalf("load built-in profile %q: %v", id, err)
}
actualProfiles[id] = profileFixtureFromDefinition(*definition)
}
expectedProfiles := make(map[string]profileFixture, len(fixture.Profiles))
for _, expected := range fixture.Profiles {
expectedProfiles[expected.ID] = expected
}
if !mapsEqual(actualProfiles, expectedProfiles) {
t.Fatalf("built-in profiles differ from compatibility fixture: got %#v, want %#v", actualProfiles, expectedProfiles)
}
}
func loadCatalogFixture(t *testing.T) catalogFixture {
t.Helper()
path := filepath.Join("..", "..", "..", "testdata", "builtin-catalog-v1.json")
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read compatibility fixture: %v", err)
}
decoder := json.NewDecoder(strings.NewReader(string(data)))
decoder.DisallowUnknownFields()
var fixture catalogFixture
if err := decoder.Decode(&fixture); err != nil {
t.Fatalf("decode compatibility fixture: %v", err)
}
if err := ensureJSONEnd(decoder); err != nil {
t.Fatalf("decode compatibility fixture: %v", err)
}
return fixture
}
func ensureJSONEnd(decoder *json.Decoder) error {
var trailing any
if err := decoder.Decode(&trailing); errors.Is(err, io.EOF) {
return nil
} else if err != nil {
return err
}
return errors.New("fixture must contain exactly one JSON value")
}
func assertCatalogFixtureOrdering(t *testing.T, fixture catalogFixture) {
t.Helper()
for index := 1; index < len(fixture.Backends); index++ {
if fixture.Backends[index-1].ID >= fixture.Backends[index].ID {
t.Fatalf("compatibility fixture backends are not sorted by ID")
}
}
for index := 1; index < len(fixture.Profiles); index++ {
if fixture.Profiles[index-1].ID >= fixture.Profiles[index].ID {
t.Fatalf("compatibility fixture profiles are not sorted by ID")
}
}
}
func backendFixtureFromDefinition(definition domain.Backend) backendFixture {
return backendFixture{
ID: definition.ID,
Endpoint: definition.Endpoint,
APIKeyEnv: definition.APIKeyEnv,
ExtraParams: definition.ExtraParams,
ConcurrencyLimit: definition.ConcurrencyLimit,
QueueCapacity: definition.QueueCapacity,
QueueCapacitySet: definition.QueueCapacitySet,
}
}
func profileFixtureFromDefinition(definition domain.ExecutionProfile) profileFixture {
return profileFixture{
ID: definition.ID,
BaseProfileID: definition.BaseProfileID,
BackendID: definition.BackendID,
Endpoint: definition.Endpoint,
Model: definition.Model,
Temperature: definition.Temperature,
MaxTokens: definition.MaxTokens,
TopP: definition.TopP,
TimeoutSeconds: definition.TimeoutSeconds,
ServiceTier: definition.ServiceTier,
ReasoningEffort: definition.ReasoningEffort,
APIKeyEnv: definition.APIKeyEnv,
APIKeyRequired: definition.APIKeyRequired,
ExtraParams: definition.ExtraParams,
}
}
func mapsEqual[K comparable, V any](actual, expected map[K]V) bool {
return reflect.DeepEqual(actual, expected)
}
func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
repo := NewRepository()
ids := loadBuiltInProfileIDs(t)

424
testdata/builtin-catalog-v1.json vendored Normal file
View File

@@ -0,0 +1,424 @@
{
"backends": [
{
"id": "openrouter",
"endpoint": "https://openrouter.ai/api/v1",
"api_key_env": "OPENROUTER_API_KEY",
"extra_params": null,
"concurrency_limit": 16,
"queue_capacity": 1024,
"queue_capacity_set": true
},
{
"id": "rakestrawhome",
"endpoint": "https://inference.ai.rakestrawhome.com/v1",
"api_key_env": "RAKESTRAWHOME_INFERENCE_API_KEY",
"extra_params": null,
"concurrency_limit": 4,
"queue_capacity": 1024,
"queue_capacity_set": true
}
],
"profiles": [
{
"id": "aion-2",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "aion-labs/aion-2.0",
"temperature": 0.72,
"max_tokens": 0,
"top_p": 0.95,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "claude-fable-latest",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "~anthropic/claude-fable-latest",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 600,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "claude-haiku-latest",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "~anthropic/claude-haiku-latest",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "medium",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "claude-opus-latest",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "~anthropic/claude-opus-latest",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "claude-sonnet-latest",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "~anthropic/claude-sonnet-latest",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "deepseek-3-2",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "deepseek/deepseek-v3.2",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "deepseek-4-flash",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "deepseek/deepseek-v4-flash",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "deepseek-4-pro",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "deepseek/deepseek-v4-pro",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemini-2-flash",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "google/gemini-2.5-flash",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemini-2-flash-lite",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "google/gemini-2.5-flash-lite",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemini-2-pro",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "google/gemini-2.5-pro",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemini-3-flash-lite",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "google/gemini-3.1-flash-lite",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemini-flash-latest",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "~google/gemini-flash-latest",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemini-pro-latest",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "~google/gemini-pro-latest",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gemma-4-31b",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "google/gemma-4-31b-it:exacto",
"temperature": 0.15,
"max_tokens": 0,
"top_p": 0.98,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gpt-5-mini",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "openai/gpt-5.4-mini",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "gpt-5-nano",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "openai/gpt-5.4-nano",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 240,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "minimax-m2",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "minimax/minimax-m2.5",
"temperature": 0.5,
"max_tokens": 0,
"top_p": 0.95,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "minimax-m3",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "minimax/minimax-m3",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "mistral-large-2512",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "mistralai/mistral-large-2512",
"temperature": 0.15,
"max_tokens": 0,
"top_p": 0.98,
"timeout_seconds": 180,
"service_tier": "",
"reasoning_effort": "",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "mistral-medium-3-5",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "mistralai/mistral-medium-3-5",
"temperature": 0.15,
"max_tokens": 0,
"top_p": 0.98,
"timeout_seconds": 180,
"service_tier": "",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "mistral-small-3",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "mistralai/mistral-small-3.2-24b-instruct",
"temperature": 0.05,
"max_tokens": 0,
"top_p": 1,
"timeout_seconds": 180,
"service_tier": "",
"reasoning_effort": "",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "mistral-small-4",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "mistralai/mistral-small-2603",
"temperature": 0.1,
"max_tokens": 0,
"top_p": 0.98,
"timeout_seconds": 180,
"service_tier": "",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "nemotron-3-ultra",
"base_profile": "",
"backend": "openrouter",
"endpoint": "",
"model": "nvidia/nemotron-3-ultra-550b-a55b",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 180,
"service_tier": "flex",
"reasoning_effort": "high",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
},
{
"id": "rakestrawhome-gemma-4-31b",
"base_profile": "",
"backend": "rakestrawhome",
"endpoint": "",
"model": "google/gemma-4-31b-it",
"temperature": 0,
"max_tokens": 0,
"top_p": 0,
"timeout_seconds": 0,
"service_tier": "",
"reasoning_effort": "",
"api_key_env": "",
"api_key_required": false,
"extra_params": null
}
]
}