Use external catalogs for maintained backends
This commit is contained in:
@@ -1,8 +0,0 @@
|
||||
id: aion-2
|
||||
backend: openrouter
|
||||
model: aion-labs/aion-2.0
|
||||
temperature: 0.72
|
||||
reasoning_effort: high
|
||||
top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: claude-fable-latest
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-fable-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 600
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: claude-haiku-latest
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-haiku-latest"
|
||||
reasoning_effort: medium
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: claude-opus-latest
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-opus-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: claude-sonnet-latest
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-sonnet-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: deepseek-3-2
|
||||
backend: openrouter
|
||||
model: deepseek/deepseek-v3.2
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: deepseek-4-flash
|
||||
backend: openrouter
|
||||
model: deepseek/deepseek-v4-flash
|
||||
#reasoning_effort: medium
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: deepseek-4-pro
|
||||
backend: openrouter
|
||||
model: deepseek/deepseek-v4-pro
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemini-2-flash-lite
|
||||
backend: openrouter
|
||||
model: "google/gemini-2.5-flash-lite"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemini-2-flash
|
||||
backend: openrouter
|
||||
model: "google/gemini-2.5-flash"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemini-2-pro
|
||||
backend: openrouter
|
||||
model: "google/gemini-2.5-pro"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemini-3-flash-lite
|
||||
backend: openrouter
|
||||
model: "google/gemini-3.1-flash-lite"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemini-flash-latest
|
||||
backend: openrouter
|
||||
model: "~google/gemini-flash-latest"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemini-pro-latest
|
||||
backend: openrouter
|
||||
model: "~google/gemini-pro-latest"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: gemma-4-31b
|
||||
backend: openrouter
|
||||
model: google/gemma-4-31b-it:exacto
|
||||
temperature: 0.15
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,3 +0,0 @@
|
||||
id: rakestrawhome-gemma-4-31b
|
||||
backend: rakestrawhome
|
||||
model: google/gemma-4-31b-it
|
||||
@@ -1,8 +0,0 @@
|
||||
id: minimax-m2
|
||||
backend: openrouter
|
||||
model: minimax/minimax-m2.5
|
||||
temperature: 0.5
|
||||
reasoning_effort: high
|
||||
top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,8 +0,0 @@
|
||||
id: minimax-m3
|
||||
backend: openrouter
|
||||
model: minimax/minimax-m3
|
||||
#temperature: 0.5
|
||||
reasoning_effort: high
|
||||
#top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: mistral-large-2512
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-large-2512
|
||||
temperature: 0.15
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
@@ -1,7 +0,0 @@
|
||||
id: mistral-medium-3-5
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-medium-3-5
|
||||
temperature: 0.15
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
@@ -1,6 +0,0 @@
|
||||
id: mistral-small-3
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-small-3.2-24b-instruct
|
||||
temperature: 0.05
|
||||
top_p: 1.0
|
||||
timeout_seconds: 180
|
||||
@@ -1,7 +0,0 @@
|
||||
id: mistral-small-4
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-small-2603
|
||||
temperature: 0.1
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
@@ -1,6 +0,0 @@
|
||||
id: nemotron-3-ultra
|
||||
backend: openrouter
|
||||
model: nvidia/nemotron-3-ultra-550b-a55b
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: gpt-5-mini
|
||||
backend: openrouter
|
||||
model: "openai/gpt-5.4-mini"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,6 +0,0 @@
|
||||
id: gpt-5-nano
|
||||
backend: openrouter
|
||||
model: "openai/gpt-5.4-nano"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
@@ -1,16 +0,0 @@
|
||||
package builtin
|
||||
|
||||
import (
|
||||
"embed"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
|
||||
const assetRoot = "assets"
|
||||
|
||||
//go:embed assets/**/*.yml
|
||||
var assets embed.FS
|
||||
|
||||
func NewRepository() profile.Repository {
|
||||
return profile.NewFSRepository(assets, assetRoot)
|
||||
}
|
||||
@@ -1,279 +0,0 @@
|
||||
package builtin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type catalogFixture struct {
|
||||
Backends []backendFixture `json:"backends"`
|
||||
Profiles []profileFixture `json:"profiles"`
|
||||
}
|
||||
|
||||
type backendFixture struct {
|
||||
ID string `json:"id"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
APIKeyEnv string `json:"api_key_env"`
|
||||
ExtraParams map[string]any `json:"extra_params"`
|
||||
ConcurrencyLimit int `json:"concurrency_limit"`
|
||||
QueueCapacity int `json:"queue_capacity"`
|
||||
QueueCapacitySet bool `json:"queue_capacity_set"`
|
||||
}
|
||||
|
||||
type profileFixture struct {
|
||||
ID string `json:"id"`
|
||||
BaseProfileID string `json:"base_profile"`
|
||||
BackendID string `json:"backend"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Model string `json:"model"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
TopP float64 `json:"top_p"`
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
ServiceTier string `json:"service_tier"`
|
||||
ReasoningEffort string `json:"reasoning_effort"`
|
||||
APIKeyEnv string `json:"api_key_env"`
|
||||
APIKeyRequired bool `json:"api_key_required"`
|
||||
ExtraParams map[string]any `json:"extra_params"`
|
||||
}
|
||||
|
||||
func TestBuiltInCatalogMatchesCompatibilityFixture(t *testing.T) {
|
||||
fixture := loadCatalogFixture(t)
|
||||
assertCatalogFixtureOrdering(t, fixture)
|
||||
|
||||
registry, err := backend.NewRegistry(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("construct built-in backend registry: %v", err)
|
||||
}
|
||||
actualBackends := make(map[string]backendFixture)
|
||||
for id := range registry.CapacityPolicies() {
|
||||
definition, err := registry.GetBackend(id)
|
||||
if err != nil {
|
||||
t.Fatalf("load built-in backend %q: %v", id, err)
|
||||
}
|
||||
actualBackends[id] = backendFixtureFromDefinition(definition)
|
||||
}
|
||||
expectedBackends := make(map[string]backendFixture, len(fixture.Backends))
|
||||
for _, expected := range fixture.Backends {
|
||||
expectedBackends[expected.ID] = expected
|
||||
}
|
||||
if !mapsEqual(actualBackends, expectedBackends) {
|
||||
t.Fatalf("built-in backends differ from compatibility fixture: got %#v, want %#v", actualBackends, expectedBackends)
|
||||
}
|
||||
|
||||
profilePaths := loadBuiltInProfileIDs(t)
|
||||
actualProfiles := make(map[string]profileFixture, len(profilePaths))
|
||||
for id := range profilePaths {
|
||||
definition, err := NewRepository().GetProfile(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("load built-in profile %q: %v", id, err)
|
||||
}
|
||||
actualProfiles[id] = profileFixtureFromDefinition(*definition)
|
||||
}
|
||||
expectedProfiles := make(map[string]profileFixture, len(fixture.Profiles))
|
||||
for _, expected := range fixture.Profiles {
|
||||
expectedProfiles[expected.ID] = expected
|
||||
}
|
||||
if !mapsEqual(actualProfiles, expectedProfiles) {
|
||||
t.Fatalf("built-in profiles differ from compatibility fixture: got %#v, want %#v", actualProfiles, expectedProfiles)
|
||||
}
|
||||
}
|
||||
|
||||
func loadCatalogFixture(t *testing.T) catalogFixture {
|
||||
t.Helper()
|
||||
|
||||
path := filepath.Join("..", "..", "..", "testdata", "builtin-catalog-v1.json")
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read compatibility fixture: %v", err)
|
||||
}
|
||||
|
||||
decoder := json.NewDecoder(strings.NewReader(string(data)))
|
||||
decoder.DisallowUnknownFields()
|
||||
var fixture catalogFixture
|
||||
if err := decoder.Decode(&fixture); err != nil {
|
||||
t.Fatalf("decode compatibility fixture: %v", err)
|
||||
}
|
||||
if err := ensureJSONEnd(decoder); err != nil {
|
||||
t.Fatalf("decode compatibility fixture: %v", err)
|
||||
}
|
||||
return fixture
|
||||
}
|
||||
|
||||
func ensureJSONEnd(decoder *json.Decoder) error {
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); errors.Is(err, io.EOF) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
return errors.New("fixture must contain exactly one JSON value")
|
||||
}
|
||||
|
||||
func assertCatalogFixtureOrdering(t *testing.T, fixture catalogFixture) {
|
||||
t.Helper()
|
||||
for index := 1; index < len(fixture.Backends); index++ {
|
||||
if fixture.Backends[index-1].ID >= fixture.Backends[index].ID {
|
||||
t.Fatalf("compatibility fixture backends are not sorted by ID")
|
||||
}
|
||||
}
|
||||
for index := 1; index < len(fixture.Profiles); index++ {
|
||||
if fixture.Profiles[index-1].ID >= fixture.Profiles[index].ID {
|
||||
t.Fatalf("compatibility fixture profiles are not sorted by ID")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func backendFixtureFromDefinition(definition domain.Backend) backendFixture {
|
||||
return backendFixture{
|
||||
ID: definition.ID,
|
||||
Endpoint: definition.Endpoint,
|
||||
APIKeyEnv: definition.APIKeyEnv,
|
||||
ExtraParams: definition.ExtraParams,
|
||||
ConcurrencyLimit: definition.ConcurrencyLimit,
|
||||
QueueCapacity: definition.QueueCapacity,
|
||||
QueueCapacitySet: definition.QueueCapacitySet,
|
||||
}
|
||||
}
|
||||
|
||||
func profileFixtureFromDefinition(definition domain.ExecutionProfile) profileFixture {
|
||||
return profileFixture{
|
||||
ID: definition.ID,
|
||||
BaseProfileID: definition.BaseProfileID,
|
||||
BackendID: definition.BackendID,
|
||||
Endpoint: definition.Endpoint,
|
||||
Model: definition.Model,
|
||||
Temperature: definition.Temperature,
|
||||
MaxTokens: definition.MaxTokens,
|
||||
TopP: definition.TopP,
|
||||
TimeoutSeconds: definition.TimeoutSeconds,
|
||||
ServiceTier: definition.ServiceTier,
|
||||
ReasoningEffort: definition.ReasoningEffort,
|
||||
APIKeyEnv: definition.APIKeyEnv,
|
||||
APIKeyRequired: definition.APIKeyRequired,
|
||||
ExtraParams: definition.ExtraParams,
|
||||
}
|
||||
}
|
||||
|
||||
func mapsEqual[K comparable, V any](actual, expected map[K]V) bool {
|
||||
return reflect.DeepEqual(actual, expected)
|
||||
}
|
||||
|
||||
func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
|
||||
repo := NewRepository()
|
||||
ids := loadBuiltInProfileIDs(t)
|
||||
if len(ids) == 0 {
|
||||
t.Fatal("expected built-in profiles")
|
||||
}
|
||||
|
||||
for id := range ids {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
p, err := repo.GetProfile(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("expected built-in profile %q to load, got %v", id, err)
|
||||
}
|
||||
if p.ID != id {
|
||||
t.Fatalf("expected profile id %q, got %q", id, p.ID)
|
||||
}
|
||||
if !builtInBackendIDs[p.BackendID] {
|
||||
t.Fatalf("expected profile %q to select a maintained built-in, got %q", id, p.BackendID)
|
||||
}
|
||||
if p.Endpoint != "" || p.APIKeyEnv != "" {
|
||||
t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltInProfilesDoNotContainDuplicateIDsOrRawAPIKeys(t *testing.T) {
|
||||
loadBuiltInProfileIDs(t)
|
||||
}
|
||||
|
||||
func TestRakestrawhomeGemmaProfileUsesNativeDefaults(t *testing.T) {
|
||||
p, err := NewRepository().GetProfile(context.Background(), "rakestrawhome-gemma-4-31b")
|
||||
if err != nil {
|
||||
t.Fatalf("load Rakestrawhome Gemma profile: %v", err)
|
||||
}
|
||||
if p.ID != "rakestrawhome-gemma-4-31b" ||
|
||||
p.BackendID != backend.RakestrawHomeID ||
|
||||
p.Model != "google/gemma-4-31b-it" ||
|
||||
p.Endpoint != "" ||
|
||||
p.Temperature != 0 ||
|
||||
p.MaxTokens != 0 ||
|
||||
p.TopP != 0 ||
|
||||
p.TimeoutSeconds != 0 ||
|
||||
p.ServiceTier != "" ||
|
||||
p.ReasoningEffort != "" ||
|
||||
p.APIKeyEnv != "" ||
|
||||
p.APIKeyRequired ||
|
||||
p.ExtraParams != nil {
|
||||
t.Fatalf("unexpected Rakestrawhome Gemma profile: %#v", p)
|
||||
}
|
||||
}
|
||||
|
||||
var builtInBackendIDs = map[string]bool{
|
||||
backend.OpenRouterID: true,
|
||||
backend.RakestrawHomeID: true,
|
||||
}
|
||||
|
||||
func loadBuiltInProfileIDs(t *testing.T) map[string]string {
|
||||
t.Helper()
|
||||
|
||||
ids := map[string]string{}
|
||||
err := fs.WalkDir(assets, assetRoot, func(name string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() || !strings.HasSuffix(name, ".yml") {
|
||||
return nil
|
||||
}
|
||||
|
||||
data, err := assets.ReadFile(name)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read built-in profile %s: %v", name, err)
|
||||
}
|
||||
|
||||
var raw map[string]any
|
||||
if err := yaml.Unmarshal(data, &raw); err != nil {
|
||||
t.Fatalf("failed to decode built-in profile %s: %v", name, err)
|
||||
}
|
||||
if _, ok := raw["api_key"]; ok {
|
||||
t.Fatalf("built-in profile %s contains raw api_key", name)
|
||||
}
|
||||
backendID, ok := raw["backend"].(string)
|
||||
if !ok || !builtInBackendIDs[backendID] {
|
||||
t.Fatalf("built-in profile %s does not select a maintained built-in: %#v", name, raw["backend"])
|
||||
}
|
||||
if _, ok := raw["endpoint"]; ok {
|
||||
t.Fatalf("built-in profile %s repeats endpoint", name)
|
||||
}
|
||||
if _, ok := raw["api_key_env"]; ok {
|
||||
t.Fatalf("built-in profile %s repeats api_key_env", name)
|
||||
}
|
||||
id, ok := raw["id"].(string)
|
||||
if !ok || strings.TrimSpace(id) == "" {
|
||||
t.Fatalf("built-in profile %s has missing id", name)
|
||||
}
|
||||
if previous, ok := ids[id]; ok {
|
||||
t.Fatalf("duplicate built-in profile id %q in %s and %s", id, previous, name)
|
||||
}
|
||||
ids[id] = name
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to walk built-in profiles: %v", err)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
Reference in New Issue
Block a user