280 lines
8.7 KiB
Go
280 lines
8.7 KiB
Go
package builtin
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
type catalogFixture struct {
|
|
Backends []backendFixture `json:"backends"`
|
|
Profiles []profileFixture `json:"profiles"`
|
|
}
|
|
|
|
type backendFixture struct {
|
|
ID string `json:"id"`
|
|
Endpoint string `json:"endpoint"`
|
|
APIKeyEnv string `json:"api_key_env"`
|
|
ExtraParams map[string]any `json:"extra_params"`
|
|
ConcurrencyLimit int `json:"concurrency_limit"`
|
|
QueueCapacity int `json:"queue_capacity"`
|
|
QueueCapacitySet bool `json:"queue_capacity_set"`
|
|
}
|
|
|
|
type profileFixture struct {
|
|
ID string `json:"id"`
|
|
BaseProfileID string `json:"base_profile"`
|
|
BackendID string `json:"backend"`
|
|
Endpoint string `json:"endpoint"`
|
|
Model string `json:"model"`
|
|
Temperature float64 `json:"temperature"`
|
|
MaxTokens int `json:"max_tokens"`
|
|
TopP float64 `json:"top_p"`
|
|
TimeoutSeconds int `json:"timeout_seconds"`
|
|
ServiceTier string `json:"service_tier"`
|
|
ReasoningEffort string `json:"reasoning_effort"`
|
|
APIKeyEnv string `json:"api_key_env"`
|
|
APIKeyRequired bool `json:"api_key_required"`
|
|
ExtraParams map[string]any `json:"extra_params"`
|
|
}
|
|
|
|
func TestBuiltInCatalogMatchesCompatibilityFixture(t *testing.T) {
|
|
fixture := loadCatalogFixture(t)
|
|
assertCatalogFixtureOrdering(t, fixture)
|
|
|
|
registry, err := backend.NewRegistry(nil)
|
|
if err != nil {
|
|
t.Fatalf("construct built-in backend registry: %v", err)
|
|
}
|
|
actualBackends := make(map[string]backendFixture)
|
|
for id := range registry.CapacityPolicies() {
|
|
definition, err := registry.GetBackend(id)
|
|
if err != nil {
|
|
t.Fatalf("load built-in backend %q: %v", id, err)
|
|
}
|
|
actualBackends[id] = backendFixtureFromDefinition(definition)
|
|
}
|
|
expectedBackends := make(map[string]backendFixture, len(fixture.Backends))
|
|
for _, expected := range fixture.Backends {
|
|
expectedBackends[expected.ID] = expected
|
|
}
|
|
if !mapsEqual(actualBackends, expectedBackends) {
|
|
t.Fatalf("built-in backends differ from compatibility fixture: got %#v, want %#v", actualBackends, expectedBackends)
|
|
}
|
|
|
|
profilePaths := loadBuiltInProfileIDs(t)
|
|
actualProfiles := make(map[string]profileFixture, len(profilePaths))
|
|
for id := range profilePaths {
|
|
definition, err := NewRepository().GetProfile(context.Background(), id)
|
|
if err != nil {
|
|
t.Fatalf("load built-in profile %q: %v", id, err)
|
|
}
|
|
actualProfiles[id] = profileFixtureFromDefinition(*definition)
|
|
}
|
|
expectedProfiles := make(map[string]profileFixture, len(fixture.Profiles))
|
|
for _, expected := range fixture.Profiles {
|
|
expectedProfiles[expected.ID] = expected
|
|
}
|
|
if !mapsEqual(actualProfiles, expectedProfiles) {
|
|
t.Fatalf("built-in profiles differ from compatibility fixture: got %#v, want %#v", actualProfiles, expectedProfiles)
|
|
}
|
|
}
|
|
|
|
func loadCatalogFixture(t *testing.T) catalogFixture {
|
|
t.Helper()
|
|
|
|
path := filepath.Join("..", "..", "..", "testdata", "builtin-catalog-v1.json")
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatalf("read compatibility fixture: %v", err)
|
|
}
|
|
|
|
decoder := json.NewDecoder(strings.NewReader(string(data)))
|
|
decoder.DisallowUnknownFields()
|
|
var fixture catalogFixture
|
|
if err := decoder.Decode(&fixture); err != nil {
|
|
t.Fatalf("decode compatibility fixture: %v", err)
|
|
}
|
|
if err := ensureJSONEnd(decoder); err != nil {
|
|
t.Fatalf("decode compatibility fixture: %v", err)
|
|
}
|
|
return fixture
|
|
}
|
|
|
|
func ensureJSONEnd(decoder *json.Decoder) error {
|
|
var trailing any
|
|
if err := decoder.Decode(&trailing); errors.Is(err, io.EOF) {
|
|
return nil
|
|
} else if err != nil {
|
|
return err
|
|
}
|
|
return errors.New("fixture must contain exactly one JSON value")
|
|
}
|
|
|
|
func assertCatalogFixtureOrdering(t *testing.T, fixture catalogFixture) {
|
|
t.Helper()
|
|
for index := 1; index < len(fixture.Backends); index++ {
|
|
if fixture.Backends[index-1].ID >= fixture.Backends[index].ID {
|
|
t.Fatalf("compatibility fixture backends are not sorted by ID")
|
|
}
|
|
}
|
|
for index := 1; index < len(fixture.Profiles); index++ {
|
|
if fixture.Profiles[index-1].ID >= fixture.Profiles[index].ID {
|
|
t.Fatalf("compatibility fixture profiles are not sorted by ID")
|
|
}
|
|
}
|
|
}
|
|
|
|
func backendFixtureFromDefinition(definition domain.Backend) backendFixture {
|
|
return backendFixture{
|
|
ID: definition.ID,
|
|
Endpoint: definition.Endpoint,
|
|
APIKeyEnv: definition.APIKeyEnv,
|
|
ExtraParams: definition.ExtraParams,
|
|
ConcurrencyLimit: definition.ConcurrencyLimit,
|
|
QueueCapacity: definition.QueueCapacity,
|
|
QueueCapacitySet: definition.QueueCapacitySet,
|
|
}
|
|
}
|
|
|
|
func profileFixtureFromDefinition(definition domain.ExecutionProfile) profileFixture {
|
|
return profileFixture{
|
|
ID: definition.ID,
|
|
BaseProfileID: definition.BaseProfileID,
|
|
BackendID: definition.BackendID,
|
|
Endpoint: definition.Endpoint,
|
|
Model: definition.Model,
|
|
Temperature: definition.Temperature,
|
|
MaxTokens: definition.MaxTokens,
|
|
TopP: definition.TopP,
|
|
TimeoutSeconds: definition.TimeoutSeconds,
|
|
ServiceTier: definition.ServiceTier,
|
|
ReasoningEffort: definition.ReasoningEffort,
|
|
APIKeyEnv: definition.APIKeyEnv,
|
|
APIKeyRequired: definition.APIKeyRequired,
|
|
ExtraParams: definition.ExtraParams,
|
|
}
|
|
}
|
|
|
|
func mapsEqual[K comparable, V any](actual, expected map[K]V) bool {
|
|
return reflect.DeepEqual(actual, expected)
|
|
}
|
|
|
|
func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
|
|
repo := NewRepository()
|
|
ids := loadBuiltInProfileIDs(t)
|
|
if len(ids) == 0 {
|
|
t.Fatal("expected built-in profiles")
|
|
}
|
|
|
|
for id := range ids {
|
|
t.Run(id, func(t *testing.T) {
|
|
p, err := repo.GetProfile(context.Background(), id)
|
|
if err != nil {
|
|
t.Fatalf("expected built-in profile %q to load, got %v", id, err)
|
|
}
|
|
if p.ID != id {
|
|
t.Fatalf("expected profile id %q, got %q", id, p.ID)
|
|
}
|
|
if !builtInBackendIDs[p.BackendID] {
|
|
t.Fatalf("expected profile %q to select a maintained built-in, got %q", id, p.BackendID)
|
|
}
|
|
if p.Endpoint != "" || p.APIKeyEnv != "" {
|
|
t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestBuiltInProfilesDoNotContainDuplicateIDsOrRawAPIKeys(t *testing.T) {
|
|
loadBuiltInProfileIDs(t)
|
|
}
|
|
|
|
func TestRakestrawhomeGemmaProfileUsesNativeDefaults(t *testing.T) {
|
|
p, err := NewRepository().GetProfile(context.Background(), "rakestrawhome-gemma-4-31b")
|
|
if err != nil {
|
|
t.Fatalf("load Rakestrawhome Gemma profile: %v", err)
|
|
}
|
|
if p.ID != "rakestrawhome-gemma-4-31b" ||
|
|
p.BackendID != backend.RakestrawHomeID ||
|
|
p.Model != "google/gemma-4-31b-it" ||
|
|
p.Endpoint != "" ||
|
|
p.Temperature != 0 ||
|
|
p.MaxTokens != 0 ||
|
|
p.TopP != 0 ||
|
|
p.TimeoutSeconds != 0 ||
|
|
p.ServiceTier != "" ||
|
|
p.ReasoningEffort != "" ||
|
|
p.APIKeyEnv != "" ||
|
|
p.APIKeyRequired ||
|
|
p.ExtraParams != nil {
|
|
t.Fatalf("unexpected Rakestrawhome Gemma profile: %#v", p)
|
|
}
|
|
}
|
|
|
|
var builtInBackendIDs = map[string]bool{
|
|
backend.OpenRouterID: true,
|
|
backend.RakestrawHomeID: true,
|
|
}
|
|
|
|
func loadBuiltInProfileIDs(t *testing.T) map[string]string {
|
|
t.Helper()
|
|
|
|
ids := map[string]string{}
|
|
err := fs.WalkDir(assets, assetRoot, func(name string, d fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if d.IsDir() || !strings.HasSuffix(name, ".yml") {
|
|
return nil
|
|
}
|
|
|
|
data, err := assets.ReadFile(name)
|
|
if err != nil {
|
|
t.Fatalf("failed to read built-in profile %s: %v", name, err)
|
|
}
|
|
|
|
var raw map[string]any
|
|
if err := yaml.Unmarshal(data, &raw); err != nil {
|
|
t.Fatalf("failed to decode built-in profile %s: %v", name, err)
|
|
}
|
|
if _, ok := raw["api_key"]; ok {
|
|
t.Fatalf("built-in profile %s contains raw api_key", name)
|
|
}
|
|
backendID, ok := raw["backend"].(string)
|
|
if !ok || !builtInBackendIDs[backendID] {
|
|
t.Fatalf("built-in profile %s does not select a maintained built-in: %#v", name, raw["backend"])
|
|
}
|
|
if _, ok := raw["endpoint"]; ok {
|
|
t.Fatalf("built-in profile %s repeats endpoint", name)
|
|
}
|
|
if _, ok := raw["api_key_env"]; ok {
|
|
t.Fatalf("built-in profile %s repeats api_key_env", name)
|
|
}
|
|
id, ok := raw["id"].(string)
|
|
if !ok || strings.TrimSpace(id) == "" {
|
|
t.Fatalf("built-in profile %s has missing id", name)
|
|
}
|
|
if previous, ok := ids[id]; ok {
|
|
t.Fatalf("duplicate built-in profile id %q in %s and %s", id, previous, name)
|
|
}
|
|
ids[id] = name
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to walk built-in profiles: %v", err)
|
|
}
|
|
return ids
|
|
}
|