Freeze the built-in catalog compatibility baseline

docs/roadmap/implementation.md

@@ -140,7 +140,7 @@ Promptkit's private adapter:
 
 ## Stage 1: Freeze The Pre-Extraction Compatibility Baseline
 
-**Status:** Pending
+**Status:** Complete
 
 ### Repository
 

internal/profile/builtin/repository_test.go

@@ -2,14 +2,175 @@ package builtin
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
+	"io"
 	"io/fs"
+	"os"
+	"path/filepath"
+	"reflect"
 	"strings"
 	"testing"
 
 	"gitea.maximumdirect.net/eric/promptkit/internal/backend"
+	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
 	"gopkg.in/yaml.v3"
 )
 
+type catalogFixture struct {
+	Backends []backendFixture `json:"backends"`
+	Profiles []profileFixture `json:"profiles"`
+}
+
+type backendFixture struct {
+	ID               string         `json:"id"`
+	Endpoint         string         `json:"endpoint"`
+	APIKeyEnv        string         `json:"api_key_env"`
+	ExtraParams      map[string]any `json:"extra_params"`
+	ConcurrencyLimit int            `json:"concurrency_limit"`
+	QueueCapacity    int            `json:"queue_capacity"`
+	QueueCapacitySet bool           `json:"queue_capacity_set"`
+}
+
+type profileFixture struct {
+	ID              string         `json:"id"`
+	BaseProfileID   string         `json:"base_profile"`
+	BackendID       string         `json:"backend"`
+	Endpoint        string         `json:"endpoint"`
+	Model           string         `json:"model"`
+	Temperature     float64        `json:"temperature"`
+	MaxTokens       int            `json:"max_tokens"`
+	TopP            float64        `json:"top_p"`
+	TimeoutSeconds  int            `json:"timeout_seconds"`
+	ServiceTier     string         `json:"service_tier"`
+	ReasoningEffort string         `json:"reasoning_effort"`
+	APIKeyEnv       string         `json:"api_key_env"`
+	APIKeyRequired  bool           `json:"api_key_required"`
+	ExtraParams     map[string]any `json:"extra_params"`
+}
+
+func TestBuiltInCatalogMatchesCompatibilityFixture(t *testing.T) {
+	fixture := loadCatalogFixture(t)
+	assertCatalogFixtureOrdering(t, fixture)
+
+	registry, err := backend.NewRegistry(nil)
+	if err != nil {
+		t.Fatalf("construct built-in backend registry: %v", err)
+	}
+	actualBackends := make(map[string]backendFixture)
+	for id := range registry.CapacityPolicies() {
+		definition, err := registry.GetBackend(id)
+		if err != nil {
+			t.Fatalf("load built-in backend %q: %v", id, err)
+		}
+		actualBackends[id] = backendFixtureFromDefinition(definition)
+	}
+	expectedBackends := make(map[string]backendFixture, len(fixture.Backends))
+	for _, expected := range fixture.Backends {
+		expectedBackends[expected.ID] = expected
+	}
+	if !mapsEqual(actualBackends, expectedBackends) {
+		t.Fatalf("built-in backends differ from compatibility fixture: got %#v, want %#v", actualBackends, expectedBackends)
+	}
+
+	profilePaths := loadBuiltInProfileIDs(t)
+	actualProfiles := make(map[string]profileFixture, len(profilePaths))
+	for id := range profilePaths {
+		definition, err := NewRepository().GetProfile(context.Background(), id)
+		if err != nil {
+			t.Fatalf("load built-in profile %q: %v", id, err)
+		}
+		actualProfiles[id] = profileFixtureFromDefinition(*definition)
+	}
+	expectedProfiles := make(map[string]profileFixture, len(fixture.Profiles))
+	for _, expected := range fixture.Profiles {
+		expectedProfiles[expected.ID] = expected
+	}
+	if !mapsEqual(actualProfiles, expectedProfiles) {
+		t.Fatalf("built-in profiles differ from compatibility fixture: got %#v, want %#v", actualProfiles, expectedProfiles)
+	}
+}
+
+func loadCatalogFixture(t *testing.T) catalogFixture {
+	t.Helper()
+
+	path := filepath.Join("..", "..", "..", "testdata", "builtin-catalog-v1.json")
+	data, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read compatibility fixture: %v", err)
+	}
+
+	decoder := json.NewDecoder(strings.NewReader(string(data)))
+	decoder.DisallowUnknownFields()
+	var fixture catalogFixture
+	if err := decoder.Decode(&fixture); err != nil {
+		t.Fatalf("decode compatibility fixture: %v", err)
+	}
+	if err := ensureJSONEnd(decoder); err != nil {
+		t.Fatalf("decode compatibility fixture: %v", err)
+	}
+	return fixture
+}
+
+func ensureJSONEnd(decoder *json.Decoder) error {
+	var trailing any
+	if err := decoder.Decode(&trailing); errors.Is(err, io.EOF) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+	return errors.New("fixture must contain exactly one JSON value")
+}
+
+func assertCatalogFixtureOrdering(t *testing.T, fixture catalogFixture) {
+	t.Helper()
+	for index := 1; index < len(fixture.Backends); index++ {
+		if fixture.Backends[index-1].ID >= fixture.Backends[index].ID {
+			t.Fatalf("compatibility fixture backends are not sorted by ID")
+		}
+	}
+	for index := 1; index < len(fixture.Profiles); index++ {
+		if fixture.Profiles[index-1].ID >= fixture.Profiles[index].ID {
+			t.Fatalf("compatibility fixture profiles are not sorted by ID")
+		}
+	}
+}
+
+func backendFixtureFromDefinition(definition domain.Backend) backendFixture {
+	return backendFixture{
+		ID:               definition.ID,
+		Endpoint:         definition.Endpoint,
+		APIKeyEnv:        definition.APIKeyEnv,
+		ExtraParams:      definition.ExtraParams,
+		ConcurrencyLimit: definition.ConcurrencyLimit,
+		QueueCapacity:    definition.QueueCapacity,
+		QueueCapacitySet: definition.QueueCapacitySet,
+	}
+}
+
+func profileFixtureFromDefinition(definition domain.ExecutionProfile) profileFixture {
+	return profileFixture{
+		ID:              definition.ID,
+		BaseProfileID:   definition.BaseProfileID,
+		BackendID:       definition.BackendID,
+		Endpoint:        definition.Endpoint,
+		Model:           definition.Model,
+		Temperature:     definition.Temperature,
+		MaxTokens:       definition.MaxTokens,
+		TopP:            definition.TopP,
+		TimeoutSeconds:  definition.TimeoutSeconds,
+		ServiceTier:     definition.ServiceTier,
+		ReasoningEffort: definition.ReasoningEffort,
+		APIKeyEnv:       definition.APIKeyEnv,
+		APIKeyRequired:  definition.APIKeyRequired,
+		ExtraParams:     definition.ExtraParams,
+	}
+}
+
+func mapsEqual[K comparable, V any](actual, expected map[K]V) bool {
+	return reflect.DeepEqual(actual, expected)
+}
+
 func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
 	repo := NewRepository()
 	ids := loadBuiltInProfileIDs(t)

testdata/builtin-catalog-v1.json

@@ -0,0 +1,424 @@
+{
+    "backends": [
+        {
+            "id": "openrouter",
+            "endpoint": "https://openrouter.ai/api/v1",
+            "api_key_env": "OPENROUTER_API_KEY",
+            "extra_params": null,
+            "concurrency_limit": 16,
+            "queue_capacity": 1024,
+            "queue_capacity_set": true
+        },
+        {
+            "id": "rakestrawhome",
+            "endpoint": "https://inference.ai.rakestrawhome.com/v1",
+            "api_key_env": "RAKESTRAWHOME_INFERENCE_API_KEY",
+            "extra_params": null,
+            "concurrency_limit": 4,
+            "queue_capacity": 1024,
+            "queue_capacity_set": true
+        }
+    ],
+    "profiles": [
+        {
+            "id": "aion-2",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "aion-labs/aion-2.0",
+            "temperature": 0.72,
+            "max_tokens": 0,
+            "top_p": 0.95,
+            "timeout_seconds": 180,
+            "service_tier": "flex",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "claude-fable-latest",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "~anthropic/claude-fable-latest",
+            "temperature": 0,
+            "max_tokens": 0,
+            "top_p": 0,
+            "timeout_seconds": 600,
+            "service_tier": "flex",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "claude-haiku-latest",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "~anthropic/claude-haiku-latest",
+            "temperature": 0,
+            "max_tokens": 0,
+            "top_p": 0,
+            "timeout_seconds": 240,
+            "service_tier": "flex",
+            "reasoning_effort": "medium",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "claude-opus-latest",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "~anthropic/claude-opus-latest",
+            "temperature": 0,
+            "max_tokens": 0,
+            "top_p": 0,
+            "timeout_seconds": 240,
+            "service_tier": "flex",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "claude-sonnet-latest",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "~anthropic/claude-sonnet-latest",
+            "temperature": 0,
+            "max_tokens": 0,
+            "top_p": 0,
+            "timeout_seconds": 240,
+            "service_tier": "flex",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "deepseek-3-2",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "deepseek/deepseek-v3.2",
+            "temperature": 0,
+            "max_tokens": 0,
+            "top_p": 0,
+            "timeout_seconds": 180,
+            "service_tier": "flex",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "deepseek-4-flash",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "deepseek/deepseek-v4-flash",
+            "temperature": 0,
+            "max_tokens": 0,
+            "top_p": 0,
+            "timeout_seconds": 180,
+            "service_tier": "flex",
+            "reasoning_effort": "",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "deepseek-4-pro",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "deepseek/deepseek-v4-pro",
+            "temperature": 0,
+            "max_tokens": 0,
+            "top_p": 0,
+            "timeout_seconds": 180,
+            "service_tier": "flex",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "gemini-2-flash",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "google/gemini-2.5-flash",
+            "temperature": 0,
+            "max_tokens": 0,
+            "top_p": 0,
+            "timeout_seconds": 240,
+            "service_tier": "flex",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "gemini-2-flash-lite",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "google/gemini-2.5-flash-lite",
+            "temperature": 0,
+            "max_tokens": 0,
+            "top_p": 0,
+            "timeout_seconds": 240,
+            "service_tier": "flex",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "gemini-2-pro",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "google/gemini-2.5-pro",
+            "temperature": 0,
+            "max_tokens": 0,
+            "top_p": 0,
+            "timeout_seconds": 240,
+            "service_tier": "flex",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "gemini-3-flash-lite",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "google/gemini-3.1-flash-lite",
+            "temperature": 0,
+            "max_tokens": 0,
+            "top_p": 0,
+            "timeout_seconds": 240,
+            "service_tier": "flex",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "gemini-flash-latest",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "~google/gemini-flash-latest",
+            "temperature": 0,
+            "max_tokens": 0,
+            "top_p": 0,
+            "timeout_seconds": 240,
+            "service_tier": "flex",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "gemini-pro-latest",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "~google/gemini-pro-latest",
+            "temperature": 0,
+            "max_tokens": 0,
+            "top_p": 0,
+            "timeout_seconds": 240,
+            "service_tier": "flex",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "gemma-4-31b",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "google/gemma-4-31b-it:exacto",
+            "temperature": 0.15,
+            "max_tokens": 0,
+            "top_p": 0.98,
+            "timeout_seconds": 240,
+            "service_tier": "flex",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "gpt-5-mini",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "openai/gpt-5.4-mini",
+            "temperature": 0,
+            "max_tokens": 0,
+            "top_p": 0,
+            "timeout_seconds": 240,
+            "service_tier": "flex",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "gpt-5-nano",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "openai/gpt-5.4-nano",
+            "temperature": 0,
+            "max_tokens": 0,
+            "top_p": 0,
+            "timeout_seconds": 240,
+            "service_tier": "flex",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "minimax-m2",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "minimax/minimax-m2.5",
+            "temperature": 0.5,
+            "max_tokens": 0,
+            "top_p": 0.95,
+            "timeout_seconds": 180,
+            "service_tier": "flex",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "minimax-m3",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "minimax/minimax-m3",
+            "temperature": 0,
+            "max_tokens": 0,
+            "top_p": 0,
+            "timeout_seconds": 180,
+            "service_tier": "flex",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "mistral-large-2512",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "mistralai/mistral-large-2512",
+            "temperature": 0.15,
+            "max_tokens": 0,
+            "top_p": 0.98,
+            "timeout_seconds": 180,
+            "service_tier": "",
+            "reasoning_effort": "",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "mistral-medium-3-5",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "mistralai/mistral-medium-3-5",
+            "temperature": 0.15,
+            "max_tokens": 0,
+            "top_p": 0.98,
+            "timeout_seconds": 180,
+            "service_tier": "",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "mistral-small-3",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "mistralai/mistral-small-3.2-24b-instruct",
+            "temperature": 0.05,
+            "max_tokens": 0,
+            "top_p": 1,
+            "timeout_seconds": 180,
+            "service_tier": "",
+            "reasoning_effort": "",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "mistral-small-4",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "mistralai/mistral-small-2603",
+            "temperature": 0.1,
+            "max_tokens": 0,
+            "top_p": 0.98,
+            "timeout_seconds": 180,
+            "service_tier": "",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "nemotron-3-ultra",
+            "base_profile": "",
+            "backend": "openrouter",
+            "endpoint": "",
+            "model": "nvidia/nemotron-3-ultra-550b-a55b",
+            "temperature": 0,
+            "max_tokens": 0,
+            "top_p": 0,
+            "timeout_seconds": 180,
+            "service_tier": "flex",
+            "reasoning_effort": "high",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        },
+        {
+            "id": "rakestrawhome-gemma-4-31b",
+            "base_profile": "",
+            "backend": "rakestrawhome",
+            "endpoint": "",
+            "model": "google/gemma-4-31b-it",
+            "temperature": 0,
+            "max_tokens": 0,
+            "top_p": 0,
+            "timeout_seconds": 0,
+            "service_tier": "",
+            "reasoning_effort": "",
+            "api_key_env": "",
+            "api_key_required": false,
+            "extra_params": null
+        }
+    ]
+}