Measure startup configuration loading

startup_bench_test.go

@@ -0,0 +1,159 @@
+package configkit_test
+
+import (
+	"fmt"
+	"net/url"
+	"slices"
+	"strings"
+	"testing"
+
+	configkit "gitea.maximumdirect.net/eric/ConfigKit"
+)
+
+// These synthetic types retain the optional-field shape of a catalog consumer
+// without importing its adapter, template compiler, or runtime registries.
+type startupCatalog struct {
+	Profiles map[string]startupProfile           `yaml:"profiles"`
+	Prompts  map[string]map[string]startupPrompt `yaml:"prompts"`
+}
+
+type startupProfile struct {
+	Backend         *string        `yaml:"backend"`
+	Endpoint        *string        `yaml:"endpoint"`
+	Model           *string        `yaml:"model"`
+	Temperature     *float64       `yaml:"temperature"`
+	TopP            *float64       `yaml:"top_p"`
+	MaxTokens       *int           `yaml:"max_tokens"`
+	TimeoutSeconds  *int           `yaml:"timeout_seconds"`
+	ServiceTier     *string        `yaml:"service_tier"`
+	ReasoningEffort *string        `yaml:"reasoning_effort"`
+	APIKeyEnv       *string        `yaml:"api_key_env"`
+	APIKeyRequired  *bool          `yaml:"api_key_required"`
+	ExtraParams     map[string]any `yaml:"extra_params"`
+}
+
+type startupPrompt struct {
+	DefaultProfile *string          `yaml:"default_profile"`
+	Description    *string          `yaml:"description"`
+	SessionID      *string          `yaml:"session_id"`
+	Inputs         []startupInput   `yaml:"inputs"`
+	Messages       []startupMessage `yaml:"messages"`
+	Output         *startupOutput   `yaml:"output"`
+}
+
+type startupInput struct {
+	Name        string  `yaml:"name"`
+	Required    bool    `yaml:"required"`
+	ContentType *string `yaml:"content_type"`
+	Description *string `yaml:"description"`
+}
+
+type startupMessage struct {
+	Role        *string `yaml:"role"`
+	Content     *string `yaml:"content"`
+	ContentFile *string `yaml:"content_file"`
+}
+
+type startupOutput struct {
+	Format         *string `yaml:"format"`
+	ValidationMode *string `yaml:"validation_mode"`
+	SchemaPath     *string `yaml:"schema_path"`
+	RepairAttempts int     `yaml:"repair_attempts"`
+}
+
+type startupPresence struct{ explicit, null bool }
+
+var startupResult *configkit.Result[startupCatalog]
+
+func BenchmarkStartupConstruction(b *testing.B) {
+	for _, count := range []int{32, 128, 512} {
+		b.Run(fmt.Sprint(count), func(b *testing.B) {
+			profiles, prompts := make(map[string]any, count), make(map[string]any, count)
+			for index := range count {
+				profileID := fmt.Sprintf("profile-%03d", index)
+				promptID := fmt.Sprintf("prompt-%03d", index)
+				profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
+				prompts[promptID] = map[string]any{"1": map[string]any{
+					"default_profile": profileID,
+					"messages":        []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}},
+					"output":          map[string]any{"format": "text", "validation_mode": "none"},
+				}}
+			}
+			// Mapping captures the source once outside timing, matching the
+			// downstream benchmark's reusable input. Each load still owns its result.
+			main := configkit.Input{Source: configkit.Mapping("startup catalog", map[string]any{
+				"profiles": profiles, "prompts": prompts,
+			})}
+			var captured map[string]map[string]startupPresence
+			validated := false
+			options := configkit.LayeredOptions[startupCatalog]{Definition: configkit.Definition[startupCatalog]{
+				// Include a simple path-based classifier, as catalog consumers use
+				// this callback even when the representative input has no credentials.
+				Secret: func(path configkit.Path) bool {
+					return strings.HasSuffix(path.String(), "/extra_params/token")
+				},
+				Resolve: func(catalog *startupCatalog, resolution *configkit.Resolution) error {
+					captured = make(map[string]map[string]startupPresence, len(catalog.Profiles))
+					for id, profile := range catalog.Profiles {
+						fields := make(map[string]startupPresence, 12)
+						for _, field := range []string{
+							"backend", "endpoint", "model", "temperature", "top_p", "max_tokens",
+							"timeout_seconds", "service_tier", "reasoning_effort", "api_key_env",
+							"api_key_required", "extra_params",
+						} {
+							info, exists := resolution.InputInfo(configkit.Keys("profiles", id, field))
+							fields[field] = startupPresence{explicit: exists && info.Explicit, null: exists && info.Null}
+						}
+						captured[id] = fields
+						// The representative endpoints are already canonical, so this
+						// callback, like the consumer's, records no normalization change.
+						if _, err := url.Parse(*profile.Endpoint); err != nil {
+							return err
+						}
+					}
+					return nil
+				},
+				Validate: func(catalog *startupCatalog, validation *configkit.Validation) {
+					validated = true
+					ids := make([]string, 0, len(catalog.Profiles))
+					for id := range catalog.Profiles {
+						ids = append(ids, id)
+					}
+					slices.Sort(ids)
+					for _, id := range ids {
+						profile := catalog.Profiles[id]
+						if profile.Model == nil || strings.TrimSpace(*profile.Model) == "" {
+							validation.Error(configkit.Keys("profiles", id, "model"), "model is required")
+						}
+					}
+				},
+			}}
+			// Check the measured workload before timing, including numeric mapping
+			// keys, sequence paths, callback presence capture, and published metadata.
+			result, err := configkit.LoadLayered(main, options)
+			if err != nil {
+				b.Fatal(err)
+			}
+			if !validated || len(result.Config.Profiles) != count || len(result.Config.Prompts) != count || len(captured) != count {
+				b.Fatal("startup fixture did not exercise the expected lifecycle")
+			}
+			fields := captured["profile-000"]
+			if !fields["endpoint"].explicit || !fields["model"].explicit || fields["temperature"].explicit {
+				b.Fatal("startup fixture presence capture disagrees with input")
+			}
+			path := configkit.Keys("prompts", "prompt-000", "1", "messages").Index(0).Key("content")
+			if info, exists := result.Metadata().Lookup(path); !exists || !info.Explicit || info.Null {
+				b.Fatal("startup fixture message provenance is missing")
+			}
+			b.ReportAllocs()
+			for b.Loop() {
+				result, err := configkit.LoadLayered(main, options)
+				if err != nil {
+					b.Fatal(err)
+				}
+				startupResult = result
+			}
+			b.ReportMetric(float64(count*2), "definitions/op")
+		})
+	}
+}