ConfigKit/startup_bench_test.go

package configkit_test

import (
	"fmt"
	"net/url"
	"slices"
	"strings"
	"testing"

	configkit "gitea.maximumdirect.net/eric/ConfigKit"
)

// These synthetic types retain the optional-field shape of a catalog consumer
// without importing its adapter, template compiler, or runtime registries.
type startupCatalog struct {
	Profiles map[string]startupProfile           `yaml:"profiles"`
	Prompts  map[string]map[string]startupPrompt `yaml:"prompts"`
}

type startupProfile struct {
	Backend         *string        `yaml:"backend"`
	Endpoint        *string        `yaml:"endpoint"`
	Model           *string        `yaml:"model"`
	Temperature     *float64       `yaml:"temperature"`
	TopP            *float64       `yaml:"top_p"`
	MaxTokens       *int           `yaml:"max_tokens"`
	TimeoutSeconds  *int           `yaml:"timeout_seconds"`
	ServiceTier     *string        `yaml:"service_tier"`
	ReasoningEffort *string        `yaml:"reasoning_effort"`
	APIKeyEnv       *string        `yaml:"api_key_env"`
	APIKeyRequired  *bool          `yaml:"api_key_required"`
	ExtraParams     map[string]any `yaml:"extra_params"`
}

type startupPrompt struct {
	DefaultProfile *string          `yaml:"default_profile"`
	Description    *string          `yaml:"description"`
	SessionID      *string          `yaml:"session_id"`
	Inputs         []startupInput   `yaml:"inputs"`
	Messages       []startupMessage `yaml:"messages"`
	Output         *startupOutput   `yaml:"output"`
}

type startupInput struct {
	Name        string  `yaml:"name"`
	Required    bool    `yaml:"required"`
	ContentType *string `yaml:"content_type"`
	Description *string `yaml:"description"`
}

type startupMessage struct {
	Role        *string `yaml:"role"`
	Content     *string `yaml:"content"`
	ContentFile *string `yaml:"content_file"`
}

type startupOutput struct {
	Format         *string `yaml:"format"`
	ValidationMode *string `yaml:"validation_mode"`
	SchemaPath     *string `yaml:"schema_path"`
	RepairAttempts int     `yaml:"repair_attempts"`
}

type startupPresence struct{ explicit, null bool }

var startupResult *configkit.Result[startupCatalog]

func BenchmarkStartupConstruction(b *testing.B) {
	for _, count := range []int{32, 128, 512} {
		b.Run(fmt.Sprint(count), func(b *testing.B) {
			profiles, prompts := make(map[string]any, count), make(map[string]any, count)
			for index := range count {
				profileID := fmt.Sprintf("profile-%03d", index)
				promptID := fmt.Sprintf("prompt-%03d", index)
				profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
				prompts[promptID] = map[string]any{"1": map[string]any{
					"default_profile": profileID,
					"messages":        []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}},
					"output":          map[string]any{"format": "text", "validation_mode": "none"},
				}}
			}
			// Mapping captures the source once outside timing, matching the
			// downstream benchmark's reusable input. Each load still owns its result.
			main := configkit.Input{Source: configkit.Mapping("startup catalog", map[string]any{
				"profiles": profiles, "prompts": prompts,
			})}
			var captured map[string]map[string]startupPresence
			validated := false
			options := configkit.LayeredOptions[startupCatalog]{Definition: configkit.Definition[startupCatalog]{
				// Include a simple path-based classifier, as catalog consumers use
				// this callback even when the representative input has no credentials.
				Secret: func(path configkit.Path) bool {
					return strings.HasSuffix(path.String(), "/extra_params/token")
				},
				Resolve: func(catalog *startupCatalog, resolution *configkit.Resolution) error {
					captured = make(map[string]map[string]startupPresence, len(catalog.Profiles))
					for id, profile := range catalog.Profiles {
						fields := make(map[string]startupPresence, 12)
						for _, field := range []string{
							"backend", "endpoint", "model", "temperature", "top_p", "max_tokens",
							"timeout_seconds", "service_tier", "reasoning_effort", "api_key_env",
							"api_key_required", "extra_params",
						} {
							info, exists := resolution.InputInfo(configkit.Keys("profiles", id, field))
							fields[field] = startupPresence{explicit: exists && info.Explicit, null: exists && info.Null}
						}
						captured[id] = fields
						// The representative endpoints are already canonical, so this
						// callback, like the consumer's, records no normalization change.
						if _, err := url.Parse(*profile.Endpoint); err != nil {
							return err
						}
					}
					return nil
				},
				Validate: func(catalog *startupCatalog, validation *configkit.Validation) {
					validated = true
					ids := make([]string, 0, len(catalog.Profiles))
					for id := range catalog.Profiles {
						ids = append(ids, id)
					}
					slices.Sort(ids)
					for _, id := range ids {
						profile := catalog.Profiles[id]
						if profile.Model == nil || strings.TrimSpace(*profile.Model) == "" {
							validation.Error(configkit.Keys("profiles", id, "model"), "model is required")
						}
					}
				},
			}}
			// Check the measured workload before timing, including numeric mapping
			// keys, sequence paths, callback presence capture, and published metadata.
			result, err := configkit.LoadLayered(main, options)
			if err != nil {
				b.Fatal(err)
			}
			if !validated || len(result.Config.Profiles) != count || len(result.Config.Prompts) != count || len(captured) != count {
				b.Fatal("startup fixture did not exercise the expected lifecycle")
			}
			fields := captured["profile-000"]
			if !fields["endpoint"].explicit || !fields["model"].explicit || fields["temperature"].explicit {
				b.Fatal("startup fixture presence capture disagrees with input")
			}
			path := configkit.Keys("prompts", "prompt-000", "1", "messages").Index(0).Key("content")
			if info, exists := result.Metadata().Lookup(path); !exists || !info.Explicit || info.Null {
				b.Fatal("startup fixture message provenance is missing")
			}
			b.ReportAllocs()
			for b.Loop() {
				result, err := configkit.LoadLayered(main, options)
				if err != nil {
					b.Fatal(err)
				}
				startupResult = result
			}
			b.ReportMetric(float64(count*2), "definitions/op")
		})
	}
}