promptkit/internal/configuration/benchmark_test.go

package configuration

import (
	"context"
	"fmt"
	"testing"
	"testing/fstest"

	"gitea.maximumdirect.net/eric/ConfigKit"
	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)

const benchmarkDefinitionCount = 128

var (
	constructionResult *loaded
	selectedPrompt     *domain.PromptDefinition
	selectedProfile    *domain.ExecutionProfile
)

type configurationShape struct {
	main             configkit.Input
	underlays        []configkit.Underlay
	inputEntries     int
	effectiveEntries int
	tiers            int
}

func benchmarkPrompt(profileID string) map[string]any {
	return map[string]any{
		"default_profile": profileID,
		"messages":        []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}},
		"output":          map[string]any{"format": "text", "validation_mode": "none"},
	}
}

func independentDefinitions(count int) configurationShape {
	profiles := make(map[string]any, count)
	prompts := make(map[string]any, count)
	for index := range count {
		profileID := fmt.Sprintf("profile-%03d", index)
		promptID := fmt.Sprintf("prompt-%03d", index)
		profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
		prompts[promptID] = map[string]any{"1": benchmarkPrompt(profileID)}
	}
	return configurationShape{
		main:             configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": profiles, "prompts": prompts})},
		inputEntries:     count * 2,
		effectiveEntries: count * 2,
		tiers:            1,
	}
}

func fileBackedDefinitions(count int) configurationShape {
	profiles := make(map[string]any, count)
	prompts := make(map[string]any, count)
	for index := range count {
		profileID := fmt.Sprintf("profile-%03d", index)
		promptID := fmt.Sprintf("prompt-%03d", index)
		profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
		prompt := benchmarkPrompt(profileID)
		prompt["messages"] = []any{map[string]any{"role": "user", "content_file": "body.md"}}
		prompts[promptID] = map[string]any{"1": prompt}
	}
	namespace := configkit.NewFilesystem("benchmark content", fstest.MapFS{
		"assets/body.md": &fstest.MapFile{Data: []byte("Hello {{.name}}")},
	})
	return configurationShape{
		main: configkit.Input{
			Source:     configkit.Mapping("file-backed definitions", map[string]any{"profiles": profiles, "prompts": prompts}),
			Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"},
		},
		inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1,
	}
}

func sharedBaseDefinitions(count int) configurationShape {
	profiles := make(map[string]any, count+1)
	profiles["base"] = map[string]any{
		"endpoint": "https://example.test/v1", "model": "shared-model",
		"extra_params": map[string]any{"provider": map[string]any{"mode": "shared", "limit": 4}},
	}
	for index := range count {
		profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": "base", "temperature": float64(index%10) / 10}
	}
	return configurationShape{
		main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{
			"profiles": profiles,
			"prompts":  map[string]any{"shared": map[string]any{"1": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}},
		})},
		inputEntries:     count + 2,
		effectiveEntries: count + 2,
		tiers:            1,
	}
}

func layeredDefinitions(count int) configurationShape {
	base := make(map[string]any, count)
	operator := make(map[string]any, count)
	programmatic := make(map[string]any, count)
	final := make(map[string]any, count)
	for index := range count {
		id := fmt.Sprintf("profile-%03d", index)
		base[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"}
		operator[id] = map[string]any{"model": "operator-model"}
		programmatic[id] = map[string]any{"extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}}
		final[id] = map[string]any{"temperature": float64(index%10) / 10}
	}
	return configurationShape{
		underlays: []configkit.Underlay{
			{Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": base})}}},
			{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": operator})}}},
			{Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": programmatic})}}},
		},
		main: configkit.Input{Source: configkit.Mapping("main", map[string]any{
			"profiles": final,
			"prompts":  map[string]any{"layered": map[string]any{"1": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}},
		})},
		inputEntries:     count*4 + 1,
		effectiveEntries: count + 1,
		tiers:            4,
	}
}

func BenchmarkConfigurationConstruction(b *testing.B) {
	for _, scenario := range []struct {
		name  string
		shape configurationShape
	}{
		{name: "independent", shape: independentDefinitions(benchmarkDefinitionCount)},
		{name: "file_backed", shape: fileBackedDefinitions(64)},
		{name: "shared_base", shape: sharedBaseDefinitions(benchmarkDefinitionCount)},
		{name: "layered", shape: layeredDefinitions(benchmarkDefinitionCount)},
	} {
		b.Run(scenario.name, func(b *testing.B) {
			b.ReportAllocs()
			for b.Loop() {
				result, err := load(scenario.shape.main, scenario.shape.underlays)
				if err != nil {
					b.Fatal(err)
				}
				constructionResult = result
			}
			b.ReportMetric(float64(scenario.shape.inputEntries), "input-entries/op")
			b.ReportMetric(float64(scenario.shape.effectiveEntries), "effective-entries/op")
			b.ReportMetric(float64(scenario.shape.tiers), "tiers/op")
		})
	}
}

func BenchmarkConfigurationSelection(b *testing.B) {
	shape := independentDefinitions(benchmarkDefinitionCount)
	result, err := load(shape.main, nil)
	if err != nil {
		b.Fatal(err)
	}
	b.Run("prompt", func(b *testing.B) {
		b.ReportAllocs()
		for b.Loop() {
			definition, _, err := result.prompt("prompt-127", "1")
			if err != nil {
				b.Fatal(err)
			}
			selectedPrompt = definition
		}
		b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
	})
	b.Run("profile", func(b *testing.B) {
		b.ReportAllocs()
		for b.Loop() {
			profile, err := result.GetProfile(context.Background(), "profile-127")
			if err != nil {
				b.Fatal(err)
			}
			selectedProfile = profile
		}
		b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
	})
}