package configuration import ( "context" "fmt" "testing" "testing/fstest" "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit/internal/domain" ) const benchmarkDefinitionCount = 128 var ( constructionResult *loaded selectedPrompt *domain.PromptDefinition selectedProfile *domain.ExecutionProfile ) type configurationShape struct { main configkit.Input underlays []configkit.Underlay inputEntries int effectiveEntries int tiers int } func benchmarkPrompt(profileID string) map[string]any { return map[string]any{ "default_profile": profileID, "messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}, } } func independentDefinitions(count int) configurationShape { profiles := make(map[string]any, count) prompts := make(map[string]any, count) for index := range count { profileID := fmt.Sprintf("profile-%03d", index) promptID := fmt.Sprintf("prompt-%03d", index) profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID} prompts[promptID] = map[string]any{"1": benchmarkPrompt(profileID)} } return configurationShape{ main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": profiles, "prompts": prompts})}, inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1, } } func fileBackedDefinitions(count int) configurationShape { profiles := make(map[string]any, count) prompts := make(map[string]any, count) for index := range count { profileID := fmt.Sprintf("profile-%03d", index) promptID := fmt.Sprintf("prompt-%03d", index) profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID} prompt := benchmarkPrompt(profileID) prompt["messages"] = []any{map[string]any{"role": "user", "content_file": "body.md"}} prompts[promptID] = map[string]any{"1": prompt} } namespace := configkit.NewFilesystem("benchmark content", fstest.MapFS{ "assets/body.md": &fstest.MapFile{Data: []byte("Hello {{.name}}")}, }) return configurationShape{ main: configkit.Input{ Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": profiles, "prompts": prompts}), Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"}, }, inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1, } } func sharedBaseDefinitions(count int) configurationShape { profiles := make(map[string]any, count+1) profiles["base"] = map[string]any{ "endpoint": "https://example.test/v1", "model": "shared-model", "extra_params": map[string]any{"provider": map[string]any{"mode": "shared", "limit": 4}}, } for index := range count { profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": "base", "temperature": float64(index%10) / 10} } return configurationShape{ main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{ "profiles": profiles, "prompts": map[string]any{"shared": map[string]any{"1": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}}, })}, inputEntries: count + 2, effectiveEntries: count + 2, tiers: 1, } } func layeredDefinitions(count int) configurationShape { base := make(map[string]any, count) operator := make(map[string]any, count) programmatic := make(map[string]any, count) final := make(map[string]any, count) for index := range count { id := fmt.Sprintf("profile-%03d", index) base[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"} operator[id] = map[string]any{"model": "operator-model"} programmatic[id] = map[string]any{"extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}} final[id] = map[string]any{"temperature": float64(index%10) / 10} } return configurationShape{ underlays: []configkit.Underlay{ {Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": base})}}}, {Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": operator})}}}, {Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": programmatic})}}}, }, main: configkit.Input{Source: configkit.Mapping("main", map[string]any{ "profiles": final, "prompts": map[string]any{"layered": map[string]any{"1": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}}, })}, inputEntries: count*4 + 1, effectiveEntries: count + 1, tiers: 4, } } func BenchmarkConfigurationConstruction(b *testing.B) { for _, scenario := range []struct { name string shape configurationShape }{ {name: "independent", shape: independentDefinitions(benchmarkDefinitionCount)}, {name: "file_backed", shape: fileBackedDefinitions(64)}, {name: "shared_base", shape: sharedBaseDefinitions(benchmarkDefinitionCount)}, {name: "layered", shape: layeredDefinitions(benchmarkDefinitionCount)}, } { b.Run(scenario.name, func(b *testing.B) { b.ReportAllocs() for b.Loop() { result, err := load(scenario.shape.main, scenario.shape.underlays) if err != nil { b.Fatal(err) } constructionResult = result } b.ReportMetric(float64(scenario.shape.inputEntries), "input-entries/op") b.ReportMetric(float64(scenario.shape.effectiveEntries), "effective-entries/op") b.ReportMetric(float64(scenario.shape.tiers), "tiers/op") }) } } func BenchmarkConfigurationSelection(b *testing.B) { shape := independentDefinitions(benchmarkDefinitionCount) result, err := load(shape.main, nil) if err != nil { b.Fatal(err) } b.Run("prompt", func(b *testing.B) { b.ReportAllocs() for b.Loop() { definition, _, err := result.prompt("prompt-127", "1") if err != nil { b.Fatal(err) } selectedPrompt = definition } b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op") }) b.Run("profile", func(b *testing.B) { b.ReportAllocs() for b.Loop() { profile, err := result.GetProfile(context.Background(), "profile-127") if err != nil { b.Fatal(err) } selectedProfile = profile } b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op") }) }