package configuration import ( "context" "fmt" "testing" "testing/fstest" "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit/internal/domain" ) const benchmarkDefinitionCount = 128 var ( constructionResult *loaded selectedPrompt *domain.PromptDefinition selectedPromptHash string selectedPromptInspection *domain.PromptInspection selectedProfile *domain.ExecutionProfile selectedProfileIdentity string ) type configurationShape struct { main configkit.Input underlays []configkit.Underlay inputEntries int effectiveEntries int tiers int } func benchmarkPrompt(profileID string) map[string]any { return map[string]any{ "version": "1", "default_profile": profileID, "messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}, } } func independentDefinitions(count int) configurationShape { profiles := make(map[string]any, count) prompts := make(map[string]any, count) for index := range count { profileID := fmt.Sprintf("profile-%03d", index) promptID := fmt.Sprintf("prompt-%03d", index) profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID} prompts[promptID] = benchmarkPrompt(profileID) } return configurationShape{ main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": profiles, "prompts": prompts})}, inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1, } } func fileBackedDefinitions(count int) configurationShape { profiles := make(map[string]any, count) prompts := make(map[string]any, count) for index := range count { profileID := fmt.Sprintf("profile-%03d", index) promptID := fmt.Sprintf("prompt-%03d", index) profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID} prompt := benchmarkPrompt(profileID) prompt["messages"] = []any{map[string]any{"role": "user", "content_file": "body.md"}} prompts[promptID] = prompt } namespace := configkit.NewFilesystem("benchmark content", fstest.MapFS{ "assets/body.md": &fstest.MapFile{Data: []byte("Hello {{.name}}")}, }) return configurationShape{ main: configkit.Input{ Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": profiles, "prompts": prompts}), Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"}, }, inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1, } } func sharedBaseDefinitions(count int) configurationShape { profiles := make(map[string]any, count+1) profiles["base"] = map[string]any{ "endpoint": "https://example.test/v1", "model": "shared-model", "extra_params": map[string]any{"provider": map[string]any{"mode": "shared", "limit": 4}}, } for index := range count { profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": "base", "temperature": float64(index%10) / 10} } return configurationShape{ main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{ "profiles": profiles, "prompts": map[string]any{"shared": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}, })}, inputEntries: count + 2, effectiveEntries: count + 2, tiers: 1, } } func layeredDefinitions(count int) configurationShape { base := make(map[string]any, count) operator := make(map[string]any, count) programmatic := make(map[string]any, count) final := make(map[string]any, count) for index := range count { id := fmt.Sprintf("profile-%03d", index) base[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"} operator[id] = map[string]any{"model": "operator-model"} programmatic[id] = map[string]any{"extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}} final[id] = map[string]any{"temperature": float64(index%10) / 10} } return configurationShape{ underlays: []configkit.Underlay{ {Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": base})}}}, {Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": operator})}}}, {Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": programmatic})}}}, }, main: configkit.Input{Source: configkit.Mapping("main", map[string]any{ "profiles": final, "prompts": map[string]any{"layered": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}, })}, inputEntries: count*4 + 1, effectiveEntries: count + 1, tiers: 4, } } func BenchmarkCollectionFileBackedConstruction(b *testing.B) { for _, count := range []int{4, 16, 64} { b.Run(fmt.Sprintf("collections_%d", count), func(b *testing.B) { collections := make(map[string]any, count) for index := range count { collections[fmt.Sprintf("collection-%03d", index)] = map[string]any{ "prompts": map[string]any{ "example": map[string]any{ "messages": []any{map[string]any{"role": "user", "content_file": "body.md"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}, }, }, } } namespace := configkit.NewFilesystem("collection benchmark", fstest.MapFS{ "body.md": {Data: []byte("Hello")}, }) main := configkit.Input{ Source: configkit.Mapping("collections", map[string]any{"collections": collections}), Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "."}, } b.ReportAllocs() for b.Loop() { result, err := load(main, nil) if err != nil { b.Fatal(err) } constructionResult = result } b.ReportMetric(float64(count), "collections/op") }) } } func BenchmarkConfigurationConstruction(b *testing.B) { for _, scenario := range []struct { name string shape configurationShape }{ {name: "independent", shape: independentDefinitions(benchmarkDefinitionCount)}, {name: "file_backed", shape: fileBackedDefinitions(64)}, {name: "shared_base", shape: sharedBaseDefinitions(benchmarkDefinitionCount)}, {name: "layered", shape: layeredDefinitions(benchmarkDefinitionCount)}, } { b.Run(scenario.name, func(b *testing.B) { b.ReportAllocs() for b.Loop() { result, err := load(scenario.shape.main, scenario.shape.underlays) if err != nil { b.Fatal(err) } constructionResult = result } b.ReportMetric(float64(scenario.shape.inputEntries), "input-entries/op") b.ReportMetric(float64(scenario.shape.effectiveEntries), "effective-entries/op") b.ReportMetric(float64(scenario.shape.tiers), "tiers/op") }) } } func BenchmarkConfigurationSelection(b *testing.B) { shape := independentDefinitions(benchmarkDefinitionCount) result, err := load(shape.main, nil) if err != nil { b.Fatal(err) } b.Run("prompt", func(b *testing.B) { b.ReportAllocs() for b.Loop() { definition, _, err := result.prompt("prompt-127") if err != nil { b.Fatal(err) } selectedPrompt = definition } b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op") }) b.Run("profile", func(b *testing.B) { b.ReportAllocs() for b.Loop() { profile, err := result.GetProfile(context.Background(), "profile-127") if err != nil { b.Fatal(err) } selectedProfile = profile } b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op") }) } func inspectionExtraParams(entries int) map[string]any { providers := make(map[string]any, entries) for index := range entries { providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{ "enabled": true, "labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)}, "limits": map[string]any{"burst": index + 1, "period": "minute"}, "routes": []any{map[string]any{"weight": index + 1, "fallback": true}}, } } return map[string]any{"provider_options": providers} } func inspectionDefinitions(extraParams map[string]any) configurationShape { profiles := map[string]any{ "small": map[string]any{"model": "inspection-model", "extra_params": extraParams}, } prompt := benchmarkPrompt("small") prompt["inputs"] = []any{map[string]any{"name": "transcript", "required": true}} prompts := map[string]any{ "inspection": prompt, } return configurationShape{ main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{ "profiles": profiles, "prompts": prompts, })}, inputEntries: 2, effectiveEntries: 2, tiers: 1, } } func BenchmarkInspectionSelection(b *testing.B) { for _, scenario := range []struct { name string entries int }{ {name: "small_extra", entries: 2}, {name: "nested_extra", entries: 64}, } { shape := inspectionDefinitions(inspectionExtraParams(scenario.entries)) result, err := load(shape.main, nil) if err != nil { b.Fatal(err) } ctx := context.Background() definition, hash, err := result.GetPromptSnapshot(ctx, "inspection") if err != nil || definition.ID != "inspection" || definition.Version == nil || *definition.Version != "1" || hash == "" || len(definition.Inputs) != 1 || definition.Inputs[0].Name != "transcript" || !definition.Inputs[0].Required { b.Fatalf("prompt lookup fixture is invalid: definition=%+v hash=%q err=%v", definition, hash, err) } profile, err := result.GetProfile(ctx, "small") if err != nil || profile.ID != "small" || profile.ResolvedTarget.Model != "inspection-model" || len(profile.ResolvedTarget.ExtraParams["provider_options"].(map[string]any)) != scenario.entries { b.Fatalf("profile lookup fixture is invalid: profile=%+v err=%v", profile, err) } inspection, err := result.GetPromptInspection(ctx, "inspection") if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion == nil || *inspection.PromptVersion != "1" || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "transcript" { b.Fatalf("prompt inspection fixture is invalid: inspection=%+v err=%v", inspection, err) } identity, err := result.GetProfileIdentity(ctx, "small") if err != nil || identity != "small" { b.Fatalf("profile identity fixture is invalid: identity=%q err=%v", identity, err) } b.Run("prompt/"+scenario.name, func(b *testing.B) { b.ReportAllocs() for b.Loop() { definition, hash, err := result.GetPromptSnapshot(ctx, "inspection") if err != nil { b.Fatal(err) } selectedPrompt = definition selectedPromptHash = hash } b.ReportMetric(float64(scenario.entries), "extra-entries/op") }) b.Run("prompt_inspection/"+scenario.name, func(b *testing.B) { b.ReportAllocs() for b.Loop() { inspection, err := result.GetPromptInspection(ctx, "inspection") if err != nil { b.Fatal(err) } selectedPromptInspection = inspection } b.ReportMetric(float64(scenario.entries), "extra-entries/op") }) b.Run("profile/"+scenario.name, func(b *testing.B) { b.ReportAllocs() for b.Loop() { profile, err := result.GetProfile(ctx, "small") if err != nil { b.Fatal(err) } selectedProfile = profile } b.ReportMetric(float64(scenario.entries), "extra-entries/op") }) b.Run("profile_identity/"+scenario.name, func(b *testing.B) { b.ReportAllocs() for b.Loop() { identity, err := result.GetProfileIdentity(ctx, "small") if err != nil { b.Fatal(err) } selectedProfileIdentity = identity } b.ReportMetric(float64(scenario.entries), "extra-entries/op") }) } }