Benchmark configuration workflows and reuse captured content origins
This commit is contained in:
179
internal/configuration/benchmark_test.go
Normal file
179
internal/configuration/benchmark_test.go
Normal file
@@ -0,0 +1,179 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
const benchmarkDefinitionCount = 128
|
||||
|
||||
var (
|
||||
constructionResult *loaded
|
||||
selectedPrompt *domain.PromptDefinition
|
||||
selectedProfile *domain.ExecutionProfile
|
||||
)
|
||||
|
||||
type configurationShape struct {
|
||||
main configkit.Input
|
||||
underlays []configkit.Underlay
|
||||
inputEntries int
|
||||
effectiveEntries int
|
||||
tiers int
|
||||
}
|
||||
|
||||
func benchmarkPrompt(profileID string) map[string]any {
|
||||
return map[string]any{
|
||||
"default_profile": profileID,
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
}
|
||||
}
|
||||
|
||||
func independentDefinitions(count int) configurationShape {
|
||||
profiles := make(map[string]any, count)
|
||||
prompts := make(map[string]any, count)
|
||||
for index := range count {
|
||||
profileID := fmt.Sprintf("profile-%03d", index)
|
||||
promptID := fmt.Sprintf("prompt-%03d", index)
|
||||
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
|
||||
prompts[promptID] = map[string]any{"1": benchmarkPrompt(profileID)}
|
||||
}
|
||||
return configurationShape{
|
||||
main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": profiles, "prompts": prompts})},
|
||||
inputEntries: count * 2,
|
||||
effectiveEntries: count * 2,
|
||||
tiers: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func fileBackedDefinitions(count int) configurationShape {
|
||||
profiles := make(map[string]any, count)
|
||||
prompts := make(map[string]any, count)
|
||||
for index := range count {
|
||||
profileID := fmt.Sprintf("profile-%03d", index)
|
||||
promptID := fmt.Sprintf("prompt-%03d", index)
|
||||
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
|
||||
prompt := benchmarkPrompt(profileID)
|
||||
prompt["messages"] = []any{map[string]any{"role": "user", "content_file": "body.md"}}
|
||||
prompts[promptID] = map[string]any{"1": prompt}
|
||||
}
|
||||
namespace := configkit.NewFilesystem("benchmark content", fstest.MapFS{
|
||||
"assets/body.md": &fstest.MapFile{Data: []byte("Hello {{.name}}")},
|
||||
})
|
||||
return configurationShape{
|
||||
main: configkit.Input{
|
||||
Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": profiles, "prompts": prompts}),
|
||||
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"},
|
||||
},
|
||||
inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func sharedBaseDefinitions(count int) configurationShape {
|
||||
profiles := make(map[string]any, count+1)
|
||||
profiles["base"] = map[string]any{
|
||||
"endpoint": "https://example.test/v1", "model": "shared-model",
|
||||
"extra_params": map[string]any{"provider": map[string]any{"mode": "shared", "limit": 4}},
|
||||
}
|
||||
for index := range count {
|
||||
profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": "base", "temperature": float64(index%10) / 10}
|
||||
}
|
||||
return configurationShape{
|
||||
main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{
|
||||
"profiles": profiles,
|
||||
"prompts": map[string]any{"shared": map[string]any{"1": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}},
|
||||
})},
|
||||
inputEntries: count + 2,
|
||||
effectiveEntries: count + 2,
|
||||
tiers: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func layeredDefinitions(count int) configurationShape {
|
||||
base := make(map[string]any, count)
|
||||
operator := make(map[string]any, count)
|
||||
programmatic := make(map[string]any, count)
|
||||
final := make(map[string]any, count)
|
||||
for index := range count {
|
||||
id := fmt.Sprintf("profile-%03d", index)
|
||||
base[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"}
|
||||
operator[id] = map[string]any{"model": "operator-model"}
|
||||
programmatic[id] = map[string]any{"extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}}
|
||||
final[id] = map[string]any{"temperature": float64(index%10) / 10}
|
||||
}
|
||||
return configurationShape{
|
||||
underlays: []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": base})}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": operator})}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": programmatic})}}},
|
||||
},
|
||||
main: configkit.Input{Source: configkit.Mapping("main", map[string]any{
|
||||
"profiles": final,
|
||||
"prompts": map[string]any{"layered": map[string]any{"1": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}},
|
||||
})},
|
||||
inputEntries: count*4 + 1,
|
||||
effectiveEntries: count + 1,
|
||||
tiers: 4,
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkConfigurationConstruction(b *testing.B) {
|
||||
for _, scenario := range []struct {
|
||||
name string
|
||||
shape configurationShape
|
||||
}{
|
||||
{name: "independent", shape: independentDefinitions(benchmarkDefinitionCount)},
|
||||
{name: "file_backed", shape: fileBackedDefinitions(64)},
|
||||
{name: "shared_base", shape: sharedBaseDefinitions(benchmarkDefinitionCount)},
|
||||
{name: "layered", shape: layeredDefinitions(benchmarkDefinitionCount)},
|
||||
} {
|
||||
b.Run(scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
result, err := load(scenario.shape.main, scenario.shape.underlays)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
constructionResult = result
|
||||
}
|
||||
b.ReportMetric(float64(scenario.shape.inputEntries), "input-entries/op")
|
||||
b.ReportMetric(float64(scenario.shape.effectiveEntries), "effective-entries/op")
|
||||
b.ReportMetric(float64(scenario.shape.tiers), "tiers/op")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkConfigurationSelection(b *testing.B) {
|
||||
shape := independentDefinitions(benchmarkDefinitionCount)
|
||||
result, err := load(shape.main, nil)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.Run("prompt", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
definition, _, err := result.prompt("prompt-127", "1")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedPrompt = definition
|
||||
}
|
||||
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
|
||||
})
|
||||
b.Run("profile", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
profile, err := result.GetProfile(context.Background(), "profile-127")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedProfile = profile
|
||||
}
|
||||
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user