297 lines
10 KiB
Go
297 lines
10 KiB
Go
package configuration
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"testing"
|
|
"testing/fstest"
|
|
|
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
|
)
|
|
|
|
const benchmarkDefinitionCount = 128
|
|
|
|
var (
|
|
constructionResult *loaded
|
|
selectedPrompt *domain.PromptDefinition
|
|
selectedPromptHash string
|
|
selectedPromptInspection *domain.PromptInspection
|
|
selectedProfile *domain.ExecutionProfile
|
|
selectedProfileIdentity string
|
|
)
|
|
|
|
type configurationShape struct {
|
|
main configkit.Input
|
|
underlays []configkit.Underlay
|
|
inputEntries int
|
|
effectiveEntries int
|
|
tiers int
|
|
}
|
|
|
|
func benchmarkPrompt(profileID string) map[string]any {
|
|
return map[string]any{
|
|
"version": "1",
|
|
"default_profile": profileID,
|
|
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}},
|
|
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
|
}
|
|
}
|
|
|
|
func independentDefinitions(count int) configurationShape {
|
|
profiles := make(map[string]any, count)
|
|
prompts := make(map[string]any, count)
|
|
for index := range count {
|
|
profileID := fmt.Sprintf("profile-%03d", index)
|
|
promptID := fmt.Sprintf("prompt-%03d", index)
|
|
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
|
|
prompts[promptID] = benchmarkPrompt(profileID)
|
|
}
|
|
return configurationShape{
|
|
main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": profiles, "prompts": prompts})},
|
|
inputEntries: count * 2,
|
|
effectiveEntries: count * 2,
|
|
tiers: 1,
|
|
}
|
|
}
|
|
|
|
func fileBackedDefinitions(count int) configurationShape {
|
|
profiles := make(map[string]any, count)
|
|
prompts := make(map[string]any, count)
|
|
for index := range count {
|
|
profileID := fmt.Sprintf("profile-%03d", index)
|
|
promptID := fmt.Sprintf("prompt-%03d", index)
|
|
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
|
|
prompt := benchmarkPrompt(profileID)
|
|
prompt["messages"] = []any{map[string]any{"role": "user", "content_file": "body.md"}}
|
|
prompts[promptID] = prompt
|
|
}
|
|
namespace := configkit.NewFilesystem("benchmark content", fstest.MapFS{
|
|
"assets/body.md": &fstest.MapFile{Data: []byte("Hello {{.name}}")},
|
|
})
|
|
return configurationShape{
|
|
main: configkit.Input{
|
|
Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": profiles, "prompts": prompts}),
|
|
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"},
|
|
},
|
|
inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1,
|
|
}
|
|
}
|
|
|
|
func sharedBaseDefinitions(count int) configurationShape {
|
|
profiles := make(map[string]any, count+1)
|
|
profiles["base"] = map[string]any{
|
|
"endpoint": "https://example.test/v1", "model": "shared-model",
|
|
"extra_params": map[string]any{"provider": map[string]any{"mode": "shared", "limit": 4}},
|
|
}
|
|
for index := range count {
|
|
profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": "base", "temperature": float64(index%10) / 10}
|
|
}
|
|
return configurationShape{
|
|
main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{
|
|
"profiles": profiles,
|
|
"prompts": map[string]any{"shared": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))},
|
|
})},
|
|
inputEntries: count + 2,
|
|
effectiveEntries: count + 2,
|
|
tiers: 1,
|
|
}
|
|
}
|
|
|
|
func layeredDefinitions(count int) configurationShape {
|
|
base := make(map[string]any, count)
|
|
operator := make(map[string]any, count)
|
|
programmatic := make(map[string]any, count)
|
|
final := make(map[string]any, count)
|
|
for index := range count {
|
|
id := fmt.Sprintf("profile-%03d", index)
|
|
base[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"}
|
|
operator[id] = map[string]any{"model": "operator-model"}
|
|
programmatic[id] = map[string]any{"extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}}
|
|
final[id] = map[string]any{"temperature": float64(index%10) / 10}
|
|
}
|
|
return configurationShape{
|
|
underlays: []configkit.Underlay{
|
|
{Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": base})}}},
|
|
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": operator})}}},
|
|
{Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": programmatic})}}},
|
|
},
|
|
main: configkit.Input{Source: configkit.Mapping("main", map[string]any{
|
|
"profiles": final,
|
|
"prompts": map[string]any{"layered": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))},
|
|
})},
|
|
inputEntries: count*4 + 1,
|
|
effectiveEntries: count + 1,
|
|
tiers: 4,
|
|
}
|
|
}
|
|
|
|
func BenchmarkConfigurationConstruction(b *testing.B) {
|
|
for _, scenario := range []struct {
|
|
name string
|
|
shape configurationShape
|
|
}{
|
|
{name: "independent", shape: independentDefinitions(benchmarkDefinitionCount)},
|
|
{name: "file_backed", shape: fileBackedDefinitions(64)},
|
|
{name: "shared_base", shape: sharedBaseDefinitions(benchmarkDefinitionCount)},
|
|
{name: "layered", shape: layeredDefinitions(benchmarkDefinitionCount)},
|
|
} {
|
|
b.Run(scenario.name, func(b *testing.B) {
|
|
b.ReportAllocs()
|
|
for b.Loop() {
|
|
result, err := load(scenario.shape.main, scenario.shape.underlays)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
constructionResult = result
|
|
}
|
|
b.ReportMetric(float64(scenario.shape.inputEntries), "input-entries/op")
|
|
b.ReportMetric(float64(scenario.shape.effectiveEntries), "effective-entries/op")
|
|
b.ReportMetric(float64(scenario.shape.tiers), "tiers/op")
|
|
})
|
|
}
|
|
}
|
|
|
|
func BenchmarkConfigurationSelection(b *testing.B) {
|
|
shape := independentDefinitions(benchmarkDefinitionCount)
|
|
result, err := load(shape.main, nil)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
b.Run("prompt", func(b *testing.B) {
|
|
b.ReportAllocs()
|
|
for b.Loop() {
|
|
definition, _, err := result.prompt("prompt-127")
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
selectedPrompt = definition
|
|
}
|
|
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
|
|
})
|
|
b.Run("profile", func(b *testing.B) {
|
|
b.ReportAllocs()
|
|
for b.Loop() {
|
|
profile, err := result.GetProfile(context.Background(), "profile-127")
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
selectedProfile = profile
|
|
}
|
|
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
|
|
})
|
|
}
|
|
|
|
func inspectionExtraParams(entries int) map[string]any {
|
|
providers := make(map[string]any, entries)
|
|
for index := range entries {
|
|
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
|
|
"enabled": true,
|
|
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
|
|
"limits": map[string]any{"burst": index + 1, "period": "minute"},
|
|
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
|
|
}
|
|
}
|
|
return map[string]any{"provider_options": providers}
|
|
}
|
|
|
|
func inspectionDefinitions(extraParams map[string]any) configurationShape {
|
|
profiles := map[string]any{
|
|
"small": map[string]any{"model": "inspection-model", "extra_params": extraParams},
|
|
}
|
|
prompt := benchmarkPrompt("small")
|
|
prompt["inputs"] = []any{map[string]any{"name": "transcript", "required": true}}
|
|
prompts := map[string]any{
|
|
"inspection": prompt,
|
|
}
|
|
return configurationShape{
|
|
main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{
|
|
"profiles": profiles,
|
|
"prompts": prompts,
|
|
})},
|
|
inputEntries: 2, effectiveEntries: 2, tiers: 1,
|
|
}
|
|
}
|
|
|
|
func BenchmarkInspectionSelection(b *testing.B) {
|
|
for _, scenario := range []struct {
|
|
name string
|
|
entries int
|
|
}{
|
|
{name: "small_extra", entries: 2},
|
|
{name: "nested_extra", entries: 64},
|
|
} {
|
|
shape := inspectionDefinitions(inspectionExtraParams(scenario.entries))
|
|
result, err := load(shape.main, nil)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
ctx := context.Background()
|
|
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection")
|
|
if err != nil || definition.ID != "inspection" || definition.Version == nil || *definition.Version != "1" || hash == "" ||
|
|
len(definition.Inputs) != 1 || definition.Inputs[0].Name != "transcript" || !definition.Inputs[0].Required {
|
|
b.Fatalf("prompt lookup fixture is invalid: definition=%+v hash=%q err=%v", definition, hash, err)
|
|
}
|
|
profile, err := result.GetProfile(ctx, "small")
|
|
if err != nil || profile.ID != "small" || profile.ResolvedTarget.Model != "inspection-model" ||
|
|
len(profile.ResolvedTarget.ExtraParams["provider_options"].(map[string]any)) != scenario.entries {
|
|
b.Fatalf("profile lookup fixture is invalid: profile=%+v err=%v", profile, err)
|
|
}
|
|
inspection, err := result.GetPromptInspection(ctx, "inspection")
|
|
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion == nil || *inspection.PromptVersion != "1" ||
|
|
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "transcript" {
|
|
b.Fatalf("prompt inspection fixture is invalid: inspection=%+v err=%v", inspection, err)
|
|
}
|
|
identity, err := result.GetProfileIdentity(ctx, "small")
|
|
if err != nil || identity != "small" {
|
|
b.Fatalf("profile identity fixture is invalid: identity=%q err=%v", identity, err)
|
|
}
|
|
|
|
b.Run("prompt/"+scenario.name, func(b *testing.B) {
|
|
b.ReportAllocs()
|
|
for b.Loop() {
|
|
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection")
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
selectedPrompt = definition
|
|
selectedPromptHash = hash
|
|
}
|
|
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
|
})
|
|
b.Run("prompt_inspection/"+scenario.name, func(b *testing.B) {
|
|
b.ReportAllocs()
|
|
for b.Loop() {
|
|
inspection, err := result.GetPromptInspection(ctx, "inspection")
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
selectedPromptInspection = inspection
|
|
}
|
|
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
|
})
|
|
b.Run("profile/"+scenario.name, func(b *testing.B) {
|
|
b.ReportAllocs()
|
|
for b.Loop() {
|
|
profile, err := result.GetProfile(ctx, "small")
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
selectedProfile = profile
|
|
}
|
|
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
|
})
|
|
b.Run("profile_identity/"+scenario.name, func(b *testing.B) {
|
|
b.ReportAllocs()
|
|
for b.Loop() {
|
|
identity, err := result.GetProfileIdentity(ctx, "small")
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
selectedProfileIdentity = identity
|
|
}
|
|
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
|
})
|
|
}
|
|
}
|