From 04d5d695e0201a97c1070e1ff49ef4f0c9a7f5ba Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 9 Oct 2026 00:30:51 +0000 Subject: [PATCH] Add requirement inspection benchmarks --- internal/configuration/benchmark_test.go | 83 +++++++++++++++++++++++ requirements_inspection_benchmark_test.go | 72 ++++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 requirements_inspection_benchmark_test.go diff --git a/internal/configuration/benchmark_test.go b/internal/configuration/benchmark_test.go index d01433e..d0c9e6c 100644 --- a/internal/configuration/benchmark_test.go +++ b/internal/configuration/benchmark_test.go @@ -15,6 +15,7 @@ const benchmarkDefinitionCount = 128 var ( constructionResult *loaded selectedPrompt *domain.PromptDefinition + selectedPromptHash string selectedProfile *domain.ExecutionProfile ) @@ -177,3 +178,85 @@ func BenchmarkConfigurationSelection(b *testing.B) { b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op") }) } + +func inspectionExtraParams(entries int) map[string]any { + providers := make(map[string]any, entries) + for index := range entries { + providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{ + "enabled": true, + "labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)}, + "limits": map[string]any{"burst": index + 1, "period": "minute"}, + "routes": []any{map[string]any{"weight": index + 1, "fallback": true}}, + } + } + return map[string]any{"provider_options": providers} +} + +func inspectionDefinitions(extraParams map[string]any) configurationShape { + profiles := map[string]any{ + "small": map[string]any{"model": "inspection-model", "extra_params": extraParams}, + } + prompt := benchmarkPrompt("small") + prompt["inputs"] = []any{map[string]any{"name": "transcript", "required": true}} + prompts := map[string]any{ + "inspection": map[string]any{"1": prompt}, + } + return configurationShape{ + main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{ + "profiles": profiles, + "prompts": prompts, + })}, + inputEntries: 2, effectiveEntries: 2, tiers: 1, + } +} + +func BenchmarkInspectionSelection(b *testing.B) { + for _, scenario := range []struct { + name string + entries int + }{ + {name: "small_extra", entries: 2}, + {name: "nested_extra", entries: 64}, + } { + shape := inspectionDefinitions(inspectionExtraParams(scenario.entries)) + result, err := load(shape.main, nil) + if err != nil { + b.Fatal(err) + } + ctx := context.Background() + definition, hash, err := result.GetPromptSnapshot(ctx, "inspection", "1") + if err != nil || definition.ID != "inspection" || definition.Version != "1" || hash == "" || + len(definition.Inputs) != 1 || definition.Inputs[0].Name != "transcript" || !definition.Inputs[0].Required { + b.Fatalf("prompt lookup fixture is invalid: definition=%+v hash=%q err=%v", definition, hash, err) + } + profile, err := result.GetProfile(ctx, "small") + if err != nil || profile.ID != "small" || profile.ResolvedTarget.Model != "inspection-model" || + len(profile.ResolvedTarget.ExtraParams["provider_options"].(map[string]any)) != scenario.entries { + b.Fatalf("profile lookup fixture is invalid: profile=%+v err=%v", profile, err) + } + + b.Run("prompt/"+scenario.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + definition, hash, err := result.GetPromptSnapshot(ctx, "inspection", "1") + if err != nil { + b.Fatal(err) + } + selectedPrompt = definition + selectedPromptHash = hash + } + b.ReportMetric(float64(scenario.entries), "extra-entries/op") + }) + b.Run("profile/"+scenario.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + profile, err := result.GetProfile(ctx, "small") + if err != nil { + b.Fatal(err) + } + selectedProfile = profile + } + b.ReportMetric(float64(scenario.entries), "extra-entries/op") + }) + } +} diff --git a/requirements_inspection_benchmark_test.go b/requirements_inspection_benchmark_test.go new file mode 100644 index 0000000..9f653d3 --- /dev/null +++ b/requirements_inspection_benchmark_test.go @@ -0,0 +1,72 @@ +package promptkit_test + +import ( + "context" + "fmt" + "testing" + + "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/promptkit" +) + +var inspectedRequirements *promptkit.RequirementInspection + +func requirementInspectionExtras(entries int) map[string]any { + providers := make(map[string]any, entries) + for index := range entries { + providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{ + "enabled": true, + "labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)}, + "limits": map[string]any{"burst": index + 1, "period": "minute"}, + "routes": []any{map[string]any{"weight": index + 1, "fallback": true}}, + } + } + return map[string]any{"provider_options": providers} +} + +func BenchmarkRequirementInspection(b *testing.B) { + profiles := map[string]any{ + "small": map[string]any{ + "model": "inspection-model", "extra_params": requirementInspectionExtras(2), + }, + "large": map[string]any{ + "model": "inspection-model", "extra_params": requirementInspectionExtras(64), + }, + } + definition := contractPrompt("small", "Hello {{.name}} {{input \"transcript\"}}", []any{ + map[string]any{"name": "transcript", "required": true}, + }, "", noValidationOutput) + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ + Source: configkit.Mapping("requirement inspection benchmark", map[string]any{ + "prompts": oneVersionPrompt("inspection", "1", definition), + "profiles": profiles, + }), + }}}) + if err != nil { + b.Fatal(err) + } + ctx := context.Background() + for _, profileID := range []string{"small", "large"} { + request := promptkit.RequirementInspectionRequest{ + PromptID: "inspection", ProfileID: profileID, Vars: map[string]string{"name": "Ada"}, + } + inspection, err := engine.InspectRequirements(ctx, request) + if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion != "1" || + inspection.ProfileID != profileID || !inspection.Complete || len(inspection.Inputs) != 1 || + inspection.Inputs[0].Name != "transcript" || inspection.Inputs[0].Kind != promptkit.RequirementRequired || + len(inspection.Variables) != 1 || inspection.Variables[0].Name != "name" || !inspection.Variables[0].Supplied { + b.Fatalf("requirement inspection fixture for %q is invalid: inspection=%+v err=%v", profileID, inspection, err) + } + selectedRequest := request + b.Run(profileID+"_extra", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + inspection, err := engine.InspectRequirements(ctx, selectedRequest) + if err != nil { + b.Fatal(err) + } + inspectedRequirements = inspection + } + }) + } +}