Add requirement inspection benchmarks

internal/configuration/benchmark_test.go

@@ -15,6 +15,7 @@ const benchmarkDefinitionCount = 128
 var (
 	constructionResult *loaded
 	selectedPrompt     *domain.PromptDefinition
+	selectedPromptHash string
 	selectedProfile    *domain.ExecutionProfile
 )
 
@@ -177,3 +178,85 @@ func BenchmarkConfigurationSelection(b *testing.B) {
 		b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
 	})
 }
+
+func inspectionExtraParams(entries int) map[string]any {
+	providers := make(map[string]any, entries)
+	for index := range entries {
+		providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
+			"enabled": true,
+			"labels":  []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
+			"limits":  map[string]any{"burst": index + 1, "period": "minute"},
+			"routes":  []any{map[string]any{"weight": index + 1, "fallback": true}},
+		}
+	}
+	return map[string]any{"provider_options": providers}
+}
+
+func inspectionDefinitions(extraParams map[string]any) configurationShape {
+	profiles := map[string]any{
+		"small": map[string]any{"model": "inspection-model", "extra_params": extraParams},
+	}
+	prompt := benchmarkPrompt("small")
+	prompt["inputs"] = []any{map[string]any{"name": "transcript", "required": true}}
+	prompts := map[string]any{
+		"inspection": map[string]any{"1": prompt},
+	}
+	return configurationShape{
+		main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{
+			"profiles": profiles,
+			"prompts":  prompts,
+		})},
+		inputEntries: 2, effectiveEntries: 2, tiers: 1,
+	}
+}
+
+func BenchmarkInspectionSelection(b *testing.B) {
+	for _, scenario := range []struct {
+		name    string
+		entries int
+	}{
+		{name: "small_extra", entries: 2},
+		{name: "nested_extra", entries: 64},
+	} {
+		shape := inspectionDefinitions(inspectionExtraParams(scenario.entries))
+		result, err := load(shape.main, nil)
+		if err != nil {
+			b.Fatal(err)
+		}
+		ctx := context.Background()
+		definition, hash, err := result.GetPromptSnapshot(ctx, "inspection", "1")
+		if err != nil || definition.ID != "inspection" || definition.Version != "1" || hash == "" ||
+			len(definition.Inputs) != 1 || definition.Inputs[0].Name != "transcript" || !definition.Inputs[0].Required {
+			b.Fatalf("prompt lookup fixture is invalid: definition=%+v hash=%q err=%v", definition, hash, err)
+		}
+		profile, err := result.GetProfile(ctx, "small")
+		if err != nil || profile.ID != "small" || profile.ResolvedTarget.Model != "inspection-model" ||
+			len(profile.ResolvedTarget.ExtraParams["provider_options"].(map[string]any)) != scenario.entries {
+			b.Fatalf("profile lookup fixture is invalid: profile=%+v err=%v", profile, err)
+		}
+
+		b.Run("prompt/"+scenario.name, func(b *testing.B) {
+			b.ReportAllocs()
+			for b.Loop() {
+				definition, hash, err := result.GetPromptSnapshot(ctx, "inspection", "1")
+				if err != nil {
+					b.Fatal(err)
+				}
+				selectedPrompt = definition
+				selectedPromptHash = hash
+			}
+			b.ReportMetric(float64(scenario.entries), "extra-entries/op")
+		})
+		b.Run("profile/"+scenario.name, func(b *testing.B) {
+			b.ReportAllocs()
+			for b.Loop() {
+				profile, err := result.GetProfile(ctx, "small")
+				if err != nil {
+					b.Fatal(err)
+				}
+				selectedProfile = profile
+			}
+			b.ReportMetric(float64(scenario.entries), "extra-entries/op")
+		})
+	}
+}

requirements_inspection_benchmark_test.go

@@ -0,0 +1,72 @@
+package promptkit_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/ConfigKit"
+	"gitea.maximumdirect.net/eric/promptkit"
+)
+
+var inspectedRequirements *promptkit.RequirementInspection
+
+func requirementInspectionExtras(entries int) map[string]any {
+	providers := make(map[string]any, entries)
+	for index := range entries {
+		providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
+			"enabled": true,
+			"labels":  []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
+			"limits":  map[string]any{"burst": index + 1, "period": "minute"},
+			"routes":  []any{map[string]any{"weight": index + 1, "fallback": true}},
+		}
+	}
+	return map[string]any{"provider_options": providers}
+}
+
+func BenchmarkRequirementInspection(b *testing.B) {
+	profiles := map[string]any{
+		"small": map[string]any{
+			"model": "inspection-model", "extra_params": requirementInspectionExtras(2),
+		},
+		"large": map[string]any{
+			"model": "inspection-model", "extra_params": requirementInspectionExtras(64),
+		},
+	}
+	definition := contractPrompt("small", "Hello {{.name}} {{input \"transcript\"}}", []any{
+		map[string]any{"name": "transcript", "required": true},
+	}, "", noValidationOutput)
+	engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
+		Source: configkit.Mapping("requirement inspection benchmark", map[string]any{
+			"prompts":  oneVersionPrompt("inspection", "1", definition),
+			"profiles": profiles,
+		}),
+	}}})
+	if err != nil {
+		b.Fatal(err)
+	}
+	ctx := context.Background()
+	for _, profileID := range []string{"small", "large"} {
+		request := promptkit.RequirementInspectionRequest{
+			PromptID: "inspection", ProfileID: profileID, Vars: map[string]string{"name": "Ada"},
+		}
+		inspection, err := engine.InspectRequirements(ctx, request)
+		if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion != "1" ||
+			inspection.ProfileID != profileID || !inspection.Complete || len(inspection.Inputs) != 1 ||
+			inspection.Inputs[0].Name != "transcript" || inspection.Inputs[0].Kind != promptkit.RequirementRequired ||
+			len(inspection.Variables) != 1 || inspection.Variables[0].Name != "name" || !inspection.Variables[0].Supplied {
+			b.Fatalf("requirement inspection fixture for %q is invalid: inspection=%+v err=%v", profileID, inspection, err)
+		}
+		selectedRequest := request
+		b.Run(profileID+"_extra", func(b *testing.B) {
+			b.ReportAllocs()
+			for b.Loop() {
+				inspection, err := engine.InspectRequirements(ctx, selectedRequest)
+				if err != nil {
+					b.Fatal(err)
+				}
+				inspectedRequirements = inspection
+			}
+		})
+	}
+}