Add lightweight configuration lookups
This commit is contained in:
@@ -13,10 +13,12 @@ import (
|
||||
const benchmarkDefinitionCount = 128
|
||||
|
||||
var (
|
||||
constructionResult *loaded
|
||||
selectedPrompt *domain.PromptDefinition
|
||||
selectedPromptHash string
|
||||
selectedProfile *domain.ExecutionProfile
|
||||
constructionResult *loaded
|
||||
selectedPrompt *domain.PromptDefinition
|
||||
selectedPromptHash string
|
||||
selectedPromptInspection *domain.PromptInspection
|
||||
selectedProfile *domain.ExecutionProfile
|
||||
selectedProfileIdentity string
|
||||
)
|
||||
|
||||
type configurationShape struct {
|
||||
@@ -234,6 +236,15 @@ func BenchmarkInspectionSelection(b *testing.B) {
|
||||
len(profile.ResolvedTarget.ExtraParams["provider_options"].(map[string]any)) != scenario.entries {
|
||||
b.Fatalf("profile lookup fixture is invalid: profile=%+v err=%v", profile, err)
|
||||
}
|
||||
inspection, err := result.GetPromptInspection(ctx, "inspection", "1")
|
||||
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion != "1" ||
|
||||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "transcript" {
|
||||
b.Fatalf("prompt inspection fixture is invalid: inspection=%+v err=%v", inspection, err)
|
||||
}
|
||||
identity, err := result.GetProfileIdentity(ctx, "small")
|
||||
if err != nil || identity != "small" {
|
||||
b.Fatalf("profile identity fixture is invalid: identity=%q err=%v", identity, err)
|
||||
}
|
||||
|
||||
b.Run("prompt/"+scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
@@ -247,6 +258,17 @@ func BenchmarkInspectionSelection(b *testing.B) {
|
||||
}
|
||||
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||
})
|
||||
b.Run("prompt_inspection/"+scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
inspection, err := result.GetPromptInspection(ctx, "inspection", "1")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedPromptInspection = inspection
|
||||
}
|
||||
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||
})
|
||||
b.Run("profile/"+scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
@@ -258,5 +280,16 @@ func BenchmarkInspectionSelection(b *testing.B) {
|
||||
}
|
||||
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||
})
|
||||
b.Run("profile_identity/"+scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
identity, err := result.GetProfileIdentity(ctx, "small")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedProfileIdentity = identity
|
||||
}
|
||||
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user