Add request preparation baseline and boundary tests

2026-10-09 00:25:21 +00:00
parent f1e1a45a05
commit 3f157150e4
2 changed files with 176 additions and 0 deletions
--- a/internal/llm/prepare_generation_benchmark_test.go
+++ b/internal/llm/prepare_generation_benchmark_test.go
@@ -0,0 +1,119 @@
+package llm
+
+import (
+	"encoding/json"
+	"runtime"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
+)
+
+var prepareGenerationBenchmarkSink *PreparedGeneration
+
+func BenchmarkPrepareGeneration(b *testing.B) {
+	client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
+		BaseURL: "https://api.example.test/v1",
+		Model:   "benchmark-model",
+	})
+	if err != nil {
+		b.Fatalf("construct benchmark client: %v", err)
+	}
+
+	prompt := domain.RenderedPrompt{
+		Messages: []domain.RenderedMessage{{Role: "user", Content: "Summarize this request."}},
+	}
+	tests := []struct {
+		name    string
+		request domain.GenerateRequest
+	}{
+		{
+			name: "no_extras",
+			request: domain.GenerateRequest{
+				Prompt: prompt,
+				Target: domain.ExecutionTarget{Model: "benchmark-model"},
+			},
+		},
+		{
+			name: "small_extras",
+			request: domain.GenerateRequest{
+				Prompt: prompt,
+				Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
+					"priority": "low",
+					"attempt":  3,
+					"enabled":  true,
+				}},
+			},
+		},
+		{
+			name: "1MiB_string_extra",
+			request: domain.GenerateRequest{
+				Prompt: prompt,
+				Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
+					"payload": strings.Repeat("x", 1<<20),
+				}},
+			},
+		},
+		{
+			name: "nested_json_extras",
+			request: domain.GenerateRequest{
+				Prompt: prompt,
+				Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
+					"options": map[string]any{
+						"enabled": true,
+						"limits":  []any{2, 4, 8},
+						"tags":    []string{"summary", "offline"},
+					},
+				}},
+			},
+		},
+		{
+			name: "structured_output",
+			request: domain.GenerateRequest{
+				Prompt: prompt,
+				Target: domain.ExecutionTarget{Model: "benchmark-model"},
+				StructuredOutput: &domain.StructuredOutputSpec{
+					Type: domain.StructuredOutputJSONSchema,
+					JSONSchema: &domain.StructuredOutputJSONSpec{
+						Name:   "summary",
+						Strict: true,
+						Schema: map[string]any{
+							"type": "object",
+							"properties": map[string]any{
+								"summary": map[string]any{"type": "string"},
+								"count":   map[string]any{"type": "integer"},
+							},
+							"required":             []string{"summary", "count"},
+							"additionalProperties": false,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		b.Run(tc.name, func(b *testing.B) {
+			prepared, err := client.PrepareGeneration(tc.request)
+			if err != nil {
+				b.Fatalf("check prepared request: %v", err)
+			}
+			if !json.Valid(prepared.payload) {
+				b.Fatal("prepared request is not valid JSON")
+			}
+
+			b.ReportAllocs()
+			b.SetBytes(int64(len(prepared.payload)))
+			b.ResetTimer()
+			for range b.N {
+				prepared, err := client.PrepareGeneration(tc.request)
+				if err != nil {
+					b.Fatalf("prepare generation: %v", err)
+				}
+				prepareGenerationBenchmarkSink = prepared
+			}
+			b.StopTimer()
+			runtime.KeepAlive(prepareGenerationBenchmarkSink)
+		})
+	}
+}