From 3f157150e4abd21e77dca229b868e67579167359 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 9 Oct 2026 00:25:21 +0000 Subject: [PATCH] Add request preparation baseline and boundary tests --- internal/llm/openai_compatible_client_test.go | 57 +++++++++ .../llm/prepare_generation_benchmark_test.go | 119 ++++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100644 internal/llm/prepare_generation_benchmark_test.go diff --git a/internal/llm/openai_compatible_client_test.go b/internal/llm/openai_compatible_client_test.go index 65310bf..b2386b0 100644 --- a/internal/llm/openai_compatible_client_test.go +++ b/internal/llm/openai_compatible_client_test.go @@ -1113,6 +1113,13 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te }{ {name: "empty key", extraParams: map[string]any{"": "empty"}, want: "key must not be empty"}, {name: "unserializable value", extraParams: map[string]any{"bad": math.Inf(1)}, want: "JSON-serializable"}, + { + name: "nested unserializable value", + extraParams: map[string]any{ + "nested": map[string]any{"values": []any{"valid", math.NaN()}}, + }, + want: "extra_params.nested", + }, } for _, key := range []string{ "model", @@ -1161,6 +1168,56 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te } } +func TestOpenAICompatibleClientClassifiesSchemaEncodingFailures(t *testing.T) { + tests := []struct { + name string + extraParams map[string]any + want error + unwanted error + }{ + { + name: "serializable extra keeps full-request failure", + extraParams: map[string]any{"client_option": "accepted"}, + want: ErrRequestFailed, + unwanted: ErrInvalidRequest, + }, + { + name: "invalid extra keeps invalid-request failure", + extraParams: map[string]any{"nested": map[string]any{"bad": math.Inf(1)}}, + want: ErrInvalidRequest, + unwanted: ErrRequestFailed, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + provider := newRecordingProvider(t) + client := newProviderClient(t, provider, OpenAICompatibleConfig{}) + request := ordinaryGenerateRequest() + request.Target.ExtraParams = tc.extraParams + request.StructuredOutput = &domain.StructuredOutputSpec{ + Type: domain.StructuredOutputJSONSchema, + JSONSchema: &domain.StructuredOutputJSONSpec{ + Name: "invalid-schema", + Strict: true, + Schema: json.RawMessage("{"), + }, + } + + _, err := client.Generate(context.Background(), request) + if err == nil || !errors.Is(err, tc.want) { + t.Fatalf("error = %v, want identity %v", err, tc.want) + } + if errors.Is(err, tc.unwanted) { + t.Fatalf("error = %v, unexpectedly has identity %v", err, tc.unwanted) + } + if calls := provider.requestCount(); calls != 0 { + t.Fatalf("provider calls = %d, want 0", calls) + } + }) + } +} + func checkConfiguredModelFallback(t *testing.T) { provider := newRecordingProvider(t) client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "default-model"}) diff --git a/internal/llm/prepare_generation_benchmark_test.go b/internal/llm/prepare_generation_benchmark_test.go new file mode 100644 index 0000000..37c3d76 --- /dev/null +++ b/internal/llm/prepare_generation_benchmark_test.go @@ -0,0 +1,119 @@ +package llm + +import ( + "encoding/json" + "runtime" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" +) + +var prepareGenerationBenchmarkSink *PreparedGeneration + +func BenchmarkPrepareGeneration(b *testing.B) { + client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ + BaseURL: "https://api.example.test/v1", + Model: "benchmark-model", + }) + if err != nil { + b.Fatalf("construct benchmark client: %v", err) + } + + prompt := domain.RenderedPrompt{ + Messages: []domain.RenderedMessage{{Role: "user", Content: "Summarize this request."}}, + } + tests := []struct { + name string + request domain.GenerateRequest + }{ + { + name: "no_extras", + request: domain.GenerateRequest{ + Prompt: prompt, + Target: domain.ExecutionTarget{Model: "benchmark-model"}, + }, + }, + { + name: "small_extras", + request: domain.GenerateRequest{ + Prompt: prompt, + Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{ + "priority": "low", + "attempt": 3, + "enabled": true, + }}, + }, + }, + { + name: "1MiB_string_extra", + request: domain.GenerateRequest{ + Prompt: prompt, + Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{ + "payload": strings.Repeat("x", 1<<20), + }}, + }, + }, + { + name: "nested_json_extras", + request: domain.GenerateRequest{ + Prompt: prompt, + Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{ + "options": map[string]any{ + "enabled": true, + "limits": []any{2, 4, 8}, + "tags": []string{"summary", "offline"}, + }, + }}, + }, + }, + { + name: "structured_output", + request: domain.GenerateRequest{ + Prompt: prompt, + Target: domain.ExecutionTarget{Model: "benchmark-model"}, + StructuredOutput: &domain.StructuredOutputSpec{ + Type: domain.StructuredOutputJSONSchema, + JSONSchema: &domain.StructuredOutputJSONSpec{ + Name: "summary", + Strict: true, + Schema: map[string]any{ + "type": "object", + "properties": map[string]any{ + "summary": map[string]any{"type": "string"}, + "count": map[string]any{"type": "integer"}, + }, + "required": []string{"summary", "count"}, + "additionalProperties": false, + }, + }, + }, + }, + }, + } + + for _, tc := range tests { + b.Run(tc.name, func(b *testing.B) { + prepared, err := client.PrepareGeneration(tc.request) + if err != nil { + b.Fatalf("check prepared request: %v", err) + } + if !json.Valid(prepared.payload) { + b.Fatal("prepared request is not valid JSON") + } + + b.ReportAllocs() + b.SetBytes(int64(len(prepared.payload))) + b.ResetTimer() + for range b.N { + prepared, err := client.PrepareGeneration(tc.request) + if err != nil { + b.Fatalf("prepare generation: %v", err) + } + prepareGenerationBenchmarkSink = prepared + } + b.StopTimer() + runtime.KeepAlive(prepareGenerationBenchmarkSink) + }) + } +}