Add request preparation baseline and boundary tests
This commit is contained in:
@@ -1113,6 +1113,13 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te
|
|||||||
}{
|
}{
|
||||||
{name: "empty key", extraParams: map[string]any{"": "empty"}, want: "key must not be empty"},
|
{name: "empty key", extraParams: map[string]any{"": "empty"}, want: "key must not be empty"},
|
||||||
{name: "unserializable value", extraParams: map[string]any{"bad": math.Inf(1)}, want: "JSON-serializable"},
|
{name: "unserializable value", extraParams: map[string]any{"bad": math.Inf(1)}, want: "JSON-serializable"},
|
||||||
|
{
|
||||||
|
name: "nested unserializable value",
|
||||||
|
extraParams: map[string]any{
|
||||||
|
"nested": map[string]any{"values": []any{"valid", math.NaN()}},
|
||||||
|
},
|
||||||
|
want: "extra_params.nested",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
for _, key := range []string{
|
for _, key := range []string{
|
||||||
"model",
|
"model",
|
||||||
@@ -1161,6 +1168,56 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientClassifiesSchemaEncodingFailures(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
extraParams map[string]any
|
||||||
|
want error
|
||||||
|
unwanted error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "serializable extra keeps full-request failure",
|
||||||
|
extraParams: map[string]any{"client_option": "accepted"},
|
||||||
|
want: ErrRequestFailed,
|
||||||
|
unwanted: ErrInvalidRequest,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid extra keeps invalid-request failure",
|
||||||
|
extraParams: map[string]any{"nested": map[string]any{"bad": math.Inf(1)}},
|
||||||
|
want: ErrInvalidRequest,
|
||||||
|
unwanted: ErrRequestFailed,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
provider := newRecordingProvider(t)
|
||||||
|
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
||||||
|
request := ordinaryGenerateRequest()
|
||||||
|
request.Target.ExtraParams = tc.extraParams
|
||||||
|
request.StructuredOutput = &domain.StructuredOutputSpec{
|
||||||
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
|
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||||
|
Name: "invalid-schema",
|
||||||
|
Strict: true,
|
||||||
|
Schema: json.RawMessage("{"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := client.Generate(context.Background(), request)
|
||||||
|
if err == nil || !errors.Is(err, tc.want) {
|
||||||
|
t.Fatalf("error = %v, want identity %v", err, tc.want)
|
||||||
|
}
|
||||||
|
if errors.Is(err, tc.unwanted) {
|
||||||
|
t.Fatalf("error = %v, unexpectedly has identity %v", err, tc.unwanted)
|
||||||
|
}
|
||||||
|
if calls := provider.requestCount(); calls != 0 {
|
||||||
|
t.Fatalf("provider calls = %d, want 0", calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func checkConfiguredModelFallback(t *testing.T) {
|
func checkConfiguredModelFallback(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "default-model"})
|
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "default-model"})
|
||||||
|
|||||||
119
internal/llm/prepare_generation_benchmark_test.go
Normal file
119
internal/llm/prepare_generation_benchmark_test.go
Normal file
@@ -0,0 +1,119 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
var prepareGenerationBenchmarkSink *PreparedGeneration
|
||||||
|
|
||||||
|
func BenchmarkPrepareGeneration(b *testing.B) {
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
|
BaseURL: "https://api.example.test/v1",
|
||||||
|
Model: "benchmark-model",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("construct benchmark client: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prompt := domain.RenderedPrompt{
|
||||||
|
Messages: []domain.RenderedMessage{{Role: "user", Content: "Summarize this request."}},
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
request domain.GenerateRequest
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "no_extras",
|
||||||
|
request: domain.GenerateRequest{
|
||||||
|
Prompt: prompt,
|
||||||
|
Target: domain.ExecutionTarget{Model: "benchmark-model"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "small_extras",
|
||||||
|
request: domain.GenerateRequest{
|
||||||
|
Prompt: prompt,
|
||||||
|
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
||||||
|
"priority": "low",
|
||||||
|
"attempt": 3,
|
||||||
|
"enabled": true,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "1MiB_string_extra",
|
||||||
|
request: domain.GenerateRequest{
|
||||||
|
Prompt: prompt,
|
||||||
|
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
||||||
|
"payload": strings.Repeat("x", 1<<20),
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nested_json_extras",
|
||||||
|
request: domain.GenerateRequest{
|
||||||
|
Prompt: prompt,
|
||||||
|
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
||||||
|
"options": map[string]any{
|
||||||
|
"enabled": true,
|
||||||
|
"limits": []any{2, 4, 8},
|
||||||
|
"tags": []string{"summary", "offline"},
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "structured_output",
|
||||||
|
request: domain.GenerateRequest{
|
||||||
|
Prompt: prompt,
|
||||||
|
Target: domain.ExecutionTarget{Model: "benchmark-model"},
|
||||||
|
StructuredOutput: &domain.StructuredOutputSpec{
|
||||||
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
|
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||||
|
Name: "summary",
|
||||||
|
Strict: true,
|
||||||
|
Schema: map[string]any{
|
||||||
|
"type": "object",
|
||||||
|
"properties": map[string]any{
|
||||||
|
"summary": map[string]any{"type": "string"},
|
||||||
|
"count": map[string]any{"type": "integer"},
|
||||||
|
},
|
||||||
|
"required": []string{"summary", "count"},
|
||||||
|
"additionalProperties": false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
b.Run(tc.name, func(b *testing.B) {
|
||||||
|
prepared, err := client.PrepareGeneration(tc.request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("check prepared request: %v", err)
|
||||||
|
}
|
||||||
|
if !json.Valid(prepared.payload) {
|
||||||
|
b.Fatal("prepared request is not valid JSON")
|
||||||
|
}
|
||||||
|
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(prepared.payload)))
|
||||||
|
b.ResetTimer()
|
||||||
|
for range b.N {
|
||||||
|
prepared, err := client.PrepareGeneration(tc.request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("prepare generation: %v", err)
|
||||||
|
}
|
||||||
|
prepareGenerationBenchmarkSink = prepared
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
runtime.KeepAlive(prepareGenerationBenchmarkSink)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user