Add request preparation baseline and boundary tests
This commit is contained in:
@@ -1113,6 +1113,13 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te
|
||||
}{
|
||||
{name: "empty key", extraParams: map[string]any{"": "empty"}, want: "key must not be empty"},
|
||||
{name: "unserializable value", extraParams: map[string]any{"bad": math.Inf(1)}, want: "JSON-serializable"},
|
||||
{
|
||||
name: "nested unserializable value",
|
||||
extraParams: map[string]any{
|
||||
"nested": map[string]any{"values": []any{"valid", math.NaN()}},
|
||||
},
|
||||
want: "extra_params.nested",
|
||||
},
|
||||
}
|
||||
for _, key := range []string{
|
||||
"model",
|
||||
@@ -1161,6 +1168,56 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientClassifiesSchemaEncodingFailures(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
extraParams map[string]any
|
||||
want error
|
||||
unwanted error
|
||||
}{
|
||||
{
|
||||
name: "serializable extra keeps full-request failure",
|
||||
extraParams: map[string]any{"client_option": "accepted"},
|
||||
want: ErrRequestFailed,
|
||||
unwanted: ErrInvalidRequest,
|
||||
},
|
||||
{
|
||||
name: "invalid extra keeps invalid-request failure",
|
||||
extraParams: map[string]any{"nested": map[string]any{"bad": math.Inf(1)}},
|
||||
want: ErrInvalidRequest,
|
||||
unwanted: ErrRequestFailed,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
provider := newRecordingProvider(t)
|
||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
||||
request := ordinaryGenerateRequest()
|
||||
request.Target.ExtraParams = tc.extraParams
|
||||
request.StructuredOutput = &domain.StructuredOutputSpec{
|
||||
Type: domain.StructuredOutputJSONSchema,
|
||||
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||
Name: "invalid-schema",
|
||||
Strict: true,
|
||||
Schema: json.RawMessage("{"),
|
||||
},
|
||||
}
|
||||
|
||||
_, err := client.Generate(context.Background(), request)
|
||||
if err == nil || !errors.Is(err, tc.want) {
|
||||
t.Fatalf("error = %v, want identity %v", err, tc.want)
|
||||
}
|
||||
if errors.Is(err, tc.unwanted) {
|
||||
t.Fatalf("error = %v, unexpectedly has identity %v", err, tc.unwanted)
|
||||
}
|
||||
if calls := provider.requestCount(); calls != 0 {
|
||||
t.Fatalf("provider calls = %d, want 0", calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func checkConfiguredModelFallback(t *testing.T) {
|
||||
provider := newRecordingProvider(t)
|
||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "default-model"})
|
||||
|
||||
119
internal/llm/prepare_generation_benchmark_test.go
Normal file
119
internal/llm/prepare_generation_benchmark_test.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
var prepareGenerationBenchmarkSink *PreparedGeneration
|
||||
|
||||
func BenchmarkPrepareGeneration(b *testing.B) {
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||
BaseURL: "https://api.example.test/v1",
|
||||
Model: "benchmark-model",
|
||||
})
|
||||
if err != nil {
|
||||
b.Fatalf("construct benchmark client: %v", err)
|
||||
}
|
||||
|
||||
prompt := domain.RenderedPrompt{
|
||||
Messages: []domain.RenderedMessage{{Role: "user", Content: "Summarize this request."}},
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
request domain.GenerateRequest
|
||||
}{
|
||||
{
|
||||
name: "no_extras",
|
||||
request: domain.GenerateRequest{
|
||||
Prompt: prompt,
|
||||
Target: domain.ExecutionTarget{Model: "benchmark-model"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "small_extras",
|
||||
request: domain.GenerateRequest{
|
||||
Prompt: prompt,
|
||||
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
||||
"priority": "low",
|
||||
"attempt": 3,
|
||||
"enabled": true,
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "1MiB_string_extra",
|
||||
request: domain.GenerateRequest{
|
||||
Prompt: prompt,
|
||||
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
||||
"payload": strings.Repeat("x", 1<<20),
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "nested_json_extras",
|
||||
request: domain.GenerateRequest{
|
||||
Prompt: prompt,
|
||||
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
||||
"options": map[string]any{
|
||||
"enabled": true,
|
||||
"limits": []any{2, 4, 8},
|
||||
"tags": []string{"summary", "offline"},
|
||||
},
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "structured_output",
|
||||
request: domain.GenerateRequest{
|
||||
Prompt: prompt,
|
||||
Target: domain.ExecutionTarget{Model: "benchmark-model"},
|
||||
StructuredOutput: &domain.StructuredOutputSpec{
|
||||
Type: domain.StructuredOutputJSONSchema,
|
||||
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||
Name: "summary",
|
||||
Strict: true,
|
||||
Schema: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"summary": map[string]any{"type": "string"},
|
||||
"count": map[string]any{"type": "integer"},
|
||||
},
|
||||
"required": []string{"summary", "count"},
|
||||
"additionalProperties": false,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
b.Run(tc.name, func(b *testing.B) {
|
||||
prepared, err := client.PrepareGeneration(tc.request)
|
||||
if err != nil {
|
||||
b.Fatalf("check prepared request: %v", err)
|
||||
}
|
||||
if !json.Valid(prepared.payload) {
|
||||
b.Fatal("prepared request is not valid JSON")
|
||||
}
|
||||
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(prepared.payload)))
|
||||
b.ResetTimer()
|
||||
for range b.N {
|
||||
prepared, err := client.PrepareGeneration(tc.request)
|
||||
if err != nil {
|
||||
b.Fatalf("prepare generation: %v", err)
|
||||
}
|
||||
prepareGenerationBenchmarkSink = prepared
|
||||
}
|
||||
b.StopTimer()
|
||||
runtime.KeepAlive(prepareGenerationBenchmarkSink)
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user