Add private inference smoke cases

2026-10-13 17:30:45 +00:00
parent edaaecc982
commit 8d05b594e2
4 changed files with 275 additions and 0 deletions
--- a/internal/usecase/smoke_inference_case_test.go
+++ b/internal/usecase/smoke_inference_case_test.go
@@ -0,0 +1,127 @@
+package usecase
+
+import (
+	"context"
+	"reflect"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
+)
+
+func TestSmokeTextInferenceCaseBuildsRequestAndEvaluatesExactContent(t *testing.T) {
+	target := domain.SmokeTestTarget{
+		EffectiveSettings: domain.ExecutionTarget{
+			BackendID:      "selected-backend",
+			Endpoint:       "https://example.invalid/v1",
+			Model:          "selected-model",
+			TimeoutSeconds: 17,
+		},
+		SettingsPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true},
+	}
+	caseValue, err := prepareSmokeInferenceCase(context.Background(), domain.SmokeTestTextInference, target, 3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(caseValue.request.Prompt.Messages) != 1 || caseValue.request.Prompt.Messages[0].Role != "user" ||
+		!strings.Contains(caseValue.request.Prompt.Messages[0].Content, caseValue.expectedContent) {
+		t.Fatalf("private text request = %+v", caseValue.request.Prompt)
+	}
+	if caseValue.request.StructuredOutput != nil || caseValue.request.TransientRetries != 3 ||
+		caseValue.request.Generation != 1 || !reflect.DeepEqual(caseValue.request.Target, target.EffectiveSettings) ||
+		!reflect.DeepEqual(caseValue.request.TargetPresence, target.SettingsPresence) {
+		t.Fatalf("generation request = %+v", caseValue.request)
+	}
+
+	for _, tc := range []struct {
+		name             string
+		content          string
+		wantTextReturned bool
+		wantMatched      bool
+	}{
+		{name: "exact token", content: caseValue.expectedContent, wantTextReturned: true, wantMatched: true},
+		{name: "surrounding whitespace", content: " \n" + caseValue.expectedContent + "\t", wantTextReturned: true, wantMatched: true},
+		{name: "case differs", content: strings.ToLower(caseValue.expectedContent), wantTextReturned: true},
+		{name: "unexpected text", content: "another answer", wantTextReturned: true},
+		{name: "whitespace only", content: " \n\t"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			facts, err := caseValue.evaluate(context.Background(), &domain.GenerateResponse{Content: tc.content})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if facts.TextReturned == nil || *facts.TextReturned != tc.wantTextReturned ||
+				facts.ExpectedTextMatched == nil || *facts.ExpectedTextMatched != tc.wantMatched {
+				t.Fatalf("facts = %+v", facts)
+			}
+			if facts.ValidJSON != nil || facts.SchemaMatched != nil {
+				t.Fatalf("text evaluation populated structured facts: %+v", facts)
+			}
+		})
+	}
+
+	noResponseFacts, err := caseValue.evaluate(context.Background(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if noResponseFacts.TextReturned != nil || noResponseFacts.ExpectedTextMatched != nil {
+		t.Fatalf("facts were populated without generated content: %+v", noResponseFacts)
+	}
+}
+
+func TestSmokeStructuredInferenceCaseUsesPrivatePreparedSchema(t *testing.T) {
+	target := domain.SmokeTestTarget{
+		EffectiveSettings: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "selected-model"},
+		SettingsPresence:  domain.ExecutionTargetPresence{TimeoutSeconds: true},
+	}
+	caseValue, err := prepareSmokeInferenceCase(context.Background(), domain.SmokeTestStructuredOutput, target, 2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !caseValue.structuredOutputRequested() || caseValue.request.StructuredOutput.JSONSchema == nil ||
+		caseValue.request.StructuredOutput.JSONSchema.Name != "promptkit_smoke" ||
+		!caseValue.request.StructuredOutput.JSONSchema.Strict {
+		t.Fatalf("structured request = %+v", caseValue.request.StructuredOutput)
+	}
+	if len(caseValue.request.Prompt.Messages) != 1 || caseValue.request.Prompt.Messages[0].Role != "user" ||
+		!strings.Contains(caseValue.request.Prompt.Messages[0].Content, caseValue.expectedContent) {
+		t.Fatalf("private structured request = %+v", caseValue.request.Prompt)
+	}
+	if caseValue.request.TransientRetries != 2 || caseValue.request.Generation != 1 ||
+		!reflect.DeepEqual(caseValue.request.Target, target.EffectiveSettings) ||
+		!reflect.DeepEqual(caseValue.request.TargetPresence, target.SettingsPresence) {
+		t.Fatalf("generation request = %+v", caseValue.request)
+	}
+
+	additionalPropertyContent := strings.TrimSuffix(caseValue.expectedContent, "}") + `,"extra":true}`
+	for _, tc := range []struct {
+		name              string
+		content           string
+		wantValidJSON     bool
+		wantSchemaMatched *bool
+	}{
+		{name: "matches", content: caseValue.expectedContent, wantValidJSON: true, wantSchemaMatched: boolPointer(true)},
+		{name: "valid JSON misses schema", content: `{"status":"other"}`, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
+		{name: "additional property misses schema", content: additionalPropertyContent, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
+		{name: "invalid JSON", content: `{invalid`, wantSchemaMatched: nil},
+		{name: "empty content", content: "", wantSchemaMatched: nil},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			facts, err := caseValue.evaluate(context.Background(), &domain.GenerateResponse{Content: tc.content})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if facts.ValidJSON == nil || *facts.ValidJSON != tc.wantValidJSON || !sameBool(facts.SchemaMatched, tc.wantSchemaMatched) {
+				t.Fatalf("facts = %+v", facts)
+			}
+		})
+	}
+
+	noResponseFacts, err := caseValue.evaluate(context.Background(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if noResponseFacts.ValidJSON != nil || noResponseFacts.SchemaMatched != nil {
+		t.Fatalf("facts were populated without generated content: %+v", noResponseFacts)
+	}
+}