package usecase import ( "context" "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/promptkit/internal/domain" ) func TestSmokeTextInferenceCaseBuildsRequestAndEvaluatesExactContent(t *testing.T) { target := domain.SmokeTestTarget{ EffectiveSettings: domain.ExecutionTarget{ BackendID: "selected-backend", Endpoint: "https://example.invalid/v1", Model: "selected-model", TimeoutSeconds: 17, }, SettingsPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true}, } caseValue, err := prepareSmokeInferenceCase(context.Background(), domain.SmokeTestTextInference, target, 3) if err != nil { t.Fatal(err) } if len(caseValue.request.Prompt.Messages) != 1 || caseValue.request.Prompt.Messages[0].Role != "user" || !strings.Contains(caseValue.request.Prompt.Messages[0].Content, caseValue.expectedContent) { t.Fatalf("private text request = %+v", caseValue.request.Prompt) } if caseValue.request.StructuredOutput != nil || caseValue.request.TransientRetries != 3 || caseValue.request.Generation != 1 || !reflect.DeepEqual(caseValue.request.Target, target.EffectiveSettings) || !reflect.DeepEqual(caseValue.request.TargetPresence, target.SettingsPresence) { t.Fatalf("generation request = %+v", caseValue.request) } for _, tc := range []struct { name string content string wantTextReturned bool wantMatched bool }{ {name: "exact token", content: caseValue.expectedContent, wantTextReturned: true, wantMatched: true}, {name: "surrounding whitespace", content: " \n" + caseValue.expectedContent + "\t", wantTextReturned: true, wantMatched: true}, {name: "case differs", content: strings.ToLower(caseValue.expectedContent), wantTextReturned: true}, {name: "unexpected text", content: "another answer", wantTextReturned: true}, {name: "whitespace only", content: " \n\t"}, } { t.Run(tc.name, func(t *testing.T) { facts, err := caseValue.evaluate(context.Background(), &domain.GenerateResponse{Content: tc.content}) if err != nil { t.Fatal(err) } if facts.TextReturned == nil || *facts.TextReturned != tc.wantTextReturned || facts.ExpectedTextMatched == nil || *facts.ExpectedTextMatched != tc.wantMatched { t.Fatalf("facts = %+v", facts) } if facts.ValidJSON != nil || facts.SchemaMatched != nil { t.Fatalf("text evaluation populated structured facts: %+v", facts) } }) } noResponseFacts, err := caseValue.evaluate(context.Background(), nil) if err != nil { t.Fatal(err) } if noResponseFacts.TextReturned != nil || noResponseFacts.ExpectedTextMatched != nil { t.Fatalf("facts were populated without generated content: %+v", noResponseFacts) } } func TestSmokeStructuredInferenceCaseUsesPrivatePreparedSchema(t *testing.T) { target := domain.SmokeTestTarget{ EffectiveSettings: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "selected-model"}, SettingsPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true}, } caseValue, err := prepareSmokeInferenceCase(context.Background(), domain.SmokeTestStructuredOutput, target, 2) if err != nil { t.Fatal(err) } if !caseValue.structuredOutputRequested() || caseValue.request.StructuredOutput.JSONSchema == nil || caseValue.request.StructuredOutput.JSONSchema.Name != "promptkit_smoke" || !caseValue.request.StructuredOutput.JSONSchema.Strict { t.Fatalf("structured request = %+v", caseValue.request.StructuredOutput) } if len(caseValue.request.Prompt.Messages) != 1 || caseValue.request.Prompt.Messages[0].Role != "user" || !strings.Contains(caseValue.request.Prompt.Messages[0].Content, caseValue.expectedContent) { t.Fatalf("private structured request = %+v", caseValue.request.Prompt) } if caseValue.request.TransientRetries != 2 || caseValue.request.Generation != 1 || !reflect.DeepEqual(caseValue.request.Target, target.EffectiveSettings) || !reflect.DeepEqual(caseValue.request.TargetPresence, target.SettingsPresence) { t.Fatalf("generation request = %+v", caseValue.request) } additionalPropertyContent := strings.TrimSuffix(caseValue.expectedContent, "}") + `,"extra":true}` for _, tc := range []struct { name string content string wantValidJSON bool wantSchemaMatched *bool }{ {name: "matches", content: caseValue.expectedContent, wantValidJSON: true, wantSchemaMatched: boolPointer(true)}, {name: "valid JSON misses schema", content: `{"status":"other"}`, wantValidJSON: true, wantSchemaMatched: boolPointer(false)}, {name: "additional property misses schema", content: additionalPropertyContent, wantValidJSON: true, wantSchemaMatched: boolPointer(false)}, {name: "invalid JSON", content: `{invalid`, wantSchemaMatched: nil}, {name: "empty content", content: "", wantSchemaMatched: nil}, } { t.Run(tc.name, func(t *testing.T) { facts, err := caseValue.evaluate(context.Background(), &domain.GenerateResponse{Content: tc.content}) if err != nil { t.Fatal(err) } if facts.ValidJSON == nil || *facts.ValidJSON != tc.wantValidJSON || !sameBool(facts.SchemaMatched, tc.wantSchemaMatched) { t.Fatalf("facts = %+v", facts) } }) } noResponseFacts, err := caseValue.evaluate(context.Background(), nil) if err != nil { t.Fatal(err) } if noResponseFacts.ValidJSON != nil || noResponseFacts.SchemaMatched != nil { t.Fatalf("facts were populated without generated content: %+v", noResponseFacts) } }