Add private inference smoke cases

internal/usecase/smoke_inference_case.go

@@ -0,0 +1,129 @@
+package usecase
+
+import (
+	"context"
+	"embed"
+	"fmt"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
+	"gitea.maximumdirect.net/eric/promptkit/internal/validate"
+)
+
+const (
+	smokeTextExpectedContent       = "PROMPTKIT_OK"
+	smokeTextPrompt                = "Reply with exactly the token PROMPTKIT_OK and no other text."
+	smokeStructuredExpectedContent = `{"status":"ok"}`
+	smokeStructuredPromptPrefix    = "Return exactly this JSON object: "
+	smokeStructuredSchemaPath      = "smoke_schema.json"
+)
+
+//go:embed smoke_schema.json
+var smokeSchemaFS embed.FS
+
+type smokeInferenceCase struct {
+	kind             domain.SmokeTestKind
+	request          domain.GenerateRequest
+	expectedContent  string
+	jsonValidator    validate.Validator
+	schemaValidation validate.PreparedValidation
+}
+
+func prepareSmokeInferenceCase(
+	ctx context.Context,
+	kind domain.SmokeTestKind,
+	target domain.SmokeTestTarget,
+	transientRetries uint64,
+) (*smokeInferenceCase, error) {
+	switch kind {
+	case domain.SmokeTestTextInference:
+		return &smokeInferenceCase{
+			kind:            kind,
+			expectedContent: smokeTextExpectedContent,
+			request: newGenerationRequest(
+				domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: smokeTextPrompt}}},
+				"",
+				target.EffectiveSettings,
+				target.SettingsPresence,
+				nil,
+				transientRetries,
+				1,
+			),
+		}, nil
+	case domain.SmokeTestStructuredOutput:
+		validator := validate.NewFSValidator(smokeSchemaFS, ".")
+		preparer, ok := validator.(validate.ValidationPreparer)
+		if !ok {
+			return nil, fmt.Errorf("%w: private validator does not support prepared validation", ErrValidation)
+		}
+		plan, err := preparer.PrepareValidation(ctx, domain.OutputContract{
+			Format:         domain.FormatJSON,
+			ValidationMode: domain.ValidationJSONSchema,
+			SchemaPath:     smokeStructuredSchemaPath,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("%w: prepare private structured-output schema: %w", ErrValidation, err)
+		}
+		structuredOutput := structuredOutputSpec(&domain.PromptDefinition{ID: "promptkit_smoke"}, plan.SchemaDocument())
+		return &smokeInferenceCase{
+			kind:             kind,
+			expectedContent:  smokeStructuredExpectedContent,
+			jsonValidator:    validate.NewStandardValidator(""),
+			schemaValidation: plan,
+			request: newGenerationRequest(
+				domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: smokeStructuredPromptPrefix + smokeStructuredExpectedContent}}},
+				"",
+				target.EffectiveSettings,
+				target.SettingsPresence,
+				structuredOutput,
+				transientRetries,
+				1,
+			),
+		}, nil
+	default:
+		return nil, fmt.Errorf("%w: smoke inference case does not support kind %q", ErrInvalidRequest, kind)
+	}
+}
+
+func (c *smokeInferenceCase) evaluate(ctx context.Context, response *domain.GenerateResponse) (domain.SmokeTestFacts, error) {
+	var facts domain.SmokeTestFacts
+	if c == nil || response == nil {
+		return facts, nil
+	}
+
+	switch c.kind {
+	case domain.SmokeTestTextInference:
+		content := strings.TrimSpace(response.Content)
+		textReturned := content != ""
+		matched := content == c.expectedContent
+		facts.TextReturned = boolPointer(textReturned)
+		facts.ExpectedTextMatched = boolPointer(matched)
+		return facts, nil
+	case domain.SmokeTestStructuredOutput:
+		artifact := domain.Artifact{Body: []byte(response.Content)}
+		jsonResult, err := c.jsonValidator.Validate(ctx, &artifact, domain.OutputContract{
+			Format:         domain.FormatJSON,
+			ValidationMode: domain.ValidationJSON,
+		})
+		if err != nil {
+			return facts, fmt.Errorf("%w: evaluate private smoke-test JSON: %w", ErrValidation, err)
+		}
+		validJSON := jsonResult.IsValid
+		facts.ValidJSON = boolPointer(validJSON)
+		if !validJSON {
+			return facts, nil
+		}
+		schemaResult, err := c.schemaValidation.Validate(ctx, &artifact)
+		if err != nil {
+			return facts, fmt.Errorf("%w: evaluate private smoke-test schema: %w", ErrValidation, err)
+		}
+		facts.SchemaMatched = boolPointer(schemaResult.IsValid)
+		return facts, nil
+	default:
+		return facts, fmt.Errorf("%w: smoke inference case does not support kind %q", ErrInvalidRequest, c.kind)
+	}
+}
+
+func (c *smokeInferenceCase) structuredOutputRequested() bool {
+	return c != nil && c.request.StructuredOutput != nil
+}

internal/usecase/smoke_inference_case_test.go

@@ -0,0 +1,127 @@
+package usecase
+
+import (
+	"context"
+	"reflect"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
+)
+
+func TestSmokeTextInferenceCaseBuildsRequestAndEvaluatesExactContent(t *testing.T) {
+	target := domain.SmokeTestTarget{
+		EffectiveSettings: domain.ExecutionTarget{
+			BackendID:      "selected-backend",
+			Endpoint:       "https://example.invalid/v1",
+			Model:          "selected-model",
+			TimeoutSeconds: 17,
+		},
+		SettingsPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true},
+	}
+	caseValue, err := prepareSmokeInferenceCase(context.Background(), domain.SmokeTestTextInference, target, 3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(caseValue.request.Prompt.Messages) != 1 || caseValue.request.Prompt.Messages[0].Role != "user" ||
+		!strings.Contains(caseValue.request.Prompt.Messages[0].Content, caseValue.expectedContent) {
+		t.Fatalf("private text request = %+v", caseValue.request.Prompt)
+	}
+	if caseValue.request.StructuredOutput != nil || caseValue.request.TransientRetries != 3 ||
+		caseValue.request.Generation != 1 || !reflect.DeepEqual(caseValue.request.Target, target.EffectiveSettings) ||
+		!reflect.DeepEqual(caseValue.request.TargetPresence, target.SettingsPresence) {
+		t.Fatalf("generation request = %+v", caseValue.request)
+	}
+
+	for _, tc := range []struct {
+		name             string
+		content          string
+		wantTextReturned bool
+		wantMatched      bool
+	}{
+		{name: "exact token", content: caseValue.expectedContent, wantTextReturned: true, wantMatched: true},
+		{name: "surrounding whitespace", content: " \n" + caseValue.expectedContent + "\t", wantTextReturned: true, wantMatched: true},
+		{name: "case differs", content: strings.ToLower(caseValue.expectedContent), wantTextReturned: true},
+		{name: "unexpected text", content: "another answer", wantTextReturned: true},
+		{name: "whitespace only", content: " \n\t"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			facts, err := caseValue.evaluate(context.Background(), &domain.GenerateResponse{Content: tc.content})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if facts.TextReturned == nil || *facts.TextReturned != tc.wantTextReturned ||
+				facts.ExpectedTextMatched == nil || *facts.ExpectedTextMatched != tc.wantMatched {
+				t.Fatalf("facts = %+v", facts)
+			}
+			if facts.ValidJSON != nil || facts.SchemaMatched != nil {
+				t.Fatalf("text evaluation populated structured facts: %+v", facts)
+			}
+		})
+	}
+
+	noResponseFacts, err := caseValue.evaluate(context.Background(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if noResponseFacts.TextReturned != nil || noResponseFacts.ExpectedTextMatched != nil {
+		t.Fatalf("facts were populated without generated content: %+v", noResponseFacts)
+	}
+}
+
+func TestSmokeStructuredInferenceCaseUsesPrivatePreparedSchema(t *testing.T) {
+	target := domain.SmokeTestTarget{
+		EffectiveSettings: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "selected-model"},
+		SettingsPresence:  domain.ExecutionTargetPresence{TimeoutSeconds: true},
+	}
+	caseValue, err := prepareSmokeInferenceCase(context.Background(), domain.SmokeTestStructuredOutput, target, 2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !caseValue.structuredOutputRequested() || caseValue.request.StructuredOutput.JSONSchema == nil ||
+		caseValue.request.StructuredOutput.JSONSchema.Name != "promptkit_smoke" ||
+		!caseValue.request.StructuredOutput.JSONSchema.Strict {
+		t.Fatalf("structured request = %+v", caseValue.request.StructuredOutput)
+	}
+	if len(caseValue.request.Prompt.Messages) != 1 || caseValue.request.Prompt.Messages[0].Role != "user" ||
+		!strings.Contains(caseValue.request.Prompt.Messages[0].Content, caseValue.expectedContent) {
+		t.Fatalf("private structured request = %+v", caseValue.request.Prompt)
+	}
+	if caseValue.request.TransientRetries != 2 || caseValue.request.Generation != 1 ||
+		!reflect.DeepEqual(caseValue.request.Target, target.EffectiveSettings) ||
+		!reflect.DeepEqual(caseValue.request.TargetPresence, target.SettingsPresence) {
+		t.Fatalf("generation request = %+v", caseValue.request)
+	}
+
+	additionalPropertyContent := strings.TrimSuffix(caseValue.expectedContent, "}") + `,"extra":true}`
+	for _, tc := range []struct {
+		name              string
+		content           string
+		wantValidJSON     bool
+		wantSchemaMatched *bool
+	}{
+		{name: "matches", content: caseValue.expectedContent, wantValidJSON: true, wantSchemaMatched: boolPointer(true)},
+		{name: "valid JSON misses schema", content: `{"status":"other"}`, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
+		{name: "additional property misses schema", content: additionalPropertyContent, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
+		{name: "invalid JSON", content: `{invalid`, wantSchemaMatched: nil},
+		{name: "empty content", content: "", wantSchemaMatched: nil},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			facts, err := caseValue.evaluate(context.Background(), &domain.GenerateResponse{Content: tc.content})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if facts.ValidJSON == nil || *facts.ValidJSON != tc.wantValidJSON || !sameBool(facts.SchemaMatched, tc.wantSchemaMatched) {
+				t.Fatalf("facts = %+v", facts)
+			}
+		})
+	}
+
+	noResponseFacts, err := caseValue.evaluate(context.Background(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if noResponseFacts.ValidJSON != nil || noResponseFacts.SchemaMatched != nil {
+		t.Fatalf("facts were populated without generated content: %+v", noResponseFacts)
+	}
+}

internal/usecase/smoke_schema.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "status": {
+      "type": "string",
+      "const": "ok"
+    }
+  },
+  "required": ["status"],
+  "additionalProperties": false
+}