Add private inference smoke cases

internal/usecase/smoke_inference_case.go

@@ -0,0 +1,129 @@
+package usecase
+
+import (
+	"context"
+	"embed"
+	"fmt"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
+	"gitea.maximumdirect.net/eric/promptkit/internal/validate"
+)
+
+const (
+	smokeTextExpectedContent       = "PROMPTKIT_OK"
+	smokeTextPrompt                = "Reply with exactly the token PROMPTKIT_OK and no other text."
+	smokeStructuredExpectedContent = `{"status":"ok"}`
+	smokeStructuredPromptPrefix    = "Return exactly this JSON object: "
+	smokeStructuredSchemaPath      = "smoke_schema.json"
+)
+
+//go:embed smoke_schema.json
+var smokeSchemaFS embed.FS
+
+type smokeInferenceCase struct {
+	kind             domain.SmokeTestKind
+	request          domain.GenerateRequest
+	expectedContent  string
+	jsonValidator    validate.Validator
+	schemaValidation validate.PreparedValidation
+}
+
+func prepareSmokeInferenceCase(
+	ctx context.Context,
+	kind domain.SmokeTestKind,
+	target domain.SmokeTestTarget,
+	transientRetries uint64,
+) (*smokeInferenceCase, error) {
+	switch kind {
+	case domain.SmokeTestTextInference:
+		return &smokeInferenceCase{
+			kind:            kind,
+			expectedContent: smokeTextExpectedContent,
+			request: newGenerationRequest(
+				domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: smokeTextPrompt}}},
+				"",
+				target.EffectiveSettings,
+				target.SettingsPresence,
+				nil,
+				transientRetries,
+				1,
+			),
+		}, nil
+	case domain.SmokeTestStructuredOutput:
+		validator := validate.NewFSValidator(smokeSchemaFS, ".")
+		preparer, ok := validator.(validate.ValidationPreparer)
+		if !ok {
+			return nil, fmt.Errorf("%w: private validator does not support prepared validation", ErrValidation)
+		}
+		plan, err := preparer.PrepareValidation(ctx, domain.OutputContract{
+			Format:         domain.FormatJSON,
+			ValidationMode: domain.ValidationJSONSchema,
+			SchemaPath:     smokeStructuredSchemaPath,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("%w: prepare private structured-output schema: %w", ErrValidation, err)
+		}
+		structuredOutput := structuredOutputSpec(&domain.PromptDefinition{ID: "promptkit_smoke"}, plan.SchemaDocument())
+		return &smokeInferenceCase{
+			kind:             kind,
+			expectedContent:  smokeStructuredExpectedContent,
+			jsonValidator:    validate.NewStandardValidator(""),
+			schemaValidation: plan,
+			request: newGenerationRequest(
+				domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: smokeStructuredPromptPrefix + smokeStructuredExpectedContent}}},
+				"",
+				target.EffectiveSettings,
+				target.SettingsPresence,
+				structuredOutput,
+				transientRetries,
+				1,
+			),
+		}, nil
+	default:
+		return nil, fmt.Errorf("%w: smoke inference case does not support kind %q", ErrInvalidRequest, kind)
+	}
+}
+
+func (c *smokeInferenceCase) evaluate(ctx context.Context, response *domain.GenerateResponse) (domain.SmokeTestFacts, error) {
+	var facts domain.SmokeTestFacts
+	if c == nil || response == nil {
+		return facts, nil
+	}
+
+	switch c.kind {
+	case domain.SmokeTestTextInference:
+		content := strings.TrimSpace(response.Content)
+		textReturned := content != ""
+		matched := content == c.expectedContent
+		facts.TextReturned = boolPointer(textReturned)
+		facts.ExpectedTextMatched = boolPointer(matched)
+		return facts, nil
+	case domain.SmokeTestStructuredOutput:
+		artifact := domain.Artifact{Body: []byte(response.Content)}
+		jsonResult, err := c.jsonValidator.Validate(ctx, &artifact, domain.OutputContract{
+			Format:         domain.FormatJSON,
+			ValidationMode: domain.ValidationJSON,
+		})
+		if err != nil {
+			return facts, fmt.Errorf("%w: evaluate private smoke-test JSON: %w", ErrValidation, err)
+		}
+		validJSON := jsonResult.IsValid
+		facts.ValidJSON = boolPointer(validJSON)
+		if !validJSON {
+			return facts, nil
+		}
+		schemaResult, err := c.schemaValidation.Validate(ctx, &artifact)
+		if err != nil {
+			return facts, fmt.Errorf("%w: evaluate private smoke-test schema: %w", ErrValidation, err)
+		}
+		facts.SchemaMatched = boolPointer(schemaResult.IsValid)
+		return facts, nil
+	default:
+		return facts, fmt.Errorf("%w: smoke inference case does not support kind %q", ErrInvalidRequest, c.kind)
+	}
+}
+
+func (c *smokeInferenceCase) structuredOutputRequested() bool {
+	return c != nil && c.request.StructuredOutput != nil
+}