Add private inference smoke cases

docs/roadmap/implementation.md

@@ -566,6 +566,13 @@ schema failure, and success using real validators. Derive generated success
 fixtures from the private case rather than spread private literals through tests.
 No domain criteria, configurable prompts, public fixtures, or repair loop.
 
+**Completed: 2026-10-13.** Added private text and structured inference cases,
+including an embedded schema prepared through `NewFSValidator`. Generation
+requests retain the selected target and retry allowance; local evaluation keeps
+JSON syntax and schema matching distinct. Validation passed: `go test -race
+./internal/usecase ./internal/validate`, `go vet ./...`, `go test ./...`, and
+`git diff --check`.
+
 ## Stage 14: Orchestrate HTTP Diagnostic Reports
 
 Scope: `internal/usecase.SmokeTester` and its HTTP operations.

internal/usecase/smoke_inference_case.go

@@ -0,0 +1,129 @@
+package usecase
+
+import (
+	"context"
+	"embed"
+	"fmt"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
+	"gitea.maximumdirect.net/eric/promptkit/internal/validate"
+)
+
+const (
+	smokeTextExpectedContent       = "PROMPTKIT_OK"
+	smokeTextPrompt                = "Reply with exactly the token PROMPTKIT_OK and no other text."
+	smokeStructuredExpectedContent = `{"status":"ok"}`
+	smokeStructuredPromptPrefix    = "Return exactly this JSON object: "
+	smokeStructuredSchemaPath      = "smoke_schema.json"
+)
+
+//go:embed smoke_schema.json
+var smokeSchemaFS embed.FS
+
+type smokeInferenceCase struct {
+	kind             domain.SmokeTestKind
+	request          domain.GenerateRequest
+	expectedContent  string
+	jsonValidator    validate.Validator
+	schemaValidation validate.PreparedValidation
+}
+
+func prepareSmokeInferenceCase(
+	ctx context.Context,
+	kind domain.SmokeTestKind,
+	target domain.SmokeTestTarget,
+	transientRetries uint64,
+) (*smokeInferenceCase, error) {
+	switch kind {
+	case domain.SmokeTestTextInference:
+		return &smokeInferenceCase{
+			kind:            kind,
+			expectedContent: smokeTextExpectedContent,
+			request: newGenerationRequest(
+				domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: smokeTextPrompt}}},
+				"",
+				target.EffectiveSettings,
+				target.SettingsPresence,
+				nil,
+				transientRetries,
+				1,
+			),
+		}, nil
+	case domain.SmokeTestStructuredOutput:
+		validator := validate.NewFSValidator(smokeSchemaFS, ".")
+		preparer, ok := validator.(validate.ValidationPreparer)
+		if !ok {
+			return nil, fmt.Errorf("%w: private validator does not support prepared validation", ErrValidation)
+		}
+		plan, err := preparer.PrepareValidation(ctx, domain.OutputContract{
+			Format:         domain.FormatJSON,
+			ValidationMode: domain.ValidationJSONSchema,
+			SchemaPath:     smokeStructuredSchemaPath,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("%w: prepare private structured-output schema: %w", ErrValidation, err)
+		}
+		structuredOutput := structuredOutputSpec(&domain.PromptDefinition{ID: "promptkit_smoke"}, plan.SchemaDocument())
+		return &smokeInferenceCase{
+			kind:             kind,
+			expectedContent:  smokeStructuredExpectedContent,
+			jsonValidator:    validate.NewStandardValidator(""),
+			schemaValidation: plan,
+			request: newGenerationRequest(
+				domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: smokeStructuredPromptPrefix + smokeStructuredExpectedContent}}},
+				"",
+				target.EffectiveSettings,
+				target.SettingsPresence,
+				structuredOutput,
+				transientRetries,
+				1,
+			),
+		}, nil
+	default:
+		return nil, fmt.Errorf("%w: smoke inference case does not support kind %q", ErrInvalidRequest, kind)
+	}
+}
+
+func (c *smokeInferenceCase) evaluate(ctx context.Context, response *domain.GenerateResponse) (domain.SmokeTestFacts, error) {
+	var facts domain.SmokeTestFacts
+	if c == nil || response == nil {
+		return facts, nil
+	}
+
+	switch c.kind {
+	case domain.SmokeTestTextInference:
+		content := strings.TrimSpace(response.Content)
+		textReturned := content != ""
+		matched := content == c.expectedContent
+		facts.TextReturned = boolPointer(textReturned)
+		facts.ExpectedTextMatched = boolPointer(matched)
+		return facts, nil
+	case domain.SmokeTestStructuredOutput:
+		artifact := domain.Artifact{Body: []byte(response.Content)}
+		jsonResult, err := c.jsonValidator.Validate(ctx, &artifact, domain.OutputContract{
+			Format:         domain.FormatJSON,
+			ValidationMode: domain.ValidationJSON,
+		})
+		if err != nil {
+			return facts, fmt.Errorf("%w: evaluate private smoke-test JSON: %w", ErrValidation, err)
+		}
+		validJSON := jsonResult.IsValid
+		facts.ValidJSON = boolPointer(validJSON)
+		if !validJSON {
+			return facts, nil
+		}
+		schemaResult, err := c.schemaValidation.Validate(ctx, &artifact)
+		if err != nil {
+			return facts, fmt.Errorf("%w: evaluate private smoke-test schema: %w", ErrValidation, err)
+		}
+		facts.SchemaMatched = boolPointer(schemaResult.IsValid)
+		return facts, nil
+	default:
+		return facts, fmt.Errorf("%w: smoke inference case does not support kind %q", ErrInvalidRequest, c.kind)
+	}
+}
+
+func (c *smokeInferenceCase) structuredOutputRequested() bool {
+	return c != nil && c.request.StructuredOutput != nil
+}

internal/usecase/smoke_inference_case_test.go

@@ -0,0 +1,127 @@
+package usecase
+
+import (
+	"context"
+	"reflect"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
+)
+
+func TestSmokeTextInferenceCaseBuildsRequestAndEvaluatesExactContent(t *testing.T) {
+	target := domain.SmokeTestTarget{
+		EffectiveSettings: domain.ExecutionTarget{
+			BackendID:      "selected-backend",
+			Endpoint:       "https://example.invalid/v1",
+			Model:          "selected-model",
+			TimeoutSeconds: 17,
+		},
+		SettingsPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true},
+	}
+	caseValue, err := prepareSmokeInferenceCase(context.Background(), domain.SmokeTestTextInference, target, 3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(caseValue.request.Prompt.Messages) != 1 || caseValue.request.Prompt.Messages[0].Role != "user" ||
+		!strings.Contains(caseValue.request.Prompt.Messages[0].Content, caseValue.expectedContent) {
+		t.Fatalf("private text request = %+v", caseValue.request.Prompt)
+	}
+	if caseValue.request.StructuredOutput != nil || caseValue.request.TransientRetries != 3 ||
+		caseValue.request.Generation != 1 || !reflect.DeepEqual(caseValue.request.Target, target.EffectiveSettings) ||
+		!reflect.DeepEqual(caseValue.request.TargetPresence, target.SettingsPresence) {
+		t.Fatalf("generation request = %+v", caseValue.request)
+	}
+
+	for _, tc := range []struct {
+		name             string
+		content          string
+		wantTextReturned bool
+		wantMatched      bool
+	}{
+		{name: "exact token", content: caseValue.expectedContent, wantTextReturned: true, wantMatched: true},
+		{name: "surrounding whitespace", content: " \n" + caseValue.expectedContent + "\t", wantTextReturned: true, wantMatched: true},
+		{name: "case differs", content: strings.ToLower(caseValue.expectedContent), wantTextReturned: true},
+		{name: "unexpected text", content: "another answer", wantTextReturned: true},
+		{name: "whitespace only", content: " \n\t"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			facts, err := caseValue.evaluate(context.Background(), &domain.GenerateResponse{Content: tc.content})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if facts.TextReturned == nil || *facts.TextReturned != tc.wantTextReturned ||
+				facts.ExpectedTextMatched == nil || *facts.ExpectedTextMatched != tc.wantMatched {
+				t.Fatalf("facts = %+v", facts)
+			}
+			if facts.ValidJSON != nil || facts.SchemaMatched != nil {
+				t.Fatalf("text evaluation populated structured facts: %+v", facts)
+			}
+		})
+	}
+
+	noResponseFacts, err := caseValue.evaluate(context.Background(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if noResponseFacts.TextReturned != nil || noResponseFacts.ExpectedTextMatched != nil {
+		t.Fatalf("facts were populated without generated content: %+v", noResponseFacts)
+	}
+}
+
+func TestSmokeStructuredInferenceCaseUsesPrivatePreparedSchema(t *testing.T) {
+	target := domain.SmokeTestTarget{
+		EffectiveSettings: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "selected-model"},
+		SettingsPresence:  domain.ExecutionTargetPresence{TimeoutSeconds: true},
+	}
+	caseValue, err := prepareSmokeInferenceCase(context.Background(), domain.SmokeTestStructuredOutput, target, 2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !caseValue.structuredOutputRequested() || caseValue.request.StructuredOutput.JSONSchema == nil ||
+		caseValue.request.StructuredOutput.JSONSchema.Name != "promptkit_smoke" ||
+		!caseValue.request.StructuredOutput.JSONSchema.Strict {
+		t.Fatalf("structured request = %+v", caseValue.request.StructuredOutput)
+	}
+	if len(caseValue.request.Prompt.Messages) != 1 || caseValue.request.Prompt.Messages[0].Role != "user" ||
+		!strings.Contains(caseValue.request.Prompt.Messages[0].Content, caseValue.expectedContent) {
+		t.Fatalf("private structured request = %+v", caseValue.request.Prompt)
+	}
+	if caseValue.request.TransientRetries != 2 || caseValue.request.Generation != 1 ||
+		!reflect.DeepEqual(caseValue.request.Target, target.EffectiveSettings) ||
+		!reflect.DeepEqual(caseValue.request.TargetPresence, target.SettingsPresence) {
+		t.Fatalf("generation request = %+v", caseValue.request)
+	}
+
+	additionalPropertyContent := strings.TrimSuffix(caseValue.expectedContent, "}") + `,"extra":true}`
+	for _, tc := range []struct {
+		name              string
+		content           string
+		wantValidJSON     bool
+		wantSchemaMatched *bool
+	}{
+		{name: "matches", content: caseValue.expectedContent, wantValidJSON: true, wantSchemaMatched: boolPointer(true)},
+		{name: "valid JSON misses schema", content: `{"status":"other"}`, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
+		{name: "additional property misses schema", content: additionalPropertyContent, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
+		{name: "invalid JSON", content: `{invalid`, wantSchemaMatched: nil},
+		{name: "empty content", content: "", wantSchemaMatched: nil},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			facts, err := caseValue.evaluate(context.Background(), &domain.GenerateResponse{Content: tc.content})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if facts.ValidJSON == nil || *facts.ValidJSON != tc.wantValidJSON || !sameBool(facts.SchemaMatched, tc.wantSchemaMatched) {
+				t.Fatalf("facts = %+v", facts)
+			}
+		})
+	}
+
+	noResponseFacts, err := caseValue.evaluate(context.Background(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if noResponseFacts.ValidJSON != nil || noResponseFacts.SchemaMatched != nil {
+		t.Fatalf("facts were populated without generated content: %+v", noResponseFacts)
+	}
+}

internal/usecase/smoke_schema.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "status": {
+      "type": "string",
+      "const": "ok"
+    }
+  },
+  "required": ["status"],
+  "additionalProperties": false
+}