diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 76efa11..d3bfd7a 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -566,6 +566,13 @@ schema failure, and success using real validators. Derive generated success fixtures from the private case rather than spread private literals through tests. No domain criteria, configurable prompts, public fixtures, or repair loop. +**Completed: 2026-10-13.** Added private text and structured inference cases, +including an embedded schema prepared through `NewFSValidator`. Generation +requests retain the selected target and retry allowance; local evaluation keeps +JSON syntax and schema matching distinct. Validation passed: `go test -race +./internal/usecase ./internal/validate`, `go vet ./...`, `go test ./...`, and +`git diff --check`. + ## Stage 14: Orchestrate HTTP Diagnostic Reports Scope: `internal/usecase.SmokeTester` and its HTTP operations. diff --git a/internal/usecase/smoke_inference_case.go b/internal/usecase/smoke_inference_case.go new file mode 100644 index 0000000..6e4bab8 --- /dev/null +++ b/internal/usecase/smoke_inference_case.go @@ -0,0 +1,129 @@ +package usecase + +import ( + "context" + "embed" + "fmt" + "strings" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" + "gitea.maximumdirect.net/eric/promptkit/internal/validate" +) + +const ( + smokeTextExpectedContent = "PROMPTKIT_OK" + smokeTextPrompt = "Reply with exactly the token PROMPTKIT_OK and no other text." + smokeStructuredExpectedContent = `{"status":"ok"}` + smokeStructuredPromptPrefix = "Return exactly this JSON object: " + smokeStructuredSchemaPath = "smoke_schema.json" +) + +//go:embed smoke_schema.json +var smokeSchemaFS embed.FS + +type smokeInferenceCase struct { + kind domain.SmokeTestKind + request domain.GenerateRequest + expectedContent string + jsonValidator validate.Validator + schemaValidation validate.PreparedValidation +} + +func prepareSmokeInferenceCase( + ctx context.Context, + kind domain.SmokeTestKind, + target domain.SmokeTestTarget, + transientRetries uint64, +) (*smokeInferenceCase, error) { + switch kind { + case domain.SmokeTestTextInference: + return &smokeInferenceCase{ + kind: kind, + expectedContent: smokeTextExpectedContent, + request: newGenerationRequest( + domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: smokeTextPrompt}}}, + "", + target.EffectiveSettings, + target.SettingsPresence, + nil, + transientRetries, + 1, + ), + }, nil + case domain.SmokeTestStructuredOutput: + validator := validate.NewFSValidator(smokeSchemaFS, ".") + preparer, ok := validator.(validate.ValidationPreparer) + if !ok { + return nil, fmt.Errorf("%w: private validator does not support prepared validation", ErrValidation) + } + plan, err := preparer.PrepareValidation(ctx, domain.OutputContract{ + Format: domain.FormatJSON, + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: smokeStructuredSchemaPath, + }) + if err != nil { + return nil, fmt.Errorf("%w: prepare private structured-output schema: %w", ErrValidation, err) + } + structuredOutput := structuredOutputSpec(&domain.PromptDefinition{ID: "promptkit_smoke"}, plan.SchemaDocument()) + return &smokeInferenceCase{ + kind: kind, + expectedContent: smokeStructuredExpectedContent, + jsonValidator: validate.NewStandardValidator(""), + schemaValidation: plan, + request: newGenerationRequest( + domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: smokeStructuredPromptPrefix + smokeStructuredExpectedContent}}}, + "", + target.EffectiveSettings, + target.SettingsPresence, + structuredOutput, + transientRetries, + 1, + ), + }, nil + default: + return nil, fmt.Errorf("%w: smoke inference case does not support kind %q", ErrInvalidRequest, kind) + } +} + +func (c *smokeInferenceCase) evaluate(ctx context.Context, response *domain.GenerateResponse) (domain.SmokeTestFacts, error) { + var facts domain.SmokeTestFacts + if c == nil || response == nil { + return facts, nil + } + + switch c.kind { + case domain.SmokeTestTextInference: + content := strings.TrimSpace(response.Content) + textReturned := content != "" + matched := content == c.expectedContent + facts.TextReturned = boolPointer(textReturned) + facts.ExpectedTextMatched = boolPointer(matched) + return facts, nil + case domain.SmokeTestStructuredOutput: + artifact := domain.Artifact{Body: []byte(response.Content)} + jsonResult, err := c.jsonValidator.Validate(ctx, &artifact, domain.OutputContract{ + Format: domain.FormatJSON, + ValidationMode: domain.ValidationJSON, + }) + if err != nil { + return facts, fmt.Errorf("%w: evaluate private smoke-test JSON: %w", ErrValidation, err) + } + validJSON := jsonResult.IsValid + facts.ValidJSON = boolPointer(validJSON) + if !validJSON { + return facts, nil + } + schemaResult, err := c.schemaValidation.Validate(ctx, &artifact) + if err != nil { + return facts, fmt.Errorf("%w: evaluate private smoke-test schema: %w", ErrValidation, err) + } + facts.SchemaMatched = boolPointer(schemaResult.IsValid) + return facts, nil + default: + return facts, fmt.Errorf("%w: smoke inference case does not support kind %q", ErrInvalidRequest, c.kind) + } +} + +func (c *smokeInferenceCase) structuredOutputRequested() bool { + return c != nil && c.request.StructuredOutput != nil +} diff --git a/internal/usecase/smoke_inference_case_test.go b/internal/usecase/smoke_inference_case_test.go new file mode 100644 index 0000000..15672b4 --- /dev/null +++ b/internal/usecase/smoke_inference_case_test.go @@ -0,0 +1,127 @@ +package usecase + +import ( + "context" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" +) + +func TestSmokeTextInferenceCaseBuildsRequestAndEvaluatesExactContent(t *testing.T) { + target := domain.SmokeTestTarget{ + EffectiveSettings: domain.ExecutionTarget{ + BackendID: "selected-backend", + Endpoint: "https://example.invalid/v1", + Model: "selected-model", + TimeoutSeconds: 17, + }, + SettingsPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true}, + } + caseValue, err := prepareSmokeInferenceCase(context.Background(), domain.SmokeTestTextInference, target, 3) + if err != nil { + t.Fatal(err) + } + if len(caseValue.request.Prompt.Messages) != 1 || caseValue.request.Prompt.Messages[0].Role != "user" || + !strings.Contains(caseValue.request.Prompt.Messages[0].Content, caseValue.expectedContent) { + t.Fatalf("private text request = %+v", caseValue.request.Prompt) + } + if caseValue.request.StructuredOutput != nil || caseValue.request.TransientRetries != 3 || + caseValue.request.Generation != 1 || !reflect.DeepEqual(caseValue.request.Target, target.EffectiveSettings) || + !reflect.DeepEqual(caseValue.request.TargetPresence, target.SettingsPresence) { + t.Fatalf("generation request = %+v", caseValue.request) + } + + for _, tc := range []struct { + name string + content string + wantTextReturned bool + wantMatched bool + }{ + {name: "exact token", content: caseValue.expectedContent, wantTextReturned: true, wantMatched: true}, + {name: "surrounding whitespace", content: " \n" + caseValue.expectedContent + "\t", wantTextReturned: true, wantMatched: true}, + {name: "case differs", content: strings.ToLower(caseValue.expectedContent), wantTextReturned: true}, + {name: "unexpected text", content: "another answer", wantTextReturned: true}, + {name: "whitespace only", content: " \n\t"}, + } { + t.Run(tc.name, func(t *testing.T) { + facts, err := caseValue.evaluate(context.Background(), &domain.GenerateResponse{Content: tc.content}) + if err != nil { + t.Fatal(err) + } + if facts.TextReturned == nil || *facts.TextReturned != tc.wantTextReturned || + facts.ExpectedTextMatched == nil || *facts.ExpectedTextMatched != tc.wantMatched { + t.Fatalf("facts = %+v", facts) + } + if facts.ValidJSON != nil || facts.SchemaMatched != nil { + t.Fatalf("text evaluation populated structured facts: %+v", facts) + } + }) + } + + noResponseFacts, err := caseValue.evaluate(context.Background(), nil) + if err != nil { + t.Fatal(err) + } + if noResponseFacts.TextReturned != nil || noResponseFacts.ExpectedTextMatched != nil { + t.Fatalf("facts were populated without generated content: %+v", noResponseFacts) + } +} + +func TestSmokeStructuredInferenceCaseUsesPrivatePreparedSchema(t *testing.T) { + target := domain.SmokeTestTarget{ + EffectiveSettings: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "selected-model"}, + SettingsPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true}, + } + caseValue, err := prepareSmokeInferenceCase(context.Background(), domain.SmokeTestStructuredOutput, target, 2) + if err != nil { + t.Fatal(err) + } + if !caseValue.structuredOutputRequested() || caseValue.request.StructuredOutput.JSONSchema == nil || + caseValue.request.StructuredOutput.JSONSchema.Name != "promptkit_smoke" || + !caseValue.request.StructuredOutput.JSONSchema.Strict { + t.Fatalf("structured request = %+v", caseValue.request.StructuredOutput) + } + if len(caseValue.request.Prompt.Messages) != 1 || caseValue.request.Prompt.Messages[0].Role != "user" || + !strings.Contains(caseValue.request.Prompt.Messages[0].Content, caseValue.expectedContent) { + t.Fatalf("private structured request = %+v", caseValue.request.Prompt) + } + if caseValue.request.TransientRetries != 2 || caseValue.request.Generation != 1 || + !reflect.DeepEqual(caseValue.request.Target, target.EffectiveSettings) || + !reflect.DeepEqual(caseValue.request.TargetPresence, target.SettingsPresence) { + t.Fatalf("generation request = %+v", caseValue.request) + } + + additionalPropertyContent := strings.TrimSuffix(caseValue.expectedContent, "}") + `,"extra":true}` + for _, tc := range []struct { + name string + content string + wantValidJSON bool + wantSchemaMatched *bool + }{ + {name: "matches", content: caseValue.expectedContent, wantValidJSON: true, wantSchemaMatched: boolPointer(true)}, + {name: "valid JSON misses schema", content: `{"status":"other"}`, wantValidJSON: true, wantSchemaMatched: boolPointer(false)}, + {name: "additional property misses schema", content: additionalPropertyContent, wantValidJSON: true, wantSchemaMatched: boolPointer(false)}, + {name: "invalid JSON", content: `{invalid`, wantSchemaMatched: nil}, + {name: "empty content", content: "", wantSchemaMatched: nil}, + } { + t.Run(tc.name, func(t *testing.T) { + facts, err := caseValue.evaluate(context.Background(), &domain.GenerateResponse{Content: tc.content}) + if err != nil { + t.Fatal(err) + } + if facts.ValidJSON == nil || *facts.ValidJSON != tc.wantValidJSON || !sameBool(facts.SchemaMatched, tc.wantSchemaMatched) { + t.Fatalf("facts = %+v", facts) + } + }) + } + + noResponseFacts, err := caseValue.evaluate(context.Background(), nil) + if err != nil { + t.Fatal(err) + } + if noResponseFacts.ValidJSON != nil || noResponseFacts.SchemaMatched != nil { + t.Fatalf("facts were populated without generated content: %+v", noResponseFacts) + } +} diff --git a/internal/usecase/smoke_schema.json b/internal/usecase/smoke_schema.json new file mode 100644 index 0000000..85754cf --- /dev/null +++ b/internal/usecase/smoke_schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "status": { + "type": "string", + "const": "ok" + } + }, + "required": ["status"], + "additionalProperties": false +}