From b7a951685b5454f6f2e1b440e8e430a231f652a4 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 13 Oct 2026 18:24:57 +0000 Subject: [PATCH] Verify smoke-test integration boundaries --- docs/roadmap/implementation.md | 9 + smoke_integration_contract_test.go | 426 +++++++++++++++++++++++++++++ 2 files changed, 435 insertions(+) create mode 100644 smoke_integration_contract_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 81db195..1c11b10 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -678,6 +678,15 @@ Scope: a small number of consequential integration regressions. Validate: focused race-enabled tests for these assembled paths. Keep cases small and each risk owned once; do not introduce a second end-to-end state-matrix suite. +**Completed: 2026-10-13.** Added public-boundary regressions for shared Run and +SmokeTest admission, cancellation/failure permit recovery, retained HTTP and +decoded-usage evidence, application-resource and observer isolation, disclosure, +and cross-backend default routing with an endpoint override. Existing focused +generation/use-case tests continue to cover deterministic interrupted backoff +and undispatched reacquisition aggregation. Validation passed: +`go test -race . ./internal/usecase ./internal/generation ./internal/engine`, +`go vet ./...`, and `git diff --check`. + ## Stage 18: Complete Canonical Documentation Scope: GoDoc and existing consumer, format, integration, and internal owners. diff --git a/smoke_integration_contract_test.go b/smoke_integration_contract_test.go new file mode 100644 index 0000000..19d3db3 --- /dev/null +++ b/smoke_integration_contract_test.go @@ -0,0 +1,426 @@ +package promptkit_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "net/http" + "reflect" + "strings" + "sync" + "testing" + "time" + + "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/promptkit" +) + +func TestSmokeTestSharesBackendCapacityWithRunAndReleasesAfterCancellation(t *testing.T) { + release := make(chan struct{}) + client := newCapacityGateClient(release, 2) + engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil) + ctx, cancel := context.WithCancel(context.Background()) + runResults := make(chan capacityRunResult, 1) + go runCapacityRequest(engine, ctx, promptkit.RunRequest{PromptID: "prompt"}, runResults) + select { + case <-client.started: + case <-time.After(5 * time.Second): + cancel() + close(release) + t.Fatal("ordinary Run did not reach the model client") + } + + report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{ + Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", + }) + if err != nil || report == nil || !report.Completed || report.Failure != promptkit.ExecutionFailureCapacity || len(report.Attempts) != 0 { + t.Fatalf("concurrent smoke test=(%+v, %v), want concluded capacity finding", report, err) + } + + cancel() + runOutcome := awaitCapacityRun(t, runResults) + if runOutcome.result != nil || !errors.Is(runOutcome.err, context.Canceled) { + t.Fatalf("canceled Run=(%+v, %v), want context cancellation", runOutcome.result, runOutcome.err) + } + close(release) + + report, err = engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{ + Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", + }) + if err != nil || report == nil || !report.Completed { + t.Fatalf("smoke test after cancellation=(%+v, %v), want conclusion", report, err) + } + _, peak, calls := client.snapshot() + if calls != 2 || peak != 1 { + t.Fatalf("shared model client calls=%d peak=%d, want 2 calls with bound 1", calls, peak) + } +} + +type failOnceSmokeClient struct { + mu sync.Mutex + calls int +} + +func (client *failOnceSmokeClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + client.mu.Lock() + defer client.mu.Unlock() + client.calls++ + if client.calls == 1 { + return nil, errors.New("synthetic-provider-message-marker") + } + return &promptkit.GenerateResponse{Content: "ordinary output marker"}, nil +} + +func (client *failOnceSmokeClient) callCount() int { + client.mu.Lock() + defer client.mu.Unlock() + return client.calls +} + +func TestSmokeTestFailureReportIsPrivateAndReleasesAdmission(t *testing.T) { + client := &failOnceSmokeClient{} + engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil) + request := promptkit.SmokeTestRequest{ + Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", APIKey: "synthetic-smoke-key-marker", + } + report, err := engine.SmokeTest(context.Background(), request) + if err != nil || report == nil || !report.Completed || report.Failure != promptkit.ExecutionFailureGeneration { + t.Fatalf("generation failure report=(%+v, %v)", report, err) + } + encoded, err := json.Marshal(report) + if err != nil { + t.Fatal(err) + } + for _, private := range []string{"synthetic-provider-message-marker", "synthetic-smoke-key-marker", "ordinary output marker"} { + if strings.Contains(fmt.Sprintf("%+v %s", report, encoded), private) { + t.Errorf("smoke report exposed %q", private) + } + } + + result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) + if err != nil || result == nil { + t.Fatalf("Run after concluded smoke failure=(%+v, %v), want success", result, err) + } + if calls := client.callCount(); calls != 2 { + t.Fatalf("model calls=%d, want diagnostic failure followed by Run", calls) + } + _, selectionErr := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{ + Kind: promptkit.SmokeTestTextInference, ProfileID: "missing-profile", APIKey: request.APIKey, + }) + if selectionErr == nil || strings.Contains(selectionErr.Error(), request.APIKey) { + t.Fatalf("selection error=%v, want an operational error without the request credential", selectionErr) + } +} + +func TestSmokeTestCancellationRetainsLastHTTPAttemptWithoutExtraAttempt(t *testing.T) { + responseReady := make(chan struct{}) + client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { + header := make(http.Header) + header.Set("Retry-After", "60") + close(responseReady) + return &http.Response{ + StatusCode: http.StatusServiceUnavailable, + Header: header, + Body: io.NopCloser(strings.NewReader("synthetic-http-body-marker")), + Request: request, + }, nil + })} + cfg := simpleConfig("unused") + cfg.HTTPClient = client + engine, err := promptkit.NewEngine(cfg) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + retries := 1 + type result struct { + report *promptkit.SmokeTestReport + err error + } + finished := make(chan result, 1) + go func() { + report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{ + Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", TransientRetries: &retries, + }) + finished <- result{report: report, err: err} + }() + select { + case <-responseReady: + cancel() + case <-time.After(5 * time.Second): + cancel() + t.Fatal("inference did not observe the controlled HTTP response") + } + select { + case outcome := <-finished: + if !errors.Is(outcome.err, context.Canceled) || outcome.report == nil || outcome.report.Completed { + t.Fatalf("interrupted smoke test=(%+v, %v), want partial report and context.Canceled", outcome.report, outcome.err) + } + if len(outcome.report.Attempts) != 1 || outcome.report.Attempts[0].HTTPStatus == nil || *outcome.report.Attempts[0].HTTPStatus != http.StatusServiceUnavailable { + t.Fatalf("attempts after cancellation=%+v, want only the observed 503 attempt", outcome.report.Attempts) + } + for name, fact := range map[string]*bool{ + "response received": outcome.report.Facts.ResponseReceived, + "endpoint reachable": outcome.report.Facts.EndpointReachable, + } { + if fact == nil || !*fact { + t.Errorf("%s fact=%v, want true", name, fact) + } + } + if strings.Contains(fmt.Sprintf("%+v", outcome.report), "synthetic-http-body-marker") { + t.Fatal("smoke report exposed the HTTP response body") + } + case <-time.After(5 * time.Second): + t.Fatal("smoke test did not return after cancellation") + } +} + +type cancelAfterGenerationClient struct{ cancel context.CancelFunc } + +func (client cancelAfterGenerationClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + client.cancel() + return &promptkit.GenerateResponse{ + Content: "synthetic-generated-content-marker", + Usage: promptkit.TokenUsage{PromptTokens: 17, CompletionTokens: 9, TotalTokens: 26}, + }, nil +} + +func TestSmokeTestCancellationRetainsDecodedUsageWithoutExposingContent(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + engine := newSimpleEngine(t, "unused", promptkit.WithLLMClient(cancelAfterGenerationClient{cancel: cancel})) + report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{ + Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", APIKey: "synthetic-smoke-key-marker", + }) + if !errors.Is(err, context.Canceled) || report == nil || report.Completed { + t.Fatalf("canceled diagnostic=(%+v, %v), want partial report and context.Canceled", report, err) + } + if !report.Usage.PromptTokens.Known || report.Usage.PromptTokens.Value != 17 || + !report.Usage.CompletionTokens.Known || report.Usage.CompletionTokens.Value != 9 { + t.Fatalf("decoded usage was not retained: %+v", report.Usage) + } + if report.Facts.ExpectedTextMatched != nil || report.Facts.TextReturned != nil { + t.Fatalf("canceled output was evaluated: %+v", report.Facts) + } + encoded, err := json.Marshal(report) + if err != nil { + t.Fatal(err) + } + for _, private := range []string{"synthetic-generated-content-marker", "synthetic-smoke-key-marker"} { + if strings.Contains(fmt.Sprintf("%+v %s", report, encoded), private) { + t.Errorf("partial report exposed %q", private) + } + } +} + +type failingArtifactReader struct{ calls int } + +func (reader *failingArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) { + reader.calls++ + return nil, errors.New("synthetic-artifact-content-marker") +} + +type failingSchemaFS struct { + mu sync.Mutex + calls int +} + +func (source *failingSchemaFS) Open(string) (fs.File, error) { + source.mu.Lock() + source.calls++ + source.mu.Unlock() + return nil, errors.New("synthetic-schema-content-marker") +} + +func (source *failingSchemaFS) openCount() int { + source.mu.Lock() + defer source.mu.Unlock() + return source.calls +} + +func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChanges(t *testing.T) { + const ( + promptMarker = "synthetic-application-prompt-marker" + artifactMarker = "synthetic-application-artifact-marker" + schemaMarker = "synthetic-application-schema-marker" + outputMarker = "synthetic-generated-content-marker" + ) + reader := &failingArtifactReader{} + schemaSource := &failingSchemaFS{} + client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: outputMarker}} + input := configkit.Input{Source: configkit.Mapping("private diagnostic test", map[string]any{ + "profiles": map[string]any{"profile": map[string]any{"endpoint": "http://example.test/v1", "model": "model"}}, + "prompts": map[string]any{"application-prompt": map[string]any{"1": map[string]any{ + "default_profile": "profile", + "session_id": `application-session {{input "document"}}`, + "inputs": []any{map[string]any{"name": "document", "required": true}}, + "messages": []any{ + map[string]any{"role": "system", "content": promptMarker}, + map[string]any{"role": "user", "content": `{{input "document"}}`}, + }, + "output": map[string]any{ + "format": "json", "validation_mode": "json_schema", "schema_path": schemaMarker, + }, + }}}, + })} + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, + promptkit.WithLLMClient(client), promptkit.WithArtifactReader(reader), promptkit.WithSchemaFS(schemaSource, "schemas")) + if err != nil { + t.Fatalf("construct engine: %v", err) + } + profilesBefore, err := engine.ListProfiles(context.Background()) + if err != nil { + t.Fatal(err) + } + promptsBefore, err := engine.ListPrompts(context.Background()) + if err != nil { + t.Fatal(err) + } + inspectionBefore, err := engine.InspectPrompt(context.Background(), "application-prompt", "") + if err != nil { + t.Fatal(err) + } + schemaOpensBefore := schemaSource.openCount() + + report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{ + Kind: promptkit.SmokeTestStructuredOutput, PromptID: "application-prompt", APIKey: "synthetic-smoke-key-marker", + }) + if err != nil || report == nil || !report.Completed { + t.Fatalf("structured smoke test=(%+v, %v)", report, err) + } + if reader.calls != 0 || schemaSource.openCount() != schemaOpensBefore { + t.Fatalf("application resources used: artifact reads=%d schema opens before=%d after=%d", reader.calls, schemaOpensBefore, schemaSource.openCount()) + } + if len(client.requests) != 1 { + t.Fatalf("injected calls=%d, want one private inference call", len(client.requests)) + } + generationRequest := client.requests[0] + if generationRequest.Prompt.SessionID != "" { + t.Errorf("application session template was rendered: %q", generationRequest.Prompt.SessionID) + } + for _, message := range generationRequest.Prompt.Messages { + if strings.Contains(message.Content, promptMarker) || strings.Contains(message.Content, artifactMarker) { + t.Errorf("application prompt content reached diagnostic generation: %+v", message) + } + } + structuredRequest, err := json.Marshal(generationRequest.StructuredOutput) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(structuredRequest), schemaMarker) { + t.Fatal("application schema path reached diagnostic generation") + } + encodedReport, err := json.Marshal(report) + if err != nil { + t.Fatal(err) + } + for _, private := range []string{promptMarker, artifactMarker, schemaMarker, outputMarker, "synthetic-smoke-key-marker"} { + if strings.Contains(fmt.Sprintf("%+v %s", report, encodedReport), private) { + t.Errorf("smoke report exposed %q", private) + } + } + profilesAfter, err := engine.ListProfiles(context.Background()) + if err != nil { + t.Fatal(err) + } + promptsAfter, err := engine.ListPrompts(context.Background()) + if err != nil { + t.Fatal(err) + } + inspectionAfter, err := engine.InspectPrompt(context.Background(), "application-prompt", "") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(profilesBefore, profilesAfter) || !reflect.DeepEqual(promptsBefore, promptsAfter) || + inspectionBefore.PromptHash != inspectionAfter.PromptHash || inspectionAfter.OutputContract.SchemaPath != schemaMarker { + t.Fatalf("diagnostic changed captured listings or application inspection: profiles %v -> %v, prompts %v -> %v, inspection %+v -> %+v", + profilesBefore, profilesAfter, promptsBefore, promptsAfter, inspectionBefore, inspectionAfter) + } +} + +type nestedSmokeObserverClient struct { + engine *promptkit.Engine + calls int + mu sync.Mutex + report *promptkit.SmokeTestReport + err error +} + +func (client *nestedSmokeObserverClient) Generate(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + client.mu.Lock() + client.calls++ + call := client.calls + client.mu.Unlock() + if call == 1 { + client.report, client.err = client.engine.SmokeTest(ctx, promptkit.SmokeTestRequest{ + Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", + }) + } + return &promptkit.GenerateResponse{Content: "an unexpected answer"}, nil +} + +func TestSmokeTestDoesNotEnterInheritedExecutionObserver(t *testing.T) { + client := &nestedSmokeObserverClient{} + engine := newSimpleEngine(t, "unused", promptkit.WithLLMClient(client)) + client.engine = engine + var events []promptkit.ExecutionEvent + result, err := engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }, + }) + if err != nil || result == nil { + t.Fatalf("outer Run=(%+v, %v)", result, err) + } + if client.err != nil || client.report == nil || !client.report.Completed { + t.Fatalf("nested smoke test=(%+v, %v)", client.report, client.err) + } + if len(events) != 3 || events[0].Kind != promptkit.ExecutionStarted || events[1].Kind != promptkit.ExecutionAttempt || events[2].Kind != promptkit.ExecutionFinished { + t.Fatalf("outer observer received diagnostic traffic: %+v", events) + } +} + +func TestSmokeTestCrossBackendDefaultUsesEndpointOverrideThroughFacade(t *testing.T) { + const endpointOverride = "http://override.example/v1" + var observedURL string + httpClient := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { + observedURL = request.URL.String() + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"role":"assistant","content":"unexpected"}}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`)), + Request: request, + }, nil + })} + input := configkit.Input{Source: configkit.Mapping("cross-backend smoke test", map[string]any{ + "backends": map[string]any{ + "backend-a": map[string]any{ + "endpoint": "http://backend-a.example/v1", + "smoke_tests": map[string]any{"default_profile": "profile-b"}, + }, + "backend-b": map[string]any{"endpoint": "http://backend-b.example/v1", "concurrency_limit": 1}, + }, + "profiles": map[string]any{"profile-b": map[string]any{"backend": "backend-b", "model": "model"}}, + })} + engine, err := promptkit.NewEngine(promptkit.Config{ + Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: httpClient, + }) + if err != nil { + t.Fatalf("construct cross-backend engine: %v", err) + } + report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{ + Kind: promptkit.SmokeTestTextInference, BackendID: "backend-a", + Execution: &promptkit.ExecutionTargetOverride{Endpoint: endpointOverride}, + }) + if err != nil || report == nil || !report.Completed { + t.Fatalf("cross-backend smoke test=(%+v, %v)", report, err) + } + if report.Target.DefaultSourceBackendID != "backend-a" || report.Target.BackendID != "backend-b" || + report.Target.ProfileID != "profile-b" || report.Target.SelectionSource != "backend_default" { + t.Fatalf("unexpected cross-backend target: %+v", report.Target) + } + if report.Target.Endpoint != endpointOverride || report.Target.EffectiveSettings.Endpoint != endpointOverride || + observedURL != endpointOverride+"/chat/completions" || len(report.Attempts) != 1 || report.Attempts[0].Endpoint != observedURL { + t.Fatalf("endpoint override not used by facade: target=%+v attempt=%+v URL=%q", report.Target, report.Attempts, observedURL) + } +}