package promptkit_test import ( "context" "encoding/json" "errors" "fmt" "io" "io/fs" "net/http" "reflect" "strings" "sync" "testing" "time" "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit" ) func TestSmokeTestSharesBackendCapacityWithRunAndReleasesAfterCancellation(t *testing.T) { release := make(chan struct{}) client := newCapacityGateClient(release, 2) engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil) ctx, cancel := context.WithCancel(context.Background()) runResults := make(chan capacityRunResult, 1) go runCapacityRequest(engine, ctx, promptkit.RunRequest{PromptID: "prompt"}, runResults) select { case <-client.started: case <-time.After(5 * time.Second): cancel() close(release) t.Fatal("ordinary Run did not reach the model client") } report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{ Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", }) if err != nil || report == nil || !report.Completed || report.Failure != promptkit.ExecutionFailureCapacity || len(report.Attempts) != 0 { t.Fatalf("concurrent smoke test=(%+v, %v), want concluded capacity finding", report, err) } cancel() runOutcome := awaitCapacityRun(t, runResults) if runOutcome.result != nil || !errors.Is(runOutcome.err, context.Canceled) { t.Fatalf("canceled Run=(%+v, %v), want context cancellation", runOutcome.result, runOutcome.err) } close(release) report, err = engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{ Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", }) if err != nil || report == nil || !report.Completed { t.Fatalf("smoke test after cancellation=(%+v, %v), want conclusion", report, err) } _, peak, calls := client.snapshot() if calls != 2 || peak != 1 { t.Fatalf("shared model client calls=%d peak=%d, want 2 calls with bound 1", calls, peak) } } type failOnceSmokeClient struct { mu sync.Mutex calls int } func (client *failOnceSmokeClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { client.mu.Lock() defer client.mu.Unlock() client.calls++ if client.calls == 1 { return nil, errors.New("synthetic-provider-message-marker") } return &promptkit.GenerateResponse{Content: "ordinary output marker"}, nil } func (client *failOnceSmokeClient) callCount() int { client.mu.Lock() defer client.mu.Unlock() return client.calls } func TestSmokeTestFailureReportIsPrivateAndReleasesAdmission(t *testing.T) { client := &failOnceSmokeClient{} engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil) request := promptkit.SmokeTestRequest{ Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", APIKey: "synthetic-smoke-key-marker", } report, err := engine.SmokeTest(context.Background(), request) if err != nil || report == nil || !report.Completed || report.Failure != promptkit.ExecutionFailureGeneration { t.Fatalf("generation failure report=(%+v, %v)", report, err) } encoded, err := json.Marshal(report) if err != nil { t.Fatal(err) } for _, private := range []string{"synthetic-provider-message-marker", "synthetic-smoke-key-marker", "ordinary output marker"} { if strings.Contains(fmt.Sprintf("%+v %s", report, encoded), private) { t.Errorf("smoke report exposed %q", private) } } result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) if err != nil || result == nil { t.Fatalf("Run after concluded smoke failure=(%+v, %v), want success", result, err) } if calls := client.callCount(); calls != 2 { t.Fatalf("model calls=%d, want diagnostic failure followed by Run", calls) } _, selectionErr := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{ Kind: promptkit.SmokeTestTextInference, ProfileID: "missing-profile", APIKey: request.APIKey, }) if selectionErr == nil || strings.Contains(selectionErr.Error(), request.APIKey) { t.Fatalf("selection error=%v, want an operational error without the request credential", selectionErr) } } func TestSmokeTestCancellationRetainsLastHTTPAttemptWithoutExtraAttempt(t *testing.T) { responseReady := make(chan struct{}) client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { header := make(http.Header) header.Set("Retry-After", "60") close(responseReady) return &http.Response{ StatusCode: http.StatusServiceUnavailable, Header: header, Body: io.NopCloser(strings.NewReader("synthetic-http-body-marker")), Request: request, }, nil })} cfg := simpleConfig("unused") cfg.HTTPClient = client engine, err := promptkit.NewEngine(cfg) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) retries := 1 type result struct { report *promptkit.SmokeTestReport err error } finished := make(chan result, 1) go func() { report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{ Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", TransientRetries: &retries, }) finished <- result{report: report, err: err} }() select { case <-responseReady: cancel() case <-time.After(5 * time.Second): cancel() t.Fatal("inference did not observe the controlled HTTP response") } select { case outcome := <-finished: if !errors.Is(outcome.err, context.Canceled) || outcome.report == nil || outcome.report.Completed { t.Fatalf("interrupted smoke test=(%+v, %v), want partial report and context.Canceled", outcome.report, outcome.err) } if len(outcome.report.Attempts) != 1 || outcome.report.Attempts[0].HTTPStatus == nil || *outcome.report.Attempts[0].HTTPStatus != http.StatusServiceUnavailable { t.Fatalf("attempts after cancellation=%+v, want only the observed 503 attempt", outcome.report.Attempts) } for name, fact := range map[string]*bool{ "response received": outcome.report.Facts.ResponseReceived, "endpoint reachable": outcome.report.Facts.EndpointReachable, } { if fact == nil || !*fact { t.Errorf("%s fact=%v, want true", name, fact) } } if strings.Contains(fmt.Sprintf("%+v", outcome.report), "synthetic-http-body-marker") { t.Fatal("smoke report exposed the HTTP response body") } case <-time.After(5 * time.Second): t.Fatal("smoke test did not return after cancellation") } } type cancelAfterGenerationClient struct{ cancel context.CancelFunc } func (client cancelAfterGenerationClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { client.cancel() return &promptkit.GenerateResponse{ Content: "synthetic-generated-content-marker", Usage: promptkit.TokenUsage{PromptTokens: 17, CompletionTokens: 9, TotalTokens: 26}, }, nil } func TestSmokeTestCancellationRetainsDecodedUsageWithoutExposingContent(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) engine := newSimpleEngine(t, "unused", promptkit.WithLLMClient(cancelAfterGenerationClient{cancel: cancel})) report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{ Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", APIKey: "synthetic-smoke-key-marker", }) if !errors.Is(err, context.Canceled) || report == nil || report.Completed { t.Fatalf("canceled diagnostic=(%+v, %v), want partial report and context.Canceled", report, err) } if !report.Usage.PromptTokens.Known || report.Usage.PromptTokens.Value != 17 || !report.Usage.CompletionTokens.Known || report.Usage.CompletionTokens.Value != 9 { t.Fatalf("decoded usage was not retained: %+v", report.Usage) } if report.Facts.ExpectedTextMatched != nil || report.Facts.TextReturned != nil { t.Fatalf("canceled output was evaluated: %+v", report.Facts) } encoded, err := json.Marshal(report) if err != nil { t.Fatal(err) } for _, private := range []string{"synthetic-generated-content-marker", "synthetic-smoke-key-marker"} { if strings.Contains(fmt.Sprintf("%+v %s", report, encoded), private) { t.Errorf("partial report exposed %q", private) } } } type failingArtifactReader struct{ calls int } func (reader *failingArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) { reader.calls++ return nil, errors.New("synthetic-artifact-content-marker") } type failingSchemaFS struct { mu sync.Mutex calls int } func (source *failingSchemaFS) Open(string) (fs.File, error) { source.mu.Lock() source.calls++ source.mu.Unlock() return nil, errors.New("synthetic-schema-content-marker") } func (source *failingSchemaFS) openCount() int { source.mu.Lock() defer source.mu.Unlock() return source.calls } func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChanges(t *testing.T) { const ( promptMarker = "synthetic-application-prompt-marker" artifactMarker = "synthetic-application-artifact-marker" schemaMarker = "synthetic-application-schema-marker" outputMarker = "synthetic-generated-content-marker" ) reader := &failingArtifactReader{} schemaSource := &failingSchemaFS{} client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: outputMarker}} input := configkit.Input{Source: configkit.Mapping("private diagnostic test", map[string]any{ "profiles": map[string]any{"profile": map[string]any{"endpoint": "http://example.test/v1", "model": "model"}}, "prompts": map[string]any{"application-prompt": map[string]any{"1": map[string]any{ "default_profile": "profile", "session_id": `application-session {{input "document"}}`, "inputs": []any{map[string]any{"name": "document", "required": true}}, "messages": []any{ map[string]any{"role": "system", "content": promptMarker}, map[string]any{"role": "user", "content": `{{input "document"}}`}, }, "output": map[string]any{ "format": "json", "validation_mode": "json_schema", "schema_path": schemaMarker, }, }}}, })} engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithLLMClient(client), promptkit.WithArtifactReader(reader), promptkit.WithSchemaFS(schemaSource, "schemas")) if err != nil { t.Fatalf("construct engine: %v", err) } profilesBefore, err := engine.ListProfiles(context.Background()) if err != nil { t.Fatal(err) } promptsBefore, err := engine.ListPrompts(context.Background()) if err != nil { t.Fatal(err) } inspectionBefore, err := engine.InspectPrompt(context.Background(), "application-prompt", "") if err != nil { t.Fatal(err) } schemaOpensBefore := schemaSource.openCount() report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{ Kind: promptkit.SmokeTestStructuredOutput, PromptID: "application-prompt", APIKey: "synthetic-smoke-key-marker", }) if err != nil || report == nil || !report.Completed { t.Fatalf("structured smoke test=(%+v, %v)", report, err) } if reader.calls != 0 || schemaSource.openCount() != schemaOpensBefore { t.Fatalf("application resources used: artifact reads=%d schema opens before=%d after=%d", reader.calls, schemaOpensBefore, schemaSource.openCount()) } if len(client.requests) != 1 { t.Fatalf("injected calls=%d, want one private inference call", len(client.requests)) } generationRequest := client.requests[0] if generationRequest.Prompt.SessionID != "" { t.Errorf("application session template was rendered: %q", generationRequest.Prompt.SessionID) } for _, message := range generationRequest.Prompt.Messages { if strings.Contains(message.Content, promptMarker) || strings.Contains(message.Content, artifactMarker) { t.Errorf("application prompt content reached diagnostic generation: %+v", message) } } structuredRequest, err := json.Marshal(generationRequest.StructuredOutput) if err != nil { t.Fatal(err) } if strings.Contains(string(structuredRequest), schemaMarker) { t.Fatal("application schema path reached diagnostic generation") } encodedReport, err := json.Marshal(report) if err != nil { t.Fatal(err) } for _, private := range []string{promptMarker, artifactMarker, schemaMarker, outputMarker, "synthetic-smoke-key-marker"} { if strings.Contains(fmt.Sprintf("%+v %s", report, encodedReport), private) { t.Errorf("smoke report exposed %q", private) } } profilesAfter, err := engine.ListProfiles(context.Background()) if err != nil { t.Fatal(err) } promptsAfter, err := engine.ListPrompts(context.Background()) if err != nil { t.Fatal(err) } inspectionAfter, err := engine.InspectPrompt(context.Background(), "application-prompt", "") if err != nil { t.Fatal(err) } if !reflect.DeepEqual(profilesBefore, profilesAfter) || !reflect.DeepEqual(promptsBefore, promptsAfter) || inspectionBefore.PromptHash != inspectionAfter.PromptHash || inspectionAfter.OutputContract.SchemaPath != schemaMarker { t.Fatalf("diagnostic changed captured listings or application inspection: profiles %v -> %v, prompts %v -> %v, inspection %+v -> %+v", profilesBefore, profilesAfter, promptsBefore, promptsAfter, inspectionBefore, inspectionAfter) } } type nestedSmokeObserverClient struct { engine *promptkit.Engine calls int mu sync.Mutex report *promptkit.SmokeTestReport err error } func (client *nestedSmokeObserverClient) Generate(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { client.mu.Lock() client.calls++ call := client.calls client.mu.Unlock() if call == 1 { client.report, client.err = client.engine.SmokeTest(ctx, promptkit.SmokeTestRequest{ Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", }) } return &promptkit.GenerateResponse{Content: "an unexpected answer"}, nil } func TestSmokeTestDoesNotEnterInheritedExecutionObserver(t *testing.T) { client := &nestedSmokeObserverClient{} engine := newSimpleEngine(t, "unused", promptkit.WithLLMClient(client)) client.engine = engine var events []promptkit.ExecutionEvent result, err := engine.Run(context.Background(), promptkit.RunRequest{ PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }, }) if err != nil || result == nil { t.Fatalf("outer Run=(%+v, %v)", result, err) } if client.err != nil || client.report == nil || !client.report.Completed { t.Fatalf("nested smoke test=(%+v, %v)", client.report, client.err) } if len(events) != 3 || events[0].Kind != promptkit.ExecutionStarted || events[1].Kind != promptkit.ExecutionAttempt || events[2].Kind != promptkit.ExecutionFinished { t.Fatalf("outer observer received diagnostic traffic: %+v", events) } } func TestSmokeTestCrossBackendDefaultUsesEndpointOverrideThroughFacade(t *testing.T) { const endpointOverride = "http://override.example/v1" var observedURL string httpClient := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { observedURL = request.URL.String() return &http.Response{ StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"role":"assistant","content":"unexpected"}}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`)), Request: request, }, nil })} input := configkit.Input{Source: configkit.Mapping("cross-backend smoke test", map[string]any{ "backends": map[string]any{ "backend-a": map[string]any{ "endpoint": "http://backend-a.example/v1", "smoke_tests": map[string]any{"default_profile": "profile-b"}, }, "backend-b": map[string]any{"endpoint": "http://backend-b.example/v1", "concurrency_limit": 1}, }, "profiles": map[string]any{"profile-b": map[string]any{"backend": "backend-b", "model": "model"}}, })} engine, err := promptkit.NewEngine(promptkit.Config{ Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: httpClient, }) if err != nil { t.Fatalf("construct cross-backend engine: %v", err) } report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{ Kind: promptkit.SmokeTestTextInference, BackendID: "backend-a", Execution: &promptkit.ExecutionTargetOverride{Endpoint: endpointOverride}, }) if err != nil || report == nil || !report.Completed { t.Fatalf("cross-backend smoke test=(%+v, %v)", report, err) } if report.Target.DefaultSourceBackendID != "backend-a" || report.Target.BackendID != "backend-b" || report.Target.ProfileID != "profile-b" || report.Target.SelectionSource != "backend_default" { t.Fatalf("unexpected cross-backend target: %+v", report.Target) } if report.Target.Endpoint != observedURL || report.Target.EffectiveSettings.Endpoint != endpointOverride || observedURL != endpointOverride+"/chat/completions" || len(report.Attempts) != 1 || report.Attempts[0].InitialEndpoint != observedURL || report.Attempts[0].Endpoint != observedURL { t.Fatalf("endpoint override not used by facade: target=%+v attempt=%+v URL=%q", report.Target, report.Attempts, observedURL) } } func TestSmokeTestInferenceEndpointsDistinguishRedirectFromRequestComposition(t *testing.T) { const base = "http://initial.example/v1" const redirected = "http://redirect.example/completion" var observed []string client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { observed = append(observed, request.URL.String()) if len(observed) == 1 { return &http.Response{ StatusCode: http.StatusTemporaryRedirect, Header: http.Header{"Location": []string{redirected}}, Body: io.NopCloser(strings.NewReader("")), Request: request, }, nil } return &http.Response{ StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"unexpected"}}]}`)), Request: request, }, nil })} input := configkit.Input{Source: configkit.Mapping("redirect diagnostic", map[string]any{ "profiles": map[string]any{"profile": map[string]any{"endpoint": base, "model": "model"}}, })} engine, err := promptkit.NewEngine(promptkit.Config{ Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client, }) if err != nil { t.Fatal(err) } report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{ Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", }) if err != nil || report == nil || !report.Completed || len(report.Attempts) != 1 || len(observed) != 2 { t.Fatalf("report=%+v error=%v observed=%v", report, err, observed) } if observed[0] != base+"/chat/completions" || observed[1] != redirected || report.Target.Endpoint != observed[0] || report.Target.EffectiveSettings.Endpoint != base || report.Attempts[0].InitialEndpoint != observed[0] || report.Attempts[0].Endpoint != observed[1] { t.Fatalf("incorrect redirect attribution: target=%+v attempt=%+v observed=%v", report.Target, report.Attempts[0], observed) } }