promptkit/smoke_integration_contract_test.go

package promptkit_test

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"net/http"
	"reflect"
	"strings"
	"sync"
	"testing"
	"time"

	"gitea.maximumdirect.net/eric/ConfigKit"
	"gitea.maximumdirect.net/eric/promptkit"
)

func TestSmokeTestSharesBackendCapacityWithRunAndReleasesAfterCancellation(t *testing.T) {
	release := make(chan struct{})
	client := newCapacityGateClient(release, 2)
	engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil)
	ctx, cancel := context.WithCancel(context.Background())
	runResults := make(chan capacityRunResult, 1)
	go runCapacityRequest(engine, ctx, promptkit.RunRequest{PromptID: "prompt"}, runResults)
	select {
	case <-client.started:
	case <-time.After(5 * time.Second):
		cancel()
		close(release)
		t.Fatal("ordinary Run did not reach the model client")
	}

	report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{
		Kind: promptkit.SmokeTestTextInference, ProfileID: "profile",
	})
	if err != nil || report == nil || !report.Completed || report.Failure != promptkit.ExecutionFailureCapacity || len(report.Attempts) != 0 {
		t.Fatalf("concurrent smoke test=(%+v, %v), want concluded capacity finding", report, err)
	}

	cancel()
	runOutcome := awaitCapacityRun(t, runResults)
	if runOutcome.result != nil || !errors.Is(runOutcome.err, context.Canceled) {
		t.Fatalf("canceled Run=(%+v, %v), want context cancellation", runOutcome.result, runOutcome.err)
	}
	close(release)

	report, err = engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{
		Kind: promptkit.SmokeTestTextInference, ProfileID: "profile",
	})
	if err != nil || report == nil || !report.Completed {
		t.Fatalf("smoke test after cancellation=(%+v, %v), want conclusion", report, err)
	}
	_, peak, calls := client.snapshot()
	if calls != 2 || peak != 1 {
		t.Fatalf("shared model client calls=%d peak=%d, want 2 calls with bound 1", calls, peak)
	}
}

type failOnceSmokeClient struct {
	mu    sync.Mutex
	calls int
}

func (client *failOnceSmokeClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
	client.mu.Lock()
	defer client.mu.Unlock()
	client.calls++
	if client.calls == 1 {
		return nil, errors.New("synthetic-provider-message-marker")
	}
	return &promptkit.GenerateResponse{Content: "ordinary output marker"}, nil
}

func (client *failOnceSmokeClient) callCount() int {
	client.mu.Lock()
	defer client.mu.Unlock()
	return client.calls
}

func TestSmokeTestFailureReportIsPrivateAndReleasesAdmission(t *testing.T) {
	client := &failOnceSmokeClient{}
	engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil)
	request := promptkit.SmokeTestRequest{
		Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", APIKey: "synthetic-smoke-key-marker",
	}
	report, err := engine.SmokeTest(context.Background(), request)
	if err != nil || report == nil || !report.Completed || report.Failure != promptkit.ExecutionFailureGeneration {
		t.Fatalf("generation failure report=(%+v, %v)", report, err)
	}
	encoded, err := json.Marshal(report)
	if err != nil {
		t.Fatal(err)
	}
	for _, private := range []string{"synthetic-provider-message-marker", "synthetic-smoke-key-marker", "ordinary output marker"} {
		if strings.Contains(fmt.Sprintf("%+v %s", report, encoded), private) {
			t.Errorf("smoke report exposed %q", private)
		}
	}

	result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
	if err != nil || result == nil {
		t.Fatalf("Run after concluded smoke failure=(%+v, %v), want success", result, err)
	}
	if calls := client.callCount(); calls != 2 {
		t.Fatalf("model calls=%d, want diagnostic failure followed by Run", calls)
	}
	_, selectionErr := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{
		Kind: promptkit.SmokeTestTextInference, ProfileID: "missing-profile", APIKey: request.APIKey,
	})
	if selectionErr == nil || strings.Contains(selectionErr.Error(), request.APIKey) {
		t.Fatalf("selection error=%v, want an operational error without the request credential", selectionErr)
	}
}

func TestSmokeTestCancellationRetainsLastHTTPAttemptWithoutExtraAttempt(t *testing.T) {
	responseReady := make(chan struct{})
	client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
		header := make(http.Header)
		header.Set("Retry-After", "60")
		close(responseReady)
		return &http.Response{
			StatusCode: http.StatusServiceUnavailable,
			Header:     header,
			Body:       io.NopCloser(strings.NewReader("synthetic-http-body-marker")),
			Request:    request,
		}, nil
	})}
	cfg := simpleConfig("unused")
	cfg.HTTPClient = client
	engine, err := promptkit.NewEngine(cfg)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	retries := 1
	type result struct {
		report *promptkit.SmokeTestReport
		err    error
	}
	finished := make(chan result, 1)
	go func() {
		report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{
			Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", TransientRetries: &retries,
		})
		finished <- result{report: report, err: err}
	}()
	select {
	case <-responseReady:
		cancel()
	case <-time.After(5 * time.Second):
		cancel()
		t.Fatal("inference did not observe the controlled HTTP response")
	}
	select {
	case outcome := <-finished:
		if !errors.Is(outcome.err, context.Canceled) || outcome.report == nil || outcome.report.Completed {
			t.Fatalf("interrupted smoke test=(%+v, %v), want partial report and context.Canceled", outcome.report, outcome.err)
		}
		if len(outcome.report.Attempts) != 1 || outcome.report.Attempts[0].HTTPStatus == nil || *outcome.report.Attempts[0].HTTPStatus != http.StatusServiceUnavailable {
			t.Fatalf("attempts after cancellation=%+v, want only the observed 503 attempt", outcome.report.Attempts)
		}
		for name, fact := range map[string]*bool{
			"response received":  outcome.report.Facts.ResponseReceived,
			"endpoint reachable": outcome.report.Facts.EndpointReachable,
		} {
			if fact == nil || !*fact {
				t.Errorf("%s fact=%v, want true", name, fact)
			}
		}
		if strings.Contains(fmt.Sprintf("%+v", outcome.report), "synthetic-http-body-marker") {
			t.Fatal("smoke report exposed the HTTP response body")
		}
	case <-time.After(5 * time.Second):
		t.Fatal("smoke test did not return after cancellation")
	}
}

type cancelAfterGenerationClient struct{ cancel context.CancelFunc }

func (client cancelAfterGenerationClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
	client.cancel()
	return &promptkit.GenerateResponse{
		Content: "synthetic-generated-content-marker",
		Usage:   promptkit.TokenUsage{PromptTokens: 17, CompletionTokens: 9, TotalTokens: 26},
	}, nil
}

func TestSmokeTestCancellationRetainsDecodedUsageWithoutExposingContent(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	engine := newSimpleEngine(t, "unused", promptkit.WithLLMClient(cancelAfterGenerationClient{cancel: cancel}))
	report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{
		Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", APIKey: "synthetic-smoke-key-marker",
	})
	if !errors.Is(err, context.Canceled) || report == nil || report.Completed {
		t.Fatalf("canceled diagnostic=(%+v, %v), want partial report and context.Canceled", report, err)
	}
	if !report.Usage.PromptTokens.Known || report.Usage.PromptTokens.Value != 17 ||
		!report.Usage.CompletionTokens.Known || report.Usage.CompletionTokens.Value != 9 {
		t.Fatalf("decoded usage was not retained: %+v", report.Usage)
	}
	if report.Facts.ExpectedTextMatched != nil || report.Facts.TextReturned != nil {
		t.Fatalf("canceled output was evaluated: %+v", report.Facts)
	}
	encoded, err := json.Marshal(report)
	if err != nil {
		t.Fatal(err)
	}
	for _, private := range []string{"synthetic-generated-content-marker", "synthetic-smoke-key-marker"} {
		if strings.Contains(fmt.Sprintf("%+v %s", report, encoded), private) {
			t.Errorf("partial report exposed %q", private)
		}
	}
}

type failingArtifactReader struct{ calls int }

func (reader *failingArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
	reader.calls++
	return nil, errors.New("synthetic-artifact-content-marker")
}

type failingSchemaFS struct {
	mu    sync.Mutex
	calls int
}

func (source *failingSchemaFS) Open(string) (fs.File, error) {
	source.mu.Lock()
	source.calls++
	source.mu.Unlock()
	return nil, errors.New("synthetic-schema-content-marker")
}

func (source *failingSchemaFS) openCount() int {
	source.mu.Lock()
	defer source.mu.Unlock()
	return source.calls
}

func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChanges(t *testing.T) {
	const (
		promptMarker   = "synthetic-application-prompt-marker"
		artifactMarker = "synthetic-application-artifact-marker"
		schemaMarker   = "synthetic-application-schema-marker"
		outputMarker   = "synthetic-generated-content-marker"
	)
	reader := &failingArtifactReader{}
	schemaSource := &failingSchemaFS{}
	client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: outputMarker}}
	input := configkit.Input{Source: configkit.Mapping("private diagnostic test", map[string]any{
		"profiles": map[string]any{"profile": map[string]any{"endpoint": "http://example.test/v1", "model": "model"}},
		"prompts": map[string]any{"application-prompt": map[string]any{"1": map[string]any{
			"default_profile": "profile",
			"session_id":      `application-session {{input "document"}}`,
			"inputs":          []any{map[string]any{"name": "document", "required": true}},
			"messages": []any{
				map[string]any{"role": "system", "content": promptMarker},
				map[string]any{"role": "user", "content": `{{input "document"}}`},
			},
			"output": map[string]any{
				"format": "json", "validation_mode": "json_schema", "schema_path": schemaMarker,
			},
		}}},
	})}
	engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}},
		promptkit.WithLLMClient(client), promptkit.WithArtifactReader(reader), promptkit.WithSchemaFS(schemaSource, "schemas"))
	if err != nil {
		t.Fatalf("construct engine: %v", err)
	}
	profilesBefore, err := engine.ListProfiles(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	promptsBefore, err := engine.ListPrompts(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	inspectionBefore, err := engine.InspectPrompt(context.Background(), "application-prompt", "")
	if err != nil {
		t.Fatal(err)
	}
	schemaOpensBefore := schemaSource.openCount()

	report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{
		Kind: promptkit.SmokeTestStructuredOutput, PromptID: "application-prompt", APIKey: "synthetic-smoke-key-marker",
	})
	if err != nil || report == nil || !report.Completed {
		t.Fatalf("structured smoke test=(%+v, %v)", report, err)
	}
	if reader.calls != 0 || schemaSource.openCount() != schemaOpensBefore {
		t.Fatalf("application resources used: artifact reads=%d schema opens before=%d after=%d", reader.calls, schemaOpensBefore, schemaSource.openCount())
	}
	if len(client.requests) != 1 {
		t.Fatalf("injected calls=%d, want one private inference call", len(client.requests))
	}
	generationRequest := client.requests[0]
	if generationRequest.Prompt.SessionID != "" {
		t.Errorf("application session template was rendered: %q", generationRequest.Prompt.SessionID)
	}
	for _, message := range generationRequest.Prompt.Messages {
		if strings.Contains(message.Content, promptMarker) || strings.Contains(message.Content, artifactMarker) {
			t.Errorf("application prompt content reached diagnostic generation: %+v", message)
		}
	}
	structuredRequest, err := json.Marshal(generationRequest.StructuredOutput)
	if err != nil {
		t.Fatal(err)
	}
	if strings.Contains(string(structuredRequest), schemaMarker) {
		t.Fatal("application schema path reached diagnostic generation")
	}
	encodedReport, err := json.Marshal(report)
	if err != nil {
		t.Fatal(err)
	}
	for _, private := range []string{promptMarker, artifactMarker, schemaMarker, outputMarker, "synthetic-smoke-key-marker"} {
		if strings.Contains(fmt.Sprintf("%+v %s", report, encodedReport), private) {
			t.Errorf("smoke report exposed %q", private)
		}
	}
	profilesAfter, err := engine.ListProfiles(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	promptsAfter, err := engine.ListPrompts(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	inspectionAfter, err := engine.InspectPrompt(context.Background(), "application-prompt", "")
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(profilesBefore, profilesAfter) || !reflect.DeepEqual(promptsBefore, promptsAfter) ||
		inspectionBefore.PromptHash != inspectionAfter.PromptHash || inspectionAfter.OutputContract.SchemaPath != schemaMarker {
		t.Fatalf("diagnostic changed captured listings or application inspection: profiles %v -> %v, prompts %v -> %v, inspection %+v -> %+v",
			profilesBefore, profilesAfter, promptsBefore, promptsAfter, inspectionBefore, inspectionAfter)
	}
}

type nestedSmokeObserverClient struct {
	engine *promptkit.Engine
	calls  int
	mu     sync.Mutex
	report *promptkit.SmokeTestReport
	err    error
}

func (client *nestedSmokeObserverClient) Generate(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
	client.mu.Lock()
	client.calls++
	call := client.calls
	client.mu.Unlock()
	if call == 1 {
		client.report, client.err = client.engine.SmokeTest(ctx, promptkit.SmokeTestRequest{
			Kind: promptkit.SmokeTestTextInference, ProfileID: "profile",
		})
	}
	return &promptkit.GenerateResponse{Content: "an unexpected answer"}, nil
}

func TestSmokeTestDoesNotEnterInheritedExecutionObserver(t *testing.T) {
	client := &nestedSmokeObserverClient{}
	engine := newSimpleEngine(t, "unused", promptkit.WithLLMClient(client))
	client.engine = engine
	var events []promptkit.ExecutionEvent
	result, err := engine.Run(context.Background(), promptkit.RunRequest{
		PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) },
	})
	if err != nil || result == nil {
		t.Fatalf("outer Run=(%+v, %v)", result, err)
	}
	if client.err != nil || client.report == nil || !client.report.Completed {
		t.Fatalf("nested smoke test=(%+v, %v)", client.report, client.err)
	}
	if len(events) != 3 || events[0].Kind != promptkit.ExecutionStarted || events[1].Kind != promptkit.ExecutionAttempt || events[2].Kind != promptkit.ExecutionFinished {
		t.Fatalf("outer observer received diagnostic traffic: %+v", events)
	}
}

func TestSmokeTestCrossBackendDefaultUsesEndpointOverrideThroughFacade(t *testing.T) {
	const endpointOverride = "http://override.example/v1"
	var observedURL string
	httpClient := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
		observedURL = request.URL.String()
		return &http.Response{
			StatusCode: http.StatusOK,
			Header:     make(http.Header),
			Body:       io.NopCloser(strings.NewReader(`{"choices":[{"message":{"role":"assistant","content":"unexpected"}}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`)),
			Request:    request,
		}, nil
	})}
	input := configkit.Input{Source: configkit.Mapping("cross-backend smoke test", map[string]any{
		"backends": map[string]any{
			"backend-a": map[string]any{
				"endpoint":    "http://backend-a.example/v1",
				"smoke_tests": map[string]any{"default_profile": "profile-b"},
			},
			"backend-b": map[string]any{"endpoint": "http://backend-b.example/v1", "concurrency_limit": 1},
		},
		"profiles": map[string]any{"profile-b": map[string]any{"backend": "backend-b", "model": "model"}},
	})}
	engine, err := promptkit.NewEngine(promptkit.Config{
		Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: httpClient,
	})
	if err != nil {
		t.Fatalf("construct cross-backend engine: %v", err)
	}
	report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{
		Kind: promptkit.SmokeTestTextInference, BackendID: "backend-a",
		Execution: &promptkit.ExecutionTargetOverride{Endpoint: endpointOverride},
	})
	if err != nil || report == nil || !report.Completed {
		t.Fatalf("cross-backend smoke test=(%+v, %v)", report, err)
	}
	if report.Target.DefaultSourceBackendID != "backend-a" || report.Target.BackendID != "backend-b" ||
		report.Target.ProfileID != "profile-b" || report.Target.SelectionSource != "backend_default" {
		t.Fatalf("unexpected cross-backend target: %+v", report.Target)
	}
	if report.Target.Endpoint != endpointOverride || report.Target.EffectiveSettings.Endpoint != endpointOverride ||
		observedURL != endpointOverride+"/chat/completions" || len(report.Attempts) != 1 || report.Attempts[0].Endpoint != observedURL {
		t.Fatalf("endpoint override not used by facade: target=%+v attempt=%+v URL=%q", report.Target, report.Attempts, observedURL)
	}
}