promptkit/smoke_test_contract_test.go

package promptkit_test

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"testing"
	"time"

	"gitea.maximumdirect.net/eric/promptkit"
)

type gatedSmokeClient struct {
	started chan promptkit.GenerateRequest
	release <-chan struct{}
}

func (client gatedSmokeClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
	client.started <- request
	<-client.release
	return &promptkit.GenerateResponse{Content: "a deliberately unexpected answer"}, nil
}

type cancelingSmokeClient struct{ started chan struct{} }

func (client cancelingSmokeClient) Generate(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
	close(client.started)
	<-ctx.Done()
	return nil, ctx.Err()
}

func TestSmokeTestNilEngineAndInvalidKindReturnPublicErrors(t *testing.T) {
	var nilEngine *promptkit.Engine
	if report, err := nilEngine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference}); report != nil || !errors.Is(err, promptkit.ErrInvalidRequest) {
		t.Fatalf("nil engine result = (%+v, %v), want nil report and ErrInvalidRequest", report, err)
	}

	engine := newSimpleEngine(t, "unused")
	if report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: "unknown"}); report != nil || !errors.Is(err, promptkit.ErrInvalidRequest) {
		t.Fatalf("invalid kind result = (%+v, %v), want nil report and ErrInvalidRequest", report, err)
	}
}

func TestSmokeTestConcludedInferenceMismatchHasNilGoErrorAndDetachedValues(t *testing.T) {
	release := make(chan struct{})
	client := gatedSmokeClient{started: make(chan promptkit.GenerateRequest, 1), release: release}
	engine := newSimpleEngine(t, "unused", promptkit.WithLLMClient(client))

	callerExtraParams := map[string]any{"nested": map[string]any{"value": "before"}}
	request := promptkit.SmokeTestRequest{
		Kind:      promptkit.SmokeTestTextInference,
		ProfileID: "profile",
		Execution: &promptkit.ExecutionTargetOverride{ExtraParams: callerExtraParams},
	}
	type result struct {
		report *promptkit.SmokeTestReport
		err    error
	}
	finished := make(chan result, 1)
	go func() {
		report, err := engine.SmokeTest(context.Background(), request)
		finished <- result{report: report, err: err}
	}()

	var generationRequest promptkit.GenerateRequest
	select {
	case generationRequest = <-client.started:
	case <-time.After(5 * time.Second):
		close(release)
		t.Fatal("inference did not reach the injected client")
	}
	callerExtraParams["nested"].(map[string]any)["value"] = "after"
	close(release)

	var outcome result
	select {
	case outcome = <-finished:
	case <-time.After(5 * time.Second):
		t.Fatal("smoke test did not finish")
	}
	if outcome.err != nil {
		t.Fatalf("concluded diagnostic returned Go error: %v", outcome.err)
	}
	if outcome.report == nil || !outcome.report.Completed || outcome.report.Facts.ExpectedTextMatched == nil || *outcome.report.Facts.ExpectedTextMatched {
		t.Fatalf("unexpected mismatch report: %+v", outcome.report)
	}
	if got := generationRequest.Target.ExtraParams["nested"].(map[string]any)["value"]; got != "before" {
		t.Fatalf("request observed caller mutation: nested value = %v", got)
	}
	if got := outcome.report.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"]; got != "before" {
		t.Fatalf("report does not own its effective settings: nested value = %v", got)
	}
	outcome.report.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"] = "report mutation"
	if got := callerExtraParams["nested"].(map[string]any)["value"]; got != "after" {
		t.Fatalf("report mutation affected caller values: nested value = %v", got)
	}
}

func TestSmokeTestCallerCancellationReturnsPartialReportAndIdentity(t *testing.T) {
	client := cancelingSmokeClient{started: make(chan struct{})}
	engine := newSimpleEngine(t, "unused", promptkit.WithLLMClient(client))
	ctx, cancel := context.WithCancel(context.Background())
	type result struct {
		report *promptkit.SmokeTestReport
		err    error
	}
	finished := make(chan result, 1)
	go func() {
		report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{
			Kind: promptkit.SmokeTestTextInference, ProfileID: "profile",
		})
		finished <- result{report: report, err: err}
	}()
	select {
	case <-client.started:
	case <-time.After(5 * time.Second):
		cancel()
		t.Fatal("inference did not reach the injected client")
	}
	cancel()
	select {
	case outcome := <-finished:
		if outcome.report == nil || outcome.report.Completed {
			t.Fatalf("cancellation report = %+v, want a partial report", outcome.report)
		}
		if !errors.Is(outcome.err, context.Canceled) {
			t.Fatalf("cancellation error = %v, want errors.Is(context.Canceled)", outcome.err)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("smoke test did not return after caller cancellation")
	}
}

func TestSmokeTestRequestFormattingAndJSONRedactAPIKey(t *testing.T) {
	request := promptkit.SmokeTestRequest{
		Kind:   promptkit.SmokeTestAuthentication,
		APIKey: "synthetic-smoke-secret",
	}
	formatted := fmt.Sprintf("%s %#v", request, request)
	if strings.Contains(formatted, request.APIKey) || !strings.Contains(formatted, "APIKeySet:true") {
		t.Fatalf("request formatting is not safely redacted: %s", formatted)
	}
	encoded, err := json.Marshal(request)
	if err != nil {
		t.Fatalf("marshal request: %v", err)
	}
	if strings.Contains(string(encoded), request.APIKey) || strings.Contains(string(encoded), "APIKey") {
		t.Fatalf("JSON exposed the request credential: %s", encoded)
	}
}