Expose the public smoke-test API

smoke_test_contract_test.go

@@ -0,0 +1,152 @@
+package promptkit_test
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/promptkit"
+)
+
+type gatedSmokeClient struct {
+	started chan promptkit.GenerateRequest
+	release <-chan struct{}
+}
+
+func (client gatedSmokeClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
+	client.started <- request
+	<-client.release
+	return &promptkit.GenerateResponse{Content: "a deliberately unexpected answer"}, nil
+}
+
+type cancelingSmokeClient struct{ started chan struct{} }
+
+func (client cancelingSmokeClient) Generate(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
+	close(client.started)
+	<-ctx.Done()
+	return nil, ctx.Err()
+}
+
+func TestSmokeTestNilEngineAndInvalidKindReturnPublicErrors(t *testing.T) {
+	var nilEngine *promptkit.Engine
+	if report, err := nilEngine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference}); report != nil || !errors.Is(err, promptkit.ErrInvalidRequest) {
+		t.Fatalf("nil engine result = (%+v, %v), want nil report and ErrInvalidRequest", report, err)
+	}
+
+	engine := newSimpleEngine(t, "unused")
+	if report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: "unknown"}); report != nil || !errors.Is(err, promptkit.ErrInvalidRequest) {
+		t.Fatalf("invalid kind result = (%+v, %v), want nil report and ErrInvalidRequest", report, err)
+	}
+}
+
+func TestSmokeTestConcludedInferenceMismatchHasNilGoErrorAndDetachedValues(t *testing.T) {
+	release := make(chan struct{})
+	client := gatedSmokeClient{started: make(chan promptkit.GenerateRequest, 1), release: release}
+	engine := newSimpleEngine(t, "unused", promptkit.WithLLMClient(client))
+
+	callerExtraParams := map[string]any{"nested": map[string]any{"value": "before"}}
+	request := promptkit.SmokeTestRequest{
+		Kind:      promptkit.SmokeTestTextInference,
+		ProfileID: "profile",
+		Execution: &promptkit.ExecutionTargetOverride{ExtraParams: callerExtraParams},
+	}
+	type result struct {
+		report *promptkit.SmokeTestReport
+		err    error
+	}
+	finished := make(chan result, 1)
+	go func() {
+		report, err := engine.SmokeTest(context.Background(), request)
+		finished <- result{report: report, err: err}
+	}()
+
+	var generationRequest promptkit.GenerateRequest
+	select {
+	case generationRequest = <-client.started:
+	case <-time.After(5 * time.Second):
+		close(release)
+		t.Fatal("inference did not reach the injected client")
+	}
+	callerExtraParams["nested"].(map[string]any)["value"] = "after"
+	close(release)
+
+	var outcome result
+	select {
+	case outcome = <-finished:
+	case <-time.After(5 * time.Second):
+		t.Fatal("smoke test did not finish")
+	}
+	if outcome.err != nil {
+		t.Fatalf("concluded diagnostic returned Go error: %v", outcome.err)
+	}
+	if outcome.report == nil || !outcome.report.Completed || outcome.report.Facts.ExpectedTextMatched == nil || *outcome.report.Facts.ExpectedTextMatched {
+		t.Fatalf("unexpected mismatch report: %+v", outcome.report)
+	}
+	if got := generationRequest.Target.ExtraParams["nested"].(map[string]any)["value"]; got != "before" {
+		t.Fatalf("request observed caller mutation: nested value = %v", got)
+	}
+	if got := outcome.report.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"]; got != "before" {
+		t.Fatalf("report does not own its effective settings: nested value = %v", got)
+	}
+	outcome.report.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"] = "report mutation"
+	if got := callerExtraParams["nested"].(map[string]any)["value"]; got != "after" {
+		t.Fatalf("report mutation affected caller values: nested value = %v", got)
+	}
+}
+
+func TestSmokeTestCallerCancellationReturnsPartialReportAndIdentity(t *testing.T) {
+	client := cancelingSmokeClient{started: make(chan struct{})}
+	engine := newSimpleEngine(t, "unused", promptkit.WithLLMClient(client))
+	ctx, cancel := context.WithCancel(context.Background())
+	type result struct {
+		report *promptkit.SmokeTestReport
+		err    error
+	}
+	finished := make(chan result, 1)
+	go func() {
+		report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{
+			Kind: promptkit.SmokeTestTextInference, ProfileID: "profile",
+		})
+		finished <- result{report: report, err: err}
+	}()
+	select {
+	case <-client.started:
+	case <-time.After(5 * time.Second):
+		cancel()
+		t.Fatal("inference did not reach the injected client")
+	}
+	cancel()
+	select {
+	case outcome := <-finished:
+		if outcome.report == nil || outcome.report.Completed {
+			t.Fatalf("cancellation report = %+v, want a partial report", outcome.report)
+		}
+		if !errors.Is(outcome.err, context.Canceled) {
+			t.Fatalf("cancellation error = %v, want errors.Is(context.Canceled)", outcome.err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("smoke test did not return after caller cancellation")
+	}
+}
+
+func TestSmokeTestRequestFormattingAndJSONRedactAPIKey(t *testing.T) {
+	request := promptkit.SmokeTestRequest{
+		Kind:   promptkit.SmokeTestAuthentication,
+		APIKey: "synthetic-smoke-secret",
+	}
+	formatted := fmt.Sprintf("%s %#v", request, request)
+	if strings.Contains(formatted, request.APIKey) || !strings.Contains(formatted, "APIKeySet:true") {
+		t.Fatalf("request formatting is not safely redacted: %s", formatted)
+	}
+	encoded, err := json.Marshal(request)
+	if err != nil {
+		t.Fatalf("marshal request: %v", err)
+	}
+	if strings.Contains(string(encoded), request.APIKey) || strings.Contains(string(encoded), "APIKey") {
+		t.Fatalf("JSON exposed the request credential: %s", encoded)
+	}
+}