Integrate inference smoke diagnostics

internal/engine/engine.go

@@ -81,9 +81,10 @@ func (o *Options) SetSchemaFile(path string) error {
 
 // Engine forwards domain operations to the single assembled use-case runner.
 type Engine struct {
-	runner     *usecase.Runner
-	inspection configuration.Inspection
-	listing    identityInventory
+	runner      *usecase.Runner
+	smokeTester *usecase.SmokeTester
+	inspection  configuration.Inspection
+	listing     identityInventory
 }
 
 type identityInventory interface {
@@ -124,11 +125,13 @@ func New(cfg Config, options Options) (*Engine, error) {
 		validator = validate.NewStandardValidator(schemaDir)
 	}
 	var client llm.Client
+	var diagnosticHTTPClient *llm.OpenAICompatibleClient
 	if options.llmClient == nil {
 		builtIn, newErr := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient})
 		if newErr != nil {
 			return nil, newErr
 		}
+		diagnosticHTTPClient = builtIn
 		client = generation.NewBuiltIn(manager, builtIn)
 	} else {
 		client = generation.NewInjected(manager, options.llmClient)
@@ -138,10 +141,12 @@ func New(cfg Config, options Options) (*Engine, error) {
 		reader = artifact.NewCompositeReader()
 	}
 
+	runner := usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries)
 	return &Engine{
-		runner:     usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries),
-		inspection: snapshot.Inspection(),
-		listing:    snapshot,
+		runner:      runner,
+		smokeTester: usecase.NewSmokeTester(runner, diagnosticHTTPClient),
+		inspection:  snapshot.Inspection(),
+		listing:     snapshot,
 	}, nil
 }
 
@@ -214,3 +219,10 @@ func (e *Engine) Run(ctx context.Context, request domain.RunRequest) (*domain.Ru
 func (e *Engine) RunPrepared(ctx context.Context, prepared *usecase.PreparedExecution) (*domain.RunResult, error) {
 	return e.runner.RunPrepared(ctx, prepared)
 }
+
+func (e *Engine) SmokeTest(ctx context.Context, request domain.SmokeTestRequest) (*domain.SmokeTestReport, error) {
+	if e == nil || e.smokeTester == nil {
+		return nil, fmt.Errorf("%w: smoke tester is not configured", usecase.ErrInvalidRequest)
+	}
+	return e.smokeTester.Run(ctx, request)
+}

internal/engine/smoke_test.go

@@ -0,0 +1,101 @@
+package engine
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
+)
+
+func TestEngineSmokeTestUsesInjectedGenerationWithoutHTTPSupport(t *testing.T) {
+	calls := 0
+	options := Options{}
+	options.SetLLMClient(engineSmokeClientFunc(func(_ context.Context, request domain.GenerateRequest) (*domain.GenerateResponse, error) {
+		calls++
+		if len(request.Prompt.Messages) != 1 || request.Prompt.Messages[0].Role != "user" {
+			t.Fatalf("injected generation request = %+v", request)
+		}
+		return &domain.GenerateResponse{Content: "engine response"}, nil
+	}))
+	assembled, err := New(Config{}, options)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	report, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
+		Kind: domain.SmokeTestTextInference, BackendID: "rakestrawhome",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if calls != 1 || !report.Completed || !report.Supported || report.Applicable == nil || !*report.Applicable ||
+		report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted ||
+		report.Facts.TextReturned == nil || !*report.Facts.TextReturned ||
+		report.Facts.ExpectedTextMatched == nil || *report.Facts.ExpectedTextMatched ||
+		report.Facts.EndpointReachable != nil || report.Target.BackendID != "rakestrawhome" {
+		t.Fatalf("injected smoke report calls=%d: %+v", calls, report)
+	}
+
+	authentication, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
+		Kind: domain.SmokeTestAuthentication, BackendID: "rakestrawhome",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !authentication.Supported || authentication.Applicable == nil || *authentication.Applicable ||
+		authentication.Reason != domain.SmokeTestReasonAuthenticationNotApplicable || len(authentication.Attempts) != 0 {
+		t.Fatalf("injected authentication report = %+v", authentication)
+	}
+}
+
+func TestEngineSmokeTestUsesBuiltInHTTPClientForInferenceEvidence(t *testing.T) {
+	var observedURL string
+	var authorization string
+	options := Options{}
+	assembled, err := New(Config{HTTPClient: &http.Client{Transport: engineRoundTripperFunc(func(request *http.Request) (*http.Response, error) {
+		observedURL = request.URL.String()
+		authorization = request.Header.Get("Authorization")
+		return &http.Response{
+			StatusCode: http.StatusOK,
+			Header:     make(http.Header),
+			Body:       io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"engine response"}}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`)),
+		}, nil
+	})}}, options)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	report, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
+		Kind: domain.SmokeTestTextInference, BackendID: "rakestrawhome",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if observedURL == "" || authorization != "" {
+		t.Fatalf("built-in request URL=%q authorization=%q", observedURL, authorization)
+	}
+	if !report.Completed || report.Facts.EndpointReachable == nil || !*report.Facts.EndpointReachable ||
+		report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived ||
+		report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted ||
+		report.Facts.CredentialsSupplied == nil || *report.Facts.CredentialsSupplied ||
+		report.Facts.TextReturned == nil || !*report.Facts.TextReturned ||
+		report.Facts.ExpectedTextMatched == nil || *report.Facts.ExpectedTextMatched || len(report.Attempts) != 1 ||
+		report.Attempts[0].Endpoint != observedURL {
+		t.Fatalf("built-in smoke report = %+v", report)
+	}
+}
+
+type engineSmokeClientFunc func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error)
+
+func (f engineSmokeClientFunc) Generate(ctx context.Context, request domain.GenerateRequest) (*domain.GenerateResponse, error) {
+	return f(ctx, request)
+}
+
+type engineRoundTripperFunc func(*http.Request) (*http.Response, error)
+
+func (f engineRoundTripperFunc) RoundTrip(request *http.Request) (*http.Response, error) {
+	return f(request)
+}