Expose opt-in HTTP execution reports

2026-09-26 15:51:47 +00:00
parent edc1f8abd9
commit c60813e490
8 changed files with 304 additions and 44 deletions
--- a/internal/adapter/http/handler_test.go
+++ b/internal/adapter/http/handler_test.go
@@ -38,6 +38,208 @@ func (f *fakeRunner) Run(ctx context.Context, req promptkit.RunRequest) (*prompt
 	return f.result, nil
 }

+func TestHandlerExecutionReportOptInAndResultShapes(t *testing.T) {
+	for _, tc := range []struct {
+		name, body string
+		wantReport bool
+		invalid    bool
+	}{
+		{"omitted", `{"prompt_id":"p"}`, false, false},
+		{"null", `{"prompt_id":"p","include_execution_report":null}`, false, false},
+		{"false", `{"prompt_id":"p","include_execution_report":false}`, false, false},
+		{"success", `{"prompt_id":"p","include_execution_report":true}`, true, false},
+		{"invalid output", `{"prompt_id":"p","include_execution_report":true}`, true, true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			runner := runnerFunc(func(_ context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
+				if (req.Observer != nil) != tc.wantReport {
+					t.Fatalf("unexpected observer presence: %v", req.Observer != nil)
+				}
+				status := promptkit.ValidationPassed
+				outcome := promptkit.ExecutionOutcomeSuccess
+				if tc.invalid {
+					status = promptkit.ValidationFailed
+					outcome = promptkit.ExecutionOutcomeInvalidOutput
+				}
+				if req.Observer != nil {
+					req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, RunID: "observed", Outcome: outcome, Failure: promptkit.ExecutionFailureNone, Attempts: 2})
+				}
+				return &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("body")}, Validation: promptkit.ValidationResult{Status: status}, Usage: promptkit.TokenUsage{PromptTokens: 7}}, nil
+			})
+			w := httptest.NewRecorder()
+			NewHandler(runner).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(tc.body)))
+			if w.Code != http.StatusOK {
+				t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
+			}
+			var response map[string]any
+			if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
+				t.Fatal(err)
+			}
+			if (response["execution"] != nil) != tc.wantReport {
+				t.Fatalf("unexpected report: %#v", response["execution"])
+			}
+			if tc.wantReport {
+				report := response["execution"].(map[string]any)
+				wantOutcome := "success"
+				if tc.invalid {
+					wantOutcome = "invalid_output"
+				}
+				if report["outcome"] != wantOutcome || report["attempts"] != float64(2) {
+					t.Fatalf("unexpected report: %#v", report)
+				}
+			}
+			if response["metadata"].(map[string]any)["usage"].(map[string]any)["prompt_tokens"] != float64(7) {
+				t.Fatalf("metadata usage changed: %#v", response["metadata"])
+			}
+		})
+	}
+
+	w := httptest.NewRecorder()
+	NewHandler(&fakeRunner{}).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","include_execution_report":"true"}`)))
+	assertHTTPErrorCode(t, w, http.StatusBadRequest, "invalid_json")
+}
+
+func TestHandlerExecutionReportFailureAndAbsence(t *testing.T) {
+	secret := "private provider diagnostic"
+	runner := runnerFunc(func(_ context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
+		req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, Outcome: promptkit.ExecutionOutcomeError, Failure: promptkit.ExecutionFailureProviderStatus, StatusCode: 503, RetriesExhausted: true, Attempts: 3})
+		return nil, fmt.Errorf("%w: %s", promptkit.ErrLLMGenerate, secret)
+	})
+	w := httptest.NewRecorder()
+	NewHandler(runner).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","include_execution_report":true}`)))
+	assertHTTPErrorCode(t, w, http.StatusBadGateway, "llm_failed")
+	var response map[string]any
+	if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
+		t.Fatal(err)
+	}
+	report := response["execution"].(map[string]any)
+	if report["failure"] != "provider_status" || report["retries_exhausted"] != true || report["attempts"] != float64(3) {
+		t.Fatalf("unexpected failure report: %#v", report)
+	}
+	if strings.Contains(w.Body.String(), secret) {
+		t.Fatalf("response leaked provider diagnostic: %s", w.Body.String())
+	}
+
+	noTerminal := runnerFunc(func(_ context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
+		if req.Observer == nil {
+			t.Fatal("missing opted-in observer")
+		}
+		return nil, promptkit.ErrPromptLoad
+	})
+	w = httptest.NewRecorder()
+	NewHandler(noTerminal).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","include_execution_report":true}`)))
+	assertHTTPErrorCode(t, w, http.StatusBadRequest, "prompt_load_failed")
+	if strings.Contains(w.Body.String(), "execution") {
+		t.Fatalf("unexpected absent report: %s", w.Body.String())
+	}
+
+	w = httptest.NewRecorder()
+	NewHandler(noTerminal).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"include_execution_report":true}`)))
+	assertHTTPErrorCode(t, w, http.StatusBadRequest, "invalid_request")
+	if strings.Contains(w.Body.String(), "execution") {
+		t.Fatalf("pre-invocation failure has report: %s", w.Body.String())
+	}
+}
+
+func TestHandlerExecutionReportResponseLimit(t *testing.T) {
+	for _, failure := range []bool{false, true} {
+		runner := runnerFunc(func(_ context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
+			if req.Observer != nil {
+				req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, Outcome: promptkit.ExecutionOutcomeError, Failure: promptkit.ExecutionFailureGeneration})
+			}
+			if failure {
+				return nil, promptkit.ErrLLMGenerate
+			}
+			return &promptkit.RunResult{}, nil
+		})
+		body := `{"prompt_id":"p"}`
+		base := httptest.NewRecorder()
+		NewHandler(runner).ServeHTTP(base, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(body)))
+		body = `{"prompt_id":"p","include_execution_report":true}`
+		limited := httptest.NewRecorder()
+		NewHandlerWithOptions(runner, HandlerOptions{MaxResponseBytes: int64(base.Body.Len())}).ServeHTTP(limited, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(body)))
+		assertHTTPErrorCode(t, limited, http.StatusRequestEntityTooLarge, "response_too_large")
+		if strings.Contains(limited.Body.String(), "execution") {
+			t.Fatalf("overflow fallback included report: %s", limited.Body.String())
+		}
+	}
+}
+
+func TestHandlerRealEngineFailureIncludesTerminalReport(t *testing.T) {
+	provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "private provider detail", http.StatusServiceUnavailable)
+	}))
+	defer provider.Close()
+	h := newRetryHandler(t, provider.URL+"/v1", intPointer(0), 0, nil)
+	w := httptest.NewRecorder()
+	h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","include_execution_report":true}`)))
+	assertHTTPErrorCode(t, w, http.StatusBadGateway, "llm_failed")
+	var response map[string]any
+	if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
+		t.Fatal(err)
+	}
+	report := response["execution"].(map[string]any)
+	if report["outcome"] != "error" || report["failure"] != "provider_status" || report["attempts"] != float64(1) {
+		t.Fatalf("unexpected terminal report: %#v", report)
+	}
+	if strings.Contains(w.Body.String(), "private provider detail") {
+		t.Fatalf("provider diagnostic leaked: %s", w.Body.String())
+	}
+}
+
+func TestHandlerConcurrentExecutionReportsAreIsolated(t *testing.T) {
+	started := make(chan struct{}, 2)
+	release := make(chan struct{})
+	defer func() {
+		select {
+		case <-release:
+		default:
+			close(release)
+		}
+	}()
+	runner := runnerFunc(func(_ context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
+		started <- struct{}{}
+		<-release
+		req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, RunID: req.SessionID, Outcome: promptkit.ExecutionOutcomeSuccess, Attempts: uint64(len(req.SessionID))})
+		return &promptkit.RunResult{}, nil
+	})
+	h := NewHandler(runner)
+	responses := make([]*httptest.ResponseRecorder, 2)
+	done := make(chan struct{}, 2)
+	for i, id := range []string{"one", "second"} {
+		responses[i] = httptest.NewRecorder()
+		go func(w *httptest.ResponseRecorder, session string) {
+			h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(fmt.Sprintf(`{"prompt_id":"p","session_id":%q,"include_execution_report":true}`, session))))
+			done <- struct{}{}
+		}(responses[i], id)
+	}
+	for range 2 {
+		select {
+		case <-started:
+		case <-time.After(3 * time.Second):
+			t.Fatal("concurrent runs did not enter runner")
+		}
+	}
+	close(release)
+	for range 2 {
+		select {
+		case <-done:
+		case <-time.After(3 * time.Second):
+			t.Fatal("concurrent run did not complete")
+		}
+	}
+	for i, id := range []string{"one", "second"} {
+		var response map[string]any
+		if err := json.Unmarshal(responses[i].Body.Bytes(), &response); err != nil {
+			t.Fatal(err)
+		}
+		report := response["execution"].(map[string]any)
+		if report["run_id"] != id || report["attempts"] != float64(len(id)) {
+			t.Fatalf("report crossed requests: %#v", report)
+		}
+	}
+}
+
 func TestMaintainedHTTPRunExampleMatchesRequestContract(t *testing.T) {
 	body, err := os.ReadFile(filepath.Join("..", "..", "..", "examples", "http-run.json"))
 	if err != nil {