Expose opt-in HTTP execution reports

2026-09-26 15:51:47 +00:00
parent edc1f8abd9
commit c60813e490
8 changed files with 304 additions and 44 deletions
--- a/docs/api.md
+++ b/docs/api.md
@@ -20,7 +20,7 @@ of their `Content-Type`; successful JSON responses use

 The configured request-body limit includes inline artifact bodies. The artifact
 limit applies to HTTP `file` inputs. The response limit applies to the encoded
-response, including the artifact body and optional raw output. A limit of zero
+response, including the artifact body, optional raw output, and optional execution report. A limit of zero
 disables that limit.

 A request body over its limit returns `413 request_too_large`; an oversized
@@ -50,6 +50,7 @@ copyable shape. At the HTTP adapter boundary, the smallest valid shape is:
 | `vars` | no | Object mapping template-variable names to strings. |
 | `model` | no | Runtime model-override object. |
 | `include_raw_output` | no | Include `raw_model_output` when true. |
+| `include_execution_report` | no | Include a terminal `execution` report when true and Promptkit supplies a terminal observation. Omitted, `null`, or false leaves reporting off. |
 | `transient_retries` | no | Nonnegative additional transient attempt count for this run. Omitted or `null` inherits the configured engine value and then Promptkit's default; `0` disables additional attempts. |
 | `run_timeout_seconds` | no | Optional signed 64-bit whole-second limit for engine execution. Omitted, `null`, or `0` adds no deadline; positive values shorten the caller context deadline. |

@@ -118,6 +119,7 @@ validation contract. The response contains:
 - `metadata`: run, prompt, rendered-prompt, profile, optional backend identity, model, input-hash, usage,
  timing, validation, and repair-attempt metadata; and
 - optional `raw_model_output` when requested.
+- optional `execution` when requested and a terminal observation exists.

 `metadata.model_params` has `endpoint`, `model`, `temperature`,
 `max_tokens`, `top_p`, and `timeout_seconds`, plus optional
@@ -139,6 +141,38 @@ A validation failure has `validation.status: "failed"`, `is_valid: false`,
 and any available diagnostic errors, while still returning the artifact and
 metadata.

+### Execution Report
+
+The opt-in `execution` object projects one Promptkit terminal observation. It
+is independent of `metadata.usage`, which retains its existing result-based
+meaning. A terminal event can describe a failed invocation even when no provider
+call was dispatched; if there is no terminal event, `execution` is omitted.
+
+| Field | Meaning |
+| --- | --- |
+| `run_id`, `prompt_id`, `prompt_version`, `profile_id`, `backend_id`, `model` | Terminal identifiers; absent values are omitted. |
+| `outcome` | `success`, `invalid_output`, `error`, or `other` for an unrecognized upstream value. |
+| `failure` | The known terminal failure category, `none`, or `other` for an unrecognized value. |
+| `status_code` | Provider status, omitted when unavailable. |
+| `retries_exhausted` | Boolean independent of the failure category. |
+| `attempts`, `transient_retries`, `repair_calls` | Unsigned terminal aggregate counts. |
+| `duration_ms`, `capacity_wait_ms`, `generation_duration_ms`, `retry_delay_ms` | Terminal observed durations in integer milliseconds; their sum need not equal wall time. |
+| `usage` | Object containing `prompt_tokens`, `completion_tokens`, `total_tokens`, `cached_tokens`, and `cache_write_tokens`. |
+
+Each `usage` count contains `value` (integer), `known` (boolean), and
+`complete` (boolean). Reported zero differs from an unknown count; an
+incomplete total does not establish complete billing. All counts, durations,
+`retries_exhausted`, and `usage` are present even when zero or unknown. The
+recognized failure values are `none`, `canceled`, `deadline`, `transport`,
+`provider_status`, `malformed_response`, `capacity`, `invalid_config`,
+`invalid_request`, `prompt_not_found`, `profile_not_found`, `prompt_load`,
+`profile_load`, `artifact_load`, `prompt_render`, `validation`, `generation`,
+and `other`.
+
+The report excludes content, credentials, URLs, headers, and raw errors, but
+its identifiers can still be sensitive. Apply appropriate access and retention
+controls to responses that include it.
+
 ## Error Responses

 Errors have this shape:
@@ -148,6 +182,12 @@ Errors have this shape:
 ```

 Messages are concise and do not expose wrapped internal causes.
+For an opted-in failed invocation, the error envelope can also include the
+optional top-level `execution` object described above. Decode and validation
+failures before engine invocation do not have a report. Report-bearing error
+responses are subject to the complete encoded response-size limit; if they
+exceed it, the response is the compact `413 response_too_large` error without
+`execution`. The existing compact error fallback itself is not re-limited.

 | Status | Code | Meaning |
 | --- | --- | --- |
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -129,28 +129,9 @@ successful run, and report writing is attempted even if artifact writing fails.
 Any output or report write failure exits 1; already-written output is not
 undone. Without a write failure, the actual run keeps its usual exit status.

-The JSON object contains the following fields:
-
-| Field | Meaning |
-| --- | --- |
-| `run_id`, `prompt_id`, `prompt_version`, `profile_id`, `backend_id`, `model` | Terminal identifiers; absent values are omitted. |
-| `outcome` | `success`, `invalid_output`, `error`, or `other` for an unrecognized upstream value. |
-| `failure` | The known terminal failure category, `none`, or `other` for an unrecognized value. |
-| `status_code` | Provider status, omitted when unavailable. |
-| `retries_exhausted` | Boolean independent of the failure category. |
-| `attempts`, `transient_retries`, `repair_calls` | Unsigned terminal aggregate counts. |
-| `duration_ms`, `capacity_wait_ms`, `generation_duration_ms`, `retry_delay_ms` | Terminal observed durations in integer milliseconds; their sum need not equal wall time. |
-| `usage` | Object containing `prompt_tokens`, `completion_tokens`, `total_tokens`, `cached_tokens`, and `cache_write_tokens`. |
-
-Each `usage` count contains `value` (integer), `known` (boolean), and
-`complete` (boolean). Reported zero differs from an unknown count; an
-incomplete total does not establish complete billing. All counts, durations,
-`retries_exhausted`, and `usage` are present even when zero or unknown. The
-recognized failure values are `none`, `canceled`, `deadline`, `transport`,
-`provider_status`, `malformed_response`, `capacity`, `invalid_config`,
-`invalid_request`, `prompt_not_found`, `profile_not_found`, `prompt_load`,
-`profile_load`, `artifact_load`, `prompt_render`, `validation`, `generation`,
-and `other`.
+The [HTTP API reference](api.md#execution-report) defines the shared report
+fields and usage-presence semantics. The file contains that JSON object followed
+by a newline.

 The report excludes content, credentials, URLs, headers, and raw errors, but
 its identifiers can still be sensitive. The caller owns file retention and
--- a/docs/internal/adapters.md
+++ b/docs/internal/adapters.md
@@ -81,6 +81,13 @@ CLI `run` attaches one only when a report path is selected, writes generated
 output first on success, and then writes the terminal snapshot when present.
 Its report path remains untouched for pre-invocation failures or absent
 terminal observations; report and artifact writes are independent.
+HTTP `POST /v1/runs` attaches a separate collector to each invocation only when
+`include_execution_report` is true. Its optional top-level `execution` field
+appears in success or error envelopes only when Promptkit emits a terminal
+event. Pre-invocation adapter errors and absent terminal events leave it out.
+The complete serialized success or report-bearing error response is subject to
+the configured response-size limit; overflow returns the compact 413 error
+without a report. Field meanings are owned by the [API contract](../api.md#execution-report).

 ## Package-Local Guarantees

--- a/docs/operations.md
+++ b/docs/operations.md
@@ -31,6 +31,11 @@ contain sensitive identifiers even though they omit prompt and response bodies.
 Keep report paths separate from generated output and apply the same access and
 retention controls. Output and report writes are not transactional; an older
 report may remain when no terminal observation was produced.
+HTTP callers may also request a [terminal execution report](api.md#execution-report).
+It shares the same identifier sensitivity and should receive response access
+and retention controls. An opted-in report increases the encoded response size;
+an oversized success or report-bearing error becomes the compact 413 response
+without a report.

 ## Deploy The Filesystem And Process

--- a/docs/roadmap/implementation.md
+++ b/docs/roadmap/implementation.md
@@ -349,7 +349,7 @@ do not erase the original run failure. Render and inspect remain unchanged.

 ## Stage 8: Expose HTTP Execution Reports

-**Status: Pending.**
+**Status: Complete.**

 Selected interface: optional top-level request boolean
 `include_execution_report`. Omission, null, or false leaves reporting off,
--- a/internal/adapter/http/dto.go
+++ b/internal/adapter/http/dto.go
@@ -2,6 +2,8 @@ package httpadapter

 import (
 	"time"
+
+	executionreport "gitea.maximumdirect.net/eric/scriptorium/internal/report"
 )

 type runRequestDTO struct {
@@ -15,6 +17,7 @@ type runRequestDTO struct {
 	IncludeRawOutput       bool                     `json:"include_raw_output,omitempty"`
 	TransientRetries       *int                     `json:"transient_retries,omitempty"`
 	RunTimeoutSeconds      *int64                   `json:"run_timeout_seconds,omitempty"`
+	IncludeExecutionReport bool                     `json:"include_execution_report,omitempty"`
 }

 type inputRefDTO struct {
@@ -41,6 +44,7 @@ type runResponseDTO struct {
 	Validation     validationDTO              `json:"validation"`
 	Metadata       metadataDTO                `json:"metadata"`
 	RawModelOutput *string                    `json:"raw_model_output,omitempty"`
+	Execution      *executionreport.Execution `json:"execution,omitempty"`
 }

 type artifactDTO struct {
@@ -107,6 +111,7 @@ type validationDTO struct {

 type errorResponse struct {
 	Error     errorBody                  `json:"error"`
+	Execution *executionreport.Execution `json:"execution,omitempty"`
 }

 type errorBody struct {
--- a/internal/adapter/http/handler.go
+++ b/internal/adapter/http/handler.go
@@ -12,6 +12,7 @@ import (

 	"gitea.maximumdirect.net/eric/promptkit"
 	"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
+	executionreport "gitea.maximumdirect.net/eric/scriptorium/internal/report"
 )

 type Runner interface {
@@ -111,7 +112,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		runContext, cancel = context.WithTimeout(runContext, time.Duration(*req.RunTimeoutSeconds)*time.Second)
 		defer cancel()
 	}
-	res, err := h.runner.Run(runContext, promptkit.RunRequest{
+	mappedRequest := promptkit.RunRequest{
 		PromptID:         req.PromptID,
 		PromptVersion:    req.PromptVersion,
 		ProfileID:        req.ProfileID,
@@ -120,18 +121,28 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		Vars:             req.Vars,
 		Execution:        model,
 		TransientRetries: req.TransientRetries,
-	})
+	}
+	var collector *executionreport.Collector
+	if req.IncludeExecutionReport {
+		collector = executionreport.NewCollector()
+		mappedRequest.Observer = collector.Observe
+	}
+	res, err := h.runner.Run(runContext, mappedRequest)
+	var execution *executionreport.Execution
+	if collector != nil {
+		execution = collector.Snapshot()
+	}
 	if err != nil {
 		switch runContext.Err() {
 		case context.DeadlineExceeded:
-			writeError(w, http.StatusGatewayTimeout, "deadline_exceeded", "run execution deadline exceeded")
+			writeRunFailure(w, http.StatusGatewayTimeout, "deadline_exceeded", "run execution deadline exceeded", execution, h.options.MaxResponseBytes)
 			return
 		case context.Canceled:
-			writeError(w, http.StatusRequestTimeout, "request_canceled", "run request canceled")
+			writeRunFailure(w, http.StatusRequestTimeout, "request_canceled", "run request canceled", execution, h.options.MaxResponseBytes)
 			return
 		}
 		status, code, message := mapRunError(err)
-		writeError(w, status, code, message)
+		writeRunFailure(w, status, code, message, execution, h.options.MaxResponseBytes)
 		return
 	}

@@ -172,6 +183,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			ValidationStatus:   string(res.Validation.Status),
 			RepairAttemptsUsed: res.Validation.RepairAttempts,
 		},
+		Execution: execution,
 	}
 	if req.IncludeRawOutput {
 		raw := res.RawOutput
@@ -180,6 +192,14 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	writeLimitedJSON(w, http.StatusOK, resp, h.options.MaxResponseBytes)
 }

+func writeRunFailure(w http.ResponseWriter, status int, code, message string, execution *executionreport.Execution, maxBytes int64) {
+	if execution == nil {
+		writeError(w, status, code, message)
+		return
+	}
+	writeLimitedJSON(w, status, errorResponse{Error: errorBody{Code: code, Message: message}, Execution: execution}, maxBytes)
+}
+
 func executionTargetOverrideFromModelOverrideDTO(dto *modelOverrideRequestDTO) *promptkit.ExecutionTargetOverride {
 	if dto == nil {
 		return nil
--- a/internal/adapter/http/handler_test.go
+++ b/internal/adapter/http/handler_test.go
@@ -38,6 +38,208 @@ func (f *fakeRunner) Run(ctx context.Context, req promptkit.RunRequest) (*prompt
 	return f.result, nil
 }

+func TestHandlerExecutionReportOptInAndResultShapes(t *testing.T) {
+	for _, tc := range []struct {
+		name, body string
+		wantReport bool
+		invalid    bool
+	}{
+		{"omitted", `{"prompt_id":"p"}`, false, false},
+		{"null", `{"prompt_id":"p","include_execution_report":null}`, false, false},
+		{"false", `{"prompt_id":"p","include_execution_report":false}`, false, false},
+		{"success", `{"prompt_id":"p","include_execution_report":true}`, true, false},
+		{"invalid output", `{"prompt_id":"p","include_execution_report":true}`, true, true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			runner := runnerFunc(func(_ context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
+				if (req.Observer != nil) != tc.wantReport {
+					t.Fatalf("unexpected observer presence: %v", req.Observer != nil)
+				}
+				status := promptkit.ValidationPassed
+				outcome := promptkit.ExecutionOutcomeSuccess
+				if tc.invalid {
+					status = promptkit.ValidationFailed
+					outcome = promptkit.ExecutionOutcomeInvalidOutput
+				}
+				if req.Observer != nil {
+					req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, RunID: "observed", Outcome: outcome, Failure: promptkit.ExecutionFailureNone, Attempts: 2})
+				}
+				return &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("body")}, Validation: promptkit.ValidationResult{Status: status}, Usage: promptkit.TokenUsage{PromptTokens: 7}}, nil
+			})
+			w := httptest.NewRecorder()
+			NewHandler(runner).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(tc.body)))
+			if w.Code != http.StatusOK {
+				t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
+			}
+			var response map[string]any
+			if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
+				t.Fatal(err)
+			}
+			if (response["execution"] != nil) != tc.wantReport {
+				t.Fatalf("unexpected report: %#v", response["execution"])
+			}
+			if tc.wantReport {
+				report := response["execution"].(map[string]any)
+				wantOutcome := "success"
+				if tc.invalid {
+					wantOutcome = "invalid_output"
+				}
+				if report["outcome"] != wantOutcome || report["attempts"] != float64(2) {
+					t.Fatalf("unexpected report: %#v", report)
+				}
+			}
+			if response["metadata"].(map[string]any)["usage"].(map[string]any)["prompt_tokens"] != float64(7) {
+				t.Fatalf("metadata usage changed: %#v", response["metadata"])
+			}
+		})
+	}
+
+	w := httptest.NewRecorder()
+	NewHandler(&fakeRunner{}).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","include_execution_report":"true"}`)))
+	assertHTTPErrorCode(t, w, http.StatusBadRequest, "invalid_json")
+}
+
+func TestHandlerExecutionReportFailureAndAbsence(t *testing.T) {
+	secret := "private provider diagnostic"
+	runner := runnerFunc(func(_ context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
+		req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, Outcome: promptkit.ExecutionOutcomeError, Failure: promptkit.ExecutionFailureProviderStatus, StatusCode: 503, RetriesExhausted: true, Attempts: 3})
+		return nil, fmt.Errorf("%w: %s", promptkit.ErrLLMGenerate, secret)
+	})
+	w := httptest.NewRecorder()
+	NewHandler(runner).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","include_execution_report":true}`)))
+	assertHTTPErrorCode(t, w, http.StatusBadGateway, "llm_failed")
+	var response map[string]any
+	if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
+		t.Fatal(err)
+	}
+	report := response["execution"].(map[string]any)
+	if report["failure"] != "provider_status" || report["retries_exhausted"] != true || report["attempts"] != float64(3) {
+		t.Fatalf("unexpected failure report: %#v", report)
+	}
+	if strings.Contains(w.Body.String(), secret) {
+		t.Fatalf("response leaked provider diagnostic: %s", w.Body.String())
+	}
+
+	noTerminal := runnerFunc(func(_ context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
+		if req.Observer == nil {
+			t.Fatal("missing opted-in observer")
+		}
+		return nil, promptkit.ErrPromptLoad
+	})
+	w = httptest.NewRecorder()
+	NewHandler(noTerminal).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","include_execution_report":true}`)))
+	assertHTTPErrorCode(t, w, http.StatusBadRequest, "prompt_load_failed")
+	if strings.Contains(w.Body.String(), "execution") {
+		t.Fatalf("unexpected absent report: %s", w.Body.String())
+	}
+
+	w = httptest.NewRecorder()
+	NewHandler(noTerminal).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"include_execution_report":true}`)))
+	assertHTTPErrorCode(t, w, http.StatusBadRequest, "invalid_request")
+	if strings.Contains(w.Body.String(), "execution") {
+		t.Fatalf("pre-invocation failure has report: %s", w.Body.String())
+	}
+}
+
+func TestHandlerExecutionReportResponseLimit(t *testing.T) {
+	for _, failure := range []bool{false, true} {
+		runner := runnerFunc(func(_ context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
+			if req.Observer != nil {
+				req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, Outcome: promptkit.ExecutionOutcomeError, Failure: promptkit.ExecutionFailureGeneration})
+			}
+			if failure {
+				return nil, promptkit.ErrLLMGenerate
+			}
+			return &promptkit.RunResult{}, nil
+		})
+		body := `{"prompt_id":"p"}`
+		base := httptest.NewRecorder()
+		NewHandler(runner).ServeHTTP(base, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(body)))
+		body = `{"prompt_id":"p","include_execution_report":true}`
+		limited := httptest.NewRecorder()
+		NewHandlerWithOptions(runner, HandlerOptions{MaxResponseBytes: int64(base.Body.Len())}).ServeHTTP(limited, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(body)))
+		assertHTTPErrorCode(t, limited, http.StatusRequestEntityTooLarge, "response_too_large")
+		if strings.Contains(limited.Body.String(), "execution") {
+			t.Fatalf("overflow fallback included report: %s", limited.Body.String())
+		}
+	}
+}
+
+func TestHandlerRealEngineFailureIncludesTerminalReport(t *testing.T) {
+	provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "private provider detail", http.StatusServiceUnavailable)
+	}))
+	defer provider.Close()
+	h := newRetryHandler(t, provider.URL+"/v1", intPointer(0), 0, nil)
+	w := httptest.NewRecorder()
+	h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","include_execution_report":true}`)))
+	assertHTTPErrorCode(t, w, http.StatusBadGateway, "llm_failed")
+	var response map[string]any
+	if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
+		t.Fatal(err)
+	}
+	report := response["execution"].(map[string]any)
+	if report["outcome"] != "error" || report["failure"] != "provider_status" || report["attempts"] != float64(1) {
+		t.Fatalf("unexpected terminal report: %#v", report)
+	}
+	if strings.Contains(w.Body.String(), "private provider detail") {
+		t.Fatalf("provider diagnostic leaked: %s", w.Body.String())
+	}
+}
+
+func TestHandlerConcurrentExecutionReportsAreIsolated(t *testing.T) {
+	started := make(chan struct{}, 2)
+	release := make(chan struct{})
+	defer func() {
+		select {
+		case <-release:
+		default:
+			close(release)
+		}
+	}()
+	runner := runnerFunc(func(_ context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
+		started <- struct{}{}
+		<-release
+		req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, RunID: req.SessionID, Outcome: promptkit.ExecutionOutcomeSuccess, Attempts: uint64(len(req.SessionID))})
+		return &promptkit.RunResult{}, nil
+	})
+	h := NewHandler(runner)
+	responses := make([]*httptest.ResponseRecorder, 2)
+	done := make(chan struct{}, 2)
+	for i, id := range []string{"one", "second"} {
+		responses[i] = httptest.NewRecorder()
+		go func(w *httptest.ResponseRecorder, session string) {
+			h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(fmt.Sprintf(`{"prompt_id":"p","session_id":%q,"include_execution_report":true}`, session))))
+			done <- struct{}{}
+		}(responses[i], id)
+	}
+	for range 2 {
+		select {
+		case <-started:
+		case <-time.After(3 * time.Second):
+			t.Fatal("concurrent runs did not enter runner")
+		}
+	}
+	close(release)
+	for range 2 {
+		select {
+		case <-done:
+		case <-time.After(3 * time.Second):
+			t.Fatal("concurrent run did not complete")
+		}
+	}
+	for i, id := range []string{"one", "second"} {
+		var response map[string]any
+		if err := json.Unmarshal(responses[i].Body.Bytes(), &response); err != nil {
+			t.Fatal(err)
+		}
+		report := response["execution"].(map[string]any)
+		if report["run_id"] != id || report["attempts"] != float64(len(id)) {
+			t.Fatalf("report crossed requests: %#v", report)
+		}
+	}
+}
+
 func TestMaintainedHTTPRunExampleMatchesRequestContract(t *testing.T) {
 	body, err := os.ReadFile(filepath.Join("..", "..", "..", "examples", "http-run.json"))
 	if err != nil {