eric/scriptorium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 9 Wiki Activity

Expose opt-in HTTP execution reports

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-26 15:51:47 +00:00
parent edc1f8abd9
commit c60813e490
8 changed files with 304 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
internal/adapter/http/dto.go
View File

@@ -2,19 +2,22 @@ package httpadapter
import (
"time"
executionreport "gitea.maximumdirect.net/eric/scriptorium/internal/report"
)
type runRequestDTO struct {
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
SessionID string `json:"session_id,omitempty"`
Inputs map[string]inputRefDTO `json:"inputs"`
Vars map[string]string `json:"vars,omitempty"`
Model *modelOverrideRequestDTO `json:"model,omitempty"`
IncludeRawOutput bool `json:"include_raw_output,omitempty"`
TransientRetries *int `json:"transient_retries,omitempty"`
RunTimeoutSeconds *int64 `json:"run_timeout_seconds,omitempty"`
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
SessionID string `json:"session_id,omitempty"`
Inputs map[string]inputRefDTO `json:"inputs"`
Vars map[string]string `json:"vars,omitempty"`
Model *modelOverrideRequestDTO `json:"model,omitempty"`
IncludeRawOutput bool `json:"include_raw_output,omitempty"`
TransientRetries *int `json:"transient_retries,omitempty"`
RunTimeoutSeconds *int64 `json:"run_timeout_seconds,omitempty"`
IncludeExecutionReport bool `json:"include_execution_report,omitempty"`
}
type inputRefDTO struct {
@@ -37,10 +40,11 @@ type modelOverrideRequestDTO struct {
}
type runResponseDTO struct {
Artifact artifactDTO `json:"artifact"`
Validation validationDTO `json:"validation"`
Metadata metadataDTO `json:"metadata"`
RawModelOutput *string `json:"raw_model_output,omitempty"`
Artifact artifactDTO `json:"artifact"`
Validation validationDTO `json:"validation"`
Metadata metadataDTO `json:"metadata"`
RawModelOutput *string `json:"raw_model_output,omitempty"`
Execution *executionreport.Execution `json:"execution,omitempty"`
}
type artifactDTO struct {
@@ -106,7 +110,8 @@ type validationDTO struct {
}
type errorResponse struct {
Error errorBody `json:"error"`
Error errorBody `json:"error"`
Execution *executionreport.Execution `json:"execution,omitempty"`
}
type errorBody struct {

30
internal/adapter/http/handler.go
View File

@@ -12,6 +12,7 @@ import (
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
executionreport "gitea.maximumdirect.net/eric/scriptorium/internal/report"
)
type Runner interface {
@@ -111,7 +112,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
runContext, cancel = context.WithTimeout(runContext, time.Duration(*req.RunTimeoutSeconds)*time.Second)
defer cancel()
}
res, err := h.runner.Run(runContext, promptkit.RunRequest{
mappedRequest := promptkit.RunRequest{
PromptID: req.PromptID,
PromptVersion: req.PromptVersion,
ProfileID: req.ProfileID,
@@ -120,18 +121,28 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
Vars: req.Vars,
Execution: model,
TransientRetries: req.TransientRetries,
})
}
var collector *executionreport.Collector
if req.IncludeExecutionReport {
collector = executionreport.NewCollector()
mappedRequest.Observer = collector.Observe
}
res, err := h.runner.Run(runContext, mappedRequest)
var execution *executionreport.Execution
if collector != nil {
execution = collector.Snapshot()
}
if err != nil {
switch runContext.Err() {
case context.DeadlineExceeded:
writeError(w, http.StatusGatewayTimeout, "deadline_exceeded", "run execution deadline exceeded")
writeRunFailure(w, http.StatusGatewayTimeout, "deadline_exceeded", "run execution deadline exceeded", execution, h.options.MaxResponseBytes)
return
case context.Canceled:
writeError(w, http.StatusRequestTimeout, "request_canceled", "run request canceled")
writeRunFailure(w, http.StatusRequestTimeout, "request_canceled", "run request canceled", execution, h.options.MaxResponseBytes)
return
}
status, code, message := mapRunError(err)
writeError(w, status, code, message)
writeRunFailure(w, status, code, message, execution, h.options.MaxResponseBytes)
return
}
@@ -172,6 +183,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ValidationStatus: string(res.Validation.Status),
RepairAttemptsUsed: res.Validation.RepairAttempts,
},
Execution: execution,
}
if req.IncludeRawOutput {
raw := res.RawOutput
@@ -180,6 +192,14 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
writeLimitedJSON(w, http.StatusOK, resp, h.options.MaxResponseBytes)
}
func writeRunFailure(w http.ResponseWriter, status int, code, message string, execution *executionreport.Execution, maxBytes int64) {
if execution == nil {
writeError(w, status, code, message)
return
}
writeLimitedJSON(w, status, errorResponse{Error: errorBody{Code: code, Message: message}, Execution: execution}, maxBytes)
}
func executionTargetOverrideFromModelOverrideDTO(dto *modelOverrideRequestDTO) *promptkit.ExecutionTargetOverride {
if dto == nil {
return nil

202
internal/adapter/http/handler_test.go
View File

@@ -38,6 +38,208 @@ func (f *fakeRunner) Run(ctx context.Context, req promptkit.RunRequest) (*prompt
return f.result, nil
}
func TestHandlerExecutionReportOptInAndResultShapes(t *testing.T) {
for _, tc := range []struct {
name, body string
wantReport bool
invalid bool
}{
{"omitted", `{"prompt_id":"p"}`, false, false},
{"null", `{"prompt_id":"p","include_execution_report":null}`, false, false},
{"false", `{"prompt_id":"p","include_execution_report":false}`, false, false},
{"success", `{"prompt_id":"p","include_execution_report":true}`, true, false},
{"invalid output", `{"prompt_id":"p","include_execution_report":true}`, true, true},
} {
t.Run(tc.name, func(t *testing.T) {
runner := runnerFunc(func(_ context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
if (req.Observer != nil) != tc.wantReport {
t.Fatalf("unexpected observer presence: %v", req.Observer != nil)
}
status := promptkit.ValidationPassed
outcome := promptkit.ExecutionOutcomeSuccess
if tc.invalid {
status = promptkit.ValidationFailed
outcome = promptkit.ExecutionOutcomeInvalidOutput
}
if req.Observer != nil {
req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, RunID: "observed", Outcome: outcome, Failure: promptkit.ExecutionFailureNone, Attempts: 2})
}
return &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("body")}, Validation: promptkit.ValidationResult{Status: status}, Usage: promptkit.TokenUsage{PromptTokens: 7}}, nil
})
w := httptest.NewRecorder()
NewHandler(runner).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(tc.body)))
if w.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
}
var response map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
t.Fatal(err)
}
if (response["execution"] != nil) != tc.wantReport {
t.Fatalf("unexpected report: %#v", response["execution"])
}
if tc.wantReport {
report := response["execution"].(map[string]any)
wantOutcome := "success"
if tc.invalid {
wantOutcome = "invalid_output"
}
if report["outcome"] != wantOutcome || report["attempts"] != float64(2) {
t.Fatalf("unexpected report: %#v", report)
}
}
if response["metadata"].(map[string]any)["usage"].(map[string]any)["prompt_tokens"] != float64(7) {
t.Fatalf("metadata usage changed: %#v", response["metadata"])
}
})
}
w := httptest.NewRecorder()
NewHandler(&fakeRunner{}).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","include_execution_report":"true"}`)))
assertHTTPErrorCode(t, w, http.StatusBadRequest, "invalid_json")
}
func TestHandlerExecutionReportFailureAndAbsence(t *testing.T) {
secret := "private provider diagnostic"
runner := runnerFunc(func(_ context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, Outcome: promptkit.ExecutionOutcomeError, Failure: promptkit.ExecutionFailureProviderStatus, StatusCode: 503, RetriesExhausted: true, Attempts: 3})
return nil, fmt.Errorf("%w: %s", promptkit.ErrLLMGenerate, secret)
})
w := httptest.NewRecorder()
NewHandler(runner).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","include_execution_report":true}`)))
assertHTTPErrorCode(t, w, http.StatusBadGateway, "llm_failed")
var response map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
t.Fatal(err)
}
report := response["execution"].(map[string]any)
if report["failure"] != "provider_status" || report["retries_exhausted"] != true || report["attempts"] != float64(3) {
t.Fatalf("unexpected failure report: %#v", report)
}
if strings.Contains(w.Body.String(), secret) {
t.Fatalf("response leaked provider diagnostic: %s", w.Body.String())
}
noTerminal := runnerFunc(func(_ context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
if req.Observer == nil {
t.Fatal("missing opted-in observer")
}
return nil, promptkit.ErrPromptLoad
})
w = httptest.NewRecorder()
NewHandler(noTerminal).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","include_execution_report":true}`)))
assertHTTPErrorCode(t, w, http.StatusBadRequest, "prompt_load_failed")
if strings.Contains(w.Body.String(), "execution") {
t.Fatalf("unexpected absent report: %s", w.Body.String())
}
w = httptest.NewRecorder()
NewHandler(noTerminal).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"include_execution_report":true}`)))
assertHTTPErrorCode(t, w, http.StatusBadRequest, "invalid_request")
if strings.Contains(w.Body.String(), "execution") {
t.Fatalf("pre-invocation failure has report: %s", w.Body.String())
}
}
func TestHandlerExecutionReportResponseLimit(t *testing.T) {
for _, failure := range []bool{false, true} {
runner := runnerFunc(func(_ context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
if req.Observer != nil {
req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, Outcome: promptkit.ExecutionOutcomeError, Failure: promptkit.ExecutionFailureGeneration})
}
if failure {
return nil, promptkit.ErrLLMGenerate
}
return &promptkit.RunResult{}, nil
})
body := `{"prompt_id":"p"}`
base := httptest.NewRecorder()
NewHandler(runner).ServeHTTP(base, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(body)))
body = `{"prompt_id":"p","include_execution_report":true}`
limited := httptest.NewRecorder()
NewHandlerWithOptions(runner, HandlerOptions{MaxResponseBytes: int64(base.Body.Len())}).ServeHTTP(limited, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(body)))
assertHTTPErrorCode(t, limited, http.StatusRequestEntityTooLarge, "response_too_large")
if strings.Contains(limited.Body.String(), "execution") {
t.Fatalf("overflow fallback included report: %s", limited.Body.String())
}
}
}
func TestHandlerRealEngineFailureIncludesTerminalReport(t *testing.T) {
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "private provider detail", http.StatusServiceUnavailable)
}))
defer provider.Close()
h := newRetryHandler(t, provider.URL+"/v1", intPointer(0), 0, nil)
w := httptest.NewRecorder()
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","include_execution_report":true}`)))
assertHTTPErrorCode(t, w, http.StatusBadGateway, "llm_failed")
var response map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
t.Fatal(err)
}
report := response["execution"].(map[string]any)
if report["outcome"] != "error" || report["failure"] != "provider_status" || report["attempts"] != float64(1) {
t.Fatalf("unexpected terminal report: %#v", report)
}
if strings.Contains(w.Body.String(), "private provider detail") {
t.Fatalf("provider diagnostic leaked: %s", w.Body.String())
}
}
func TestHandlerConcurrentExecutionReportsAreIsolated(t *testing.T) {
started := make(chan struct{}, 2)
release := make(chan struct{})
defer func() {
select {
case <-release:
default:
close(release)
}
}()
runner := runnerFunc(func(_ context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
started <- struct{}{}
<-release
req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, RunID: req.SessionID, Outcome: promptkit.ExecutionOutcomeSuccess, Attempts: uint64(len(req.SessionID))})
return &promptkit.RunResult{}, nil
})
h := NewHandler(runner)
responses := make([]*httptest.ResponseRecorder, 2)
done := make(chan struct{}, 2)
for i, id := range []string{"one", "second"} {
responses[i] = httptest.NewRecorder()
go func(w *httptest.ResponseRecorder, session string) {
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(fmt.Sprintf(`{"prompt_id":"p","session_id":%q,"include_execution_report":true}`, session))))
done <- struct{}{}
}(responses[i], id)
}
for range 2 {
select {
case <-started:
case <-time.After(3 * time.Second):
t.Fatal("concurrent runs did not enter runner")
}
}
close(release)
for range 2 {
select {
case <-done:
case <-time.After(3 * time.Second):
t.Fatal("concurrent run did not complete")
}
}
for i, id := range []string{"one", "second"} {
var response map[string]any
if err := json.Unmarshal(responses[i].Body.Bytes(), &response); err != nil {
t.Fatal(err)
}
report := response["execution"].(map[string]any)
if report["run_id"] != id || report["attempts"] != float64(len(id)) {
t.Fatalf("report crossed requests: %#v", report)
}
}
}
func TestMaintainedHTTPRunExampleMatchesRequestContract(t *testing.T) {
body, err := os.ReadFile(filepath.Join("..", "..", "..", "examples", "http-run.json"))
if err != nil {