933 lines
33 KiB
Go
933 lines
33 KiB
Go
package httpadapter
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
|
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
|
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
|
)
|
|
|
|
type fakeRunner struct {
|
|
result *domain.RunResult
|
|
err error
|
|
last domain.RunRequest
|
|
}
|
|
|
|
func (f *fakeRunner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) {
|
|
f.last = req
|
|
if f.err != nil {
|
|
return nil, f.err
|
|
}
|
|
return f.result, nil
|
|
}
|
|
|
|
func TestMaintainedHTTPRunExampleMatchesRequestContract(t *testing.T) {
|
|
body, err := os.ReadFile(filepath.Join("..", "..", "..", "examples", "http-run.json"))
|
|
if err != nil {
|
|
t.Fatalf("read maintained HTTP request example: %v", err)
|
|
}
|
|
|
|
runner := &fakeRunner{result: &domain.RunResult{}}
|
|
h := NewHandler(runner)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewReader(body))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected maintained HTTP request example to be accepted, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
|
|
var invalidExample map[string]json.RawMessage
|
|
if err := json.Unmarshal(body, &invalidExample); err != nil {
|
|
t.Fatalf("decode maintained HTTP request example: %v", err)
|
|
}
|
|
invalidExample["unexpected"] = json.RawMessage(`true`)
|
|
invalidBody, err := json.Marshal(invalidExample)
|
|
if err != nil {
|
|
t.Fatalf("encode structurally invalid request example: %v", err)
|
|
}
|
|
invalidReq := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewReader(invalidBody))
|
|
invalidW := httptest.NewRecorder()
|
|
h.ServeHTTP(invalidW, invalidReq)
|
|
assertHTTPErrorCode(t, invalidW, http.StatusBadRequest, "invalid_json")
|
|
}
|
|
|
|
type handlerPromptRepo struct {
|
|
def *domain.PromptDefinition
|
|
}
|
|
|
|
func (r handlerPromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
|
return r.def, nil
|
|
}
|
|
|
|
type handlerProfileRepo struct {
|
|
profile *domain.ExecutionProfile
|
|
}
|
|
|
|
func (r handlerProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
|
return r.profile, nil
|
|
}
|
|
|
|
type handlerArtifactReader struct{}
|
|
|
|
func (handlerArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
|
return &domain.Artifact{Name: "input", Body: []byte("input"), Hash: "hash"}, nil
|
|
}
|
|
|
|
type handlerRenderer struct{}
|
|
|
|
func (handlerRenderer) Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
|
|
return &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, nil
|
|
}
|
|
|
|
type handlerLLMClient struct{}
|
|
|
|
func (handlerLLMClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
|
return &domain.GenerateResponse{Content: "ok"}, nil
|
|
}
|
|
|
|
func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|
start := time.Now().UTC()
|
|
end := start.Add(2 * time.Second)
|
|
const envName = "SCRIPTORIUM_API_KEY"
|
|
const secret = "never-include-me"
|
|
|
|
r := &fakeRunner{result: &domain.RunResult{
|
|
RunID: "11111111-1111-4111-8111-111111111111",
|
|
Artifact: domain.Artifact{
|
|
Name: "output",
|
|
ContentType: "text/plain",
|
|
Body: []byte("hello"),
|
|
Size: 5,
|
|
Hash: "abc",
|
|
},
|
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
|
PromptID: "prompt-1",
|
|
PromptVersion: "1.0.0",
|
|
PromptHash: "phash",
|
|
RenderedPromptHash: "rhash",
|
|
SelectedProfileID: "exec-default",
|
|
ModelName: "m1",
|
|
Endpoint: "http://llm/v1",
|
|
EffectiveModelParams: domain.ExecutionTarget{
|
|
Endpoint: "http://llm/v1",
|
|
Model: "m1",
|
|
Temperature: 0.2,
|
|
MaxTokens: 42,
|
|
TopP: 0.9,
|
|
TimeoutSeconds: 120,
|
|
ServiceTier: "priority",
|
|
APIKeyEnv: envName,
|
|
},
|
|
InputHashes: map[string]string{"transcript": "h1"},
|
|
Usage: domain.TokenUsage{
|
|
PromptTokens: 1,
|
|
CompletionTokens: 2,
|
|
TotalTokens: 3,
|
|
CachedTokens: 4,
|
|
CacheWriteTokens: 5,
|
|
},
|
|
StartTime: start,
|
|
EndTime: end,
|
|
Duration: 2 * time.Second,
|
|
RawOutput: "hello",
|
|
}}
|
|
|
|
h := NewHandler(r)
|
|
|
|
body := []byte(`{
|
|
"prompt_id": "prompt-1",
|
|
"profile_id": "exec-default",
|
|
"inputs": {
|
|
"transcript": {"type": "file", "uri": "./t.md"}
|
|
},
|
|
"vars": {"k": "v"},
|
|
"model": {"model": "gpt-x", "timeout_seconds": 120, "service_tier": "flex", "api_key_env": "SCRIPTORIUM_API_KEY"}
|
|
}`)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewReader(body))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
|
|
var resp map[string]any
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("invalid JSON response: %v", err)
|
|
}
|
|
|
|
metadata := resp["metadata"].(map[string]any)
|
|
if metadata["prompt_id"] != "prompt-1" {
|
|
t.Fatalf("unexpected metadata.prompt_id: %#v", metadata["prompt_id"])
|
|
}
|
|
if metadata["prompt_version"] != "1.0.0" {
|
|
t.Fatalf("unexpected metadata.prompt_version: %#v", metadata["prompt_version"])
|
|
}
|
|
if metadata["selected_profile_id"] != "exec-default" {
|
|
t.Fatalf("unexpected metadata.selected_profile_id: %#v", metadata["selected_profile_id"])
|
|
}
|
|
if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" {
|
|
t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"])
|
|
}
|
|
usage := metadata["usage"].(map[string]any)
|
|
if usage["prompt_tokens"] != float64(1) || usage["completion_tokens"] != float64(2) || usage["total_tokens"] != float64(3) {
|
|
t.Fatalf("unexpected base usage metadata: %#v", usage)
|
|
}
|
|
if usage["cached_tokens"] != float64(4) || usage["cache_write_tokens"] != float64(5) {
|
|
t.Fatalf("unexpected cache usage metadata: %#v", usage)
|
|
}
|
|
modelParams := metadata["model_params"].(map[string]any)
|
|
if modelParams["api_key_env"] != envName {
|
|
t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"])
|
|
}
|
|
if modelParams["service_tier"] != "priority" {
|
|
t.Fatalf("expected model_params.service_tier=priority, got %#v", modelParams["service_tier"])
|
|
}
|
|
if strings.Contains(w.Body.String(), secret) {
|
|
t.Fatalf("response leaked raw API key value: %s", w.Body.String())
|
|
}
|
|
if _, ok := resp["raw_model_output"]; ok {
|
|
t.Fatalf("expected raw_model_output to be omitted by default, got %#v", resp["raw_model_output"])
|
|
}
|
|
|
|
if r.last.PromptID != "prompt-1" {
|
|
t.Fatalf("expected request prompt_id prompt-1, got %q", r.last.PromptID)
|
|
}
|
|
if r.last.ProfileID != "exec-default" {
|
|
t.Fatalf("expected request profile_id exec-default, got %q", r.last.ProfileID)
|
|
}
|
|
if r.last.Execution == nil || r.last.Execution.Model != "gpt-x" {
|
|
t.Fatalf("expected model override, got %#v", r.last.Execution)
|
|
}
|
|
if r.last.Execution.TimeoutSeconds == nil || *r.last.Execution.TimeoutSeconds != 120 {
|
|
t.Fatalf("expected timeout_seconds override 120, got %#v", r.last.Execution)
|
|
}
|
|
if r.last.Execution.ServiceTier != "flex" {
|
|
t.Fatalf("expected service_tier override flex, got %#v", r.last.Execution)
|
|
}
|
|
}
|
|
|
|
func TestHandlerInlineRefsWorkWithoutArtifactRoot(t *testing.T) {
|
|
h := newArtifactRootHandler(t, "")
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
|
"prompt_id":"p",
|
|
"inputs":{"x":{"type":"inline","body":"inline body"}}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestHandlerFileRefsWithoutArtifactRootAreRejected(t *testing.T) {
|
|
h := newArtifactRootHandler(t, "")
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
|
"prompt_id":"p",
|
|
"inputs":{"x":{"type":"file","uri":"input.txt"}}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
assertHTTPErrorCode(t, w, http.StatusBadRequest, "artifact_not_allowed")
|
|
}
|
|
|
|
func TestHandlerFileRefsUnderArtifactRootWork(t *testing.T) {
|
|
root := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(root, "input.txt"), []byte("allowed"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h := newArtifactRootHandler(t, root)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
|
"prompt_id":"p",
|
|
"inputs":{"x":{"type":"file","uri":"input.txt"}}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestHandlerFileRefsAboveArtifactLimitAreRejected(t *testing.T) {
|
|
root := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(root, "large.txt"), []byte("123456"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h := newArtifactRootHandlerWithLimit(t, root, 5)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
|
"prompt_id":"p",
|
|
"inputs":{"x":{"type":"file","uri":"large.txt"}}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
assertHTTPErrorCode(t, w, http.StatusRequestEntityTooLarge, "artifact_too_large")
|
|
}
|
|
|
|
func TestHandlerFileRefsOutsideArtifactRootAreRejected(t *testing.T) {
|
|
root := t.TempDir()
|
|
outside := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(outside, "secret.txt"), []byte("denied"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h := newArtifactRootHandler(t, root)
|
|
|
|
tests := []struct {
|
|
name string
|
|
uri string
|
|
}{
|
|
{name: "relative traversal", uri: filepath.Join("..", filepath.Base(outside), "secret.txt")},
|
|
{name: "absolute outside root", uri: filepath.Join(outside, "secret.txt")},
|
|
}
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
body := fmt.Sprintf(`{
|
|
"prompt_id":"p",
|
|
"inputs":{"x":{"type":"file","uri":%q}}
|
|
}`, tc.uri)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(body))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
assertHTTPErrorCode(t, w, http.StatusBadRequest, "artifact_not_allowed")
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
|
|
r := &fakeRunner{result: &domain.RunResult{
|
|
Artifact: domain.Artifact{Body: []byte("ok")},
|
|
PromptID: "prompt-1",
|
|
PromptVersion: "1.0.0",
|
|
SelectedProfileID: "prompt-default",
|
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
|
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
|
}}
|
|
h := NewHandler(r)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"prompt-1","inputs":{"x":{"type":"file","uri":"a"}}}`))
|
|
w := httptest.NewRecorder()
|
|
h.ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if r.last.ProfileID != "" {
|
|
t.Fatalf("expected empty request profile_id when omitted, got %q", r.last.ProfileID)
|
|
}
|
|
var resp map[string]any
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("invalid JSON response: %v", err)
|
|
}
|
|
metadata := resp["metadata"].(map[string]any)
|
|
if metadata["selected_profile_id"] != "prompt-default" {
|
|
t.Fatalf("expected selected_profile_id from result, got %#v", metadata["selected_profile_id"])
|
|
}
|
|
usage := metadata["usage"].(map[string]any)
|
|
if usage["cached_tokens"] != float64(0) || usage["cache_write_tokens"] != float64(0) {
|
|
t.Fatalf("expected zero cache usage fields to be included, got %#v", usage)
|
|
}
|
|
}
|
|
|
|
func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
|
|
r := &fakeRunner{result: &domain.RunResult{
|
|
Artifact: domain.Artifact{Body: []byte("ok")},
|
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
|
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
|
}}
|
|
h := NewHandler(r)
|
|
|
|
reqBody := `{
|
|
"prompt_id": "prompt-1",
|
|
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
|
|
"model": {
|
|
"endpoint": "http://override/v1",
|
|
"model": "override-model",
|
|
"temperature": 0.6,
|
|
"max_tokens": 250,
|
|
"top_p": 0.85,
|
|
"timeout_seconds": 33,
|
|
"service_tier": "flex",
|
|
"reasoning_effort": "medium",
|
|
"api_key_env": "SCRIPTORIUM_API_KEY",
|
|
"extra_params": {"provider_option":"on"}
|
|
}
|
|
}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(reqBody))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if r.last.Execution == nil {
|
|
t.Fatalf("expected execution override in run request")
|
|
}
|
|
got := r.last.Execution
|
|
if got.Endpoint != "http://override/v1" ||
|
|
got.Model != "override-model" ||
|
|
got.ServiceTier != "flex" ||
|
|
got.ReasoningEffort != "medium" ||
|
|
got.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
|
|
t.Fatalf("unexpected mapped execution target: %+v", got)
|
|
}
|
|
if got.Temperature == nil || *got.Temperature != 0.6 {
|
|
t.Fatalf("unexpected mapped temperature: %#v", got.Temperature)
|
|
}
|
|
if got.MaxTokens == nil || *got.MaxTokens != 250 {
|
|
t.Fatalf("unexpected mapped max_tokens: %#v", got.MaxTokens)
|
|
}
|
|
if got.TopP == nil || *got.TopP != 0.85 {
|
|
t.Fatalf("unexpected mapped top_p: %#v", got.TopP)
|
|
}
|
|
if got.TimeoutSeconds == nil || *got.TimeoutSeconds != 33 {
|
|
t.Fatalf("unexpected mapped timeout_seconds: %#v", got.TimeoutSeconds)
|
|
}
|
|
if !reflect.DeepEqual(got.ExtraParams, map[string]any{"provider_option": "on"}) {
|
|
t.Fatalf("unexpected mapped extra_params: %#v", got.ExtraParams)
|
|
}
|
|
}
|
|
|
|
func TestHandlerModelOverrideAcceptsJSONCompatibleExtraParams(t *testing.T) {
|
|
r := &fakeRunner{result: &domain.RunResult{
|
|
Artifact: domain.Artifact{Body: []byte("ok")},
|
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
|
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
|
}}
|
|
h := NewHandler(r)
|
|
|
|
reqBody := `{
|
|
"prompt_id": "prompt-1",
|
|
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
|
|
"model": {
|
|
"extra_params": {
|
|
"string_value": "enabled",
|
|
"number_value": 42,
|
|
"boolean_value": true,
|
|
"object_value": {"nested": "value", "count": 2},
|
|
"array_value": ["first", 3, false]
|
|
}
|
|
}
|
|
}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(reqBody))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if r.last.Execution == nil {
|
|
t.Fatal("expected execution override in run request")
|
|
}
|
|
want := map[string]any{
|
|
"string_value": "enabled",
|
|
"number_value": float64(42),
|
|
"boolean_value": true,
|
|
"object_value": map[string]any{"nested": "value", "count": float64(2)},
|
|
"array_value": []any{"first", float64(3), false},
|
|
}
|
|
if !reflect.DeepEqual(r.last.Execution.ExtraParams, want) {
|
|
t.Fatalf("unexpected mapped extra_params:\ngot=%#v\nwant=%#v", r.last.Execution.ExtraParams, want)
|
|
}
|
|
}
|
|
|
|
func TestHandlerModelOverrideExplicitZeroTemperatureMapsAsPresent(t *testing.T) {
|
|
r := &fakeRunner{result: &domain.RunResult{
|
|
Artifact: domain.Artifact{Body: []byte("ok")},
|
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
|
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0},
|
|
}}
|
|
h := NewHandler(r)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
|
"prompt_id": "prompt-1",
|
|
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
|
|
"model": {"temperature": 0}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
h.ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if r.last.Execution == nil || r.last.Execution.Temperature == nil {
|
|
t.Fatalf("expected temperature override to be present, got %#v", r.last.Execution)
|
|
}
|
|
if *r.last.Execution.Temperature != 0 {
|
|
t.Fatalf("expected zero temperature override, got %v", *r.last.Execution.Temperature)
|
|
}
|
|
}
|
|
|
|
func TestHandlerModelOverrideOmittedTemperatureMapsAsAbsent(t *testing.T) {
|
|
r := &fakeRunner{result: &domain.RunResult{
|
|
Artifact: domain.Artifact{Body: []byte("ok")},
|
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
|
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0.7},
|
|
}}
|
|
h := NewHandler(r)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
|
"prompt_id": "prompt-1",
|
|
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
|
|
"model": {"model": "override-model"}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
h.ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if r.last.Execution == nil {
|
|
t.Fatal("expected model override")
|
|
}
|
|
if r.last.Execution.Temperature != nil {
|
|
t.Fatalf("expected omitted temperature to remain absent, got %#v", r.last.Execution.Temperature)
|
|
}
|
|
|
|
var resp map[string]any
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("invalid JSON response: %v", err)
|
|
}
|
|
metadata := resp["metadata"].(map[string]any)
|
|
params := metadata["model_params"].(map[string]any)
|
|
if params["temperature"] != 0.7 {
|
|
t.Fatalf("expected effective profile/default temperature in response, got %#v", params["temperature"])
|
|
}
|
|
}
|
|
|
|
func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing.T) {
|
|
r := &fakeRunner{result: &domain.RunResult{
|
|
Artifact: domain.Artifact{
|
|
Name: "output",
|
|
ContentType: "text/plain",
|
|
Body: []byte("ok"),
|
|
Size: 2,
|
|
Hash: "abc",
|
|
},
|
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
|
EffectiveModelParams: domain.ExecutionTarget{
|
|
Endpoint: "http://llm/v1",
|
|
Model: "gpt-test",
|
|
Temperature: 0.4,
|
|
MaxTokens: 321,
|
|
TopP: 0.7,
|
|
TimeoutSeconds: 45,
|
|
ServiceTier: "priority",
|
|
ReasoningEffort: "high",
|
|
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
|
ExtraParams: map[string]any{
|
|
"provider_option": "on",
|
|
"number_value": 42,
|
|
"object_value": map[string]any{"nested": "value"},
|
|
},
|
|
},
|
|
}}
|
|
h := NewHandler(r)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
|
|
w := httptest.NewRecorder()
|
|
h.ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
var resp map[string]any
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("invalid JSON response: %v", err)
|
|
}
|
|
metadata := resp["metadata"].(map[string]any)
|
|
params := metadata["model_params"].(map[string]any)
|
|
|
|
if params["endpoint"] != "http://llm/v1" {
|
|
t.Fatalf("unexpected endpoint: %#v", params["endpoint"])
|
|
}
|
|
if params["model"] != "gpt-test" {
|
|
t.Fatalf("unexpected model: %#v", params["model"])
|
|
}
|
|
if params["temperature"] != 0.4 {
|
|
t.Fatalf("unexpected temperature: %#v", params["temperature"])
|
|
}
|
|
if params["max_tokens"] != float64(321) {
|
|
t.Fatalf("unexpected max_tokens: %#v", params["max_tokens"])
|
|
}
|
|
if params["top_p"] != 0.7 {
|
|
t.Fatalf("unexpected top_p: %#v", params["top_p"])
|
|
}
|
|
if params["timeout_seconds"] != float64(45) {
|
|
t.Fatalf("unexpected timeout_seconds: %#v", params["timeout_seconds"])
|
|
}
|
|
if params["service_tier"] != "priority" {
|
|
t.Fatalf("unexpected service_tier: %#v", params["service_tier"])
|
|
}
|
|
if params["reasoning_effort"] != "high" {
|
|
t.Fatalf("unexpected reasoning_effort: %#v", params["reasoning_effort"])
|
|
}
|
|
if params["api_key_env"] != "SCRIPTORIUM_API_KEY" {
|
|
t.Fatalf("unexpected api_key_env: %#v", params["api_key_env"])
|
|
}
|
|
extraParams, ok := params["extra_params"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected extra_params object, got %#v", params["extra_params"])
|
|
}
|
|
if extraParams["provider_option"] != "on" {
|
|
t.Fatalf("unexpected extra_params.provider_option: %#v", extraParams["provider_option"])
|
|
}
|
|
if extraParams["number_value"] != float64(42) {
|
|
t.Fatalf("unexpected extra_params.number_value: %#v", extraParams["number_value"])
|
|
}
|
|
objectValue, ok := extraParams["object_value"].(map[string]any)
|
|
if !ok || objectValue["nested"] != "value" {
|
|
t.Fatalf("unexpected extra_params.object_value: %#v", extraParams["object_value"])
|
|
}
|
|
}
|
|
|
|
func TestHandlerInvalidJSON(t *testing.T) {
|
|
h := NewHandler(&fakeRunner{})
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString("{"))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("expected 400, got %d", w.Code)
|
|
}
|
|
}
|
|
|
|
func TestHandlerRejectsTrailingJSON(t *testing.T) {
|
|
h := NewHandler(&fakeRunner{})
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}} {}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
assertHTTPErrorCode(t, w, http.StatusBadRequest, "invalid_json")
|
|
}
|
|
|
|
func TestHandlerRequestTooLarge(t *testing.T) {
|
|
h := NewHandlerWithOptions(&fakeRunner{}, HandlerOptions{MaxRequestBytes: 12})
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
assertHTTPErrorCode(t, w, http.StatusRequestEntityTooLarge, "request_too_large")
|
|
}
|
|
|
|
func TestHandlerMalformedJSONBelowLimitStillBadRequest(t *testing.T) {
|
|
h := NewHandlerWithOptions(&fakeRunner{}, HandlerOptions{MaxRequestBytes: 1024})
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString("{"))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
assertHTTPErrorCode(t, w, http.StatusBadRequest, "invalid_json")
|
|
}
|
|
|
|
func TestHandlerResponseTooLarge(t *testing.T) {
|
|
h := NewHandlerWithOptions(&fakeRunner{result: &domain.RunResult{
|
|
Artifact: domain.Artifact{Body: []byte(strings.Repeat("x", 128))},
|
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
|
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
|
}}, HandlerOptions{MaxRequestBytes: 1024, MaxResponseBytes: 64})
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
assertHTTPErrorCode(t, w, http.StatusRequestEntityTooLarge, "response_too_large")
|
|
}
|
|
|
|
func TestHandlerRawOutputDoesNotBypassResponseLimit(t *testing.T) {
|
|
h := NewHandlerWithOptions(&fakeRunner{result: &domain.RunResult{
|
|
Artifact: domain.Artifact{Body: []byte("ok")},
|
|
RawOutput: strings.Repeat("raw", 80),
|
|
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
|
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
|
}}, HandlerOptions{MaxRequestBytes: 1024, MaxResponseBytes: 128})
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
|
"prompt_id":"p",
|
|
"inputs":{"x":{"type":"file","uri":"a"}},
|
|
"include_raw_output":true
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
assertHTTPErrorCode(t, w, http.StatusRequestEntityTooLarge, "response_too_large")
|
|
}
|
|
|
|
func TestHandlerMissingPromptID(t *testing.T) {
|
|
h := NewHandler(&fakeRunner{})
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"inputs":{"x":{"type":"file","uri":"a"}}}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("expected 400, got %d", w.Code)
|
|
}
|
|
var resp map[string]any
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("invalid JSON response: %v", err)
|
|
}
|
|
errBody := resp["error"].(map[string]any)
|
|
if errBody["code"] != "invalid_request" {
|
|
t.Fatalf("expected invalid_request code, got %#v", errBody["code"])
|
|
}
|
|
}
|
|
|
|
func TestHandlerReservedExtraParamsThroughRunnerMapsToInvalidRequest(t *testing.T) {
|
|
llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runner := usecase.NewRunner(
|
|
handlerPromptRepo{def: &domain.PromptDefinition{
|
|
ID: "p",
|
|
Version: "1",
|
|
DefaultProfile: "exec",
|
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "hi"}},
|
|
OutputFormat: domain.FormatText,
|
|
Validation: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone},
|
|
}},
|
|
handlerProfileRepo{profile: &domain.ExecutionProfile{
|
|
ID: "exec",
|
|
Endpoint: "http://example.invalid/v1",
|
|
Model: "model",
|
|
}},
|
|
handlerArtifactReader{},
|
|
handlerRenderer{},
|
|
llmClient,
|
|
nil,
|
|
)
|
|
h := NewHandler(runner)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
|
"prompt_id":"p",
|
|
"inputs":{"x":{"type":"file","uri":"a"}},
|
|
"model":{"extra_params":{"model":"collision"}}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("expected 400, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("invalid JSON response: %v", err)
|
|
}
|
|
errBody := resp["error"].(map[string]any)
|
|
if errBody["code"] != "invalid_request" {
|
|
t.Fatalf("expected invalid_request code, got %#v", errBody["code"])
|
|
}
|
|
}
|
|
|
|
func TestHandlerUsecaseErrorMapping(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
err error
|
|
status int
|
|
code string
|
|
message string
|
|
avoidCause string
|
|
}{
|
|
{name: "prompt not found", err: wrap(usecase.ErrPromptLoad, promptdef.ErrPromptDefinitionNotFound), status: http.StatusNotFound, code: "prompt_not_found", message: "prompt definition not found"},
|
|
{name: "prompt load invalid", err: wrap(usecase.ErrPromptLoad, promptdef.ErrInvalidPromptDefinition), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition"},
|
|
{name: "prompt load generic", err: wrap(usecase.ErrPromptLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition", avoidCause: "read failed"},
|
|
{name: "missing profile/default", err: wrap(usecase.ErrInvalidRequest, usecase.ErrProfileRequired), status: http.StatusBadRequest, code: "profile_required", message: "profile_id is required when prompt default_profile is not set"},
|
|
{name: "profile not found", err: wrap(usecase.ErrProfileLoad, profile.ErrProfileNotFound), status: http.StatusNotFound, code: "profile_not_found", message: "execution profile not found"},
|
|
{name: "profile invalid", err: wrap(usecase.ErrProfileLoad, profile.ErrInvalidProfile), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile"},
|
|
{name: "profile load generic", err: wrap(usecase.ErrProfileLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile", avoidCause: "read failed"},
|
|
{name: "api key env missing", err: wrap(usecase.ErrInvalidRequest, usecase.ErrAPIKeyEnvMissing), status: http.StatusBadRequest, code: "api_key_env_missing", message: "api_key_env is set but the environment variable is missing"},
|
|
{name: "artifact", err: wrap(usecase.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"},
|
|
{name: "prompt render", err: wrap(usecase.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"},
|
|
{name: "llm", err: wrap(usecase.ErrLLMGenerate, fmt.Errorf("llm failed")), status: http.StatusBadGateway, code: "llm_failed", message: "model generation request failed", avoidCause: "llm failed"},
|
|
{name: "validation runtime", err: wrap(usecase.ErrValidation, fmt.Errorf("validator broke")), status: http.StatusInternalServerError, code: "validation_runtime_failed", message: "validation runtime failed", avoidCause: "validator broke"},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
h := NewHandler(&fakeRunner{err: tc.err})
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
if w.Code != tc.status {
|
|
t.Fatalf("expected %d, got %d body=%s", tc.status, w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("invalid JSON response: %v", err)
|
|
}
|
|
errBody := resp["error"].(map[string]any)
|
|
if errBody["code"] != tc.code {
|
|
t.Fatalf("expected code %q, got %#v", tc.code, errBody["code"])
|
|
}
|
|
if errBody["message"] != tc.message {
|
|
t.Fatalf("expected message %q, got %#v", tc.message, errBody["message"])
|
|
}
|
|
if tc.avoidCause != "" && strings.Contains(w.Body.String(), tc.avoidCause) {
|
|
t.Fatalf("expected response to avoid leaking cause details, got %s", w.Body.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestHandlerRawAPIKeyRejectedByStrictJSON(t *testing.T) {
|
|
h := NewHandler(&fakeRunner{})
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}},"model":{"model":"m","api_key":"secret"}}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("expected 400, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("invalid JSON response: %v", err)
|
|
}
|
|
errBody := resp["error"].(map[string]any)
|
|
if errBody["code"] != "invalid_json" {
|
|
t.Fatalf("expected invalid_json code, got %#v", errBody["code"])
|
|
}
|
|
}
|
|
|
|
func TestHandlerValidationFailureStillSuccessAndRawOutputOptIn(t *testing.T) {
|
|
h := NewHandler(&fakeRunner{result: &domain.RunResult{
|
|
Artifact: domain.Artifact{Body: []byte("bad json")},
|
|
RawOutput: "bad json",
|
|
Validation: domain.ValidationResult{
|
|
Status: domain.ValidationFailed,
|
|
Mode: domain.ValidationJSON,
|
|
Errors: []string{"invalid JSON"},
|
|
},
|
|
}})
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
h.ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("invalid JSON response: %v", err)
|
|
}
|
|
validation := resp["validation"].(map[string]any)
|
|
if status, ok := validation["status"].(string); !ok || status != "failed" {
|
|
t.Fatalf("expected validation status=failed, got %#v", validation["status"])
|
|
}
|
|
if _, ok := resp["raw_model_output"]; ok {
|
|
t.Fatalf("expected raw_model_output omitted by default, got %#v", resp["raw_model_output"])
|
|
}
|
|
|
|
reqWithRaw := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}},"include_raw_output":true}`))
|
|
wWithRaw := httptest.NewRecorder()
|
|
h.ServeHTTP(wWithRaw, reqWithRaw)
|
|
|
|
if wWithRaw.Code != http.StatusOK {
|
|
t.Fatalf("expected 200, got %d: %s", wWithRaw.Code, wWithRaw.Body.String())
|
|
}
|
|
var respWithRaw map[string]any
|
|
if err := json.Unmarshal(wWithRaw.Body.Bytes(), &respWithRaw); err != nil {
|
|
t.Fatalf("invalid JSON response: %v", err)
|
|
}
|
|
if got, ok := respWithRaw["raw_model_output"].(string); !ok || got != "bad json" {
|
|
t.Fatalf("expected raw_model_output to be included when requested, got %#v", respWithRaw["raw_model_output"])
|
|
}
|
|
}
|
|
|
|
func wrap(stage error, cause error) error {
|
|
return fmt.Errorf("%w: %w", stage, cause)
|
|
}
|
|
|
|
func newArtifactRootHandler(t *testing.T, root string) *Handler {
|
|
t.Helper()
|
|
|
|
return newArtifactRootHandlerWithLimit(t, root, 0)
|
|
}
|
|
|
|
func newArtifactRootHandlerWithLimit(t *testing.T, root string, maxArtifactBytes int64) *Handler {
|
|
t.Helper()
|
|
|
|
reader, err := artifact.NewRestrictedCompositeReaderWithLimit(root, maxArtifactBytes)
|
|
if err != nil {
|
|
t.Fatalf("expected restricted artifact reader: %v", err)
|
|
}
|
|
runner := usecase.NewRunner(
|
|
handlerPromptRepo{def: &domain.PromptDefinition{
|
|
ID: "p",
|
|
Version: "1",
|
|
DefaultProfile: "exec",
|
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "hi"}},
|
|
OutputFormat: domain.FormatText,
|
|
Validation: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone},
|
|
}},
|
|
handlerProfileRepo{profile: &domain.ExecutionProfile{
|
|
ID: "exec",
|
|
Endpoint: "http://example.invalid/v1",
|
|
Model: "model",
|
|
}},
|
|
reader,
|
|
handlerRenderer{},
|
|
handlerLLMClient{},
|
|
nil,
|
|
)
|
|
return NewHandler(runner)
|
|
}
|
|
|
|
func assertHTTPErrorCode(t *testing.T, w *httptest.ResponseRecorder, status int, code string) {
|
|
t.Helper()
|
|
|
|
if w.Code != status {
|
|
t.Fatalf("expected %d, got %d body=%s", status, w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("invalid JSON response: %v", err)
|
|
}
|
|
errBody := resp["error"].(map[string]any)
|
|
if errBody["code"] != code {
|
|
t.Fatalf("expected code %q, got %#v", code, errBody["code"])
|
|
}
|
|
}
|