Add public run API with injectable LLM

2026-07-04 14:21:37 +00:00
parent 14a7e7e04c
commit 4ac2038331
6 changed files with 538 additions and 27 deletions
--- a/engine_test.go
+++ b/engine_test.go
@@ -139,6 +139,243 @@ func TestPreparePreservesExplicitZeroExecutionOverrides(t *testing.T) {
 	}
 }

+func TestRunSucceedsWithInjectedLLMClient(t *testing.T) {
+	const envName = "SCRIPTORIUM_API_KEY"
+	const secret = "run-secret-value"
+	t.Setenv(envName, secret)
+
+	fake := &fakeLLMClient{
+		response: &scriptorium.GenerateResponse{
+			Content: "# Summary\n\nDone.",
+			Usage: scriptorium.TokenUsage{
+				PromptTokens:     10,
+				CompletionTokens: 5,
+				TotalTokens:      15,
+				CachedTokens:     3,
+				CacheWriteTokens: 2,
+			},
+		},
+	}
+	engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake))
+
+	result, err := engine.Run(context.Background(), scriptorium.RunRequest{
+		PromptID: "generic.markdown_summary",
+		Inputs: map[string]scriptorium.ArtifactRef{
+			"transcript": scriptorium.Inline("Rin opens the gate."),
+			"glossary":   scriptorium.Inline("gate: A guarded passage."),
+		},
+		Execution: &scriptorium.ExecutionTargetOverride{
+			APIKeyEnv: envName,
+		},
+	})
+	if err != nil {
+		t.Fatalf("expected run to succeed, got %v", err)
+	}
+	if result.RunID == "" {
+		t.Fatalf("expected run id")
+	}
+	if result.RawOutput != fake.response.Content {
+		t.Fatalf("unexpected raw output: %q", result.RawOutput)
+	}
+	if string(result.Artifact.Body) != fake.response.Content {
+		t.Fatalf("unexpected artifact body: %q", string(result.Artifact.Body))
+	}
+	if result.Artifact.ContentType != "text/markdown" {
+		t.Fatalf("unexpected artifact content type: %q", result.Artifact.ContentType)
+	}
+	if result.Validation.Status != scriptorium.ValidationPassed || !result.Validation.IsValid {
+		t.Fatalf("expected passed validation, got %+v", result.Validation)
+	}
+	if result.PromptID != "generic.markdown_summary" || result.SelectedProfileID != "local-fast" || result.ModelName != "gpt-4o-mini" {
+		t.Fatalf("unexpected run metadata: %+v", result)
+	}
+	if result.Usage.TotalTokens != 15 || result.Usage.CachedTokens != 3 || result.Usage.CacheWriteTokens != 2 {
+		t.Fatalf("unexpected usage: %+v", result.Usage)
+	}
+
+	payload, err := json.Marshal(result)
+	if err != nil {
+		t.Fatalf("expected run result to marshal, got %v", err)
+	}
+	if strings.Contains(string(payload), secret) {
+		t.Fatalf("run result JSON leaked raw API key value: %s", payload)
+	}
+}
+
+func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) {
+	fake := &fakeLLMClient{
+		response: &scriptorium.GenerateResponse{Content: "ok"},
+	}
+	engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake))
+	zeroFloat := 0.0
+	zeroInt := 0
+
+	_, err := engine.Run(context.Background(), scriptorium.RunRequest{
+		PromptID: "generic.markdown_summary",
+		Inputs: map[string]scriptorium.ArtifactRef{
+			"transcript": scriptorium.Inline("Rin opens the gate."),
+			"glossary":   scriptorium.Inline("gate: A guarded passage."),
+		},
+		Execution: &scriptorium.ExecutionTargetOverride{
+			Temperature:    &zeroFloat,
+			MaxTokens:      &zeroInt,
+			TopP:           &zeroFloat,
+			TimeoutSeconds: &zeroInt,
+		},
+	})
+	if err != nil {
+		t.Fatalf("expected run to succeed, got %v", err)
+	}
+	if len(fake.requests) != 1 {
+		t.Fatalf("expected one generate request, got %d", len(fake.requests))
+	}
+	req := fake.requests[0]
+	if len(req.Prompt.Messages) != 2 || !strings.Contains(req.Prompt.Messages[1].Content, "Rin opens the gate.") {
+		t.Fatalf("expected rendered prompt in generate request, got %+v", req.Prompt)
+	}
+	if req.Target.Model != "gpt-4o-mini" || req.Target.Temperature != 0 || req.Target.MaxTokens != 0 || req.Target.TopP != 0 || req.Target.TimeoutSeconds != 0 {
+		t.Fatalf("unexpected effective target: %+v", req.Target)
+	}
+	if !req.TargetPresence.Temperature || !req.TargetPresence.MaxTokens || !req.TargetPresence.TopP || !req.TargetPresence.TimeoutSeconds {
+		t.Fatalf("expected explicit zero target presence, got %+v", req.TargetPresence)
+	}
+	if req.StructuredOutput != nil {
+		t.Fatalf("did not expect structured output for markdown prompt: %+v", req.StructuredOutput)
+	}
+}
+
+func TestRunValidationFailureReturnsResult(t *testing.T) {
+	fake := &fakeLLMClient{
+		response: &scriptorium.GenerateResponse{Content: ""},
+	}
+	engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake))
+
+	result, err := engine.Run(context.Background(), scriptorium.RunRequest{
+		PromptID: "generic.markdown_summary",
+		Inputs: map[string]scriptorium.ArtifactRef{
+			"transcript": scriptorium.Inline("Rin opens the gate."),
+			"glossary":   scriptorium.Inline("gate: A guarded passage."),
+		},
+	})
+	if err != nil {
+		t.Fatalf("expected validation failure as successful result, got %v", err)
+	}
+	if result.Validation.Status != scriptorium.ValidationFailed || result.Validation.IsValid {
+		t.Fatalf("expected failed validation result, got %+v", result.Validation)
+	}
+	if len(result.Validation.Errors) == 0 {
+		t.Fatalf("expected validation errors")
+	}
+}
+
+func TestPublicErrorsSupportErrorsIs(t *testing.T) {
+	llmErr := errors.New("llm failed")
+
+	tests := []struct {
+		name      string
+		req       scriptorium.RunRequest
+		client    scriptorium.LLMClient
+		schemaDir string
+		want      error
+	}{
+		{
+			name:   "invalid request",
+			req:    scriptorium.RunRequest{},
+			client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}},
+			want:   scriptorium.ErrInvalidRequest,
+		},
+		{
+			name:   "prompt not found",
+			req:    scriptorium.RunRequest{PromptID: "missing.prompt"},
+			client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}},
+			want:   scriptorium.ErrPromptNotFound,
+		},
+		{
+			name: "profile not found",
+			req: scriptorium.RunRequest{
+				PromptID:  "generic.markdown_summary",
+				ProfileID: "missing-profile",
+				Inputs: map[string]scriptorium.ArtifactRef{
+					"transcript": scriptorium.Inline("Rin opens the gate."),
+				},
+			},
+			client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}},
+			want:   scriptorium.ErrProfileNotFound,
+		},
+		{
+			name: "artifact load",
+			req: scriptorium.RunRequest{
+				PromptID: "generic.markdown_summary",
+				Inputs: map[string]scriptorium.ArtifactRef{
+					"transcript": scriptorium.File("./examples/fixtures/does-not-exist.md"),
+				},
+			},
+			client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}},
+			want:   scriptorium.ErrArtifactLoad,
+		},
+		{
+			name: "prompt render",
+			req: scriptorium.RunRequest{
+				PromptID: "generic.markdown_summary",
+			},
+			client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}},
+			want:   scriptorium.ErrPromptRender,
+		},
+		{
+			name: "llm failure",
+			req: scriptorium.RunRequest{
+				PromptID: "generic.markdown_summary",
+				Inputs: map[string]scriptorium.ArtifactRef{
+					"transcript": scriptorium.Inline("Rin opens the gate."),
+					"glossary":   scriptorium.Inline("gate: A guarded passage."),
+				},
+			},
+			client: &fakeLLMClient{err: llmErr},
+			want:   scriptorium.ErrLLMGenerate,
+		},
+		{
+			name: "validation runtime failure",
+			req: scriptorium.RunRequest{
+				PromptID: "generic.structured_events",
+				Inputs: map[string]scriptorium.ArtifactRef{
+					"transcript": scriptorium.Inline("Rin opens the gate."),
+				},
+			},
+			client:    &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: `{"events":[]}`}},
+			schemaDir: t.TempDir(),
+			want:      scriptorium.ErrValidation,
+		},
+	}
+
+	t.Setenv("SCRIPTORIUM_API_KEY", "test-secret")
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			schemaDir := tc.schemaDir
+			if schemaDir == "" {
+				schemaDir = "./examples/schemas"
+			}
+			engine := newExampleEngineWithOptions(t, schemaDir, scriptorium.WithLLMClient(tc.client))
+			_, err := engine.Run(context.Background(), tc.req)
+			if !errors.Is(err, tc.want) {
+				t.Fatalf("expected errors.Is(%v), got %v", tc.want, err)
+			}
+		})
+	}
+}
+
+func TestWithLLMClientRejectsNilClient(t *testing.T) {
+	_, err := scriptorium.NewEngine(exampleConfig("./examples/schemas"), scriptorium.WithLLMClient(nil))
+	if !errors.Is(err, scriptorium.ErrInvalidConfig) {
+		t.Fatalf("expected ErrInvalidConfig, got %v", err)
+	}
+}
+
+func TestNewEngineConstructsDefaultLLMClientWithoutCredentials(t *testing.T) {
+	if _, err := scriptorium.NewEngine(exampleConfig("./examples/schemas")); err != nil {
+		t.Fatalf("expected default engine construction without credentials to succeed, got %v", err)
+	}
+}
+
 func newExampleEngine(t *testing.T) *scriptorium.Engine {
 	t.Helper()

@@ -152,13 +389,41 @@ func newExampleEngine(t *testing.T) *scriptorium.Engine {
 		}
 	}

-	engine, err := scriptorium.NewEngine(scriptorium.Config{
-		PromptDir:  "./examples/prompts",
-		ProfileDir: "./examples/profiles",
-		SchemaDir:  "./examples/schemas",
-	})
+	engine, err := scriptorium.NewEngine(exampleConfig("./examples/schemas"))
 	if err != nil {
 		t.Fatalf("expected engine construction to succeed, got %v", err)
 	}
 	return engine
 }
+
+func newExampleEngineWithOptions(t *testing.T, schemaDir string, opts ...scriptorium.Option) *scriptorium.Engine {
+	t.Helper()
+
+	engine, err := scriptorium.NewEngine(exampleConfig(schemaDir), opts...)
+	if err != nil {
+		t.Fatalf("expected engine construction to succeed, got %v", err)
+	}
+	return engine
+}
+
+func exampleConfig(schemaDir string) scriptorium.Config {
+	return scriptorium.Config{
+		PromptDir:  "./examples/prompts",
+		ProfileDir: "./examples/profiles",
+		SchemaDir:  schemaDir,
+	}
+}
+
+type fakeLLMClient struct {
+	response *scriptorium.GenerateResponse
+	err      error
+	requests []scriptorium.GenerateRequest
+}
+
+func (f *fakeLLMClient) Generate(_ context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
+	f.requests = append(f.requests, req)
+	if f.err != nil {
+		return nil, f.err
+	}
+	return f.response, nil
+}