From 4ac203833111a8d138cbb02dfa4924aca24c1163 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 4 Jul 2026 14:21:37 +0000 Subject: [PATCH] Add public run API with injectable LLM --- convert.go | 61 +++++++++++ engine.go | 60 +++++++++-- engine_test.go | 275 ++++++++++++++++++++++++++++++++++++++++++++++++- errors.go | 71 +++++++++++++ llm_adapter.go | 23 +++++ types.go | 75 +++++++++++--- 6 files changed, 538 insertions(+), 27 deletions(-) create mode 100644 errors.go create mode 100644 llm_adapter.go diff --git a/convert.go b/convert.go index 4a888aa..7be61ff 100644 --- a/convert.go +++ b/convert.go @@ -37,6 +37,57 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun { } } +func fromDomainRunResult(result *domain.RunResult) *RunResult { + if result == nil { + return nil + } + return &RunResult{ + RunID: result.RunID, + Artifact: fromDomainArtifact(result.Artifact), + RawOutput: result.RawOutput, + Validation: fromDomainValidationResult(result.Validation), + PromptID: result.PromptID, + PromptVersion: result.PromptVersion, + PromptHash: result.PromptHash, + RenderedPromptHash: result.RenderedPromptHash, + SelectedProfileID: result.SelectedProfileID, + ModelName: result.ModelName, + Endpoint: result.Endpoint, + EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams), + InputHashes: copyStringMap(result.InputHashes), + Usage: fromDomainTokenUsage(result.Usage), + StartTime: result.StartTime, + EndTime: result.EndTime, + Duration: result.Duration, + } +} + +func fromDomainGenerateRequest(req domain.GenerateRequest) GenerateRequest { + return GenerateRequest{ + Prompt: fromDomainRenderedPrompt(req.Prompt), + Target: fromDomainExecutionTarget(req.Target), + TargetPresence: fromDomainExecutionTargetPresence(req.TargetPresence), + StructuredOutput: fromDomainStructuredOutputSpec(req.StructuredOutput), + } +} + +func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse { + if resp == nil { + return nil + } + return &domain.GenerateResponse{ + Content: resp.Content, + Usage: toDomainTokenUsage(resp.Usage), + } +} + +func fromDomainRenderedPrompt(prompt domain.RenderedPrompt) RenderedPrompt { + return RenderedPrompt{ + SessionID: prompt.SessionID, + Messages: fromDomainRenderedMessages(prompt.Messages), + } +} + func toDomainArtifactRefMap(src map[string]ArtifactRef) map[string]domain.ArtifactRef { if src == nil { return nil @@ -156,6 +207,16 @@ func fromDomainTokenUsage(usage domain.TokenUsage) TokenUsage { } } +func toDomainTokenUsage(usage TokenUsage) domain.TokenUsage { + return domain.TokenUsage{ + PromptTokens: usage.PromptTokens, + CompletionTokens: usage.CompletionTokens, + TotalTokens: usage.TotalTokens, + CachedTokens: usage.CachedTokens, + CacheWriteTokens: usage.CacheWriteTokens, + } +} + func fromDomainRenderedMessages(messages []domain.RenderedMessage) []RenderedMessage { if messages == nil { return nil diff --git a/engine.go b/engine.go index 8353e7c..b7281f0 100644 --- a/engine.go +++ b/engine.go @@ -21,7 +21,19 @@ import ( // ErrInvalidConfig indicates invalid public engine configuration. var ErrInvalidConfig = errors.New("invalid engine configuration") -// Engine prepares Scriptorium prompt requests. +var ( + ErrInvalidRequest = errors.New("invalid run request") + ErrPromptNotFound = errors.New("prompt not found") + ErrProfileNotFound = errors.New("profile not found") + ErrPromptLoad = errors.New("failed to load prompt definition") + ErrProfileLoad = errors.New("failed to load execution profile") + ErrArtifactLoad = errors.New("failed to load artifact") + ErrPromptRender = errors.New("failed to render prompt") + ErrLLMGenerate = errors.New("failed to generate output") + ErrValidation = errors.New("failed to validate output") +) + +// Engine prepares and runs Scriptorium prompt requests. type Engine struct { runner *usecase.Runner } @@ -38,7 +50,20 @@ type Config struct { // Option customizes engine construction. type Option func(*engineOptions) error -type engineOptions struct{} +type engineOptions struct { + llmClient llm.Client +} + +// WithLLMClient injects a custom LLM client for execution. +func WithLLMClient(client LLMClient) Option { + return func(options *engineOptions) error { + if client == nil { + return ErrInvalidConfig + } + options.llmClient = publicLLMClientAdapter{client: client} + return nil + } +} // NewEngine constructs an Engine using the same default internal components as // the CLI and HTTP adapters. @@ -65,12 +90,16 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) { schemaDir = defaults.SchemaDirDefault } - llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{ - Timeout: cfg.Timeout, - HTTPClient: cfg.HTTPClient, - }) - if err != nil { - return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err) + llmClient := options.llmClient + if llmClient == nil { + var err error + llmClient, err = llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{ + Timeout: cfg.Timeout, + HTTPClient: cfg.HTTPClient, + }) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err) + } } return &Engine{ @@ -93,7 +122,20 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err prepared, err := e.runner.Prepare(ctx, toDomainRunRequest(req)) if err != nil { - return nil, err + return nil, mapPublicError(err) } return fromDomainPreparedRun(prepared), nil } + +// Run executes a prompt request and returns the generated artifact and metadata. +func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) { + if e == nil || e.runner == nil { + return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig) + } + + result, err := e.runner.Run(ctx, toDomainRunRequest(req)) + if err != nil { + return nil, mapPublicError(err) + } + return fromDomainRunResult(result), nil +} diff --git a/engine_test.go b/engine_test.go index 682820a..420c302 100644 --- a/engine_test.go +++ b/engine_test.go @@ -139,6 +139,243 @@ func TestPreparePreservesExplicitZeroExecutionOverrides(t *testing.T) { } } +func TestRunSucceedsWithInjectedLLMClient(t *testing.T) { + const envName = "SCRIPTORIUM_API_KEY" + const secret = "run-secret-value" + t.Setenv(envName, secret) + + fake := &fakeLLMClient{ + response: &scriptorium.GenerateResponse{ + Content: "# Summary\n\nDone.", + Usage: scriptorium.TokenUsage{ + PromptTokens: 10, + CompletionTokens: 5, + TotalTokens: 15, + CachedTokens: 3, + CacheWriteTokens: 2, + }, + }, + } + engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake)) + + result, err := engine.Run(context.Background(), scriptorium.RunRequest{ + PromptID: "generic.markdown_summary", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.Inline("Rin opens the gate."), + "glossary": scriptorium.Inline("gate: A guarded passage."), + }, + Execution: &scriptorium.ExecutionTargetOverride{ + APIKeyEnv: envName, + }, + }) + if err != nil { + t.Fatalf("expected run to succeed, got %v", err) + } + if result.RunID == "" { + t.Fatalf("expected run id") + } + if result.RawOutput != fake.response.Content { + t.Fatalf("unexpected raw output: %q", result.RawOutput) + } + if string(result.Artifact.Body) != fake.response.Content { + t.Fatalf("unexpected artifact body: %q", string(result.Artifact.Body)) + } + if result.Artifact.ContentType != "text/markdown" { + t.Fatalf("unexpected artifact content type: %q", result.Artifact.ContentType) + } + if result.Validation.Status != scriptorium.ValidationPassed || !result.Validation.IsValid { + t.Fatalf("expected passed validation, got %+v", result.Validation) + } + if result.PromptID != "generic.markdown_summary" || result.SelectedProfileID != "local-fast" || result.ModelName != "gpt-4o-mini" { + t.Fatalf("unexpected run metadata: %+v", result) + } + if result.Usage.TotalTokens != 15 || result.Usage.CachedTokens != 3 || result.Usage.CacheWriteTokens != 2 { + t.Fatalf("unexpected usage: %+v", result.Usage) + } + + payload, err := json.Marshal(result) + if err != nil { + t.Fatalf("expected run result to marshal, got %v", err) + } + if strings.Contains(string(payload), secret) { + t.Fatalf("run result JSON leaked raw API key value: %s", payload) + } +} + +func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) { + fake := &fakeLLMClient{ + response: &scriptorium.GenerateResponse{Content: "ok"}, + } + engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake)) + zeroFloat := 0.0 + zeroInt := 0 + + _, err := engine.Run(context.Background(), scriptorium.RunRequest{ + PromptID: "generic.markdown_summary", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.Inline("Rin opens the gate."), + "glossary": scriptorium.Inline("gate: A guarded passage."), + }, + Execution: &scriptorium.ExecutionTargetOverride{ + Temperature: &zeroFloat, + MaxTokens: &zeroInt, + TopP: &zeroFloat, + TimeoutSeconds: &zeroInt, + }, + }) + if err != nil { + t.Fatalf("expected run to succeed, got %v", err) + } + if len(fake.requests) != 1 { + t.Fatalf("expected one generate request, got %d", len(fake.requests)) + } + req := fake.requests[0] + if len(req.Prompt.Messages) != 2 || !strings.Contains(req.Prompt.Messages[1].Content, "Rin opens the gate.") { + t.Fatalf("expected rendered prompt in generate request, got %+v", req.Prompt) + } + if req.Target.Model != "gpt-4o-mini" || req.Target.Temperature != 0 || req.Target.MaxTokens != 0 || req.Target.TopP != 0 || req.Target.TimeoutSeconds != 0 { + t.Fatalf("unexpected effective target: %+v", req.Target) + } + if !req.TargetPresence.Temperature || !req.TargetPresence.MaxTokens || !req.TargetPresence.TopP || !req.TargetPresence.TimeoutSeconds { + t.Fatalf("expected explicit zero target presence, got %+v", req.TargetPresence) + } + if req.StructuredOutput != nil { + t.Fatalf("did not expect structured output for markdown prompt: %+v", req.StructuredOutput) + } +} + +func TestRunValidationFailureReturnsResult(t *testing.T) { + fake := &fakeLLMClient{ + response: &scriptorium.GenerateResponse{Content: ""}, + } + engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake)) + + result, err := engine.Run(context.Background(), scriptorium.RunRequest{ + PromptID: "generic.markdown_summary", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.Inline("Rin opens the gate."), + "glossary": scriptorium.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("expected validation failure as successful result, got %v", err) + } + if result.Validation.Status != scriptorium.ValidationFailed || result.Validation.IsValid { + t.Fatalf("expected failed validation result, got %+v", result.Validation) + } + if len(result.Validation.Errors) == 0 { + t.Fatalf("expected validation errors") + } +} + +func TestPublicErrorsSupportErrorsIs(t *testing.T) { + llmErr := errors.New("llm failed") + + tests := []struct { + name string + req scriptorium.RunRequest + client scriptorium.LLMClient + schemaDir string + want error + }{ + { + name: "invalid request", + req: scriptorium.RunRequest{}, + client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}, + want: scriptorium.ErrInvalidRequest, + }, + { + name: "prompt not found", + req: scriptorium.RunRequest{PromptID: "missing.prompt"}, + client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}, + want: scriptorium.ErrPromptNotFound, + }, + { + name: "profile not found", + req: scriptorium.RunRequest{ + PromptID: "generic.markdown_summary", + ProfileID: "missing-profile", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.Inline("Rin opens the gate."), + }, + }, + client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}, + want: scriptorium.ErrProfileNotFound, + }, + { + name: "artifact load", + req: scriptorium.RunRequest{ + PromptID: "generic.markdown_summary", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.File("./examples/fixtures/does-not-exist.md"), + }, + }, + client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}, + want: scriptorium.ErrArtifactLoad, + }, + { + name: "prompt render", + req: scriptorium.RunRequest{ + PromptID: "generic.markdown_summary", + }, + client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}, + want: scriptorium.ErrPromptRender, + }, + { + name: "llm failure", + req: scriptorium.RunRequest{ + PromptID: "generic.markdown_summary", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.Inline("Rin opens the gate."), + "glossary": scriptorium.Inline("gate: A guarded passage."), + }, + }, + client: &fakeLLMClient{err: llmErr}, + want: scriptorium.ErrLLMGenerate, + }, + { + name: "validation runtime failure", + req: scriptorium.RunRequest{ + PromptID: "generic.structured_events", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.Inline("Rin opens the gate."), + }, + }, + client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: `{"events":[]}`}}, + schemaDir: t.TempDir(), + want: scriptorium.ErrValidation, + }, + } + + t.Setenv("SCRIPTORIUM_API_KEY", "test-secret") + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + schemaDir := tc.schemaDir + if schemaDir == "" { + schemaDir = "./examples/schemas" + } + engine := newExampleEngineWithOptions(t, schemaDir, scriptorium.WithLLMClient(tc.client)) + _, err := engine.Run(context.Background(), tc.req) + if !errors.Is(err, tc.want) { + t.Fatalf("expected errors.Is(%v), got %v", tc.want, err) + } + }) + } +} + +func TestWithLLMClientRejectsNilClient(t *testing.T) { + _, err := scriptorium.NewEngine(exampleConfig("./examples/schemas"), scriptorium.WithLLMClient(nil)) + if !errors.Is(err, scriptorium.ErrInvalidConfig) { + t.Fatalf("expected ErrInvalidConfig, got %v", err) + } +} + +func TestNewEngineConstructsDefaultLLMClientWithoutCredentials(t *testing.T) { + if _, err := scriptorium.NewEngine(exampleConfig("./examples/schemas")); err != nil { + t.Fatalf("expected default engine construction without credentials to succeed, got %v", err) + } +} + func newExampleEngine(t *testing.T) *scriptorium.Engine { t.Helper() @@ -152,13 +389,41 @@ func newExampleEngine(t *testing.T) *scriptorium.Engine { } } - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", - ProfileDir: "./examples/profiles", - SchemaDir: "./examples/schemas", - }) + engine, err := scriptorium.NewEngine(exampleConfig("./examples/schemas")) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } return engine } + +func newExampleEngineWithOptions(t *testing.T, schemaDir string, opts ...scriptorium.Option) *scriptorium.Engine { + t.Helper() + + engine, err := scriptorium.NewEngine(exampleConfig(schemaDir), opts...) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + return engine +} + +func exampleConfig(schemaDir string) scriptorium.Config { + return scriptorium.Config{ + PromptDir: "./examples/prompts", + ProfileDir: "./examples/profiles", + SchemaDir: schemaDir, + } +} + +type fakeLLMClient struct { + response *scriptorium.GenerateResponse + err error + requests []scriptorium.GenerateRequest +} + +func (f *fakeLLMClient) Generate(_ context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) { + f.requests = append(f.requests, req) + if f.err != nil { + return nil, f.err + } + return f.response, nil +} diff --git a/errors.go b/errors.go new file mode 100644 index 0000000..f324375 --- /dev/null +++ b/errors.go @@ -0,0 +1,71 @@ +package scriptorium + +import ( + "errors" + "fmt" + + "gitea.maximumdirect.net/eric/scriptorium/internal/profile" + "gitea.maximumdirect.net/eric/scriptorium/internal/promptdef" + "gitea.maximumdirect.net/eric/scriptorium/internal/usecase" +) + +func mapPublicError(err error) error { + if err == nil { + return nil + } + if hasPublicError(err) { + return err + } + publicErr := publicErrorFor(err) + if publicErr == nil { + return err + } + return fmt.Errorf("%w: %w", publicErr, err) +} + +func hasPublicError(err error) bool { + for _, publicErr := range []error{ + ErrInvalidConfig, + ErrInvalidRequest, + ErrPromptNotFound, + ErrProfileNotFound, + ErrPromptLoad, + ErrProfileLoad, + ErrArtifactLoad, + ErrPromptRender, + ErrLLMGenerate, + ErrValidation, + } { + if errors.Is(err, publicErr) { + return true + } + } + return false +} + +func publicErrorFor(err error) error { + switch { + case errors.Is(err, promptdef.ErrPromptDefinitionNotFound): + return ErrPromptNotFound + case errors.Is(err, profile.ErrProfileNotFound): + return ErrProfileNotFound + case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition): + return ErrPromptLoad + case errors.Is(err, profile.ErrInvalidYAML), errors.Is(err, profile.ErrInvalidProfile): + return ErrProfileLoad + case errors.Is(err, usecase.ErrArtifactLoad): + return ErrArtifactLoad + case errors.Is(err, usecase.ErrPromptRender): + return ErrPromptRender + case errors.Is(err, usecase.ErrLLMGenerate): + return ErrLLMGenerate + case errors.Is(err, usecase.ErrValidation): + return ErrValidation + case errors.Is(err, usecase.ErrInvalidRequest): + return ErrInvalidRequest + case errors.Is(err, usecase.ErrProfileLoad): + return ErrPromptLoad + default: + return nil + } +} diff --git a/llm_adapter.go b/llm_adapter.go new file mode 100644 index 0000000..f5b9672 --- /dev/null +++ b/llm_adapter.go @@ -0,0 +1,23 @@ +package scriptorium + +import ( + "context" + "fmt" + + "gitea.maximumdirect.net/eric/scriptorium/internal/domain" +) + +type publicLLMClientAdapter struct { + client LLMClient +} + +func (a publicLLMClientAdapter) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) { + resp, err := a.client.Generate(ctx, fromDomainGenerateRequest(req)) + if err != nil { + return nil, err + } + if resp == nil { + return nil, fmt.Errorf("%w: llm client returned nil response", ErrLLMGenerate) + } + return toDomainGenerateResponse(resp), nil +} diff --git a/types.go b/types.go index d548757..866e824 100644 --- a/types.go +++ b/types.go @@ -1,6 +1,9 @@ package scriptorium -import "time" +import ( + "context" + "time" +) // ArtifactRefType defines how an artifact is referenced. type ArtifactRefType string @@ -52,7 +55,7 @@ const ( StructuredOutputJSONSchema StructuredOutputType = "json_schema" ) -// RunRequest represents a request to prepare a single prompt. +// RunRequest represents a request to prepare or run a single prompt. type RunRequest struct { PromptID string PromptVersion string @@ -84,6 +87,27 @@ type PreparedRun struct { DurationMS int64 `json:"duration_ms,omitempty"` } +// RunResult contains generated output, validation state, and run metadata. +type RunResult struct { + RunID string `json:"run_id"` + Artifact Artifact `json:"artifact"` + RawOutput string `json:"raw_output"` + Validation ValidationResult `json:"validation"` + PromptID string `json:"prompt_id"` + PromptVersion string `json:"prompt_version,omitempty"` + PromptHash string `json:"prompt_hash,omitempty"` + RenderedPromptHash string `json:"rendered_prompt_hash"` + SelectedProfileID string `json:"selected_profile_id"` + ModelName string `json:"model_name"` + Endpoint string `json:"endpoint"` + EffectiveModelParams ExecutionTarget `json:"effective_model_params"` + InputHashes map[string]string `json:"input_hashes,omitempty"` + Usage TokenUsage `json:"usage"` + StartTime time.Time `json:"start_time,omitempty"` + EndTime time.Time `json:"end_time,omitempty"` + Duration time.Duration `json:"duration,omitempty"` +} + // ArtifactRef represents a reference to prompt input content. type ArtifactRef struct { Type ArtifactRefType @@ -148,21 +172,27 @@ type OutputContract struct { // ValidationResult represents output validation state. type ValidationResult struct { - Status ValidationStatus - Mode ValidationMode - Errors []string - SchemaPath string - RepairAttempts int - IsValid bool + Status ValidationStatus `json:"status"` + Mode ValidationMode `json:"mode"` + Errors []string `json:"errors,omitempty"` + SchemaPath string `json:"schema_path,omitempty"` + RepairAttempts int `json:"repair_attempts"` + IsValid bool `json:"is_valid"` } // TokenUsage tracks token consumption. type TokenUsage struct { - PromptTokens int - CompletionTokens int - TotalTokens int - CachedTokens int - CacheWriteTokens int + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + CachedTokens int `json:"cached_tokens"` + CacheWriteTokens int `json:"cache_write_tokens"` +} + +// RenderedPrompt is the fully rendered prompt passed to an LLM client. +type RenderedPrompt struct { + SessionID string `json:"session_id,omitempty"` + Messages []RenderedMessage `json:"messages"` } // RenderedMessage is a rendered chat message. @@ -191,6 +221,25 @@ type StructuredOutputJSONSpec struct { Schema any `json:"schema"` } +// LLMClient executes rendered prompts for Engine.Run. +type LLMClient interface { + Generate(context.Context, GenerateRequest) (*GenerateResponse, error) +} + +// GenerateRequest is passed to an injected LLM client. +type GenerateRequest struct { + Prompt RenderedPrompt `json:"prompt"` + Target ExecutionTarget `json:"target"` + TargetPresence ExecutionTargetPresence `json:"target_presence"` + StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"` +} + +// GenerateResponse is returned by an injected LLM client. +type GenerateResponse struct { + Content string `json:"content"` + Usage TokenUsage `json:"usage"` +} + // File returns a file-backed artifact reference. func File(path string) ArtifactRef { return ArtifactRef{Type: ArtifactRefFile, URI: path}