package scriptorium_test import ( "context" "encoding/json" "errors" "os" "strings" "testing" "gitea.maximumdirect.net/eric/scriptorium" ) func TestNewEngineRejectsMissingPromptDir(t *testing.T) { _, err := scriptorium.NewEngine(scriptorium.Config{ProfileDir: "./examples/profiles"}) if !errors.Is(err, scriptorium.ErrInvalidConfig) { t.Fatalf("expected ErrInvalidConfig, got %v", err) } } func TestNewEngineRejectsMissingProfileDir(t *testing.T) { _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: "./examples/prompts"}) if !errors.Is(err, scriptorium.ErrInvalidConfig) { t.Fatalf("expected ErrInvalidConfig, got %v", err) } } func TestPrepareWorksWithExampleDirectoriesAndFileInputs(t *testing.T) { engine := newExampleEngine(t) prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ PromptID: "generic.markdown_summary", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.File("./examples/fixtures/transcript.md"), "glossary": scriptorium.File("./examples/fixtures/glossary.yml"), }, }) if err != nil { t.Fatalf("expected prepare to succeed, got %v", err) } if prepared.PromptID != "generic.markdown_summary" { t.Fatalf("unexpected prompt id: %q", prepared.PromptID) } if prepared.SelectedProfileID != "local-fast" { t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID) } if prepared.EffectiveModelParams.Model != "gpt-4o-mini" { t.Fatalf("unexpected effective model: %q", prepared.EffectiveModelParams.Model) } if len(prepared.Messages) != 2 { t.Fatalf("expected rendered messages, got %d", len(prepared.Messages)) } if prepared.InputHashes["transcript"] == "" || prepared.InputHashes["glossary"] == "" { t.Fatalf("expected input hashes, got %#v", prepared.InputHashes) } } func TestPrepareWorksWithInlineInputs(t *testing.T) { engine := newExampleEngine(t) prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ PromptID: "generic.markdown_summary", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin scouts the tower.\nKara lights a lantern."), "glossary": scriptorium.InlineWithURI("memory://glossary.yml", "party:\n - Rin\n - Kara\n"), }, }) if err != nil { t.Fatalf("expected prepare to succeed, got %v", err) } if len(prepared.Messages) != 2 { t.Fatalf("expected rendered messages, got %d", len(prepared.Messages)) } rendered := prepared.Messages[1].Content if !strings.Contains(rendered, "Rin scouts the tower.") || !strings.Contains(rendered, "party:") { t.Fatalf("expected inline inputs in rendered prompt, got %q", rendered) } } func TestPreparedRunJSONDoesNotExposeSecretOrTargetPresence(t *testing.T) { const envName = "SCRIPTORIUM_API_KEY" const secret = "public-api-test-secret" t.Setenv(envName, secret) engine := newExampleEngine(t) prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ PromptID: "generic.structured_events", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.File("./examples/fixtures/transcript.md"), "glossary": scriptorium.File("./examples/fixtures/glossary.yml"), }, }) if err != nil { t.Fatalf("expected prepare to succeed, got %v", err) } payload, err := json.Marshal(prepared) if err != nil { t.Fatalf("expected prepared run to marshal, got %v", err) } out := string(payload) if strings.Contains(out, secret) { t.Fatalf("prepared run JSON leaked raw API key value: %s", out) } if !strings.Contains(out, envName) { t.Fatalf("prepared run JSON should retain api_key_env name, got %s", out) } for _, forbidden := range []string{"TargetPresence", "target_presence"} { if strings.Contains(out, forbidden) { t.Fatalf("prepared run JSON exposed internal target presence metadata %q: %s", forbidden, out) } } } func TestPreparePreservesExplicitZeroExecutionOverrides(t *testing.T) { engine := newExampleEngine(t) zeroFloat := 0.0 zeroInt := 0 prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ PromptID: "generic.markdown_summary", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.File("./examples/fixtures/transcript.md"), "glossary": scriptorium.File("./examples/fixtures/glossary.yml"), }, Execution: &scriptorium.ExecutionTargetOverride{ Temperature: &zeroFloat, MaxTokens: &zeroInt, TopP: &zeroFloat, TimeoutSeconds: &zeroInt, }, }) if err != nil { t.Fatalf("expected prepare to succeed, got %v", err) } target := prepared.EffectiveModelParams if target.Temperature != 0 || target.MaxTokens != 0 || target.TopP != 0 || target.TimeoutSeconds != 0 { t.Fatalf("expected explicit zero overrides in effective target, got %+v", target) } } func TestRunSucceedsWithInjectedLLMClient(t *testing.T) { const envName = "SCRIPTORIUM_API_KEY" const secret = "run-secret-value" t.Setenv(envName, secret) fake := &fakeLLMClient{ response: &scriptorium.GenerateResponse{ Content: "# Summary\n\nDone.", Usage: scriptorium.TokenUsage{ PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15, CachedTokens: 3, CacheWriteTokens: 2, }, }, } engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake)) result, err := engine.Run(context.Background(), scriptorium.RunRequest{ PromptID: "generic.markdown_summary", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), "glossary": scriptorium.Inline("gate: A guarded passage."), }, Execution: &scriptorium.ExecutionTargetOverride{ APIKeyEnv: envName, }, }) if err != nil { t.Fatalf("expected run to succeed, got %v", err) } if result.RunID == "" { t.Fatalf("expected run id") } if result.RawOutput != fake.response.Content { t.Fatalf("unexpected raw output: %q", result.RawOutput) } if string(result.Artifact.Body) != fake.response.Content { t.Fatalf("unexpected artifact body: %q", string(result.Artifact.Body)) } if result.Artifact.ContentType != "text/markdown" { t.Fatalf("unexpected artifact content type: %q", result.Artifact.ContentType) } if result.Validation.Status != scriptorium.ValidationPassed || !result.Validation.IsValid { t.Fatalf("expected passed validation, got %+v", result.Validation) } if result.PromptID != "generic.markdown_summary" || result.SelectedProfileID != "local-fast" || result.ModelName != "gpt-4o-mini" { t.Fatalf("unexpected run metadata: %+v", result) } if result.Usage.TotalTokens != 15 || result.Usage.CachedTokens != 3 || result.Usage.CacheWriteTokens != 2 { t.Fatalf("unexpected usage: %+v", result.Usage) } payload, err := json.Marshal(result) if err != nil { t.Fatalf("expected run result to marshal, got %v", err) } if strings.Contains(string(payload), secret) { t.Fatalf("run result JSON leaked raw API key value: %s", payload) } } func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) { fake := &fakeLLMClient{ response: &scriptorium.GenerateResponse{Content: "ok"}, } engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake)) zeroFloat := 0.0 zeroInt := 0 _, err := engine.Run(context.Background(), scriptorium.RunRequest{ PromptID: "generic.markdown_summary", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), "glossary": scriptorium.Inline("gate: A guarded passage."), }, Execution: &scriptorium.ExecutionTargetOverride{ Temperature: &zeroFloat, MaxTokens: &zeroInt, TopP: &zeroFloat, TimeoutSeconds: &zeroInt, }, }) if err != nil { t.Fatalf("expected run to succeed, got %v", err) } if len(fake.requests) != 1 { t.Fatalf("expected one generate request, got %d", len(fake.requests)) } req := fake.requests[0] if len(req.Prompt.Messages) != 2 || !strings.Contains(req.Prompt.Messages[1].Content, "Rin opens the gate.") { t.Fatalf("expected rendered prompt in generate request, got %+v", req.Prompt) } if req.Target.Model != "gpt-4o-mini" || req.Target.Temperature != 0 || req.Target.MaxTokens != 0 || req.Target.TopP != 0 || req.Target.TimeoutSeconds != 0 { t.Fatalf("unexpected effective target: %+v", req.Target) } if !req.TargetPresence.Temperature || !req.TargetPresence.MaxTokens || !req.TargetPresence.TopP || !req.TargetPresence.TimeoutSeconds { t.Fatalf("expected explicit zero target presence, got %+v", req.TargetPresence) } if req.StructuredOutput != nil { t.Fatalf("did not expect structured output for markdown prompt: %+v", req.StructuredOutput) } } func TestRunValidationFailureReturnsResult(t *testing.T) { fake := &fakeLLMClient{ response: &scriptorium.GenerateResponse{Content: ""}, } engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake)) result, err := engine.Run(context.Background(), scriptorium.RunRequest{ PromptID: "generic.markdown_summary", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), "glossary": scriptorium.Inline("gate: A guarded passage."), }, }) if err != nil { t.Fatalf("expected validation failure as successful result, got %v", err) } if result.Validation.Status != scriptorium.ValidationFailed || result.Validation.IsValid { t.Fatalf("expected failed validation result, got %+v", result.Validation) } if len(result.Validation.Errors) == 0 { t.Fatalf("expected validation errors") } } func TestPublicErrorsSupportErrorsIs(t *testing.T) { llmErr := errors.New("llm failed") tests := []struct { name string req scriptorium.RunRequest client scriptorium.LLMClient schemaDir string want error }{ { name: "invalid request", req: scriptorium.RunRequest{}, client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}, want: scriptorium.ErrInvalidRequest, }, { name: "prompt not found", req: scriptorium.RunRequest{PromptID: "missing.prompt"}, client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}, want: scriptorium.ErrPromptNotFound, }, { name: "profile not found", req: scriptorium.RunRequest{ PromptID: "generic.markdown_summary", ProfileID: "missing-profile", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), }, }, client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}, want: scriptorium.ErrProfileNotFound, }, { name: "artifact load", req: scriptorium.RunRequest{ PromptID: "generic.markdown_summary", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.File("./examples/fixtures/does-not-exist.md"), }, }, client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}, want: scriptorium.ErrArtifactLoad, }, { name: "prompt render", req: scriptorium.RunRequest{ PromptID: "generic.markdown_summary", }, client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}, want: scriptorium.ErrPromptRender, }, { name: "llm failure", req: scriptorium.RunRequest{ PromptID: "generic.markdown_summary", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), "glossary": scriptorium.Inline("gate: A guarded passage."), }, }, client: &fakeLLMClient{err: llmErr}, want: scriptorium.ErrLLMGenerate, }, { name: "validation runtime failure", req: scriptorium.RunRequest{ PromptID: "generic.structured_events", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), }, }, client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: `{"events":[]}`}}, schemaDir: t.TempDir(), want: scriptorium.ErrValidation, }, } t.Setenv("SCRIPTORIUM_API_KEY", "test-secret") for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { schemaDir := tc.schemaDir if schemaDir == "" { schemaDir = "./examples/schemas" } engine := newExampleEngineWithOptions(t, schemaDir, scriptorium.WithLLMClient(tc.client)) _, err := engine.Run(context.Background(), tc.req) if !errors.Is(err, tc.want) { t.Fatalf("expected errors.Is(%v), got %v", tc.want, err) } }) } } func TestWithLLMClientRejectsNilClient(t *testing.T) { _, err := scriptorium.NewEngine(exampleConfig("./examples/schemas"), scriptorium.WithLLMClient(nil)) if !errors.Is(err, scriptorium.ErrInvalidConfig) { t.Fatalf("expected ErrInvalidConfig, got %v", err) } } func TestNewEngineConstructsDefaultLLMClientWithoutCredentials(t *testing.T) { if _, err := scriptorium.NewEngine(exampleConfig("./examples/schemas")); err != nil { t.Fatalf("expected default engine construction without credentials to succeed, got %v", err) } } func newExampleEngine(t *testing.T) *scriptorium.Engine { t.Helper() for _, path := range []string{ "./examples/prompts", "./examples/profiles", "./examples/schemas", } { if _, err := os.Stat(path); err != nil { t.Fatalf("expected example path %s to exist: %v", path, err) } } engine, err := scriptorium.NewEngine(exampleConfig("./examples/schemas")) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } return engine } func newExampleEngineWithOptions(t *testing.T, schemaDir string, opts ...scriptorium.Option) *scriptorium.Engine { t.Helper() engine, err := scriptorium.NewEngine(exampleConfig(schemaDir), opts...) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } return engine } func exampleConfig(schemaDir string) scriptorium.Config { return scriptorium.Config{ PromptDir: "./examples/prompts", ProfileDir: "./examples/profiles", SchemaDir: schemaDir, } } type fakeLLMClient struct { response *scriptorium.GenerateResponse err error requests []scriptorium.GenerateRequest } func (f *fakeLLMClient) Generate(_ context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) { f.requests = append(f.requests, req) if f.err != nil { return nil, f.err } return f.response, nil }