package llm import ( "context" "errors" "testing" "testing/fstest" "time" "gitea.maximumdirect.net/eric/scriptorium" ) func TestScriptoriumPublicAPIGrounding(t *testing.T) { // Keep this compile-time grounding close to the future Notarius adapter so // dependency upgrades reveal API drift before the runtime cutover. engine, err := scriptorium.NewEngine( scriptorium.Config{ PromptDir: "unused-when-prompt-option-is-set", ProfileDir: "", SchemaDir: "", Timeout: time.Second, }, scriptorium.WithPromptFS(fstest.MapFS{}, "."), scriptorium.WithProfileFS(fstest.MapFS{}, "."), scriptorium.WithSchemaFS(fstest.MapFS{}, "."), scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ ID: "test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "test-model", APIKeyRequired: true, ExtraParams: map[string]any{"mode": "test"}, })), scriptorium.WithLLMClient(scriptoriumGroundingLLMClient{}), ) if err != nil { t.Fatalf("NewEngine() error = %v, want nil", err) } if engine == nil { t.Fatalf("NewEngine() = nil, want engine") } var ( _ func(string) scriptorium.Option = scriptorium.WithPromptFile _ func(string) scriptorium.Option = scriptorium.WithProfileFile _ func(string) scriptorium.Option = scriptorium.WithSchemaFile ) req := scriptorium.RunRequest{ PromptID: "dnd.spells", PromptVersion: "v1", ProfileID: "test-profile", APIKey: "request-scoped-secret", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.InlineWithURI("file:///tmp/transcript.json", `{"segments":[]}`), "glossary": scriptorium.Inline(""), "roster": scriptorium.File("/tmp/roster.txt"), }, Vars: map[string]string{ "session_id": "session-1", }, Execution: &scriptorium.ExecutionTargetOverride{ Model: "override-model", Temperature: ptr(0.2), MaxTokens: ptr(100), TopP: ptr(0.9), TimeoutSeconds: ptr(30), ServiceTier: "standard", ReasoningEffort: "low", APIKeyEnv: "SCRIPTORIUM_API_KEY", ExtraParams: map[string]any{"provider_option": "value"}, }, Validation: &scriptorium.OutputContract{ Format: scriptorium.FormatJSON, ValidationMode: scriptorium.ValidationJSONSchema, SchemaPath: "schemas/dnd_spells.v1.json", RepairAttempts: 1, }, Metadata: map[string]string{ "artifact_kind": "dnd_spell", }, } if req.Inputs["transcript"].Type != scriptorium.ArtifactRefInline { t.Fatalf("inline input type = %q, want %q", req.Inputs["transcript"].Type, scriptorium.ArtifactRefInline) } if req.Inputs["roster"].Type != scriptorium.ArtifactRefFile { t.Fatalf("file input type = %q, want %q", req.Inputs["roster"].Type, scriptorium.ArtifactRefFile) } result := scriptorium.RunResult{ RunID: "run-1", Artifact: scriptorium.Artifact{ Name: "output", ContentType: "application/json", Body: []byte(`{"ok":true}`), URI: "inline://output", Size: int64(len(`{"ok":true}`)), Hash: "sha256:abc", }, RawOutput: `{"ok":true}`, PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-prompt-hash", SelectedProfileID: req.ProfileID, ModelName: "test-model", Endpoint: "http://127.0.0.1:1/v1", EffectiveModelParams: scriptorium.ExecutionTarget{ Model: "test-model", APIKeyEnv: "SCRIPTORIUM_API_KEY", ExtraParams: map[string]any{"provider_option": "value"}, ReasoningEffort: "low", }, InputHashes: map[string]string{ "transcript": "sha256:def", }, Validation: scriptorium.ValidationResult{ Status: scriptorium.ValidationPassed, Mode: scriptorium.ValidationJSONSchema, SchemaPath: req.Validation.SchemaPath, RepairAttempts: 1, IsValid: true, }, Usage: scriptorium.TokenUsage{ PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15, CachedTokens: 3, CacheWriteTokens: 2, }, StartTime: time.Unix(1, 0), EndTime: time.Unix(2, 0), Duration: time.Second, } if result.Validation.Status != scriptorium.ValidationPassed { t.Fatalf("validation status = %q, want %q", result.Validation.Status, scriptorium.ValidationPassed) } if result.Usage.TotalTokens != 15 { t.Fatalf("total tokens = %d, want 15", result.Usage.TotalTokens) } publicErrors := []error{ scriptorium.ErrInvalidConfig, scriptorium.ErrInvalidRequest, scriptorium.ErrPromptNotFound, scriptorium.ErrProfileNotFound, scriptorium.ErrPromptLoad, scriptorium.ErrProfileLoad, scriptorium.ErrArtifactLoad, scriptorium.ErrPromptRender, scriptorium.ErrLLMGenerate, scriptorium.ErrValidation, } for _, publicErr := range publicErrors { if !errors.Is(publicErr, publicErr) { t.Fatalf("sentinel error does not match itself: %v", publicErr) } } } type scriptoriumGroundingLLMClient struct{} func (scriptoriumGroundingLLMClient) Generate(context.Context, scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) { return &scriptorium.GenerateResponse{ Content: `{"ok":true}`, Usage: scriptorium.TokenUsage{ PromptTokens: 1, CompletionTokens: 1, TotalTokens: 2, }, }, nil } func ptr[T any](v T) *T { return &v }