From 291298cf7b9f2203c69eecbd658dce9bad763d2a Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 5 Jul 2026 17:46:05 +0000 Subject: [PATCH] Add Scriptorium API grounding --- go.mod | 12 +- go.sum | 8 + .../framework/llm/scriptorium_api_test.go | 175 ++++++++++++++++++ 3 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 internal/framework/llm/scriptorium_api_test.go diff --git a/go.mod b/go.mod index 11018a1..cbc8beb 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,13 @@ module gitea.maximumdirect.net/eric/notarius -go 1.24.0 +go 1.25.5 -require gopkg.in/yaml.v3 v3.0.1 +require ( + gitea.maximumdirect.net/eric/scriptorium v0.11.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect + golang.org/x/text v0.14.0 // indirect +) diff --git a/go.sum b/go.sum index a62c313..c4a118e 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,11 @@ +gitea.maximumdirect.net/eric/scriptorium v0.11.0 h1:rjvbt9FTaWHxYlHq7QlUzmMVUt3QdbTmeCkmH81N//o= +gitea.maximumdirect.net/eric/scriptorium v0.11.0/go.mod h1:FQ5lEuNxmrQyNgIomkpZdxvfTC0jWjbXYuq3tbJWF64= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/framework/llm/scriptorium_api_test.go b/internal/framework/llm/scriptorium_api_test.go new file mode 100644 index 0000000..66fae15 --- /dev/null +++ b/internal/framework/llm/scriptorium_api_test.go @@ -0,0 +1,175 @@ +package llm + +import ( + "context" + "errors" + "testing" + "testing/fstest" + "time" + + "gitea.maximumdirect.net/eric/scriptorium" +) + +func TestScriptoriumPublicAPIGrounding(t *testing.T) { + // Keep this compile-time grounding close to the future Notarius adapter so + // dependency upgrades reveal API drift before the runtime cutover. + engine, err := scriptorium.NewEngine( + scriptorium.Config{ + PromptDir: "unused-when-prompt-option-is-set", + ProfileDir: "", + SchemaDir: "", + Timeout: time.Second, + }, + scriptorium.WithPromptFS(fstest.MapFS{}, "."), + scriptorium.WithProfileFS(fstest.MapFS{}, "."), + scriptorium.WithSchemaFS(fstest.MapFS{}, "."), + scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ + ID: "test-profile", + Endpoint: "http://127.0.0.1:1/v1", + Model: "test-model", + APIKeyRequired: true, + ExtraParams: map[string]any{"mode": "test"}, + })), + scriptorium.WithLLMClient(scriptoriumGroundingLLMClient{}), + ) + if err != nil { + t.Fatalf("NewEngine() error = %v, want nil", err) + } + if engine == nil { + t.Fatalf("NewEngine() = nil, want engine") + } + + var ( + _ func(string) scriptorium.Option = scriptorium.WithPromptFile + _ func(string) scriptorium.Option = scriptorium.WithProfileFile + _ func(string) scriptorium.Option = scriptorium.WithSchemaFile + ) + + req := scriptorium.RunRequest{ + PromptID: "dnd.spells", + PromptVersion: "v1", + ProfileID: "test-profile", + APIKey: "request-scoped-secret", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.InlineWithURI("file:///tmp/transcript.json", `{"segments":[]}`), + "glossary": scriptorium.Inline(""), + "roster": scriptorium.File("/tmp/roster.txt"), + }, + Vars: map[string]string{ + "session_id": "session-1", + }, + Execution: &scriptorium.ExecutionTargetOverride{ + Model: "override-model", + Temperature: ptr(0.2), + MaxTokens: ptr(100), + TopP: ptr(0.9), + TimeoutSeconds: ptr(30), + ServiceTier: "standard", + ReasoningEffort: "low", + APIKeyEnv: "SCRIPTORIUM_API_KEY", + ExtraParams: map[string]any{"provider_option": "value"}, + }, + Validation: &scriptorium.OutputContract{ + Format: scriptorium.FormatJSON, + ValidationMode: scriptorium.ValidationJSONSchema, + SchemaPath: "schemas/dnd_spells.v1.json", + RepairAttempts: 1, + }, + Metadata: map[string]string{ + "artifact_kind": "dnd_spell", + }, + } + if req.Inputs["transcript"].Type != scriptorium.ArtifactRefInline { + t.Fatalf("inline input type = %q, want %q", req.Inputs["transcript"].Type, scriptorium.ArtifactRefInline) + } + if req.Inputs["roster"].Type != scriptorium.ArtifactRefFile { + t.Fatalf("file input type = %q, want %q", req.Inputs["roster"].Type, scriptorium.ArtifactRefFile) + } + + result := scriptorium.RunResult{ + RunID: "run-1", + Artifact: scriptorium.Artifact{ + Name: "output", + ContentType: "application/json", + Body: []byte(`{"ok":true}`), + URI: "inline://output", + Size: int64(len(`{"ok":true}`)), + Hash: "sha256:abc", + }, + RawOutput: `{"ok":true}`, + PromptID: req.PromptID, + PromptVersion: req.PromptVersion, + PromptHash: "prompt-hash", + RenderedPromptHash: "rendered-prompt-hash", + SelectedProfileID: req.ProfileID, + ModelName: "test-model", + Endpoint: "http://127.0.0.1:1/v1", + EffectiveModelParams: scriptorium.ExecutionTarget{ + Model: "test-model", + APIKeyEnv: "SCRIPTORIUM_API_KEY", + ExtraParams: map[string]any{"provider_option": "value"}, + ReasoningEffort: "low", + }, + InputHashes: map[string]string{ + "transcript": "sha256:def", + }, + Validation: scriptorium.ValidationResult{ + Status: scriptorium.ValidationPassed, + Mode: scriptorium.ValidationJSONSchema, + SchemaPath: req.Validation.SchemaPath, + RepairAttempts: 1, + IsValid: true, + }, + Usage: scriptorium.TokenUsage{ + PromptTokens: 10, + CompletionTokens: 5, + TotalTokens: 15, + CachedTokens: 3, + CacheWriteTokens: 2, + }, + StartTime: time.Unix(1, 0), + EndTime: time.Unix(2, 0), + Duration: time.Second, + } + if result.Validation.Status != scriptorium.ValidationPassed { + t.Fatalf("validation status = %q, want %q", result.Validation.Status, scriptorium.ValidationPassed) + } + if result.Usage.TotalTokens != 15 { + t.Fatalf("total tokens = %d, want 15", result.Usage.TotalTokens) + } + + publicErrors := []error{ + scriptorium.ErrInvalidConfig, + scriptorium.ErrInvalidRequest, + scriptorium.ErrPromptNotFound, + scriptorium.ErrProfileNotFound, + scriptorium.ErrPromptLoad, + scriptorium.ErrProfileLoad, + scriptorium.ErrArtifactLoad, + scriptorium.ErrPromptRender, + scriptorium.ErrLLMGenerate, + scriptorium.ErrValidation, + } + for _, publicErr := range publicErrors { + if !errors.Is(publicErr, publicErr) { + t.Fatalf("sentinel error does not match itself: %v", publicErr) + } + } +} + +type scriptoriumGroundingLLMClient struct{} + +func (scriptoriumGroundingLLMClient) Generate(context.Context, scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) { + return &scriptorium.GenerateResponse{ + Content: `{"ok":true}`, + Usage: scriptorium.TokenUsage{ + PromptTokens: 1, + CompletionTokens: 1, + TotalTokens: 2, + }, + }, nil +} + +func ptr[T any](v T) *T { + return &v +}