eric/scriptorium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Added support for OpenAI-compatible structured output

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-08 07:32:54 -05:00
parent b52e3252f3
commit f3e8c960af
11 changed files with 493 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
internal/usecase/integration_test.go
View File

@@ -16,6 +16,7 @@ import (
type integrationLLM struct{}
func (f *integrationLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
lastIntegrationRequest = req
return &domain.GenerateResponse{
Content: `{"summary":"Party discovered a captive scout beneath the tower.","events":[{"title":"Scout found in cellar","type":"discovery","notes":"Scout requested rescue from goblin raiders."}]}`,
Usage: domain.TokenUsage{
@@ -26,6 +27,8 @@ func (f *integrationLLM) Generate(ctx context.Context, req domain.GenerateReques
}, nil
}
var lastIntegrationRequest domain.GenerateRequest
func TestRunnerIntegrationWithPromptAndProfileFixturesAndValidation(t *testing.T) {
root, err := filepath.Abs(filepath.Join("..", ".."))
if err != nil {
@@ -85,6 +88,15 @@ func TestRunnerIntegrationWithPromptAndProfileFixturesAndValidation(t *testing.T
if res.Validation.Mode != domain.ValidationJSONSchema {
t.Fatalf("expected json_schema mode, got %q", res.Validation.Mode)
}
if lastIntegrationRequest.StructuredOutput == nil {
t.Fatal("expected provider-level structured output request for json_schema prompt")
}
if lastIntegrationRequest.StructuredOutput.Type != domain.StructuredOutputJSONSchema {
t.Fatalf("expected structured output type json_schema, got %q", lastIntegrationRequest.StructuredOutput.Type)
}
if lastIntegrationRequest.StructuredOutput.JSONSchema == nil || lastIntegrationRequest.StructuredOutput.JSONSchema.Schema == nil {
t.Fatalf("expected structured output json_schema payload, got %+v", lastIntegrationRequest.StructuredOutput.JSONSchema)
}
if res.Artifact.ContentType != "application/json" {
t.Fatalf("expected application/json output, got %q", res.Artifact.ContentType)
}

7
internal/usecase/repairer.go
View File

@@ -18,6 +18,7 @@ type RepairRequest struct {
PreviousOutput string
ValidationErrors []string
Target domain.ExecutionTarget
StructuredOutput *domain.StructuredOutputSpec
Attempt int
MaxAttempts int
Mode domain.ValidationMode
@@ -59,7 +60,11 @@ func (r *defaultOutputRepairer) Repair(ctx context.Context, req RepairRequest) (
},
}}
resp, err := r.llm.Generate(ctx, domain.GenerateRequest{Prompt: prompt, Target: req.Target})
resp, err := r.llm.Generate(ctx, domain.GenerateRequest{
Prompt: prompt,
Target: req.Target,
StructuredOutput: req.StructuredOutput,
})
if err != nil {
return nil, err
}

63
internal/usecase/runner.go
View File

@@ -11,6 +11,7 @@ import (
"os"
"strings"
"time"
"unicode"
"gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
@@ -87,8 +88,9 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
}
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: prepared.Messages},
Target: prepared.EffectiveModelParams,
Prompt: domain.RenderedPrompt{Messages: prepared.Messages},
Target: prepared.EffectiveModelParams,
StructuredOutput: prepared.StructuredOutput,
})
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err)
@@ -109,6 +111,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
PreviousOutput: genResp.Content,
ValidationErrors: validationResult.Errors,
Target: prepared.EffectiveModelParams,
StructuredOutput: prepared.StructuredOutput,
Attempt: attemptsUsed,
MaxAttempts: prepared.OutputContract.RepairAttempts,
Mode: prepared.OutputContract.ValidationMode,
@@ -194,6 +197,10 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
}
effectiveContract := resolveOutputContract(def, req.Validation)
structuredOutput, err := r.resolveStructuredOutput(ctx, def, effectiveContract)
if err != nil {
return nil, err
}
resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs))
inputHashes := make(map[string]string, len(req.Inputs))
@@ -222,6 +229,7 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
SelectedProfileID: selectedProfileID,
EffectiveModelParams: effectiveModel,
OutputContract: effectiveContract,
StructuredOutput: structuredOutput,
InputHashes: inputHashes,
RenderedPromptHash: hashRenderedPrompt(*renderedPrompt),
Messages: renderedPrompt.Messages,
@@ -231,6 +239,57 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
}, nil
}
func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.PromptDefinition, contract domain.OutputContract) (*domain.StructuredOutputSpec, error) {
if contract.ValidationMode != domain.ValidationJSONSchema {
return nil, nil
}
loader, ok := r.validator.(validate.SchemaDocumentLoader)
if !ok || loader == nil {
return nil, fmt.Errorf("%w: json_schema output requires schema document loader", ErrValidation)
}
schemaDoc, err := loader.LoadSchemaDocument(ctx, contract.SchemaPath)
if err != nil {
return nil, fmt.Errorf("%w: failed to load json schema for structured output: %v", ErrValidation, err)
}
return &domain.StructuredOutputSpec{
Type: domain.StructuredOutputJSONSchema,
JSONSchema: &domain.StructuredOutputJSONSpec{
Name: deriveStructuredSchemaName(def.ID, def.Version),
Strict: true,
Schema: schemaDoc,
},
}, nil
}
func deriveStructuredSchemaName(promptID string, promptVersion string) string {
raw := strings.TrimSpace(promptID)
if v := strings.TrimSpace(promptVersion); v != "" {
if raw == "" {
raw = v
} else {
raw = raw + "_" + v
}
}
var b strings.Builder
for _, r := range raw {
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' || r == '-' {
b.WriteRune(r)
} else {
b.WriteRune('_')
}
}
name := strings.Trim(b.String(), "_-")
if name == "" {
return "scriptorium_schema"
}
return name
}
func (r *Runner) validateOutput(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, attemptsUsed int) (domain.ValidationResult, error) {
if r.validator == nil || contract.ValidationMode == domain.ValidationNone {
return domain.ValidationResult{

185
internal/usecase/runner_test.go
View File

@@ -103,8 +103,12 @@ func (f *fakeLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*do
}
type fakeValidator struct {
result domain.ValidationResult
err error
result domain.ValidationResult
err error
schemaDoc any
schemaErr error
schemaLoadPath string
schemaLoads int
}
func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
@@ -114,6 +118,18 @@ func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact,
return f.result, nil
}
func (f *fakeValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) {
f.schemaLoads++
f.schemaLoadPath = schemaPath
if f.schemaErr != nil {
return nil, f.schemaErr
}
if f.schemaDoc != nil {
return f.schemaDoc, nil
}
return map[string]any{"type": "object"}, nil
}
type fakeRepairer struct {
responses []*domain.GenerateResponse
err error
@@ -411,6 +427,115 @@ func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) {
}
}
func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) {
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
def.Validation.SchemaPath = "events.schema.json"
validator := &fakeValidator{
schemaDoc: map[string]any{
"type": "object",
"properties": map[string]any{
"events": map[string]any{"type": "array"},
},
},
}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
validator,
)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if validator.schemaLoads != 1 {
t.Fatalf("expected one schema load, got %d", validator.schemaLoads)
}
if validator.schemaLoadPath != "events.schema.json" {
t.Fatalf("expected schema path events.schema.json, got %q", validator.schemaLoadPath)
}
if prepared.StructuredOutput == nil {
t.Fatal("expected structured output spec")
}
if prepared.StructuredOutput.Type != domain.StructuredOutputJSONSchema {
t.Fatalf("expected structured output type json_schema, got %q", prepared.StructuredOutput.Type)
}
if prepared.StructuredOutput.JSONSchema == nil {
t.Fatal("expected structured output json_schema payload")
}
if prepared.StructuredOutput.JSONSchema.Name != "p_1" {
t.Fatalf("expected derived schema name p_1, got %q", prepared.StructuredOutput.JSONSchema.Name)
}
if prepared.StructuredOutput.JSONSchema.Strict != true {
t.Fatalf("expected strict=true, got %v", prepared.StructuredOutput.JSONSchema.Strict)
}
}
func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) {
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
def.Validation.SchemaPath = "missing.schema.json"
llmClient := &fakeLLM{forbid: true}
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
llmClient,
validator,
)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if !errors.Is(err, ErrValidation) {
t.Fatalf("expected ErrValidation, got %v", err)
}
if llmClient.calls != 0 {
t.Fatalf("expected llm not called when schema loading fails, calls=%d", llmClient.calls)
}
}
func TestDeriveStructuredSchemaName(t *testing.T) {
tests := []struct {
name string
id string
version string
want string
}{
{
name: "sanitizes punctuation and keeps dashes",
id: "prompt.id/alpha",
version: "1.0.0-beta",
want: "prompt_id_alpha_1_0_0-beta",
},
{
name: "fallback when empty",
id: "",
version: "",
want: "scriptorium_schema",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := deriveStructuredSchemaName(tc.id, tc.version)
if got != tc.want {
t.Fatalf("expected %q, got %q", tc.want, got)
}
})
}
}
func TestRunnerRunSuccessful(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}
@@ -923,6 +1048,62 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
}
}
func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 1)
def.Validation.SchemaPath = "events.schema.json"
validator := &fakeValidator{
result: domain.ValidationResult{
Status: domain.ValidationFailed,
Mode: domain.ValidationJSONSchema,
Errors: []string{"schema mismatch"},
IsValid: false,
},
schemaDoc: map[string]any{
"type": "object",
"properties": map[string]any{
"events": map[string]any{"type": "array"},
},
},
}
repairer := &fakeRepairer{
responses: []*domain.GenerateResponse{
{Content: `{"events":[]}`},
},
}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"events":[1]}`}}
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
llmClient,
validator,
repairer,
)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if llmClient.lastReq.StructuredOutput == nil || llmClient.lastReq.StructuredOutput.JSONSchema == nil {
t.Fatalf("expected initial llm request to include structured output, got %+v", llmClient.lastReq.StructuredOutput)
}
if len(repairer.reqs) != 1 {
t.Fatalf("expected one repair request, got %d", len(repairer.reqs))
}
if repairer.reqs[0].StructuredOutput == nil || repairer.reqs[0].StructuredOutput.JSONSchema == nil {
t.Fatalf("expected repair request structured output, got %+v", repairer.reqs[0].StructuredOutput)
}
if repairer.reqs[0].StructuredOutput.JSONSchema.Name != "p_1" {
t.Fatalf("expected derived schema name p_1, got %q", repairer.reqs[0].StructuredOutput.JSONSchema.Name)
}
}
func TestBuildOutputArtifactDefaults(t *testing.T) {
tests := []struct {
name string