Added support for OpenAI-compatible structured output
This commit is contained in:
@@ -176,7 +176,7 @@ func renderCommand(args []string, stdout, stderr io.Writer) int {
|
|||||||
artifactadapter.NewCompositeReader(),
|
artifactadapter.NewCompositeReader(),
|
||||||
prompt.NewGoRenderer(),
|
prompt.NewGoRenderer(),
|
||||||
nil,
|
nil,
|
||||||
nil,
|
validate.NewStandardValidator(cfg.schemaDir),
|
||||||
)
|
)
|
||||||
|
|
||||||
prepared, prepErr := runner.Prepare(context.Background(), req)
|
prepared, prepErr := runner.Prepare(context.Background(), req)
|
||||||
|
|||||||
@@ -78,18 +78,19 @@ type RunResult struct {
|
|||||||
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
|
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
|
||||||
// It must never include resolved API key values, model output, or validation data.
|
// It must never include resolved API key values, model output, or validation data.
|
||||||
type PreparedRun struct {
|
type PreparedRun struct {
|
||||||
PromptID string `json:"prompt_id"`
|
PromptID string `json:"prompt_id"`
|
||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
PromptVersion string `json:"prompt_version,omitempty"`
|
||||||
PromptHash string `json:"prompt_hash,omitempty"`
|
PromptHash string `json:"prompt_hash,omitempty"`
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||||
OutputContract OutputContract `json:"output_contract"`
|
OutputContract OutputContract `json:"output_contract"`
|
||||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||||
Messages []RenderedMessage `json:"messages"`
|
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||||
StartTime time.Time `json:"start_time,omitempty"`
|
Messages []RenderedMessage `json:"messages"`
|
||||||
EndTime time.Time `json:"end_time,omitempty"`
|
StartTime time.Time `json:"start_time,omitempty"`
|
||||||
DurationMS int64 `json:"duration_ms,omitempty"`
|
EndTime time.Time `json:"end_time,omitempty"`
|
||||||
|
DurationMS int64 `json:"duration_ms,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArtifactRef represents a reference to an input artifact.
|
// ArtifactRef represents a reference to an input artifact.
|
||||||
@@ -184,8 +185,29 @@ type RenderedMessage struct {
|
|||||||
|
|
||||||
// GenerateRequest is the internal request passed to the LLM client.
|
// GenerateRequest is the internal request passed to the LLM client.
|
||||||
type GenerateRequest struct {
|
type GenerateRequest struct {
|
||||||
Prompt RenderedPrompt
|
Prompt RenderedPrompt
|
||||||
Target ExecutionTarget
|
Target ExecutionTarget
|
||||||
|
StructuredOutput *StructuredOutputSpec
|
||||||
|
}
|
||||||
|
|
||||||
|
// StructuredOutputType indicates which provider-level output mode is requested.
|
||||||
|
type StructuredOutputType string
|
||||||
|
|
||||||
|
const (
|
||||||
|
StructuredOutputJSONSchema StructuredOutputType = "json_schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
// StructuredOutputSpec describes provider-level structured output requirements.
|
||||||
|
type StructuredOutputSpec struct {
|
||||||
|
Type StructuredOutputType `json:"type"`
|
||||||
|
JSONSchema *StructuredOutputJSONSpec `json:"json_schema,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// StructuredOutputJSONSpec contains json_schema output constraints.
|
||||||
|
type StructuredOutputJSONSpec struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Strict bool `json:"strict"`
|
||||||
|
Schema any `json:"schema"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// GenerateResponse is the response received from the LLM client.
|
// GenerateResponse is the response received from the LLM client.
|
||||||
|
|||||||
@@ -113,6 +113,13 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
if req.Target.TopP != 0 {
|
if req.Target.TopP != 0 {
|
||||||
wireReq.TopP = &req.Target.TopP
|
wireReq.TopP = &req.Target.TopP
|
||||||
}
|
}
|
||||||
|
if req.StructuredOutput != nil {
|
||||||
|
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
wireReq.ResponseFormat = responseFormat
|
||||||
|
}
|
||||||
|
|
||||||
payload, err := json.Marshal(wireReq)
|
payload, err := json.Marshal(wireReq)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -181,11 +188,12 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
}
|
}
|
||||||
|
|
||||||
type openAIChatRequest struct {
|
type openAIChatRequest struct {
|
||||||
Model string `json:"model"`
|
Model string `json:"model"`
|
||||||
Messages []openAIChatMessage `json:"messages"`
|
Messages []openAIChatMessage `json:"messages"`
|
||||||
Temperature *float64 `json:"temperature,omitempty"`
|
Temperature *float64 `json:"temperature,omitempty"`
|
||||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
MaxTokens *int `json:"max_tokens,omitempty"`
|
||||||
TopP *float64 `json:"top_p,omitempty"`
|
TopP *float64 `json:"top_p,omitempty"`
|
||||||
|
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type openAIChatMessage struct {
|
type openAIChatMessage struct {
|
||||||
@@ -203,3 +211,43 @@ type openAIChatResponse struct {
|
|||||||
TotalTokens int `json:"total_tokens"`
|
TotalTokens int `json:"total_tokens"`
|
||||||
} `json:"usage"`
|
} `json:"usage"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type openAIResponseFormat struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type openAIJSONSchemaEnvelope struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Strict bool `json:"strict"`
|
||||||
|
Schema any `json:"schema"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func toOpenAIResponseFormat(spec *domain.StructuredOutputSpec) (*openAIResponseFormat, error) {
|
||||||
|
if spec == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
switch spec.Type {
|
||||||
|
case domain.StructuredOutputJSONSchema:
|
||||||
|
if spec.JSONSchema == nil {
|
||||||
|
return nil, errors.New("json_schema structured output requires schema payload")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(spec.JSONSchema.Name) == "" {
|
||||||
|
return nil, errors.New("json_schema structured output requires non-empty schema name")
|
||||||
|
}
|
||||||
|
if spec.JSONSchema.Schema == nil {
|
||||||
|
return nil, errors.New("json_schema structured output requires schema document")
|
||||||
|
}
|
||||||
|
return &openAIResponseFormat{
|
||||||
|
Type: "json_schema",
|
||||||
|
JSONSchema: &openAIJSONSchemaEnvelope{
|
||||||
|
Name: spec.JSONSchema.Name,
|
||||||
|
Strict: spec.JSONSchema.Strict,
|
||||||
|
Schema: spec.JSONSchema.Schema,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unsupported structured output type %q", spec.Type)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -63,6 +63,20 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
|
|||||||
TopP: 0.7,
|
TopP: 0.7,
|
||||||
APIKeyEnv: "SCRIPTORIUM_TEST_API_KEY",
|
APIKeyEnv: "SCRIPTORIUM_TEST_API_KEY",
|
||||||
},
|
},
|
||||||
|
StructuredOutput: &domain.StructuredOutputSpec{
|
||||||
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
|
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||||
|
Name: "weather_schema",
|
||||||
|
Strict: true,
|
||||||
|
Schema: map[string]any{
|
||||||
|
"type": "object",
|
||||||
|
"properties": map[string]any{
|
||||||
|
"location": map[string]any{"type": "string"},
|
||||||
|
},
|
||||||
|
"required": []any{"location"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -94,6 +108,55 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
|
|||||||
if msg1["role"] != "user" || msg1["content"] != "Say hello" {
|
if msg1["role"] != "user" || msg1["content"] != "Say hello" {
|
||||||
t.Fatalf("unexpected second message: %#v", msg1)
|
t.Fatalf("unexpected second message: %#v", msg1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
responseFormat, ok := obs.Body["response_format"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected response_format payload, got %#v", obs.Body["response_format"])
|
||||||
|
}
|
||||||
|
if responseFormat["type"] != "json_schema" {
|
||||||
|
t.Fatalf("expected response_format.type=json_schema, got %#v", responseFormat["type"])
|
||||||
|
}
|
||||||
|
jsonSchema, ok := responseFormat["json_schema"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected response_format.json_schema map, got %#v", responseFormat["json_schema"])
|
||||||
|
}
|
||||||
|
if jsonSchema["name"] != "weather_schema" {
|
||||||
|
t.Fatalf("expected json_schema.name weather_schema, got %#v", jsonSchema["name"])
|
||||||
|
}
|
||||||
|
if jsonSchema["strict"] != true {
|
||||||
|
t.Fatalf("expected json_schema.strict=true, got %#v", jsonSchema["strict"])
|
||||||
|
}
|
||||||
|
if _, ok := jsonSchema["schema"].(map[string]any); !ok {
|
||||||
|
t.Fatalf("expected json_schema.schema object, got %#v", jsonSchema["schema"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *testing.T) {
|
||||||
|
var observedBody map[string]any
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
defer r.Body.Close()
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
||||||
|
t.Fatalf("failed to decode request body: %v", err)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Model: "model"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if _, exists := observedBody["response_format"]; exists {
|
||||||
|
t.Fatalf("expected response_format omitted, got %#v", observedBody["response_format"])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOpenAICompatibleClientNoAuthorizationHeaderWhenNoAPIKey(t *testing.T) {
|
func TestOpenAICompatibleClientNoAuthorizationHeaderWhenNoAPIKey(t *testing.T) {
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
type integrationLLM struct{}
|
type integrationLLM struct{}
|
||||||
|
|
||||||
func (f *integrationLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
func (f *integrationLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
|
lastIntegrationRequest = req
|
||||||
return &domain.GenerateResponse{
|
return &domain.GenerateResponse{
|
||||||
Content: `{"summary":"Party discovered a captive scout beneath the tower.","events":[{"title":"Scout found in cellar","type":"discovery","notes":"Scout requested rescue from goblin raiders."}]}`,
|
Content: `{"summary":"Party discovered a captive scout beneath the tower.","events":[{"title":"Scout found in cellar","type":"discovery","notes":"Scout requested rescue from goblin raiders."}]}`,
|
||||||
Usage: domain.TokenUsage{
|
Usage: domain.TokenUsage{
|
||||||
@@ -26,6 +27,8 @@ func (f *integrationLLM) Generate(ctx context.Context, req domain.GenerateReques
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var lastIntegrationRequest domain.GenerateRequest
|
||||||
|
|
||||||
func TestRunnerIntegrationWithPromptAndProfileFixturesAndValidation(t *testing.T) {
|
func TestRunnerIntegrationWithPromptAndProfileFixturesAndValidation(t *testing.T) {
|
||||||
root, err := filepath.Abs(filepath.Join("..", ".."))
|
root, err := filepath.Abs(filepath.Join("..", ".."))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -85,6 +88,15 @@ func TestRunnerIntegrationWithPromptAndProfileFixturesAndValidation(t *testing.T
|
|||||||
if res.Validation.Mode != domain.ValidationJSONSchema {
|
if res.Validation.Mode != domain.ValidationJSONSchema {
|
||||||
t.Fatalf("expected json_schema mode, got %q", res.Validation.Mode)
|
t.Fatalf("expected json_schema mode, got %q", res.Validation.Mode)
|
||||||
}
|
}
|
||||||
|
if lastIntegrationRequest.StructuredOutput == nil {
|
||||||
|
t.Fatal("expected provider-level structured output request for json_schema prompt")
|
||||||
|
}
|
||||||
|
if lastIntegrationRequest.StructuredOutput.Type != domain.StructuredOutputJSONSchema {
|
||||||
|
t.Fatalf("expected structured output type json_schema, got %q", lastIntegrationRequest.StructuredOutput.Type)
|
||||||
|
}
|
||||||
|
if lastIntegrationRequest.StructuredOutput.JSONSchema == nil || lastIntegrationRequest.StructuredOutput.JSONSchema.Schema == nil {
|
||||||
|
t.Fatalf("expected structured output json_schema payload, got %+v", lastIntegrationRequest.StructuredOutput.JSONSchema)
|
||||||
|
}
|
||||||
if res.Artifact.ContentType != "application/json" {
|
if res.Artifact.ContentType != "application/json" {
|
||||||
t.Fatalf("expected application/json output, got %q", res.Artifact.ContentType)
|
t.Fatalf("expected application/json output, got %q", res.Artifact.ContentType)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ type RepairRequest struct {
|
|||||||
PreviousOutput string
|
PreviousOutput string
|
||||||
ValidationErrors []string
|
ValidationErrors []string
|
||||||
Target domain.ExecutionTarget
|
Target domain.ExecutionTarget
|
||||||
|
StructuredOutput *domain.StructuredOutputSpec
|
||||||
Attempt int
|
Attempt int
|
||||||
MaxAttempts int
|
MaxAttempts int
|
||||||
Mode domain.ValidationMode
|
Mode domain.ValidationMode
|
||||||
@@ -59,7 +60,11 @@ func (r *defaultOutputRepairer) Repair(ctx context.Context, req RepairRequest) (
|
|||||||
},
|
},
|
||||||
}}
|
}}
|
||||||
|
|
||||||
resp, err := r.llm.Generate(ctx, domain.GenerateRequest{Prompt: prompt, Target: req.Target})
|
resp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
||||||
|
Prompt: prompt,
|
||||||
|
Target: req.Target,
|
||||||
|
StructuredOutput: req.StructuredOutput,
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
"unicode"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
@@ -87,8 +88,9 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
}
|
}
|
||||||
|
|
||||||
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: prepared.Messages},
|
Prompt: domain.RenderedPrompt{Messages: prepared.Messages},
|
||||||
Target: prepared.EffectiveModelParams,
|
Target: prepared.EffectiveModelParams,
|
||||||
|
StructuredOutput: prepared.StructuredOutput,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err)
|
return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err)
|
||||||
@@ -109,6 +111,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
PreviousOutput: genResp.Content,
|
PreviousOutput: genResp.Content,
|
||||||
ValidationErrors: validationResult.Errors,
|
ValidationErrors: validationResult.Errors,
|
||||||
Target: prepared.EffectiveModelParams,
|
Target: prepared.EffectiveModelParams,
|
||||||
|
StructuredOutput: prepared.StructuredOutput,
|
||||||
Attempt: attemptsUsed,
|
Attempt: attemptsUsed,
|
||||||
MaxAttempts: prepared.OutputContract.RepairAttempts,
|
MaxAttempts: prepared.OutputContract.RepairAttempts,
|
||||||
Mode: prepared.OutputContract.ValidationMode,
|
Mode: prepared.OutputContract.ValidationMode,
|
||||||
@@ -194,6 +197,10 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
|||||||
}
|
}
|
||||||
|
|
||||||
effectiveContract := resolveOutputContract(def, req.Validation)
|
effectiveContract := resolveOutputContract(def, req.Validation)
|
||||||
|
structuredOutput, err := r.resolveStructuredOutput(ctx, def, effectiveContract)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs))
|
resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs))
|
||||||
inputHashes := make(map[string]string, len(req.Inputs))
|
inputHashes := make(map[string]string, len(req.Inputs))
|
||||||
@@ -222,6 +229,7 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
|||||||
SelectedProfileID: selectedProfileID,
|
SelectedProfileID: selectedProfileID,
|
||||||
EffectiveModelParams: effectiveModel,
|
EffectiveModelParams: effectiveModel,
|
||||||
OutputContract: effectiveContract,
|
OutputContract: effectiveContract,
|
||||||
|
StructuredOutput: structuredOutput,
|
||||||
InputHashes: inputHashes,
|
InputHashes: inputHashes,
|
||||||
RenderedPromptHash: hashRenderedPrompt(*renderedPrompt),
|
RenderedPromptHash: hashRenderedPrompt(*renderedPrompt),
|
||||||
Messages: renderedPrompt.Messages,
|
Messages: renderedPrompt.Messages,
|
||||||
@@ -231,6 +239,57 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.PromptDefinition, contract domain.OutputContract) (*domain.StructuredOutputSpec, error) {
|
||||||
|
if contract.ValidationMode != domain.ValidationJSONSchema {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
loader, ok := r.validator.(validate.SchemaDocumentLoader)
|
||||||
|
if !ok || loader == nil {
|
||||||
|
return nil, fmt.Errorf("%w: json_schema output requires schema document loader", ErrValidation)
|
||||||
|
}
|
||||||
|
|
||||||
|
schemaDoc, err := loader.LoadSchemaDocument(ctx, contract.SchemaPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: failed to load json schema for structured output: %v", ErrValidation, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &domain.StructuredOutputSpec{
|
||||||
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
|
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||||
|
Name: deriveStructuredSchemaName(def.ID, def.Version),
|
||||||
|
Strict: true,
|
||||||
|
Schema: schemaDoc,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func deriveStructuredSchemaName(promptID string, promptVersion string) string {
|
||||||
|
raw := strings.TrimSpace(promptID)
|
||||||
|
if v := strings.TrimSpace(promptVersion); v != "" {
|
||||||
|
if raw == "" {
|
||||||
|
raw = v
|
||||||
|
} else {
|
||||||
|
raw = raw + "_" + v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var b strings.Builder
|
||||||
|
for _, r := range raw {
|
||||||
|
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' || r == '-' {
|
||||||
|
b.WriteRune(r)
|
||||||
|
} else {
|
||||||
|
b.WriteRune('_')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
name := strings.Trim(b.String(), "_-")
|
||||||
|
if name == "" {
|
||||||
|
return "scriptorium_schema"
|
||||||
|
}
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
|
||||||
func (r *Runner) validateOutput(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, attemptsUsed int) (domain.ValidationResult, error) {
|
func (r *Runner) validateOutput(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, attemptsUsed int) (domain.ValidationResult, error) {
|
||||||
if r.validator == nil || contract.ValidationMode == domain.ValidationNone {
|
if r.validator == nil || contract.ValidationMode == domain.ValidationNone {
|
||||||
return domain.ValidationResult{
|
return domain.ValidationResult{
|
||||||
|
|||||||
@@ -103,8 +103,12 @@ func (f *fakeLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*do
|
|||||||
}
|
}
|
||||||
|
|
||||||
type fakeValidator struct {
|
type fakeValidator struct {
|
||||||
result domain.ValidationResult
|
result domain.ValidationResult
|
||||||
err error
|
err error
|
||||||
|
schemaDoc any
|
||||||
|
schemaErr error
|
||||||
|
schemaLoadPath string
|
||||||
|
schemaLoads int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
|
func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
|
||||||
@@ -114,6 +118,18 @@ func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact,
|
|||||||
return f.result, nil
|
return f.result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f *fakeValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) {
|
||||||
|
f.schemaLoads++
|
||||||
|
f.schemaLoadPath = schemaPath
|
||||||
|
if f.schemaErr != nil {
|
||||||
|
return nil, f.schemaErr
|
||||||
|
}
|
||||||
|
if f.schemaDoc != nil {
|
||||||
|
return f.schemaDoc, nil
|
||||||
|
}
|
||||||
|
return map[string]any{"type": "object"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
type fakeRepairer struct {
|
type fakeRepairer struct {
|
||||||
responses []*domain.GenerateResponse
|
responses []*domain.GenerateResponse
|
||||||
err error
|
err error
|
||||||
@@ -411,6 +427,115 @@ func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) {
|
||||||
|
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||||
|
def.Validation.SchemaPath = "events.schema.json"
|
||||||
|
validator := &fakeValidator{
|
||||||
|
schemaDoc: map[string]any{
|
||||||
|
"type": "object",
|
||||||
|
"properties": map[string]any{
|
||||||
|
"events": map[string]any{"type": "array"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: def},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
&fakeLLM{forbid: true},
|
||||||
|
validator,
|
||||||
|
)
|
||||||
|
|
||||||
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if validator.schemaLoads != 1 {
|
||||||
|
t.Fatalf("expected one schema load, got %d", validator.schemaLoads)
|
||||||
|
}
|
||||||
|
if validator.schemaLoadPath != "events.schema.json" {
|
||||||
|
t.Fatalf("expected schema path events.schema.json, got %q", validator.schemaLoadPath)
|
||||||
|
}
|
||||||
|
if prepared.StructuredOutput == nil {
|
||||||
|
t.Fatal("expected structured output spec")
|
||||||
|
}
|
||||||
|
if prepared.StructuredOutput.Type != domain.StructuredOutputJSONSchema {
|
||||||
|
t.Fatalf("expected structured output type json_schema, got %q", prepared.StructuredOutput.Type)
|
||||||
|
}
|
||||||
|
if prepared.StructuredOutput.JSONSchema == nil {
|
||||||
|
t.Fatal("expected structured output json_schema payload")
|
||||||
|
}
|
||||||
|
if prepared.StructuredOutput.JSONSchema.Name != "p_1" {
|
||||||
|
t.Fatalf("expected derived schema name p_1, got %q", prepared.StructuredOutput.JSONSchema.Name)
|
||||||
|
}
|
||||||
|
if prepared.StructuredOutput.JSONSchema.Strict != true {
|
||||||
|
t.Fatalf("expected strict=true, got %v", prepared.StructuredOutput.JSONSchema.Strict)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) {
|
||||||
|
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||||
|
def.Validation.SchemaPath = "missing.schema.json"
|
||||||
|
llmClient := &fakeLLM{forbid: true}
|
||||||
|
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: def},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
llmClient,
|
||||||
|
validator,
|
||||||
|
)
|
||||||
|
|
||||||
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrValidation) {
|
||||||
|
t.Fatalf("expected ErrValidation, got %v", err)
|
||||||
|
}
|
||||||
|
if llmClient.calls != 0 {
|
||||||
|
t.Fatalf("expected llm not called when schema loading fails, calls=%d", llmClient.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeriveStructuredSchemaName(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
id string
|
||||||
|
version string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "sanitizes punctuation and keeps dashes",
|
||||||
|
id: "prompt.id/alpha",
|
||||||
|
version: "1.0.0-beta",
|
||||||
|
want: "prompt_id_alpha_1_0_0-beta",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fallback when empty",
|
||||||
|
id: "",
|
||||||
|
version: "",
|
||||||
|
want: "scriptorium_schema",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got := deriveStructuredSchemaName(tc.id, tc.version)
|
||||||
|
if got != tc.want {
|
||||||
|
t.Fatalf("expected %q, got %q", tc.want, got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerRunSuccessful(t *testing.T) {
|
func TestRunnerRunSuccessful(t *testing.T) {
|
||||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
|
||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}
|
||||||
@@ -923,6 +1048,62 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
|
||||||
|
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 1)
|
||||||
|
def.Validation.SchemaPath = "events.schema.json"
|
||||||
|
|
||||||
|
validator := &fakeValidator{
|
||||||
|
result: domain.ValidationResult{
|
||||||
|
Status: domain.ValidationFailed,
|
||||||
|
Mode: domain.ValidationJSONSchema,
|
||||||
|
Errors: []string{"schema mismatch"},
|
||||||
|
IsValid: false,
|
||||||
|
},
|
||||||
|
schemaDoc: map[string]any{
|
||||||
|
"type": "object",
|
||||||
|
"properties": map[string]any{
|
||||||
|
"events": map[string]any{"type": "array"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
repairer := &fakeRepairer{
|
||||||
|
responses: []*domain.GenerateResponse{
|
||||||
|
{Content: `{"events":[]}`},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"events":[1]}`}}
|
||||||
|
runner := NewRunnerWithRepairer(
|
||||||
|
&fakePromptRepo{def: def},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
llmClient,
|
||||||
|
validator,
|
||||||
|
repairer,
|
||||||
|
)
|
||||||
|
|
||||||
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if llmClient.lastReq.StructuredOutput == nil || llmClient.lastReq.StructuredOutput.JSONSchema == nil {
|
||||||
|
t.Fatalf("expected initial llm request to include structured output, got %+v", llmClient.lastReq.StructuredOutput)
|
||||||
|
}
|
||||||
|
if len(repairer.reqs) != 1 {
|
||||||
|
t.Fatalf("expected one repair request, got %d", len(repairer.reqs))
|
||||||
|
}
|
||||||
|
if repairer.reqs[0].StructuredOutput == nil || repairer.reqs[0].StructuredOutput.JSONSchema == nil {
|
||||||
|
t.Fatalf("expected repair request structured output, got %+v", repairer.reqs[0].StructuredOutput)
|
||||||
|
}
|
||||||
|
if repairer.reqs[0].StructuredOutput.JSONSchema.Name != "p_1" {
|
||||||
|
t.Fatalf("expected derived schema name p_1, got %q", repairer.reqs[0].StructuredOutput.JSONSchema.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestBuildOutputArtifactDefaults(t *testing.T) {
|
func TestBuildOutputArtifactDefaults(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
@@ -108,6 +108,30 @@ func parseJSON(body []byte) (any, error) {
|
|||||||
return v, nil
|
return v, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (v *StandardValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved, err := v.resolveSchemaPath(schemaPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
raw, err := os.ReadFile(resolved)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var doc any
|
||||||
|
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
|
||||||
|
}
|
||||||
|
return doc, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error) {
|
func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error) {
|
||||||
if strings.TrimSpace(schemaPath) == "" {
|
if strings.TrimSpace(schemaPath) == "" {
|
||||||
return "", errors.New("schema path is required for json_schema validation")
|
return "", errors.New("schema path is required for json_schema validation")
|
||||||
|
|||||||
@@ -158,3 +158,52 @@ func TestStandardValidatorJSONSchemaSchemaLoadError(t *testing.T) {
|
|||||||
t.Fatal("expected schema load error")
|
t.Fatal("expected schema load error")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStandardValidatorLoadSchemaDocumentSuccess(t *testing.T) {
|
||||||
|
tmp := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"name": {"type": "string"}
|
||||||
|
}
|
||||||
|
}`), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
v := NewStandardValidator(tmp)
|
||||||
|
loader, ok := v.(SchemaDocumentLoader)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("standard validator must implement SchemaDocumentLoader")
|
||||||
|
}
|
||||||
|
|
||||||
|
doc, err := loader.LoadSchemaDocument(context.Background(), "schema.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
obj, ok := doc.(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected object document, got %#v", doc)
|
||||||
|
}
|
||||||
|
if obj["type"] != "object" {
|
||||||
|
t.Fatalf("expected schema type=object, got %#v", obj["type"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStandardValidatorLoadSchemaDocumentInvalidJSON(t *testing.T) {
|
||||||
|
tmp := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{`), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
v := NewStandardValidator(tmp)
|
||||||
|
loader, ok := v.(SchemaDocumentLoader)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("standard validator must implement SchemaDocumentLoader")
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := loader.LoadSchemaDocument(context.Background(), "schema.json")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected decode error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -9,3 +9,8 @@ import (
|
|||||||
type Validator interface {
|
type Validator interface {
|
||||||
Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error)
|
Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SchemaDocumentLoader loads JSON schema documents using validator path semantics.
|
||||||
|
type SchemaDocumentLoader interface {
|
||||||
|
LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error)
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user