From f3e8c960af70bc4cace13aed6e1b33784bf8fe91 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 8 May 2026 07:32:54 -0500 Subject: [PATCH] Added support for OpenAI-compatible structured output --- internal/adapter/cli/run.go | 2 +- internal/domain/domain.go | 50 +++-- internal/llm/openai_compatible_client.go | 58 +++++- internal/llm/openai_compatible_client_test.go | 63 ++++++ internal/usecase/integration_test.go | 12 ++ internal/usecase/repairer.go | 7 +- internal/usecase/runner.go | 63 +++++- internal/usecase/runner_test.go | 185 +++++++++++++++++- internal/validate/standard_validator.go | 24 +++ internal/validate/standard_validator_test.go | 49 +++++ internal/validate/validator.go | 5 + 11 files changed, 493 insertions(+), 25 deletions(-) diff --git a/internal/adapter/cli/run.go b/internal/adapter/cli/run.go index 5ef9f65..91d3550 100644 --- a/internal/adapter/cli/run.go +++ b/internal/adapter/cli/run.go @@ -176,7 +176,7 @@ func renderCommand(args []string, stdout, stderr io.Writer) int { artifactadapter.NewCompositeReader(), prompt.NewGoRenderer(), nil, - nil, + validate.NewStandardValidator(cfg.schemaDir), ) prepared, prepErr := runner.Prepare(context.Background(), req) diff --git a/internal/domain/domain.go b/internal/domain/domain.go index 03c7e24..0b9935a 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -78,18 +78,19 @@ type RunResult struct { // PreparedRun contains pre-LLM execution state from the prepare/render phase. // It must never include resolved API key values, model output, or validation data. type PreparedRun struct { - PromptID string `json:"prompt_id"` - PromptVersion string `json:"prompt_version,omitempty"` - PromptHash string `json:"prompt_hash,omitempty"` - SelectedProfileID string `json:"selected_profile_id"` - EffectiveModelParams ExecutionTarget `json:"effective_model_params"` - OutputContract OutputContract `json:"output_contract"` - InputHashes map[string]string `json:"input_hashes,omitempty"` - RenderedPromptHash string `json:"rendered_prompt_hash"` - Messages []RenderedMessage `json:"messages"` - StartTime time.Time `json:"start_time,omitempty"` - EndTime time.Time `json:"end_time,omitempty"` - DurationMS int64 `json:"duration_ms,omitempty"` + PromptID string `json:"prompt_id"` + PromptVersion string `json:"prompt_version,omitempty"` + PromptHash string `json:"prompt_hash,omitempty"` + SelectedProfileID string `json:"selected_profile_id"` + EffectiveModelParams ExecutionTarget `json:"effective_model_params"` + OutputContract OutputContract `json:"output_contract"` + StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"` + InputHashes map[string]string `json:"input_hashes,omitempty"` + RenderedPromptHash string `json:"rendered_prompt_hash"` + Messages []RenderedMessage `json:"messages"` + StartTime time.Time `json:"start_time,omitempty"` + EndTime time.Time `json:"end_time,omitempty"` + DurationMS int64 `json:"duration_ms,omitempty"` } // ArtifactRef represents a reference to an input artifact. @@ -184,8 +185,29 @@ type RenderedMessage struct { // GenerateRequest is the internal request passed to the LLM client. type GenerateRequest struct { - Prompt RenderedPrompt - Target ExecutionTarget + Prompt RenderedPrompt + Target ExecutionTarget + StructuredOutput *StructuredOutputSpec +} + +// StructuredOutputType indicates which provider-level output mode is requested. +type StructuredOutputType string + +const ( + StructuredOutputJSONSchema StructuredOutputType = "json_schema" +) + +// StructuredOutputSpec describes provider-level structured output requirements. +type StructuredOutputSpec struct { + Type StructuredOutputType `json:"type"` + JSONSchema *StructuredOutputJSONSpec `json:"json_schema,omitempty"` +} + +// StructuredOutputJSONSpec contains json_schema output constraints. +type StructuredOutputJSONSpec struct { + Name string `json:"name"` + Strict bool `json:"strict"` + Schema any `json:"schema"` } // GenerateResponse is the response received from the LLM client. diff --git a/internal/llm/openai_compatible_client.go b/internal/llm/openai_compatible_client.go index c80bb2d..de65350 100644 --- a/internal/llm/openai_compatible_client.go +++ b/internal/llm/openai_compatible_client.go @@ -113,6 +113,13 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera if req.Target.TopP != 0 { wireReq.TopP = &req.Target.TopP } + if req.StructuredOutput != nil { + responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) + } + wireReq.ResponseFormat = responseFormat + } payload, err := json.Marshal(wireReq) if err != nil { @@ -181,11 +188,12 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera } type openAIChatRequest struct { - Model string `json:"model"` - Messages []openAIChatMessage `json:"messages"` - Temperature *float64 `json:"temperature,omitempty"` - MaxTokens *int `json:"max_tokens,omitempty"` - TopP *float64 `json:"top_p,omitempty"` + Model string `json:"model"` + Messages []openAIChatMessage `json:"messages"` + Temperature *float64 `json:"temperature,omitempty"` + MaxTokens *int `json:"max_tokens,omitempty"` + TopP *float64 `json:"top_p,omitempty"` + ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"` } type openAIChatMessage struct { @@ -203,3 +211,43 @@ type openAIChatResponse struct { TotalTokens int `json:"total_tokens"` } `json:"usage"` } + +type openAIResponseFormat struct { + Type string `json:"type"` + JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"` +} + +type openAIJSONSchemaEnvelope struct { + Name string `json:"name"` + Strict bool `json:"strict"` + Schema any `json:"schema"` +} + +func toOpenAIResponseFormat(spec *domain.StructuredOutputSpec) (*openAIResponseFormat, error) { + if spec == nil { + return nil, nil + } + + switch spec.Type { + case domain.StructuredOutputJSONSchema: + if spec.JSONSchema == nil { + return nil, errors.New("json_schema structured output requires schema payload") + } + if strings.TrimSpace(spec.JSONSchema.Name) == "" { + return nil, errors.New("json_schema structured output requires non-empty schema name") + } + if spec.JSONSchema.Schema == nil { + return nil, errors.New("json_schema structured output requires schema document") + } + return &openAIResponseFormat{ + Type: "json_schema", + JSONSchema: &openAIJSONSchemaEnvelope{ + Name: spec.JSONSchema.Name, + Strict: spec.JSONSchema.Strict, + Schema: spec.JSONSchema.Schema, + }, + }, nil + default: + return nil, fmt.Errorf("unsupported structured output type %q", spec.Type) + } +} diff --git a/internal/llm/openai_compatible_client_test.go b/internal/llm/openai_compatible_client_test.go index bfadffd..e5765b5 100644 --- a/internal/llm/openai_compatible_client_test.go +++ b/internal/llm/openai_compatible_client_test.go @@ -63,6 +63,20 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) { TopP: 0.7, APIKeyEnv: "SCRIPTORIUM_TEST_API_KEY", }, + StructuredOutput: &domain.StructuredOutputSpec{ + Type: domain.StructuredOutputJSONSchema, + JSONSchema: &domain.StructuredOutputJSONSpec{ + Name: "weather_schema", + Strict: true, + Schema: map[string]any{ + "type": "object", + "properties": map[string]any{ + "location": map[string]any{"type": "string"}, + }, + "required": []any{"location"}, + }, + }, + }, }) if err != nil { t.Fatalf("expected no error, got %v", err) @@ -94,6 +108,55 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) { if msg1["role"] != "user" || msg1["content"] != "Say hello" { t.Fatalf("unexpected second message: %#v", msg1) } + + responseFormat, ok := obs.Body["response_format"].(map[string]any) + if !ok { + t.Fatalf("expected response_format payload, got %#v", obs.Body["response_format"]) + } + if responseFormat["type"] != "json_schema" { + t.Fatalf("expected response_format.type=json_schema, got %#v", responseFormat["type"]) + } + jsonSchema, ok := responseFormat["json_schema"].(map[string]any) + if !ok { + t.Fatalf("expected response_format.json_schema map, got %#v", responseFormat["json_schema"]) + } + if jsonSchema["name"] != "weather_schema" { + t.Fatalf("expected json_schema.name weather_schema, got %#v", jsonSchema["name"]) + } + if jsonSchema["strict"] != true { + t.Fatalf("expected json_schema.strict=true, got %#v", jsonSchema["strict"]) + } + if _, ok := jsonSchema["schema"].(map[string]any); !ok { + t.Fatalf("expected json_schema.schema object, got %#v", jsonSchema["schema"]) + } +} + +func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *testing.T) { + var observedBody map[string]any + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil { + t.Fatalf("failed to decode request body: %v", err) + } + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) + })) + defer ts.Close() + + client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"}) + if err != nil { + t.Fatal(err) + } + + _, err = client.Generate(context.Background(), domain.GenerateRequest{ + Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, + Target: domain.ExecutionTarget{Model: "model"}, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if _, exists := observedBody["response_format"]; exists { + t.Fatalf("expected response_format omitted, got %#v", observedBody["response_format"]) + } } func TestOpenAICompatibleClientNoAuthorizationHeaderWhenNoAPIKey(t *testing.T) { diff --git a/internal/usecase/integration_test.go b/internal/usecase/integration_test.go index 9249157..8d489da 100644 --- a/internal/usecase/integration_test.go +++ b/internal/usecase/integration_test.go @@ -16,6 +16,7 @@ import ( type integrationLLM struct{} func (f *integrationLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) { + lastIntegrationRequest = req return &domain.GenerateResponse{ Content: `{"summary":"Party discovered a captive scout beneath the tower.","events":[{"title":"Scout found in cellar","type":"discovery","notes":"Scout requested rescue from goblin raiders."}]}`, Usage: domain.TokenUsage{ @@ -26,6 +27,8 @@ func (f *integrationLLM) Generate(ctx context.Context, req domain.GenerateReques }, nil } +var lastIntegrationRequest domain.GenerateRequest + func TestRunnerIntegrationWithPromptAndProfileFixturesAndValidation(t *testing.T) { root, err := filepath.Abs(filepath.Join("..", "..")) if err != nil { @@ -85,6 +88,15 @@ func TestRunnerIntegrationWithPromptAndProfileFixturesAndValidation(t *testing.T if res.Validation.Mode != domain.ValidationJSONSchema { t.Fatalf("expected json_schema mode, got %q", res.Validation.Mode) } + if lastIntegrationRequest.StructuredOutput == nil { + t.Fatal("expected provider-level structured output request for json_schema prompt") + } + if lastIntegrationRequest.StructuredOutput.Type != domain.StructuredOutputJSONSchema { + t.Fatalf("expected structured output type json_schema, got %q", lastIntegrationRequest.StructuredOutput.Type) + } + if lastIntegrationRequest.StructuredOutput.JSONSchema == nil || lastIntegrationRequest.StructuredOutput.JSONSchema.Schema == nil { + t.Fatalf("expected structured output json_schema payload, got %+v", lastIntegrationRequest.StructuredOutput.JSONSchema) + } if res.Artifact.ContentType != "application/json" { t.Fatalf("expected application/json output, got %q", res.Artifact.ContentType) } diff --git a/internal/usecase/repairer.go b/internal/usecase/repairer.go index 84c2af5..475878e 100644 --- a/internal/usecase/repairer.go +++ b/internal/usecase/repairer.go @@ -18,6 +18,7 @@ type RepairRequest struct { PreviousOutput string ValidationErrors []string Target domain.ExecutionTarget + StructuredOutput *domain.StructuredOutputSpec Attempt int MaxAttempts int Mode domain.ValidationMode @@ -59,7 +60,11 @@ func (r *defaultOutputRepairer) Repair(ctx context.Context, req RepairRequest) ( }, }} - resp, err := r.llm.Generate(ctx, domain.GenerateRequest{Prompt: prompt, Target: req.Target}) + resp, err := r.llm.Generate(ctx, domain.GenerateRequest{ + Prompt: prompt, + Target: req.Target, + StructuredOutput: req.StructuredOutput, + }) if err != nil { return nil, err } diff --git a/internal/usecase/runner.go b/internal/usecase/runner.go index cd1ef5e..657b987 100644 --- a/internal/usecase/runner.go +++ b/internal/usecase/runner.go @@ -11,6 +11,7 @@ import ( "os" "strings" "time" + "unicode" "gitea.maximumdirect.net/eric/scriptorium/internal/artifact" "gitea.maximumdirect.net/eric/scriptorium/internal/defaults" @@ -87,8 +88,9 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes } genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: prepared.Messages}, - Target: prepared.EffectiveModelParams, + Prompt: domain.RenderedPrompt{Messages: prepared.Messages}, + Target: prepared.EffectiveModelParams, + StructuredOutput: prepared.StructuredOutput, }) if err != nil { return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err) @@ -109,6 +111,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes PreviousOutput: genResp.Content, ValidationErrors: validationResult.Errors, Target: prepared.EffectiveModelParams, + StructuredOutput: prepared.StructuredOutput, Attempt: attemptsUsed, MaxAttempts: prepared.OutputContract.RepairAttempts, Mode: prepared.OutputContract.ValidationMode, @@ -194,6 +197,10 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr } effectiveContract := resolveOutputContract(def, req.Validation) + structuredOutput, err := r.resolveStructuredOutput(ctx, def, effectiveContract) + if err != nil { + return nil, err + } resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs)) inputHashes := make(map[string]string, len(req.Inputs)) @@ -222,6 +229,7 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr SelectedProfileID: selectedProfileID, EffectiveModelParams: effectiveModel, OutputContract: effectiveContract, + StructuredOutput: structuredOutput, InputHashes: inputHashes, RenderedPromptHash: hashRenderedPrompt(*renderedPrompt), Messages: renderedPrompt.Messages, @@ -231,6 +239,57 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr }, nil } +func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.PromptDefinition, contract domain.OutputContract) (*domain.StructuredOutputSpec, error) { + if contract.ValidationMode != domain.ValidationJSONSchema { + return nil, nil + } + + loader, ok := r.validator.(validate.SchemaDocumentLoader) + if !ok || loader == nil { + return nil, fmt.Errorf("%w: json_schema output requires schema document loader", ErrValidation) + } + + schemaDoc, err := loader.LoadSchemaDocument(ctx, contract.SchemaPath) + if err != nil { + return nil, fmt.Errorf("%w: failed to load json schema for structured output: %v", ErrValidation, err) + } + + return &domain.StructuredOutputSpec{ + Type: domain.StructuredOutputJSONSchema, + JSONSchema: &domain.StructuredOutputJSONSpec{ + Name: deriveStructuredSchemaName(def.ID, def.Version), + Strict: true, + Schema: schemaDoc, + }, + }, nil +} + +func deriveStructuredSchemaName(promptID string, promptVersion string) string { + raw := strings.TrimSpace(promptID) + if v := strings.TrimSpace(promptVersion); v != "" { + if raw == "" { + raw = v + } else { + raw = raw + "_" + v + } + } + + var b strings.Builder + for _, r := range raw { + if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' || r == '-' { + b.WriteRune(r) + } else { + b.WriteRune('_') + } + } + + name := strings.Trim(b.String(), "_-") + if name == "" { + return "scriptorium_schema" + } + return name +} + func (r *Runner) validateOutput(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, attemptsUsed int) (domain.ValidationResult, error) { if r.validator == nil || contract.ValidationMode == domain.ValidationNone { return domain.ValidationResult{ diff --git a/internal/usecase/runner_test.go b/internal/usecase/runner_test.go index 7f11b6e..3f2a629 100644 --- a/internal/usecase/runner_test.go +++ b/internal/usecase/runner_test.go @@ -103,8 +103,12 @@ func (f *fakeLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*do } type fakeValidator struct { - result domain.ValidationResult - err error + result domain.ValidationResult + err error + schemaDoc any + schemaErr error + schemaLoadPath string + schemaLoads int } func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) { @@ -114,6 +118,18 @@ func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, return f.result, nil } +func (f *fakeValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) { + f.schemaLoads++ + f.schemaLoadPath = schemaPath + if f.schemaErr != nil { + return nil, f.schemaErr + } + if f.schemaDoc != nil { + return f.schemaDoc, nil + } + return map[string]any{"type": "object"}, nil +} + type fakeRepairer struct { responses []*domain.GenerateResponse err error @@ -411,6 +427,115 @@ func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) { } } +func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) { + def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0) + def.Validation.SchemaPath = "events.schema.json" + validator := &fakeValidator{ + schemaDoc: map[string]any{ + "type": "object", + "properties": map[string]any{ + "events": map[string]any{"type": "array"}, + }, + }, + } + runner := NewRunner( + &fakePromptRepo{def: def}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + defaultArtifactReader(), + defaultRenderer(), + &fakeLLM{forbid: true}, + validator, + ) + + prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ + PromptID: "p", + ProfileID: "exec", + Inputs: singleInputRef(), + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if validator.schemaLoads != 1 { + t.Fatalf("expected one schema load, got %d", validator.schemaLoads) + } + if validator.schemaLoadPath != "events.schema.json" { + t.Fatalf("expected schema path events.schema.json, got %q", validator.schemaLoadPath) + } + if prepared.StructuredOutput == nil { + t.Fatal("expected structured output spec") + } + if prepared.StructuredOutput.Type != domain.StructuredOutputJSONSchema { + t.Fatalf("expected structured output type json_schema, got %q", prepared.StructuredOutput.Type) + } + if prepared.StructuredOutput.JSONSchema == nil { + t.Fatal("expected structured output json_schema payload") + } + if prepared.StructuredOutput.JSONSchema.Name != "p_1" { + t.Fatalf("expected derived schema name p_1, got %q", prepared.StructuredOutput.JSONSchema.Name) + } + if prepared.StructuredOutput.JSONSchema.Strict != true { + t.Fatalf("expected strict=true, got %v", prepared.StructuredOutput.JSONSchema.Strict) + } +} + +func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) { + def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0) + def.Validation.SchemaPath = "missing.schema.json" + llmClient := &fakeLLM{forbid: true} + validator := &fakeValidator{schemaErr: errors.New("schema unavailable")} + runner := NewRunner( + &fakePromptRepo{def: def}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + defaultArtifactReader(), + defaultRenderer(), + llmClient, + validator, + ) + + _, err := runner.Run(context.Background(), domain.RunRequest{ + PromptID: "p", + ProfileID: "exec", + Inputs: singleInputRef(), + }) + if !errors.Is(err, ErrValidation) { + t.Fatalf("expected ErrValidation, got %v", err) + } + if llmClient.calls != 0 { + t.Fatalf("expected llm not called when schema loading fails, calls=%d", llmClient.calls) + } +} + +func TestDeriveStructuredSchemaName(t *testing.T) { + tests := []struct { + name string + id string + version string + want string + }{ + { + name: "sanitizes punctuation and keeps dashes", + id: "prompt.id/alpha", + version: "1.0.0-beta", + want: "prompt_id_alpha_1_0_0-beta", + }, + { + name: "fallback when empty", + id: "", + version: "", + want: "scriptorium_schema", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := deriveStructuredSchemaName(tc.id, tc.version) + if got != tc.want { + t.Fatalf("expected %q, got %q", tc.want, got) + } + }) + } +} + func TestRunnerRunSuccessful(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}} @@ -923,6 +1048,62 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t } } +func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) { + def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 1) + def.Validation.SchemaPath = "events.schema.json" + + validator := &fakeValidator{ + result: domain.ValidationResult{ + Status: domain.ValidationFailed, + Mode: domain.ValidationJSONSchema, + Errors: []string{"schema mismatch"}, + IsValid: false, + }, + schemaDoc: map[string]any{ + "type": "object", + "properties": map[string]any{ + "events": map[string]any{"type": "array"}, + }, + }, + } + repairer := &fakeRepairer{ + responses: []*domain.GenerateResponse{ + {Content: `{"events":[]}`}, + }, + } + llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"events":[1]}`}} + runner := NewRunnerWithRepairer( + &fakePromptRepo{def: def}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + defaultArtifactReader(), + defaultRenderer(), + llmClient, + validator, + repairer, + ) + + _, err := runner.Run(context.Background(), domain.RunRequest{ + PromptID: "p", + ProfileID: "exec", + Inputs: singleInputRef(), + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if llmClient.lastReq.StructuredOutput == nil || llmClient.lastReq.StructuredOutput.JSONSchema == nil { + t.Fatalf("expected initial llm request to include structured output, got %+v", llmClient.lastReq.StructuredOutput) + } + if len(repairer.reqs) != 1 { + t.Fatalf("expected one repair request, got %d", len(repairer.reqs)) + } + if repairer.reqs[0].StructuredOutput == nil || repairer.reqs[0].StructuredOutput.JSONSchema == nil { + t.Fatalf("expected repair request structured output, got %+v", repairer.reqs[0].StructuredOutput) + } + if repairer.reqs[0].StructuredOutput.JSONSchema.Name != "p_1" { + t.Fatalf("expected derived schema name p_1, got %q", repairer.reqs[0].StructuredOutput.JSONSchema.Name) + } +} + func TestBuildOutputArtifactDefaults(t *testing.T) { tests := []struct { name string diff --git a/internal/validate/standard_validator.go b/internal/validate/standard_validator.go index 8489450..2cfeb22 100644 --- a/internal/validate/standard_validator.go +++ b/internal/validate/standard_validator.go @@ -108,6 +108,30 @@ func parseJSON(body []byte) (any, error) { return v, nil } +func (v *StandardValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + resolved, err := v.resolveSchemaPath(schemaPath) + if err != nil { + return nil, err + } + + raw, err := os.ReadFile(resolved) + if err != nil { + return nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err) + } + + var doc any + if err := json.Unmarshal(raw, &doc); err != nil { + return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err) + } + return doc, nil +} + func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error) { if strings.TrimSpace(schemaPath) == "" { return "", errors.New("schema path is required for json_schema validation") diff --git a/internal/validate/standard_validator_test.go b/internal/validate/standard_validator_test.go index 7a95070..6e1a70c 100644 --- a/internal/validate/standard_validator_test.go +++ b/internal/validate/standard_validator_test.go @@ -158,3 +158,52 @@ func TestStandardValidatorJSONSchemaSchemaLoadError(t *testing.T) { t.Fatal("expected schema load error") } } + +func TestStandardValidatorLoadSchemaDocumentSuccess(t *testing.T) { + tmp := t.TempDir() + if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{ + "type": "object", + "properties": { + "name": {"type": "string"} + } +}`), 0644); err != nil { + t.Fatal(err) + } + + v := NewStandardValidator(tmp) + loader, ok := v.(SchemaDocumentLoader) + if !ok { + t.Fatal("standard validator must implement SchemaDocumentLoader") + } + + doc, err := loader.LoadSchemaDocument(context.Background(), "schema.json") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + obj, ok := doc.(map[string]any) + if !ok { + t.Fatalf("expected object document, got %#v", doc) + } + if obj["type"] != "object" { + t.Fatalf("expected schema type=object, got %#v", obj["type"]) + } +} + +func TestStandardValidatorLoadSchemaDocumentInvalidJSON(t *testing.T) { + tmp := t.TempDir() + if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{`), 0644); err != nil { + t.Fatal(err) + } + + v := NewStandardValidator(tmp) + loader, ok := v.(SchemaDocumentLoader) + if !ok { + t.Fatal("standard validator must implement SchemaDocumentLoader") + } + + _, err := loader.LoadSchemaDocument(context.Background(), "schema.json") + if err == nil { + t.Fatal("expected decode error") + } +} diff --git a/internal/validate/validator.go b/internal/validate/validator.go index 851af44..e483df7 100644 --- a/internal/validate/validator.go +++ b/internal/validate/validator.go @@ -9,3 +9,8 @@ import ( type Validator interface { Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) } + +// SchemaDocumentLoader loads JSON schema documents using validator path semantics. +type SchemaDocumentLoader interface { + LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) +}