Added support for OpenAI-compatible structured output
This commit is contained in:
@@ -103,8 +103,12 @@ func (f *fakeLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*do
|
||||
}
|
||||
|
||||
type fakeValidator struct {
|
||||
result domain.ValidationResult
|
||||
err error
|
||||
result domain.ValidationResult
|
||||
err error
|
||||
schemaDoc any
|
||||
schemaErr error
|
||||
schemaLoadPath string
|
||||
schemaLoads int
|
||||
}
|
||||
|
||||
func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
|
||||
@@ -114,6 +118,18 @@ func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact,
|
||||
return f.result, nil
|
||||
}
|
||||
|
||||
func (f *fakeValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) {
|
||||
f.schemaLoads++
|
||||
f.schemaLoadPath = schemaPath
|
||||
if f.schemaErr != nil {
|
||||
return nil, f.schemaErr
|
||||
}
|
||||
if f.schemaDoc != nil {
|
||||
return f.schemaDoc, nil
|
||||
}
|
||||
return map[string]any{"type": "object"}, nil
|
||||
}
|
||||
|
||||
type fakeRepairer struct {
|
||||
responses []*domain.GenerateResponse
|
||||
err error
|
||||
@@ -411,6 +427,115 @@ func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) {
|
||||
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||
def.Validation.SchemaPath = "events.schema.json"
|
||||
validator := &fakeValidator{
|
||||
schemaDoc: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"events": map[string]any{"type": "array"},
|
||||
},
|
||||
},
|
||||
}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
validator,
|
||||
)
|
||||
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if validator.schemaLoads != 1 {
|
||||
t.Fatalf("expected one schema load, got %d", validator.schemaLoads)
|
||||
}
|
||||
if validator.schemaLoadPath != "events.schema.json" {
|
||||
t.Fatalf("expected schema path events.schema.json, got %q", validator.schemaLoadPath)
|
||||
}
|
||||
if prepared.StructuredOutput == nil {
|
||||
t.Fatal("expected structured output spec")
|
||||
}
|
||||
if prepared.StructuredOutput.Type != domain.StructuredOutputJSONSchema {
|
||||
t.Fatalf("expected structured output type json_schema, got %q", prepared.StructuredOutput.Type)
|
||||
}
|
||||
if prepared.StructuredOutput.JSONSchema == nil {
|
||||
t.Fatal("expected structured output json_schema payload")
|
||||
}
|
||||
if prepared.StructuredOutput.JSONSchema.Name != "p_1" {
|
||||
t.Fatalf("expected derived schema name p_1, got %q", prepared.StructuredOutput.JSONSchema.Name)
|
||||
}
|
||||
if prepared.StructuredOutput.JSONSchema.Strict != true {
|
||||
t.Fatalf("expected strict=true, got %v", prepared.StructuredOutput.JSONSchema.Strict)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) {
|
||||
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||
def.Validation.SchemaPath = "missing.schema.json"
|
||||
llmClient := &fakeLLM{forbid: true}
|
||||
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
llmClient,
|
||||
validator,
|
||||
)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if !errors.Is(err, ErrValidation) {
|
||||
t.Fatalf("expected ErrValidation, got %v", err)
|
||||
}
|
||||
if llmClient.calls != 0 {
|
||||
t.Fatalf("expected llm not called when schema loading fails, calls=%d", llmClient.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeriveStructuredSchemaName(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
id string
|
||||
version string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "sanitizes punctuation and keeps dashes",
|
||||
id: "prompt.id/alpha",
|
||||
version: "1.0.0-beta",
|
||||
want: "prompt_id_alpha_1_0_0-beta",
|
||||
},
|
||||
{
|
||||
name: "fallback when empty",
|
||||
id: "",
|
||||
version: "",
|
||||
want: "scriptorium_schema",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := deriveStructuredSchemaName(tc.id, tc.version)
|
||||
if got != tc.want {
|
||||
t.Fatalf("expected %q, got %q", tc.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunSuccessful(t *testing.T) {
|
||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}
|
||||
@@ -923,6 +1048,62 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
|
||||
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 1)
|
||||
def.Validation.SchemaPath = "events.schema.json"
|
||||
|
||||
validator := &fakeValidator{
|
||||
result: domain.ValidationResult{
|
||||
Status: domain.ValidationFailed,
|
||||
Mode: domain.ValidationJSONSchema,
|
||||
Errors: []string{"schema mismatch"},
|
||||
IsValid: false,
|
||||
},
|
||||
schemaDoc: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"events": map[string]any{"type": "array"},
|
||||
},
|
||||
},
|
||||
}
|
||||
repairer := &fakeRepairer{
|
||||
responses: []*domain.GenerateResponse{
|
||||
{Content: `{"events":[]}`},
|
||||
},
|
||||
}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"events":[1]}`}}
|
||||
runner := NewRunnerWithRepairer(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
llmClient,
|
||||
validator,
|
||||
repairer,
|
||||
)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if llmClient.lastReq.StructuredOutput == nil || llmClient.lastReq.StructuredOutput.JSONSchema == nil {
|
||||
t.Fatalf("expected initial llm request to include structured output, got %+v", llmClient.lastReq.StructuredOutput)
|
||||
}
|
||||
if len(repairer.reqs) != 1 {
|
||||
t.Fatalf("expected one repair request, got %d", len(repairer.reqs))
|
||||
}
|
||||
if repairer.reqs[0].StructuredOutput == nil || repairer.reqs[0].StructuredOutput.JSONSchema == nil {
|
||||
t.Fatalf("expected repair request structured output, got %+v", repairer.reqs[0].StructuredOutput)
|
||||
}
|
||||
if repairer.reqs[0].StructuredOutput.JSONSchema.Name != "p_1" {
|
||||
t.Fatalf("expected derived schema name p_1, got %q", repairer.reqs[0].StructuredOutput.JSONSchema.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildOutputArtifactDefaults(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
Reference in New Issue
Block a user