Add bounded structured-output repair
This commit is contained in:
@@ -33,6 +33,7 @@ type Runner struct {
|
||||
renderer prompt.Renderer
|
||||
llm llm.Client
|
||||
validator validate.Validator
|
||||
repairer OutputRepairer
|
||||
}
|
||||
|
||||
func NewRunner(
|
||||
@@ -41,6 +42,17 @@ func NewRunner(
|
||||
renderer prompt.Renderer,
|
||||
llmClient llm.Client,
|
||||
validator validate.Validator,
|
||||
) *Runner {
|
||||
return NewRunnerWithRepairer(profiles, artifacts, renderer, llmClient, validator, nil)
|
||||
}
|
||||
|
||||
func NewRunnerWithRepairer(
|
||||
profiles profile.Repository,
|
||||
artifacts artifact.Reader,
|
||||
renderer prompt.Renderer,
|
||||
llmClient llm.Client,
|
||||
validator validate.Validator,
|
||||
repairer OutputRepairer,
|
||||
) *Runner {
|
||||
return &Runner{
|
||||
profiles: profiles,
|
||||
@@ -48,6 +60,7 @@ func NewRunner(
|
||||
renderer: renderer,
|
||||
llm: llmClient,
|
||||
validator: validator,
|
||||
repairer: repairer,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,18 +109,38 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
||||
}
|
||||
|
||||
outputArtifact := buildOutputArtifact(genResp.Content, effectiveContract.Format)
|
||||
|
||||
validationResult := domain.ValidationResult{
|
||||
Status: domain.ValidationSkipped,
|
||||
Mode: effectiveContract.ValidationMode,
|
||||
SchemaPath: effectiveContract.SchemaPath,
|
||||
RepairAttempts: effectiveContract.RepairAttempts,
|
||||
IsValid: true,
|
||||
validationResult, err := r.validateOutput(ctx, &outputArtifact, effectiveContract, 0)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
||||
}
|
||||
if r.validator != nil && effectiveContract.ValidationMode != domain.ValidationNone {
|
||||
validationResult, err = r.validator.Validate(ctx, &outputArtifact, effectiveContract)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
||||
|
||||
if r.shouldAttemptRepair(effectiveContract, validationResult) {
|
||||
attemptsUsed := 0
|
||||
for attemptsUsed < effectiveContract.RepairAttempts && validationResult.Status == domain.ValidationFailed {
|
||||
attemptsUsed++
|
||||
|
||||
repairResp, repairErr := r.repairer.Repair(ctx, RepairRequest{
|
||||
PreviousOutput: genResp.Content,
|
||||
ValidationErrors: validationResult.Errors,
|
||||
Target: effectiveModel,
|
||||
Attempt: attemptsUsed,
|
||||
MaxAttempts: effectiveContract.RepairAttempts,
|
||||
Mode: effectiveContract.ValidationMode,
|
||||
})
|
||||
if repairErr != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrValidation, repairErr)
|
||||
}
|
||||
if repairResp == nil {
|
||||
return nil, fmt.Errorf("%w: repairer returned nil response", ErrValidation)
|
||||
}
|
||||
|
||||
genResp = repairResp
|
||||
outputArtifact = buildOutputArtifact(genResp.Content, effectiveContract.Format)
|
||||
|
||||
validationResult, err = r.validateOutput(ctx, &outputArtifact, effectiveContract, attemptsUsed)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,6 +162,38 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *Runner) validateOutput(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, attemptsUsed int) (domain.ValidationResult, error) {
|
||||
if r.validator == nil || contract.ValidationMode == domain.ValidationNone {
|
||||
return domain.ValidationResult{
|
||||
Status: domain.ValidationSkipped,
|
||||
Mode: contract.ValidationMode,
|
||||
SchemaPath: contract.SchemaPath,
|
||||
RepairAttempts: attemptsUsed,
|
||||
IsValid: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
res, err := r.validator.Validate(ctx, artifact, contract)
|
||||
if err != nil {
|
||||
return domain.ValidationResult{}, err
|
||||
}
|
||||
res.RepairAttempts = attemptsUsed
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationResult domain.ValidationResult) bool {
|
||||
if r.repairer == nil {
|
||||
return false
|
||||
}
|
||||
if contract.RepairAttempts <= 0 {
|
||||
return false
|
||||
}
|
||||
if validationResult.Status != domain.ValidationFailed {
|
||||
return false
|
||||
}
|
||||
return contract.ValidationMode == domain.ValidationJSON || contract.ValidationMode == domain.ValidationJSONSchema
|
||||
}
|
||||
|
||||
func mergeModelTarget(base domain.ModelTarget, override *domain.ModelTarget) domain.ModelTarget {
|
||||
if override == nil {
|
||||
return base
|
||||
|
||||
Reference in New Issue
Block a user