Improve semantic reconciliation retries
This commit is contained in:
@@ -218,6 +218,9 @@ func TestRunProducerAttemptsUsesModuleRetryBudgetAndFallback(t *testing.T) {
|
||||
calls := 0
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
calls++
|
||||
if request.Correction != nil {
|
||||
t.Fatalf("feedback-free module retry correction = %#v, want nil", request.Correction)
|
||||
}
|
||||
if request.Number == 1 {
|
||||
return producerAttemptOutput{Value: "fallback", Retry: &producerRetryDirective{}}, nil
|
||||
}
|
||||
@@ -234,6 +237,58 @@ func TestRunProducerAttemptsUsesModuleRetryBudgetAndFallback(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("feedback retry", func(t *testing.T) {
|
||||
const (
|
||||
defective = `{"duplicate_groups":[{"candidate_ids":[1,99],"canonical_candidate_id":1}]}`
|
||||
guidance = "Use only candidate IDs from the supplied candidate list. Return one complete corrected response."
|
||||
)
|
||||
var observed *contracts.SemanticCorrection
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
if request.Number == 1 {
|
||||
return producerAttemptOutput{Value: "safe fallback", Candidate: attemptCandidate(t, defective), Retry: &producerRetryDirective{CorrectionGuidance: guidance}}, nil
|
||||
}
|
||||
observed = request.Correction
|
||||
return producerAttemptOutput{Value: "corrected"}, nil
|
||||
}, approveAttempt)
|
||||
if err != nil {
|
||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||
}
|
||||
if terminal.Action != producerTerminalAccepted || terminal.Value != "corrected" {
|
||||
t.Fatalf("terminal = %#v, want corrected accepted value", terminal)
|
||||
}
|
||||
if observed == nil || string(observed.AssistantResponse) != defective || observed.UserGuidance != guidance {
|
||||
t.Fatalf("module retry correction = %#v, want exact latest response and guidance", observed)
|
||||
}
|
||||
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptModuleRetry}) {
|
||||
t.Fatalf("attempt kinds = %v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("feedback-free retry clears prior correction", func(t *testing.T) {
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
switch request.Number {
|
||||
case 1:
|
||||
return producerAttemptOutput{Value: "first fallback", Candidate: attemptCandidate(t, "first defective response"), Retry: &producerRetryDirective{CorrectionGuidance: "Correct the first response."}}, nil
|
||||
case 2:
|
||||
if request.Correction == nil || string(request.Correction.AssistantResponse) != "first defective response" {
|
||||
t.Fatalf("second attempt correction = %#v", request.Correction)
|
||||
}
|
||||
return producerAttemptOutput{Value: "second fallback", Retry: &producerRetryDirective{}}, nil
|
||||
case 3:
|
||||
if request.Correction != nil {
|
||||
t.Fatalf("third attempt retained stale correction %#v", request.Correction)
|
||||
}
|
||||
return producerAttemptOutput{Value: "accepted"}, nil
|
||||
default:
|
||||
t.Fatalf("unexpected producer attempt %d", request.Number)
|
||||
return producerAttemptOutput{}, nil
|
||||
}
|
||||
}, approveAttempt)
|
||||
if err != nil || terminal.Action != producerTerminalAccepted || terminal.Value != "accepted" {
|
||||
t.Fatalf("terminal = %#v, error = %v", terminal, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("fallback", func(t *testing.T) {
|
||||
fallbackDiagnostic := contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryFallback, ReasonCode: "fallback", OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "fallback", Message: "fallback warning"}}}
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
@@ -248,6 +303,24 @@ func TestRunProducerAttemptsUsesModuleRetryBudgetAndFallback(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunProducerAttemptsRejectsModuleCorrectionWithoutModelCandidate(t *testing.T) {
|
||||
producerCalls := 0
|
||||
validatorCalls := 0
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
producerCalls++
|
||||
return producerAttemptOutput{Value: "safe", Retry: &producerRetryDirective{CorrectionGuidance: "Return a complete corrected response."}}, nil
|
||||
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||
validatorCalls++
|
||||
return validationReport{}, nil
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "requires a model candidate") {
|
||||
t.Fatalf("runProducerAttempts() error = %v, want model-candidate contract failure", err)
|
||||
}
|
||||
if terminal.Action != producerTerminalFailed || producerCalls != 1 || validatorCalls != 0 {
|
||||
t.Fatalf("terminal = %#v, producer calls = %d, validator calls = %d", terminal, producerCalls, validatorCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProducerAttemptsRejectsInvalidDiagnosticsWithoutRetry(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
Reference in New Issue
Block a user