Improve semantic reconciliation retries
This commit is contained in:
@@ -102,11 +102,12 @@ type TypedNormalizeResult[T any] struct {
|
||||
ModelCandidate *ModelCandidate
|
||||
}
|
||||
|
||||
// Normalize retry diagnostic limits bound module-provided values before the
|
||||
// framework persists them in debug artifacts.
|
||||
// Normalize retry limits bound module-provided control and diagnostic text
|
||||
// before the framework consumes or records it.
|
||||
const (
|
||||
MaxNormalizeRetryReasonCodeBytes = 128
|
||||
MaxNormalizeRetryMessageBytes = 4096
|
||||
MaxNormalizeRetryReasonCodeBytes = 128
|
||||
MaxNormalizeRetryMessageBytes = 4096
|
||||
MaxNormalizeRetryCorrectionGuidanceBytes = 4096
|
||||
)
|
||||
|
||||
// NormalizeRetry asks the framework to retry normalization while retaining a
|
||||
@@ -114,6 +115,7 @@ const (
|
||||
type NormalizeRetry struct {
|
||||
ReasonCode string
|
||||
Message string
|
||||
CorrectionGuidance string
|
||||
FallbackDiagnostics []ProducerDiagnostic
|
||||
}
|
||||
|
||||
|
||||
@@ -30,5 +30,14 @@ func validateNormalizeRetry(retry *contracts.NormalizeRetry) error {
|
||||
if len(retry.Message) > contracts.MaxNormalizeRetryMessageBytes {
|
||||
return errors.New("normalize retry directive message exceeds maximum length")
|
||||
}
|
||||
if !utf8.ValidString(retry.CorrectionGuidance) {
|
||||
return errors.New("normalize retry directive correction guidance has invalid UTF-8")
|
||||
}
|
||||
if retry.CorrectionGuidance != "" && strings.TrimSpace(retry.CorrectionGuidance) == "" {
|
||||
return errors.New("normalize retry directive correction guidance is blank")
|
||||
}
|
||||
if len(retry.CorrectionGuidance) > contracts.MaxNormalizeRetryCorrectionGuidanceBytes {
|
||||
return errors.New("normalize retry directive correction guidance exceeds maximum length")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -101,6 +101,7 @@ func cloneNormalizeRetry(retry *contracts.NormalizeRetry) *contracts.NormalizeRe
|
||||
return &contracts.NormalizeRetry{
|
||||
ReasonCode: retry.ReasonCode,
|
||||
Message: retry.Message,
|
||||
CorrectionGuidance: retry.CorrectionGuidance,
|
||||
FallbackDiagnostics: contracts.CloneProducerDiagnostics(retry.FallbackDiagnostics),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,8 +19,9 @@ func (n retryingNotesNormalizer) Normalize(_ context.Context, req contracts.Type
|
||||
|
||||
func TestNormalizerRegistryErasureClonesRetryDirective(t *testing.T) {
|
||||
retry := &contracts.NormalizeRetry{
|
||||
ReasonCode: "retryable",
|
||||
Message: "safe fallback available",
|
||||
ReasonCode: "retryable",
|
||||
Message: "safe fallback available",
|
||||
CorrectionGuidance: "return a complete corrected proposal",
|
||||
}
|
||||
registry := NewNormalizerRegistry()
|
||||
if err := RegisterNormalizer(registry, ModuleSpec{Key: "test/retry-normalize", Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: "test/notes"}, func() (contracts.Normalizer[codecNotes], error) {
|
||||
@@ -41,7 +42,8 @@ func TestNormalizerRegistryErasureClonesRetryDirective(t *testing.T) {
|
||||
t.Fatalf("normalize() error = %v", err)
|
||||
}
|
||||
retry.Message = "mutated"
|
||||
if result.Retry == nil || result.Retry.Message != "safe fallback available" {
|
||||
retry.CorrectionGuidance = "mutated guidance"
|
||||
if result.Retry == nil || result.Retry.Message != "safe fallback available" || result.Retry.CorrectionGuidance != "return a complete corrected proposal" {
|
||||
t.Fatalf("erased retry result = %#v, want independent retry data", result)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,6 +52,7 @@ type producerAttemptRequest struct {
|
||||
// the current value as a safe fallback if its shared budget is exhausted.
|
||||
// Artifact-specific adapters are responsible for validating and populating it.
|
||||
type producerRetryDirective struct {
|
||||
CorrectionGuidance string
|
||||
FallbackDiagnostics []contracts.ProducerDiagnostic
|
||||
}
|
||||
|
||||
@@ -59,7 +60,10 @@ func (directive *producerRetryDirective) clone() *producerRetryDirective {
|
||||
if directive == nil {
|
||||
return nil
|
||||
}
|
||||
return &producerRetryDirective{FallbackDiagnostics: contracts.CloneProducerDiagnostics(directive.FallbackDiagnostics)}
|
||||
return &producerRetryDirective{
|
||||
CorrectionGuidance: directive.CorrectionGuidance,
|
||||
FallbackDiagnostics: contracts.CloneProducerDiagnostics(directive.FallbackDiagnostics),
|
||||
}
|
||||
}
|
||||
|
||||
// producerAttemptOutput is intentionally artifact-neutral. Value remains
|
||||
@@ -204,10 +208,21 @@ func runProducerAttempts(ctx context.Context, config producerAttemptConfig, prod
|
||||
return failedProducerAttempt(provenance), err
|
||||
}
|
||||
if output.Retry != nil && number < attemptLimit {
|
||||
correction, err = moduleRetryCorrection(output)
|
||||
if err != nil {
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
||||
return failedProducerAttempt(provenance), err
|
||||
}
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRetried})
|
||||
kind, correction = producerAttemptModuleRetry, nil
|
||||
kind = producerAttemptModuleRetry
|
||||
continue
|
||||
}
|
||||
if output.Retry != nil && output.Retry.CorrectionGuidance != "" {
|
||||
if _, err := moduleRetryCorrection(output); err != nil {
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
||||
return failedProducerAttempt(provenance), err
|
||||
}
|
||||
}
|
||||
if output.Retry != nil {
|
||||
output.Diagnostics = append(output.Diagnostics, contracts.CloneProducerDiagnostics(output.Retry.FallbackDiagnostics)...)
|
||||
}
|
||||
@@ -269,6 +284,23 @@ func runProducerAttempts(ctx context.Context, config producerAttemptConfig, prod
|
||||
return failedProducerAttempt(provenance), errors.New("producer attempt budget was not exhausted deterministically")
|
||||
}
|
||||
|
||||
func moduleRetryCorrection(output producerAttemptOutput) (*contracts.SemanticCorrection, error) {
|
||||
if output.Retry == nil || output.Retry.CorrectionGuidance == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if output.Candidate == nil {
|
||||
return nil, errors.New("module retry correction guidance requires a model candidate")
|
||||
}
|
||||
if output.Candidate.Protocol != contracts.CorrectionProtocolSingleResponseV1 {
|
||||
return nil, fmt.Errorf("module retry correction guidance requires protocol %q", contracts.CorrectionProtocolSingleResponseV1)
|
||||
}
|
||||
correction, err := contracts.NewSemanticCorrection(output.Candidate.Response, output.Retry.CorrectionGuidance)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("construct module retry semantic correction: %w", err)
|
||||
}
|
||||
return correction, nil
|
||||
}
|
||||
|
||||
func validateProducerAttemptDiagnostics(output producerAttemptOutput) error {
|
||||
if err := contracts.ValidateProducerDiagnostics(output.Diagnostics); err != nil {
|
||||
return fmt.Errorf("producer returned invalid diagnostics: %w", err)
|
||||
|
||||
@@ -218,6 +218,9 @@ func TestRunProducerAttemptsUsesModuleRetryBudgetAndFallback(t *testing.T) {
|
||||
calls := 0
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
calls++
|
||||
if request.Correction != nil {
|
||||
t.Fatalf("feedback-free module retry correction = %#v, want nil", request.Correction)
|
||||
}
|
||||
if request.Number == 1 {
|
||||
return producerAttemptOutput{Value: "fallback", Retry: &producerRetryDirective{}}, nil
|
||||
}
|
||||
@@ -234,6 +237,58 @@ func TestRunProducerAttemptsUsesModuleRetryBudgetAndFallback(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("feedback retry", func(t *testing.T) {
|
||||
const (
|
||||
defective = `{"duplicate_groups":[{"candidate_ids":[1,99],"canonical_candidate_id":1}]}`
|
||||
guidance = "Use only candidate IDs from the supplied candidate list. Return one complete corrected response."
|
||||
)
|
||||
var observed *contracts.SemanticCorrection
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
if request.Number == 1 {
|
||||
return producerAttemptOutput{Value: "safe fallback", Candidate: attemptCandidate(t, defective), Retry: &producerRetryDirective{CorrectionGuidance: guidance}}, nil
|
||||
}
|
||||
observed = request.Correction
|
||||
return producerAttemptOutput{Value: "corrected"}, nil
|
||||
}, approveAttempt)
|
||||
if err != nil {
|
||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||
}
|
||||
if terminal.Action != producerTerminalAccepted || terminal.Value != "corrected" {
|
||||
t.Fatalf("terminal = %#v, want corrected accepted value", terminal)
|
||||
}
|
||||
if observed == nil || string(observed.AssistantResponse) != defective || observed.UserGuidance != guidance {
|
||||
t.Fatalf("module retry correction = %#v, want exact latest response and guidance", observed)
|
||||
}
|
||||
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptModuleRetry}) {
|
||||
t.Fatalf("attempt kinds = %v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("feedback-free retry clears prior correction", func(t *testing.T) {
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
switch request.Number {
|
||||
case 1:
|
||||
return producerAttemptOutput{Value: "first fallback", Candidate: attemptCandidate(t, "first defective response"), Retry: &producerRetryDirective{CorrectionGuidance: "Correct the first response."}}, nil
|
||||
case 2:
|
||||
if request.Correction == nil || string(request.Correction.AssistantResponse) != "first defective response" {
|
||||
t.Fatalf("second attempt correction = %#v", request.Correction)
|
||||
}
|
||||
return producerAttemptOutput{Value: "second fallback", Retry: &producerRetryDirective{}}, nil
|
||||
case 3:
|
||||
if request.Correction != nil {
|
||||
t.Fatalf("third attempt retained stale correction %#v", request.Correction)
|
||||
}
|
||||
return producerAttemptOutput{Value: "accepted"}, nil
|
||||
default:
|
||||
t.Fatalf("unexpected producer attempt %d", request.Number)
|
||||
return producerAttemptOutput{}, nil
|
||||
}
|
||||
}, approveAttempt)
|
||||
if err != nil || terminal.Action != producerTerminalAccepted || terminal.Value != "accepted" {
|
||||
t.Fatalf("terminal = %#v, error = %v", terminal, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("fallback", func(t *testing.T) {
|
||||
fallbackDiagnostic := contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryFallback, ReasonCode: "fallback", OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "fallback", Message: "fallback warning"}}}
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
@@ -248,6 +303,24 @@ func TestRunProducerAttemptsUsesModuleRetryBudgetAndFallback(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunProducerAttemptsRejectsModuleCorrectionWithoutModelCandidate(t *testing.T) {
|
||||
producerCalls := 0
|
||||
validatorCalls := 0
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
producerCalls++
|
||||
return producerAttemptOutput{Value: "safe", Retry: &producerRetryDirective{CorrectionGuidance: "Return a complete corrected response."}}, nil
|
||||
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||
validatorCalls++
|
||||
return validationReport{}, nil
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "requires a model candidate") {
|
||||
t.Fatalf("runProducerAttempts() error = %v, want model-candidate contract failure", err)
|
||||
}
|
||||
if terminal.Action != producerTerminalFailed || producerCalls != 1 || validatorCalls != 0 {
|
||||
t.Fatalf("terminal = %#v, producer calls = %d, validator calls = %d", terminal, producerCalls, validatorCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProducerAttemptsRejectsInvalidDiagnosticsWithoutRetry(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -153,6 +153,60 @@ func TestRunnerHandlesRetryableNormalizeFallbacks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerForwardsModuleRequestedNormalizeCorrection(t *testing.T) {
|
||||
const (
|
||||
defective = `{"duplicate_groups":[{"candidate_ids":[1,99],"canonical_candidate_id":1}]}`
|
||||
guidance = "Duplicate group 1 must use only supplied candidate IDs. Return one complete corrected response."
|
||||
)
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
lane.resolved.Normalize.Retries = 1
|
||||
var observed *contracts.SemanticCorrection
|
||||
calls := 0
|
||||
lane.typed.normalize = func(_ context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
||||
calls++
|
||||
if request.Correction != nil {
|
||||
clone, err := contracts.CloneSemanticCorrection(request.Correction)
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
observed = clone
|
||||
}
|
||||
if calls == 1 {
|
||||
return erasedTypedResult{
|
||||
Value: codecNotes{Items: []string{"safe fallback"}},
|
||||
ModelCandidate: attemptCandidate(t, defective),
|
||||
Retry: &contracts.NormalizeRetry{
|
||||
ReasonCode: "semantic_proposal_invalid",
|
||||
Message: "operator-facing proposal diagnostic",
|
||||
CorrectionGuidance: guidance,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"corrected"}}}, nil
|
||||
}
|
||||
debug := newCapturedDebugRecorder()
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if calls != 2 || observed == nil || string(observed.AssistantResponse) != defective || observed.UserGuidance != guidance {
|
||||
t.Fatalf("normalize calls = %d correction = %#v", calls, observed)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 || len(output.Rejected) != 0 {
|
||||
t.Fatalf("run output = %#v, want corrected accepted output", output)
|
||||
}
|
||||
var retryDebug strings.Builder
|
||||
for _, name := range debug.names() {
|
||||
if strings.HasPrefix(name, "normalize/notes/attempt-") && strings.HasSuffix(name, ".json") {
|
||||
retryDebug.Write(debug.json[name])
|
||||
}
|
||||
}
|
||||
if !strings.Contains(retryDebug.String(), `"correction_available":true`) || strings.Contains(retryDebug.String(), guidance) || strings.Contains(retryDebug.String(), defective) {
|
||||
t.Fatalf("retry debug = %s, want safe correction metadata without content", retryDebug.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerValidatesNormalizeRetryDiagnostics(t *testing.T) {
|
||||
const (
|
||||
reasonSentinel = "reason-diagnostic-sentinel"
|
||||
@@ -160,9 +214,11 @@ func TestRunnerValidatesNormalizeRetryDiagnostics(t *testing.T) {
|
||||
)
|
||||
reasonOverLimit := strings.Repeat("r", contracts.MaxNormalizeRetryReasonCodeBytes-len(reasonSentinel)) + reasonSentinel + "x"
|
||||
messageOverLimit := strings.Repeat("m", contracts.MaxNormalizeRetryMessageBytes-len(messageSentinel)) + messageSentinel + "x"
|
||||
guidanceOverLimit := strings.Repeat("g", contracts.MaxNormalizeRetryCorrectionGuidanceBytes-len(messageSentinel)) + messageSentinel + "x"
|
||||
tests := []struct {
|
||||
name string
|
||||
retry contracts.NormalizeRetry
|
||||
candidate *contracts.ModelCandidate
|
||||
wantError string
|
||||
hiddenValues []string
|
||||
}{
|
||||
@@ -173,6 +229,15 @@ func TestRunnerValidatesNormalizeRetryDiagnostics(t *testing.T) {
|
||||
Message: strings.Repeat("m", contracts.MaxNormalizeRetryMessageBytes),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "accepts correction guidance byte limit with candidate",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel,
|
||||
Message: messageSentinel,
|
||||
CorrectionGuidance: strings.Repeat("g", contracts.MaxNormalizeRetryCorrectionGuidanceBytes),
|
||||
},
|
||||
candidate: attemptCandidate(t, `{"duplicate_groups":[]}`),
|
||||
},
|
||||
{
|
||||
name: "rejects oversized reason code",
|
||||
retry: contracts.NormalizeRetry{
|
||||
@@ -209,6 +274,46 @@ func TestRunnerValidatesNormalizeRetryDiagnostics(t *testing.T) {
|
||||
wantError: "message has invalid UTF-8",
|
||||
hiddenValues: []string{reasonSentinel, messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects oversized correction guidance",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel,
|
||||
Message: messageSentinel,
|
||||
CorrectionGuidance: guidanceOverLimit,
|
||||
},
|
||||
wantError: "correction guidance exceeds maximum length",
|
||||
hiddenValues: []string{reasonSentinel, messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects invalid correction guidance UTF-8",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel,
|
||||
Message: messageSentinel,
|
||||
CorrectionGuidance: messageSentinel + string([]byte{0xff}),
|
||||
},
|
||||
wantError: "correction guidance has invalid UTF-8",
|
||||
hiddenValues: []string{reasonSentinel, messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects blank correction guidance",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel,
|
||||
Message: messageSentinel,
|
||||
CorrectionGuidance: " \t\n ",
|
||||
},
|
||||
wantError: "correction guidance is blank",
|
||||
hiddenValues: []string{reasonSentinel, messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects correction guidance without candidate",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel,
|
||||
Message: messageSentinel,
|
||||
CorrectionGuidance: "Return one complete corrected response.",
|
||||
},
|
||||
wantError: "requires a model candidate",
|
||||
hiddenValues: []string{reasonSentinel, messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects blank reason code",
|
||||
retry: contracts.NormalizeRetry{
|
||||
@@ -233,7 +338,7 @@ func TestRunnerValidatesNormalizeRetryDiagnostics(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
prepared.Steps[0].lanes[0].typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"safe"}}, Retry: &tc.retry}, nil
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"safe"}}, Retry: &tc.retry, ModelCandidate: tc.candidate}, nil
|
||||
}
|
||||
debug := newCapturedDebugRecorder()
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
|
||||
@@ -451,8 +451,8 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoi
|
||||
return producerAttemptOutput{}, terminal.record(map[string]any{"output": debugCheckpointArtifact(serializedCandidate)}, fmt.Errorf("normalize lane %q returned invalid retry directive: %w", lane.ID, err))
|
||||
}
|
||||
anotherAttempt := request.Number <= lane.Normalize.Retries
|
||||
attemptValue.retry = map[string]any{"reason_code": result.Retry.ReasonCode, "message": result.Retry.Message, "another_attempt": anotherAttempt, "fallback_accepted": !anotherAttempt}
|
||||
directive = &producerRetryDirective{FallbackDiagnostics: contracts.CloneProducerDiagnostics(result.Retry.FallbackDiagnostics)}
|
||||
attemptValue.retry = map[string]any{"reason_code": result.Retry.ReasonCode, "message": result.Retry.Message, "correction_available": result.Retry.CorrectionGuidance != "", "another_attempt": anotherAttempt, "fallback_accepted": !anotherAttempt}
|
||||
directive = &producerRetryDirective{CorrectionGuidance: result.Retry.CorrectionGuidance, FallbackDiagnostics: contracts.CloneProducerDiagnostics(result.Retry.FallbackDiagnostics)}
|
||||
if anotherAttempt {
|
||||
payload := map[string]any{"output": debugCheckpointArtifact(serializedCandidate), "retry": attemptValue.retry}
|
||||
if debugErr := terminal.record(payload, nil); debugErr != nil {
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
)
|
||||
|
||||
// Policy identifies the framework-owned reconciliation and assessment rules.
|
||||
const Policy = "semantic_reconciliation.v1"
|
||||
const Policy = "semantic_reconciliation.v2"
|
||||
|
||||
var _ contracts.ManifestMetadataProvider = (*Engine)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Engine)(nil)
|
||||
|
||||
@@ -3,6 +3,10 @@ package semanticreconcile
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
// ProposalResponse is the complete private structured response contract.
|
||||
@@ -31,6 +35,28 @@ const (
|
||||
IssueOverlappingMember IssueCategory = "overlapping_member"
|
||||
)
|
||||
|
||||
var allIssueCategories = []IssueCategory{
|
||||
IssueMemberNonPositive,
|
||||
IssueMemberUnknown,
|
||||
IssueRepeatedMember,
|
||||
IssueFewerThanTwoMembers,
|
||||
IssueCanonicalNonPositive,
|
||||
IssueCanonicalUnknown,
|
||||
IssueCanonicalNotMember,
|
||||
IssueOverlappingMember,
|
||||
}
|
||||
|
||||
var issueCorrectionProse = map[IssueCategory]string{
|
||||
IssueMemberNonPositive: "Use only positive candidate IDs from the supplied candidate list.",
|
||||
IssueMemberUnknown: "Remove every candidate ID that is not present in the supplied candidate list.",
|
||||
IssueRepeatedMember: "List each candidate ID at most once within the duplicate group.",
|
||||
IssueFewerThanTwoMembers: "Include at least two distinct candidate IDs, or omit the duplicate group.",
|
||||
IssueCanonicalNonPositive: "Choose a positive canonical_candidate_id from the supplied candidate list.",
|
||||
IssueCanonicalUnknown: "Choose canonical_candidate_id from the supplied candidate list.",
|
||||
IssueCanonicalNotMember: "Make canonical_candidate_id one of the candidate_ids in the same duplicate group.",
|
||||
IssueOverlappingMember: "Place each candidate ID in at most one duplicate group.",
|
||||
}
|
||||
|
||||
// Issue identifies an unsafe proposal category at its original response group
|
||||
// index without prescribing caller diagnostic text.
|
||||
type Issue struct {
|
||||
@@ -38,8 +64,8 @@ type Issue struct {
|
||||
Category IssueCategory
|
||||
}
|
||||
|
||||
// IssueDetails renders stable, domain-neutral proposal diagnostics for an
|
||||
// adapter's retry message.
|
||||
// IssueDetails renders stable, domain-neutral proposal diagnostics for
|
||||
// operators and debug records. Its internal categories are not model guidance.
|
||||
func IssueDetails(issues []Issue) []string {
|
||||
details := make([]string, len(issues))
|
||||
for index, issue := range issues {
|
||||
@@ -48,6 +74,94 @@ func IssueDetails(issues []Issue) []string {
|
||||
return details
|
||||
}
|
||||
|
||||
// CorrectionDetails translates proposal issues into stable model-facing prose.
|
||||
// The response-local group ordinals help the model find the defective group in
|
||||
// the exact response appended to the correction request.
|
||||
func CorrectionDetails(issues []Issue) ([]string, error) {
|
||||
groupsByCategory := make(map[IssueCategory][]int)
|
||||
seen := make(map[IssueCategory]map[int]struct{})
|
||||
for _, issue := range issues {
|
||||
if issue.GroupIndex < 0 {
|
||||
return nil, fmt.Errorf("semantic reconciliation issue group index must not be negative")
|
||||
}
|
||||
if _, exists := issueCorrectionProse[issue.Category]; !exists {
|
||||
return nil, fmt.Errorf("semantic reconciliation issue category %q has no correction guidance", issue.Category)
|
||||
}
|
||||
if seen[issue.Category] == nil {
|
||||
seen[issue.Category] = make(map[int]struct{})
|
||||
}
|
||||
if _, exists := seen[issue.Category][issue.GroupIndex]; exists {
|
||||
continue
|
||||
}
|
||||
seen[issue.Category][issue.GroupIndex] = struct{}{}
|
||||
groupsByCategory[issue.Category] = append(groupsByCategory[issue.Category], issue.GroupIndex)
|
||||
}
|
||||
|
||||
details := make([]string, 0, len(groupsByCategory))
|
||||
for _, category := range allIssueCategories {
|
||||
groups := groupsByCategory[category]
|
||||
if len(groups) == 0 {
|
||||
continue
|
||||
}
|
||||
sort.Ints(groups)
|
||||
details = append(details, fmt.Sprintf("%s: %s", correctionGroupLabel(groups), issueCorrectionProse[category]))
|
||||
}
|
||||
return details, nil
|
||||
}
|
||||
|
||||
// CorrectionGuidance builds one bounded request for a complete corrected
|
||||
// proposal. Additional details let a typed owner append a domain rule without
|
||||
// weakening or duplicating the shared protocol guidance.
|
||||
func CorrectionGuidance(issues []Issue, additionalDetails ...string) (string, error) {
|
||||
details, err := CorrectionDetails(issues)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, detail := range additionalDetails {
|
||||
if !utf8.ValidString(detail) {
|
||||
return "", fmt.Errorf("semantic reconciliation additional correction detail has invalid UTF-8")
|
||||
}
|
||||
detail = strings.TrimSpace(detail)
|
||||
if detail == "" {
|
||||
return "", fmt.Errorf("semantic reconciliation additional correction detail must not be blank")
|
||||
}
|
||||
details = append(details, detail)
|
||||
}
|
||||
if len(details) == 0 {
|
||||
return "", fmt.Errorf("semantic reconciliation correction guidance requires at least one detail")
|
||||
}
|
||||
|
||||
parts := make([]string, 0, len(details)+2)
|
||||
parts = append(parts, "The previous semantic-duplicate proposal was invalid.")
|
||||
parts = append(parts, details...)
|
||||
parts = append(parts, "Return one complete corrected JSON response that follows the original instructions; do not return a patch or commentary.")
|
||||
guidance := strings.Join(parts, " ")
|
||||
if len(guidance) > contracts.MaxNormalizeRetryCorrectionGuidanceBytes {
|
||||
return "", fmt.Errorf("semantic reconciliation correction guidance exceeds maximum length")
|
||||
}
|
||||
return guidance, nil
|
||||
}
|
||||
|
||||
func correctionGroupLabel(groupIndexes []int) string {
|
||||
const maximumDisplayedGroups = 12
|
||||
displayed := groupIndexes
|
||||
if len(displayed) > maximumDisplayedGroups {
|
||||
displayed = displayed[:maximumDisplayedGroups]
|
||||
}
|
||||
ordinals := make([]string, len(displayed))
|
||||
for index, groupIndex := range displayed {
|
||||
ordinals[index] = fmt.Sprintf("%d", groupIndex+1)
|
||||
}
|
||||
if len(groupIndexes) == 1 {
|
||||
return "Duplicate group " + ordinals[0]
|
||||
}
|
||||
label := "Duplicate groups " + strings.Join(ordinals, ", ")
|
||||
if omitted := len(groupIndexes) - len(displayed); omitted > 0 {
|
||||
label += fmt.Sprintf(", and %d additional affected group(s)", omitted)
|
||||
}
|
||||
return label
|
||||
}
|
||||
|
||||
// PlanGroup identifies one validated group using original candidate positions.
|
||||
type PlanGroup struct {
|
||||
memberPositions []int
|
||||
|
||||
@@ -2,9 +2,11 @@ package semanticreconcile
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestAssessProducesAStableOriginalPositionPlan(t *testing.T) {
|
||||
@@ -50,6 +52,72 @@ func TestIssueDetailsPreservesIssueOrder(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrectionGuidanceCoversEveryIssueCategoryWithoutExposingInternalLabels(t *testing.T) {
|
||||
if len(issueCorrectionProse) != len(allIssueCategories) {
|
||||
t.Fatalf("correction prose entries = %d, categories = %d", len(issueCorrectionProse), len(allIssueCategories))
|
||||
}
|
||||
issues := make([]Issue, len(allIssueCategories))
|
||||
seen := make(map[IssueCategory]struct{}, len(allIssueCategories))
|
||||
for index, category := range allIssueCategories {
|
||||
if _, duplicate := seen[category]; duplicate {
|
||||
t.Fatalf("duplicate authoritative issue category %q", category)
|
||||
}
|
||||
seen[category] = struct{}{}
|
||||
if strings.TrimSpace(issueCorrectionProse[category]) == "" {
|
||||
t.Fatalf("issue category %q has no model-facing prose", category)
|
||||
}
|
||||
issues[index] = Issue{GroupIndex: index, Category: category}
|
||||
}
|
||||
|
||||
details, err := CorrectionDetails(issues)
|
||||
if err != nil {
|
||||
t.Fatalf("CorrectionDetails() error = %v", err)
|
||||
}
|
||||
guidance, err := CorrectionGuidance(issues)
|
||||
if err != nil {
|
||||
t.Fatalf("CorrectionGuidance() error = %v", err)
|
||||
}
|
||||
if len(details) != len(allIssueCategories) || !strings.Contains(guidance, "Duplicate group 1") || !strings.Contains(guidance, "complete corrected JSON response") || len(guidance) > contracts.MaxNormalizeRetryCorrectionGuidanceBytes {
|
||||
t.Fatalf("correction details = %#v guidance = %q", details, guidance)
|
||||
}
|
||||
for _, category := range allIssueCategories {
|
||||
if strings.Contains(guidance, string(category)) {
|
||||
t.Fatalf("model guidance exposed internal category %q: %q", category, guidance)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrectionDetailsDeduplicatesAndBoundsAffectedGroupLists(t *testing.T) {
|
||||
issues := make([]Issue, 0, 257)
|
||||
for group := 0; group < 256; group++ {
|
||||
issues = append(issues, Issue{GroupIndex: group, Category: IssueMemberUnknown})
|
||||
}
|
||||
issues = append(issues, Issue{GroupIndex: 0, Category: IssueMemberUnknown})
|
||||
|
||||
details, err := CorrectionDetails(issues)
|
||||
if err != nil {
|
||||
t.Fatalf("CorrectionDetails() error = %v", err)
|
||||
}
|
||||
if len(details) != 1 || !strings.Contains(details[0], "additional affected group") || strings.Count(details[0], "Duplicate groups") != 1 {
|
||||
t.Fatalf("CorrectionDetails() = %#v, want one bounded grouped detail", details)
|
||||
}
|
||||
guidance, err := CorrectionGuidance(issues)
|
||||
if err != nil || len(guidance) > contracts.MaxNormalizeRetryCorrectionGuidanceBytes {
|
||||
t.Fatalf("CorrectionGuidance() = %q, %v", guidance, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrectionDetailsRejectsUnknownOrInvalidIssues(t *testing.T) {
|
||||
for _, issues := range [][]Issue{
|
||||
{{GroupIndex: 0, Category: "future_unmapped_category"}},
|
||||
{{GroupIndex: -1, Category: IssueMemberUnknown}},
|
||||
} {
|
||||
if details, err := CorrectionDetails(issues); err == nil || details != nil {
|
||||
t.Fatalf("CorrectionDetails(%#v) = %#v, %v; want fail-closed error", issues, details, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssessRejectsEveryUnsafeLocalGroupShape(t *testing.T) {
|
||||
preparation := proposalPreparation(t)
|
||||
tests := []struct {
|
||||
|
||||
Reference in New Issue
Block a user