Classify registry normalization diagnostics
This commit is contained in:
@@ -31,8 +31,8 @@ const (
|
||||
ReasonCodeSourceReferencesNormalized = "source_references_normalized"
|
||||
ReasonCodeDuplicateItemCollapsed = "duplicate_item_collapsed"
|
||||
ReasonCodeItemSemanticProposalInvalid = "item_semantic_proposal_invalid"
|
||||
ReasonCodeItemSemanticRetryProposalInvalid = "item_semantic_retry_proposal_invalid"
|
||||
ReasonCodeItemSemanticReconciliationExhausted = "item_semantic_reconciliation_exhausted"
|
||||
ReasonCodeItemNormalizationWarningsOmitted = "item_normalization_warnings_omitted"
|
||||
)
|
||||
|
||||
var requiredCapabilities = []string{"merged"}
|
||||
@@ -107,10 +107,10 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
}
|
||||
|
||||
order := shared.NewSourceRefOrder(req.Source)
|
||||
records, warnings := preprocessRecords(req.MergeOutput.Value, order)
|
||||
records, findings := preprocessRecords(req.MergeOutput.Value, order)
|
||||
deterministic := recordList(records)
|
||||
if len(records) < 2 {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{Value: deterministic, Warnings: limitWarnings(warnings)}, nil
|
||||
return normalizationResult(deterministic, findings, nil, nil)
|
||||
}
|
||||
|
||||
candidates, envelopes, err := reconciliationInputs(records)
|
||||
@@ -127,48 +127,42 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
|
||||
switch reconciliation.Disposition() {
|
||||
case semanticreconcile.SkippedInsufficientCandidates:
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{Value: deterministic, Warnings: limitWarnings(warnings)}, nil
|
||||
return normalizationResult(deterministic, findings, nil, nil)
|
||||
case semanticreconcile.SkippedLimitExceeded:
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{Value: deterministic, Warnings: limitWarningsWithSemanticFallback(warnings)}, nil
|
||||
return fallbackResult(deterministic, findings, nil, semanticFallbackFinding(-1))
|
||||
case semanticreconcile.RetryableInvalidStructuredOutput:
|
||||
return n.invalidStructuredResult(deterministic, warnings), nil
|
||||
return n.invalidStructuredResult(deterministic, findings)
|
||||
case semanticreconcile.Complete, semanticreconcile.RetryableDiscardedProposalGroups:
|
||||
default:
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("unknown semantic reconciliation disposition %d", reconciliation.Disposition())
|
||||
}
|
||||
|
||||
applied, semanticWarnings, rejectedGroups, err := applyReconciliationPlan(reconciliation.Plan(), records, envelopes, order)
|
||||
applied, semanticFindings, advisoryFindings, rejectedGroups, err := applyReconciliationPlan(reconciliation.Plan(), records, envelopes, order)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("apply semantic reconciliation plan: %w", err)
|
||||
}
|
||||
warnings = append(warnings, semanticWarnings...)
|
||||
findings = append(findings, semanticFindings...)
|
||||
discardedGroups := reconciliation.DiscardedGroupCount() + rejectedGroups
|
||||
if discardedGroups == 0 {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{Value: recordList(applied), Warnings: limitWarnings(warnings), ModelCandidate: reconciliation.ModelCandidate()}, nil
|
||||
return normalizationResult(recordList(applied), findings, advisoryFindings, reconciliation.ModelCandidate())
|
||||
}
|
||||
return retryResult(recordList(applied), warnings, reconciliation, rejectedGroups), nil
|
||||
return retryResult(recordList(applied), findings, advisoryFindings, reconciliation, rejectedGroups)
|
||||
}
|
||||
|
||||
func (n *Normalizer) invalidStructuredResult(value dnd.ItemRegistry, warnings []contracts.Warning) contracts.TypedNormalizeResult[dnd.ItemRegistry] {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{Value: value, Warnings: limitWarningsForRetry(warnings), Retry: &contracts.NormalizeRetry{
|
||||
ReasonCode: ReasonCodeItemSemanticProposalInvalid, Message: "semantic proposal requires retry: invalid structured output",
|
||||
FallbackWarnings: []contracts.Warning{semanticFallbackWarning(-1)},
|
||||
}}
|
||||
func (n *Normalizer) invalidStructuredResult(value dnd.ItemRegistry, findings []contracts.Warning) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
|
||||
return retryResultWithFallback(value, findings, nil, nil, ReasonCodeItemSemanticRetryProposalInvalid, "semantic proposal requires retry: invalid structured output", semanticFallbackFinding(-1))
|
||||
}
|
||||
|
||||
func retryResult(value dnd.ItemRegistry, warnings []contracts.Warning, reconciliation semanticreconcile.Result, rejectedGroups int) contracts.TypedNormalizeResult[dnd.ItemRegistry] {
|
||||
func retryResult(value dnd.ItemRegistry, findings, advisoryFindings []contracts.Warning, reconciliation semanticreconcile.Result, rejectedGroups int) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
|
||||
details := semanticreconcile.IssueDetails(reconciliation.Issues())
|
||||
if rejectedGroups > 0 {
|
||||
details = append(details, "currency may only be consolidated with aliases of one denomination")
|
||||
}
|
||||
discardedGroups := reconciliation.DiscardedGroupCount() + rejectedGroups
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{Value: value, Warnings: limitWarningsForRetry(warnings), ModelCandidate: reconciliation.ModelCandidate(), Retry: &contracts.NormalizeRetry{
|
||||
ReasonCode: ReasonCodeItemSemanticProposalInvalid, Message: diagnostics.Aggregate("semantic proposal requires retry", details),
|
||||
FallbackWarnings: []contracts.Warning{semanticFallbackWarning(discardedGroups)},
|
||||
}}
|
||||
return retryResultWithFallback(value, findings, advisoryFindings, reconciliation.ModelCandidate(), ReasonCodeItemSemanticRetryProposalInvalid, diagnostics.Aggregate("semantic proposal requires retry", details), semanticFallbackFinding(discardedGroups))
|
||||
}
|
||||
|
||||
func semanticFallbackWarning(discarded int) contracts.Warning {
|
||||
func semanticFallbackFinding(discarded int) contracts.Warning {
|
||||
message := "semantic proposal could not be applied"
|
||||
if discarded >= 0 {
|
||||
message = fmt.Sprintf("%d proposal group(s) omitted after semantic proposal retry exhaustion", discarded)
|
||||
@@ -176,24 +170,43 @@ func semanticFallbackWarning(discarded int) contracts.Warning {
|
||||
return contracts.Warning{Scope: "items", ReasonCode: ReasonCodeItemSemanticReconciliationExhausted, Message: message}
|
||||
}
|
||||
|
||||
func limitWarnings(warnings []contracts.Warning) []contracts.Warning {
|
||||
return diagnostics.LimitWarnings(warnings, "items", ReasonCodeItemNormalizationWarningsOmitted)
|
||||
func normalizationResult(value dnd.ItemRegistry, findings, advisoryFindings []contracts.Warning, candidate *contracts.ModelCandidate) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
|
||||
diagnosticGroups, err := diagnostics.Collect(findings, contracts.DiagnosticDispositionObservation, contracts.DiagnosticCategoryNormalization)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect normalization diagnostics: %w", err)
|
||||
}
|
||||
advisoryGroups, err := diagnostics.Collect(advisoryFindings, contracts.DiagnosticDispositionAdvisory, contracts.DiagnosticCategoryDataQuality)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect data-quality diagnostics: %w", err)
|
||||
}
|
||||
diagnosticGroups = append(diagnosticGroups, advisoryGroups...)
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{Value: value, Diagnostics: diagnosticGroups, ModelCandidate: candidate}, nil
|
||||
}
|
||||
|
||||
func limitWarningsForRetry(warnings []contracts.Warning) []contracts.Warning {
|
||||
if warnings == nil {
|
||||
return nil
|
||||
func fallbackResult(value dnd.ItemRegistry, findings, advisoryFindings []contracts.Warning, fallback contracts.Warning) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
|
||||
result, err := normalizationResult(value, findings, advisoryFindings, nil)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, err
|
||||
}
|
||||
if len(warnings) < diagnostics.MaxWarnings {
|
||||
return append([]contracts.Warning(nil), warnings...)
|
||||
fallbackGroups, err := diagnostics.Collect([]contracts.Warning{fallback}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err)
|
||||
}
|
||||
displayed := diagnostics.MaxWarnings - 2
|
||||
bounded := append([]contracts.Warning(nil), warnings[:displayed]...)
|
||||
return append(bounded, contracts.Warning{Scope: "items", ReasonCode: ReasonCodeItemNormalizationWarningsOmitted, Message: fmt.Sprintf("%d additional warning(s) omitted", len(warnings)-displayed)})
|
||||
result.Diagnostics = append(result.Diagnostics, fallbackGroups...)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func limitWarningsWithSemanticFallback(warnings []contracts.Warning) []contracts.Warning {
|
||||
return append(limitWarningsForRetry(warnings), semanticFallbackWarning(-1))
|
||||
func retryResultWithFallback(value dnd.ItemRegistry, findings, advisoryFindings []contracts.Warning, candidate *contracts.ModelCandidate, reasonCode, message string, fallback contracts.Warning) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
|
||||
result, err := normalizationResult(value, findings, advisoryFindings, candidate)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, err
|
||||
}
|
||||
fallbackGroups, err := diagnostics.Collect([]contracts.Warning{fallback}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err)
|
||||
}
|
||||
result.Retry = &contracts.NormalizeRetry{ReasonCode: reasonCode, Message: message, FallbackDiagnostics: fallbackGroups}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type normalizedRecord struct {
|
||||
@@ -207,18 +220,18 @@ func preprocessRecords(input dnd.ItemRegistry, order shared.SourceRefOrder) ([]n
|
||||
return nil, nil
|
||||
}
|
||||
records := make([]normalizedRecord, len(input.Items))
|
||||
warnings := make([]contracts.Warning, 0)
|
||||
findings := make([]contracts.Warning, 0)
|
||||
for index, inputItem := range input.Items {
|
||||
item, fieldsChanged, refsChanged := normalizeRecord(inputItem, order)
|
||||
records[index] = normalizedRecord{item: item, inputIndexes: []int{index}, earliest: index}
|
||||
if fieldsChanged {
|
||||
warnings = append(warnings, contracts.Warning{Scope: itemScope(index), ReasonCode: ReasonCodeItemFieldsNormalized, Message: fmt.Sprintf("input index %d: item name normalized for %s", index, diagnostics.Quote(inputItem.Name))})
|
||||
findings = append(findings, contracts.Warning{Scope: itemScope(index), ReasonCode: ReasonCodeItemFieldsNormalized, Message: fmt.Sprintf("input index %d: item name normalized for %s", index, diagnostics.Quote(inputItem.Name))})
|
||||
}
|
||||
if refsChanged {
|
||||
warnings = append(warnings, contracts.Warning{Scope: itemScope(index), ReasonCode: ReasonCodeSourceReferencesNormalized, Message: fmt.Sprintf("input index %d: source references normalized (original count %d, final count %d)", index, len(inputItem.SourceRefs), len(item.SourceRefs))})
|
||||
findings = append(findings, contracts.Warning{Scope: itemScope(index), ReasonCode: ReasonCodeSourceReferencesNormalized, Message: fmt.Sprintf("input index %d: source references normalized (original count %d, final count %d)", index, len(inputItem.SourceRefs), len(item.SourceRefs))})
|
||||
}
|
||||
if inputItem.ID != item.ID {
|
||||
warnings = append(warnings, contracts.Warning{Scope: itemScope(index), ReasonCode: ReasonCodeItemIDRecomputed, Message: fmt.Sprintf("input index %d: item ID recomputed from %s", index, diagnostics.Quote(item.Name))})
|
||||
findings = append(findings, contracts.Warning{Scope: itemScope(index), ReasonCode: ReasonCodeItemIDRecomputed, Message: fmt.Sprintf("input index %d: item ID recomputed from %s", index, diagnostics.Quote(item.Name))})
|
||||
}
|
||||
}
|
||||
groups := comparisonNameGroups(records)
|
||||
@@ -234,10 +247,10 @@ func preprocessRecords(input dnd.ItemRegistry, order shared.SourceRefOrder) ([]n
|
||||
retained.inputIndexes = sortedUniqueIndexes(retained.inputIndexes)
|
||||
output = append(output, retained)
|
||||
if len(members) > 1 {
|
||||
warnings = append(warnings, duplicateWarning(retained.earliest, memberInputIndexes(records, members[1:])))
|
||||
findings = append(findings, duplicateWarning(retained.earliest, memberInputIndexes(records, members[1:])))
|
||||
}
|
||||
}
|
||||
return output, warnings
|
||||
return output, findings
|
||||
}
|
||||
|
||||
func normalizeRecord(input dnd.Item, order shared.SourceRefOrder) (dnd.Item, bool, bool) {
|
||||
|
||||
@@ -18,7 +18,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
|
||||
identityvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/identity"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
@@ -81,8 +80,8 @@ func TestNormalizeConsolidatesEqualNamesAcrossEvidenceWithoutMutation(t *testing
|
||||
}
|
||||
rope := result.Value.Items[0]
|
||||
wantRefs := []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}, {SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}
|
||||
if rope.Name != "Rope" || rope.ID != identity.DeriveID("Rope") || !reflect.DeepEqual(rope.SourceRefs, wantRefs) || result.Value.Items[1].Name != "Lantern" || !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) {
|
||||
t.Fatalf("items = %#v, warnings = %#v; want earliest display name, canonical evidence union, and stable placement", result.Value.Items, result.Warnings)
|
||||
if rope.Name != "Rope" || rope.ID != identity.DeriveID("Rope") || !reflect.DeepEqual(rope.SourceRefs, wantRefs) || result.Value.Items[1].Name != "Lantern" || !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) {
|
||||
t.Fatalf("items = %#v, diagnostics = %#v; want earliest display name, canonical evidence union, and stable placement", result.Value.Items, result.Diagnostics)
|
||||
}
|
||||
validation, validationErr := identityvalidator.New(identityvalidator.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Value: result.Value})
|
||||
if validationErr != nil || !validation.Approved {
|
||||
@@ -129,8 +128,8 @@ func TestNormalizeAppliesSafeAliasProposal(t *testing.T) {
|
||||
t.Fatalf("Normalize() = %#v, %v", result, err)
|
||||
}
|
||||
merged := result.Value.Items[0]
|
||||
if merged.Name != "Compass of the Stars" || merged.ID != identity.DeriveID(merged.Name) || len(merged.SourceRefs) != 2 || !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) {
|
||||
t.Fatalf("merged item = %#v, warnings = %#v", merged, result.Warnings)
|
||||
if merged.Name != "Compass of the Stars" || merged.ID != identity.DeriveID(merged.Name) || len(merged.SourceRefs) != 2 || !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) {
|
||||
t.Fatalf("merged item = %#v, diagnostics = %#v", merged, result.Diagnostics)
|
||||
}
|
||||
encoded := string(client.requests[0].Inputs["candidates"].Content) + string(client.requests[0].Inputs["transcript"].Content)
|
||||
if strings.Contains(encoded, doc.ID) || strings.Contains(encoded, "candidate-") || strings.Contains(encoded, merged.ID) || !strings.Contains(encoded, `"source_refs"`) {
|
||||
@@ -150,7 +149,8 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
wantNames []string
|
||||
wantRefCounts []int
|
||||
wantRetry bool
|
||||
warning string
|
||||
reasonCode string
|
||||
disposition contracts.DiagnosticDisposition
|
||||
}{
|
||||
{
|
||||
name: "same denomination aliases",
|
||||
@@ -162,7 +162,8 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
response: `{"duplicate_groups":[{"candidate_ids":[1,2,3],"canonical_candidate_id":2}]}`,
|
||||
wantNames: []string{"Gold Piece"},
|
||||
wantRefCounts: []int{3},
|
||||
warning: ReasonCodeDuplicateItemCollapsed,
|
||||
reasonCode: ReasonCodeDuplicateItemCollapsed,
|
||||
disposition: contracts.DiagnosticDispositionObservation,
|
||||
},
|
||||
{
|
||||
name: "different denominations",
|
||||
@@ -174,7 +175,8 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
wantNames: []string{"Gold Pieces", "Silver Pieces"},
|
||||
wantRefCounts: []int{1, 1},
|
||||
wantRetry: true,
|
||||
warning: ReasonCodeItemSemanticProposalInvalid,
|
||||
reasonCode: ReasonCodeItemSemanticProposalInvalid,
|
||||
disposition: contracts.DiagnosticDispositionAdvisory,
|
||||
},
|
||||
{
|
||||
name: "currency plus ordinary item",
|
||||
@@ -186,7 +188,8 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
wantNames: []string{"Gold Pieces", "Longsword"},
|
||||
wantRefCounts: []int{1, 1},
|
||||
wantRetry: true,
|
||||
warning: ReasonCodeItemSemanticProposalInvalid,
|
||||
reasonCode: ReasonCodeItemSemanticProposalInvalid,
|
||||
disposition: contracts.DiagnosticDispositionAdvisory,
|
||||
},
|
||||
{
|
||||
name: "ordinary items",
|
||||
@@ -197,7 +200,8 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2}]}`,
|
||||
wantNames: []string{"Compass of the Stars"},
|
||||
wantRefCounts: []int{2},
|
||||
warning: ReasonCodeDuplicateItemCollapsed,
|
||||
reasonCode: ReasonCodeDuplicateItemCollapsed,
|
||||
disposition: contracts.DiagnosticDispositionObservation,
|
||||
},
|
||||
{
|
||||
name: "currency plus ordinary canonical item",
|
||||
@@ -209,7 +213,8 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
wantNames: []string{"Gold Pieces", "Longsword"},
|
||||
wantRefCounts: []int{1, 1},
|
||||
wantRetry: true,
|
||||
warning: ReasonCodeItemSemanticProposalInvalid,
|
||||
reasonCode: ReasonCodeItemSemanticProposalInvalid,
|
||||
disposition: contracts.DiagnosticDispositionAdvisory,
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
@@ -223,7 +228,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
}
|
||||
|
||||
result, err := newNormalizer(t, &recordingNormalizerClient{response: test.response}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || (result.Retry != nil) != test.wantRetry || !reflect.DeepEqual(input, before) || !hasWarning(result.Warnings, test.warning) {
|
||||
if err != nil || (result.Retry != nil) != test.wantRetry || !reflect.DeepEqual(input, before) || !hasDiagnostic(result.Diagnostics, test.reasonCode, test.disposition) {
|
||||
t.Fatalf("Normalize() = %#v, %v", result, err)
|
||||
}
|
||||
if len(result.Value.Items) != len(test.wantNames) {
|
||||
@@ -234,7 +239,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
t.Fatalf("item %d = %#v, want name %q with %d source refs", index, item, test.wantNames[index], test.wantRefCounts[index])
|
||||
}
|
||||
}
|
||||
if test.wantRetry && (len(result.Retry.FallbackWarnings) != 1 || result.Retry.FallbackWarnings[0].ReasonCode != ReasonCodeItemSemanticReconciliationExhausted) {
|
||||
if test.wantRetry && (len(result.Retry.FallbackDiagnostics) != 1 || result.Retry.FallbackDiagnostics[0].ReasonCode != ReasonCodeItemSemanticReconciliationExhausted || result.Retry.ReasonCode != ReasonCodeItemSemanticRetryProposalInvalid) {
|
||||
t.Fatalf("retry = %#v, want preserved-group fallback", result.Retry)
|
||||
}
|
||||
})
|
||||
@@ -281,10 +286,10 @@ func TestNormalizeAppliesIndependentGroupAndCountsAllOmissions(t *testing.T) {
|
||||
t.Fatalf("item %d = %#v, want %q", index, result.Value.Items[index], name)
|
||||
}
|
||||
}
|
||||
if !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) || !hasWarning(result.Warnings, ReasonCodeItemSemanticProposalInvalid) {
|
||||
t.Fatalf("warnings = %#v, want accepted and guarded-group diagnostics", result.Warnings)
|
||||
if !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) || !hasDiagnostic(result.Diagnostics, ReasonCodeItemSemanticProposalInvalid, contracts.DiagnosticDispositionAdvisory) {
|
||||
t.Fatalf("diagnostics = %#v, want accepted and guarded-group diagnostics", result.Diagnostics)
|
||||
}
|
||||
if len(result.Retry.FallbackWarnings) != 1 || !strings.Contains(result.Retry.FallbackWarnings[0].Message, "2 proposal group(s)") {
|
||||
if len(result.Retry.FallbackDiagnostics) != 1 || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "2 proposal group(s)") {
|
||||
t.Fatalf("retry = %#v, want one guarded and one malformed group counted", result.Retry)
|
||||
}
|
||||
}
|
||||
@@ -304,8 +309,8 @@ func TestNormalizeLimitSkipDoesNotCallLLMAndAddsBoundedFallbackWarning(t *testin
|
||||
if len(client.requests) != 0 || len(result.Value.Items) != limit+1 {
|
||||
t.Fatalf("completion calls = %d, items = %d; want no call and all records", len(client.requests), len(result.Value.Items))
|
||||
}
|
||||
if !hasWarning(result.Warnings, ReasonCodeItemSemanticReconciliationExhausted) || len(result.Warnings) > diagnostics.MaxWarnings {
|
||||
t.Fatalf("warnings = %#v, want bounded reconciliation fallback", result.Warnings)
|
||||
if !hasDiagnostic(result.Diagnostics, ReasonCodeItemSemanticReconciliationExhausted, contracts.DiagnosticDispositionWarning) {
|
||||
t.Fatalf("diagnostics = %#v, want reconciliation fallback warning", result.Diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -313,16 +318,13 @@ func TestNormalizeRetryFallbackErrorsWarningsAndIdempotence(t *testing.T) {
|
||||
doc := semanticDocument()
|
||||
input := dnd.ItemRegistry{Items: []dnd.Item{{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}}
|
||||
invalid, err := newNormalizer(t, &recordingNormalizerClient{err: contracts.ErrInvalidStructuredOutput}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || invalid.Retry == nil || invalid.Retry.ReasonCode != ReasonCodeItemSemanticProposalInvalid {
|
||||
if err != nil || invalid.Retry == nil || invalid.Retry.ReasonCode != ReasonCodeItemSemanticRetryProposalInvalid {
|
||||
t.Fatalf("invalid result = %#v, %v", invalid, err)
|
||||
}
|
||||
_, err = newNormalizer(t, &recordingNormalizerClient{err: errors.New("provider unavailable")}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err == nil || !strings.Contains(err.Error(), "provider unavailable") {
|
||||
t.Fatalf("provider error = %v", err)
|
||||
}
|
||||
if bounded := limitWarningsForRetry(make([]contracts.Warning, diagnostics.MaxWarnings+5)); len(bounded) != diagnostics.MaxWarnings-1 || bounded[len(bounded)-1].ReasonCode != ReasonCodeItemNormalizationWarningsOmitted {
|
||||
t.Fatalf("retry warning limit = %#v", bounded)
|
||||
}
|
||||
first, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
second, secondErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(first.Value, doc))
|
||||
if err != nil || secondErr != nil || !reflect.DeepEqual(first.Value, second.Value) {
|
||||
@@ -407,9 +409,9 @@ func normalizeRequestWithSource(value dnd.ItemRegistry, doc *source.SourceDocume
|
||||
func semanticDocument() *source.SourceDocument {
|
||||
return &source.SourceDocument{ID: "item-session", Units: []source.SourceUnit{{ID: 10, Text: "The Star Compass points north."}, {ID: 20, Text: "The compass of the stars glows."}, {ID: 30, Text: "The chest holds gold pieces."}}}
|
||||
}
|
||||
func hasWarning(warnings []contracts.Warning, reason string) bool {
|
||||
for _, warning := range warnings {
|
||||
if warning.ReasonCode == reason {
|
||||
func hasDiagnostic(diagnostics []contracts.ProducerDiagnostic, reason string, disposition contracts.DiagnosticDisposition) bool {
|
||||
for _, diagnostic := range diagnostics {
|
||||
if diagnostic.ReasonCode == reason && diagnostic.Disposition == disposition {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ func reconciliationInputs(records []normalizedRecord) ([]semanticreconcile.Candi
|
||||
return candidates, envelopes, nil
|
||||
}
|
||||
|
||||
func applyReconciliationPlan(plan semanticreconcile.Plan, records []normalizedRecord, envelopes []semanticreconcile.Record[dnd.Item], order shared.SourceRefOrder) ([]normalizedRecord, []contracts.Warning, int, error) {
|
||||
func applyReconciliationPlan(plan semanticreconcile.Plan, records []normalizedRecord, envelopes []semanticreconcile.Record[dnd.Item], order shared.SourceRefOrder) ([]normalizedRecord, []contracts.Warning, []contracts.Warning, int, error) {
|
||||
application, err := semanticreconcile.ApplyPlan(plan, envelopes, semanticreconcile.ApplicationPolicy[dnd.Item]{
|
||||
CloneValue: cloneItem,
|
||||
RejectGroup: func(members []dnd.Item, _ dnd.Item) semanticreconcile.RejectionCategory {
|
||||
@@ -52,7 +52,7 @@ func applyReconciliationPlan(plan semanticreconcile.Plan, records []normalizedRe
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, 0, err
|
||||
return nil, nil, nil, 0, err
|
||||
}
|
||||
|
||||
applied := application.Records()
|
||||
@@ -64,35 +64,41 @@ func applyReconciliationPlan(plan semanticreconcile.Plan, records []normalizedRe
|
||||
earliest: record.EarliestInputPosition(),
|
||||
}
|
||||
}
|
||||
type orderedWarning struct {
|
||||
type orderedFinding struct {
|
||||
position int
|
||||
warning contracts.Warning
|
||||
finding contracts.Warning
|
||||
}
|
||||
orderedWarnings := make([]orderedWarning, 0, len(application.AppliedGroups())+len(application.RejectedGroups()))
|
||||
observations := make([]orderedFinding, 0, len(application.AppliedGroups()))
|
||||
advisories := make([]orderedFinding, 0, len(application.RejectedGroups()))
|
||||
for _, event := range application.AppliedGroups() {
|
||||
provenance := event.Provenance()
|
||||
orderedWarnings = append(orderedWarnings, orderedWarning{
|
||||
observations = append(observations, orderedFinding{
|
||||
position: provenance.EarliestInputPosition(),
|
||||
warning: semanticDuplicateWarning(provenance, records[provenance.CanonicalPosition()]),
|
||||
finding: semanticDuplicateFinding(provenance, records[provenance.CanonicalPosition()]),
|
||||
})
|
||||
}
|
||||
for _, event := range application.RejectedGroups() {
|
||||
provenance := event.Provenance()
|
||||
orderedWarnings = append(orderedWarnings, orderedWarning{
|
||||
advisories = append(advisories, orderedFinding{
|
||||
position: provenance.EarliestInputPosition(),
|
||||
warning: contracts.Warning{
|
||||
finding: contracts.Warning{
|
||||
Scope: itemScope(provenance.EarliestInputPosition()),
|
||||
ReasonCode: ReasonCodeItemSemanticProposalInvalid,
|
||||
Message: "proposal group preserved because currency may only be consolidated with aliases of one denomination",
|
||||
},
|
||||
})
|
||||
}
|
||||
sort.SliceStable(orderedWarnings, func(left, right int) bool { return orderedWarnings[left].position < orderedWarnings[right].position })
|
||||
warnings := make([]contracts.Warning, len(orderedWarnings))
|
||||
for index, entry := range orderedWarnings {
|
||||
warnings[index] = entry.warning
|
||||
sort.SliceStable(observations, func(left, right int) bool { return observations[left].position < observations[right].position })
|
||||
sort.SliceStable(advisories, func(left, right int) bool { return advisories[left].position < advisories[right].position })
|
||||
observationFindings := make([]contracts.Warning, len(observations))
|
||||
for index, entry := range observations {
|
||||
observationFindings[index] = entry.finding
|
||||
}
|
||||
return output, warnings, len(application.RejectedGroups()), nil
|
||||
advisoryFindings := make([]contracts.Warning, len(advisories))
|
||||
for index, entry := range advisories {
|
||||
advisoryFindings[index] = entry.finding
|
||||
}
|
||||
return output, observationFindings, advisoryFindings, len(application.RejectedGroups()), nil
|
||||
}
|
||||
|
||||
func canConsolidate(items []dnd.Item) bool {
|
||||
@@ -133,7 +139,7 @@ func currencyDenomination(name string) string {
|
||||
}
|
||||
}
|
||||
|
||||
func semanticDuplicateWarning(provenance semanticreconcile.GroupProvenance, canonical normalizedRecord) contracts.Warning {
|
||||
func semanticDuplicateFinding(provenance semanticreconcile.GroupProvenance, canonical normalizedRecord) contracts.Warning {
|
||||
inputIndexes := provenance.OriginalInputIndexes()
|
||||
details := make([]string, 0, len(inputIndexes)+1)
|
||||
for _, inputIndex := range inputIndexes {
|
||||
|
||||
Reference in New Issue
Block a user