eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Classify registry normalization diagnostics

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-27 15:57:29 +00:00
parent ccba2ce3f9
commit 6a1fd7bdb6
12 changed files with 247 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
internal/modules/dnd/normalize/itemregistry/normalizer_test.go
View File

@@ -18,7 +18,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
identityvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/identity"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -81,8 +80,8 @@ func TestNormalizeConsolidatesEqualNamesAcrossEvidenceWithoutMutation(t *testing
}
rope := result.Value.Items[0]
wantRefs := []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}, {SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}
if rope.Name != "Rope" || rope.ID != identity.DeriveID("Rope") || !reflect.DeepEqual(rope.SourceRefs, wantRefs) || result.Value.Items[1].Name != "Lantern" || !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) {
t.Fatalf("items = %#v, warnings = %#v; want earliest display name, canonical evidence union, and stable placement", result.Value.Items, result.Warnings)
if rope.Name != "Rope" || rope.ID != identity.DeriveID("Rope") || !reflect.DeepEqual(rope.SourceRefs, wantRefs) || result.Value.Items[1].Name != "Lantern" || !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) {
t.Fatalf("items = %#v, diagnostics = %#v; want earliest display name, canonical evidence union, and stable placement", result.Value.Items, result.Diagnostics)
}
validation, validationErr := identityvalidator.New(identityvalidator.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Value: result.Value})
if validationErr != nil || !validation.Approved {
@@ -129,8 +128,8 @@ func TestNormalizeAppliesSafeAliasProposal(t *testing.T) {
t.Fatalf("Normalize() = %#v, %v", result, err)
}
merged := result.Value.Items[0]
if merged.Name != "Compass of the Stars" || merged.ID != identity.DeriveID(merged.Name) || len(merged.SourceRefs) != 2 || !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) {
t.Fatalf("merged item = %#v, warnings = %#v", merged, result.Warnings)
if merged.Name != "Compass of the Stars" || merged.ID != identity.DeriveID(merged.Name) || len(merged.SourceRefs) != 2 || !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) {
t.Fatalf("merged item = %#v, diagnostics = %#v", merged, result.Diagnostics)
}
encoded := string(client.requests[0].Inputs["candidates"].Content) + string(client.requests[0].Inputs["transcript"].Content)
if strings.Contains(encoded, doc.ID) || strings.Contains(encoded, "candidate-") || strings.Contains(encoded, merged.ID) || !strings.Contains(encoded, `"source_refs"`) {
@@ -150,7 +149,8 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
wantNames []string
wantRefCounts []int
wantRetry bool
warning string
reasonCode string
disposition contracts.DiagnosticDisposition
}{
{
name: "same denomination aliases",
@@ -162,7 +162,8 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
response: `{"duplicate_groups":[{"candidate_ids":[1,2,3],"canonical_candidate_id":2}]}`,
wantNames: []string{"Gold Piece"},
wantRefCounts: []int{3},
warning: ReasonCodeDuplicateItemCollapsed,
reasonCode: ReasonCodeDuplicateItemCollapsed,
disposition: contracts.DiagnosticDispositionObservation,
},
{
name: "different denominations",
@@ -174,7 +175,8 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
wantNames: []string{"Gold Pieces", "Silver Pieces"},
wantRefCounts: []int{1, 1},
wantRetry: true,
warning: ReasonCodeItemSemanticProposalInvalid,
reasonCode: ReasonCodeItemSemanticProposalInvalid,
disposition: contracts.DiagnosticDispositionAdvisory,
},
{
name: "currency plus ordinary item",
@@ -186,7 +188,8 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
wantNames: []string{"Gold Pieces", "Longsword"},
wantRefCounts: []int{1, 1},
wantRetry: true,
warning: ReasonCodeItemSemanticProposalInvalid,
reasonCode: ReasonCodeItemSemanticProposalInvalid,
disposition: contracts.DiagnosticDispositionAdvisory,
},
{
name: "ordinary items",
@@ -197,7 +200,8 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2}]}`,
wantNames: []string{"Compass of the Stars"},
wantRefCounts: []int{2},
warning: ReasonCodeDuplicateItemCollapsed,
reasonCode: ReasonCodeDuplicateItemCollapsed,
disposition: contracts.DiagnosticDispositionObservation,
},
{
name: "currency plus ordinary canonical item",
@@ -209,7 +213,8 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
wantNames: []string{"Gold Pieces", "Longsword"},
wantRefCounts: []int{1, 1},
wantRetry: true,
warning: ReasonCodeItemSemanticProposalInvalid,
reasonCode: ReasonCodeItemSemanticProposalInvalid,
disposition: contracts.DiagnosticDispositionAdvisory,
},
} {
t.Run(test.name, func(t *testing.T) {
@@ -223,7 +228,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
}
result, err := newNormalizer(t, &recordingNormalizerClient{response: test.response}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
if err != nil || (result.Retry != nil) != test.wantRetry || !reflect.DeepEqual(input, before) || !hasWarning(result.Warnings, test.warning) {
if err != nil || (result.Retry != nil) != test.wantRetry || !reflect.DeepEqual(input, before) || !hasDiagnostic(result.Diagnostics, test.reasonCode, test.disposition) {
t.Fatalf("Normalize() = %#v, %v", result, err)
}
if len(result.Value.Items) != len(test.wantNames) {
@@ -234,7 +239,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
t.Fatalf("item %d = %#v, want name %q with %d source refs", index, item, test.wantNames[index], test.wantRefCounts[index])
}
}
if test.wantRetry && (len(result.Retry.FallbackWarnings) != 1 || result.Retry.FallbackWarnings[0].ReasonCode != ReasonCodeItemSemanticReconciliationExhausted) {
if test.wantRetry && (len(result.Retry.FallbackDiagnostics) != 1 || result.Retry.FallbackDiagnostics[0].ReasonCode != ReasonCodeItemSemanticReconciliationExhausted || result.Retry.ReasonCode != ReasonCodeItemSemanticRetryProposalInvalid) {
t.Fatalf("retry = %#v, want preserved-group fallback", result.Retry)
}
})
@@ -281,10 +286,10 @@ func TestNormalizeAppliesIndependentGroupAndCountsAllOmissions(t *testing.T) {
t.Fatalf("item %d = %#v, want %q", index, result.Value.Items[index], name)
}
}
if !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) || !hasWarning(result.Warnings, ReasonCodeItemSemanticProposalInvalid) {
t.Fatalf("warnings = %#v, want accepted and guarded-group diagnostics", result.Warnings)
if !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) || !hasDiagnostic(result.Diagnostics, ReasonCodeItemSemanticProposalInvalid, contracts.DiagnosticDispositionAdvisory) {
t.Fatalf("diagnostics = %#v, want accepted and guarded-group diagnostics", result.Diagnostics)
}
if len(result.Retry.FallbackWarnings) != 1 || !strings.Contains(result.Retry.FallbackWarnings[0].Message, "2 proposal group(s)") {
if len(result.Retry.FallbackDiagnostics) != 1 || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "2 proposal group(s)") {
t.Fatalf("retry = %#v, want one guarded and one malformed group counted", result.Retry)
}
}
@@ -304,8 +309,8 @@ func TestNormalizeLimitSkipDoesNotCallLLMAndAddsBoundedFallbackWarning(t *testin
if len(client.requests) != 0 || len(result.Value.Items) != limit+1 {
t.Fatalf("completion calls = %d, items = %d; want no call and all records", len(client.requests), len(result.Value.Items))
}
if !hasWarning(result.Warnings, ReasonCodeItemSemanticReconciliationExhausted) || len(result.Warnings) > diagnostics.MaxWarnings {
t.Fatalf("warnings = %#v, want bounded reconciliation fallback", result.Warnings)
if !hasDiagnostic(result.Diagnostics, ReasonCodeItemSemanticReconciliationExhausted, contracts.DiagnosticDispositionWarning) {
t.Fatalf("diagnostics = %#v, want reconciliation fallback warning", result.Diagnostics)
}
}
@@ -313,16 +318,13 @@ func TestNormalizeRetryFallbackErrorsWarningsAndIdempotence(t *testing.T) {
doc := semanticDocument()
input := dnd.ItemRegistry{Items: []dnd.Item{{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}}
invalid, err := newNormalizer(t, &recordingNormalizerClient{err: contracts.ErrInvalidStructuredOutput}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
if err != nil || invalid.Retry == nil || invalid.Retry.ReasonCode != ReasonCodeItemSemanticProposalInvalid {
if err != nil || invalid.Retry == nil || invalid.Retry.ReasonCode != ReasonCodeItemSemanticRetryProposalInvalid {
t.Fatalf("invalid result = %#v, %v", invalid, err)
}
_, err = newNormalizer(t, &recordingNormalizerClient{err: errors.New("provider unavailable")}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
if err == nil || !strings.Contains(err.Error(), "provider unavailable") {
t.Fatalf("provider error = %v", err)
}
if bounded := limitWarningsForRetry(make([]contracts.Warning, diagnostics.MaxWarnings+5)); len(bounded) != diagnostics.MaxWarnings-1 || bounded[len(bounded)-1].ReasonCode != ReasonCodeItemNormalizationWarningsOmitted {
t.Fatalf("retry warning limit = %#v", bounded)
}
first, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
second, secondErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(first.Value, doc))
if err != nil || secondErr != nil || !reflect.DeepEqual(first.Value, second.Value) {
@@ -407,9 +409,9 @@ func normalizeRequestWithSource(value dnd.ItemRegistry, doc *source.SourceDocume
func semanticDocument() *source.SourceDocument {
return &source.SourceDocument{ID: "item-session", Units: []source.SourceUnit{{ID: 10, Text: "The Star Compass points north."}, {ID: 20, Text: "The compass of the stars glows."}, {ID: 30, Text: "The chest holds gold pieces."}}}
}
func hasWarning(warnings []contracts.Warning, reason string) bool {
for _, warning := range warnings {
if warning.ReasonCode == reason {
func hasDiagnostic(diagnostics []contracts.ProducerDiagnostic, reason string, disposition contracts.DiagnosticDisposition) bool {
for _, diagnostic := range diagnostics {
if diagnostic.ReasonCode == reason && diagnostic.Disposition == disposition {
return true
}
}