Transport structured diagnostics through the pipeline
This commit is contained in:
132
internal/framework/diagnostics/aggregator.go
Normal file
132
internal/framework/diagnostics/aggregator.go
Normal file
@@ -0,0 +1,132 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxWarningGroups = 128
|
||||
MaxNonWarningGroups = 256
|
||||
)
|
||||
|
||||
// Aggregator merges origin-enriched diagnostics in caller-supplied canonical
|
||||
// order. Its zero value is ready for use.
|
||||
type Aggregator struct {
|
||||
groups []contracts.DiagnosticGroup
|
||||
indices map[groupKey]int
|
||||
omitted map[groupKey]struct{}
|
||||
warningGroups int
|
||||
nonWarningGroups int
|
||||
unrepresentedOccurrences int
|
||||
}
|
||||
|
||||
// Add validates and incorporates one final diagnostic group. Actionable
|
||||
// warnings cannot overflow; later non-warning groups are represented by exact
|
||||
// unrepresented-occurrence metadata once their fixed bound is reached.
|
||||
func (aggregator *Aggregator) Add(group contracts.DiagnosticGroup) error {
|
||||
if err := group.Validate(); err != nil {
|
||||
return fmt.Errorf("diagnostic group: %w", err)
|
||||
}
|
||||
if aggregator.indices == nil {
|
||||
aggregator.indices = make(map[groupKey]int)
|
||||
aggregator.omitted = make(map[groupKey]struct{})
|
||||
}
|
||||
key := groupKeyFromGroup(group)
|
||||
if index, exists := aggregator.indices[key]; exists {
|
||||
return aggregator.merge(index, group)
|
||||
}
|
||||
if _, exists := aggregator.omitted[key]; exists {
|
||||
return aggregator.addUnrepresented(group.OccurrenceCount)
|
||||
}
|
||||
if group.Disposition == contracts.DiagnosticDispositionWarning {
|
||||
if aggregator.warningGroups >= MaxWarningGroups {
|
||||
return errors.New("diagnostic warning groups exceed maximum count")
|
||||
}
|
||||
aggregator.warningGroups++
|
||||
} else if aggregator.nonWarningGroups >= MaxNonWarningGroups {
|
||||
aggregator.omitted[key] = struct{}{}
|
||||
return aggregator.addUnrepresented(group.OccurrenceCount)
|
||||
} else {
|
||||
aggregator.nonWarningGroups++
|
||||
}
|
||||
aggregator.indices[key] = len(aggregator.groups)
|
||||
aggregator.groups = append(aggregator.groups, contracts.CloneDiagnosticCollection(contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{group}}).Groups[0])
|
||||
return nil
|
||||
}
|
||||
|
||||
// Collection returns an independently owned grouped result in first-occurrence
|
||||
// order.
|
||||
func (aggregator *Aggregator) Collection() contracts.DiagnosticCollection {
|
||||
if aggregator == nil {
|
||||
return contracts.DiagnosticCollection{}
|
||||
}
|
||||
return contracts.CloneDiagnosticCollection(contracts.DiagnosticCollection{
|
||||
Groups: aggregator.groups,
|
||||
Truncated: aggregator.unrepresentedOccurrences > 0,
|
||||
UnrepresentedOccurrenceCount: aggregator.unrepresentedOccurrences,
|
||||
})
|
||||
}
|
||||
|
||||
func (aggregator *Aggregator) merge(index int, incoming contracts.DiagnosticGroup) error {
|
||||
current := &aggregator.groups[index]
|
||||
if incoming.OccurrenceCount > maximumInt()-current.OccurrenceCount {
|
||||
return errors.New("diagnostic occurrence count overflow")
|
||||
}
|
||||
current.OccurrenceCount += incoming.OccurrenceCount
|
||||
for _, sample := range incoming.Samples {
|
||||
if len(current.Samples) == contracts.MaxDiagnosticSamples || containsGroupSample(current.Samples, sample) {
|
||||
continue
|
||||
}
|
||||
current.Samples = append(current.Samples, cloneGroupSample(sample))
|
||||
}
|
||||
current.OmittedSampleCount = current.OccurrenceCount - len(current.Samples)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (aggregator *Aggregator) addUnrepresented(count int) error {
|
||||
if count > maximumInt()-aggregator.unrepresentedOccurrences {
|
||||
return errors.New("diagnostic unrepresented occurrence count overflow")
|
||||
}
|
||||
aggregator.unrepresentedOccurrences += count
|
||||
return nil
|
||||
}
|
||||
|
||||
func containsGroupSample(samples []contracts.DiagnosticSample, candidate contracts.DiagnosticSample) bool {
|
||||
for _, sample := range samples {
|
||||
if sample.Scope != candidate.Scope || sample.Message != candidate.Message || sample.ChunkID != candidate.ChunkID {
|
||||
continue
|
||||
}
|
||||
if sample.ChunkIndex == nil || candidate.ChunkIndex == nil {
|
||||
if sample.ChunkIndex == candidate.ChunkIndex {
|
||||
return true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if *sample.ChunkIndex == *candidate.ChunkIndex {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func cloneGroupSample(sample contracts.DiagnosticSample) contracts.DiagnosticSample {
|
||||
if sample.ChunkIndex != nil {
|
||||
value := *sample.ChunkIndex
|
||||
sample.ChunkIndex = &value
|
||||
}
|
||||
return sample
|
||||
}
|
||||
|
||||
type groupKey struct {
|
||||
disposition contracts.DiagnosticDisposition
|
||||
category contracts.DiagnosticCategory
|
||||
reasonCode string
|
||||
origin contracts.DiagnosticOrigin
|
||||
}
|
||||
|
||||
func groupKeyFromGroup(group contracts.DiagnosticGroup) groupKey {
|
||||
return groupKey{disposition: group.Disposition, category: group.Category, reasonCode: group.ReasonCode, origin: group.Origin}
|
||||
}
|
||||
71
internal/framework/diagnostics/aggregator_test.go
Normal file
71
internal/framework/diagnostics/aggregator_test.go
Normal file
@@ -0,0 +1,71 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestAggregatorMergesByOriginAndPreservesFirstOccurrenceOrder(t *testing.T) {
|
||||
aggregator := Aggregator{}
|
||||
for _, group := range []contracts.DiagnosticGroup{
|
||||
groupForAggregation("first", "message one", contracts.DiagnosticDispositionAdvisory, contracts.DiagnosticCategoryDataQuality, "dnd/spells"),
|
||||
groupForAggregation("second", "message two", contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback, "dnd/items"),
|
||||
groupForAggregation("third", "message three", contracts.DiagnosticDispositionAdvisory, contracts.DiagnosticCategoryDataQuality, "dnd/spells"),
|
||||
} {
|
||||
if err := aggregator.Add(group); err != nil {
|
||||
t.Fatalf("Add() error = %v", err)
|
||||
}
|
||||
}
|
||||
collection := aggregator.Collection()
|
||||
if len(collection.Groups) != 2 {
|
||||
t.Fatalf("group count = %d, want 2", len(collection.Groups))
|
||||
}
|
||||
if got := []string{collection.Groups[0].Origin.ModuleKey, collection.Groups[1].Origin.ModuleKey}; !equalStrings(got, []string{"dnd/spells", "dnd/items"}) {
|
||||
t.Fatalf("group order = %#v, want first occurrence order", got)
|
||||
}
|
||||
if collection.Groups[0].OccurrenceCount != 2 || collection.Groups[0].OmittedSampleCount != 0 {
|
||||
t.Fatalf("merged group = %#v, want two represented occurrences", collection.Groups[0])
|
||||
}
|
||||
if got := []string{collection.Groups[0].Samples[0].Scope, collection.Groups[0].Samples[1].Scope}; !equalStrings(got, []string{"first", "third"}) {
|
||||
t.Fatalf("merged samples = %#v, want first occurrence order", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregatorEnforcesWarningBoundAndTruncatesOnlyNonWarnings(t *testing.T) {
|
||||
warnings := Aggregator{}
|
||||
for index := 0; index < MaxWarningGroups; index++ {
|
||||
group := groupForAggregation("scope", "message", contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback, "module")
|
||||
group.ReasonCode = "warning-" + string(rune('a'+index))
|
||||
if err := warnings.Add(group); err != nil {
|
||||
t.Fatalf("warning Add(%d) error = %v", index, err)
|
||||
}
|
||||
}
|
||||
if err := warnings.Add(groupForAggregation("overflow", "overflow", contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback, "overflow")); err == nil {
|
||||
t.Fatal("warning overflow error = nil, want error")
|
||||
}
|
||||
|
||||
nonWarnings := Aggregator{}
|
||||
for index := 0; index < MaxNonWarningGroups+1; index++ {
|
||||
group := groupForAggregation("scope", "message", contracts.DiagnosticDispositionAdvisory, contracts.DiagnosticCategoryDataQuality, "module")
|
||||
group.ReasonCode = "advisory-" + string(rune('a'+index))
|
||||
if err := nonWarnings.Add(group); err != nil {
|
||||
t.Fatalf("non-warning Add(%d) error = %v", index, err)
|
||||
}
|
||||
}
|
||||
collection := nonWarnings.Collection()
|
||||
if len(collection.Groups) != MaxNonWarningGroups || !collection.Truncated || collection.UnrepresentedOccurrenceCount != 1 {
|
||||
t.Fatalf("collection = %#v, want bounded non-warning groups and one unrepresented occurrence", collection)
|
||||
}
|
||||
}
|
||||
|
||||
func groupForAggregation(scope string, message string, disposition contracts.DiagnosticDisposition, category contracts.DiagnosticCategory, module string) contracts.DiagnosticGroup {
|
||||
return contracts.DiagnosticGroup{
|
||||
Disposition: disposition,
|
||||
Category: category,
|
||||
ReasonCode: "source_unrelated",
|
||||
Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: "extract", LaneID: "spells", ModuleKey: module},
|
||||
OccurrenceCount: 1,
|
||||
Samples: []contracts.DiagnosticSample{{Scope: scope, Message: message}},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user