eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finalize structured diagnostic aggregation

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-27 16:40:17 +00:00
parent 480680b257
commit 4dbbf68051
112 changed files with 656 additions and 893 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
internal/framework/pipeline/runner_chunk_plan_test.go
View File

@@ -118,9 +118,9 @@ func (*retryingChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil
func (c *retryingChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
c.calls++
if c.calls == 1 {
return contracts.ChunkPlanResult{Warnings: []contracts.Warning{{Scope: "discarded", ReasonCode: "retry", Message: "discarded warning"}}}, errors.New("retry generation")
return contracts.ChunkPlanResult{Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("discarded", "discarded diagnostic")}}, errors.New("retry generation")
}
return contracts.ChunkPlanResult{Plan: source.CloneChunkPlan(c.plan), Warnings: []contracts.Warning{{Scope: "accepted", ReasonCode: "observed", Message: "accepted warning"}}}, nil
return contracts.ChunkPlanResult{Plan: source.CloneChunkPlan(c.plan), Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("accepted", "accepted diagnostic")}}, nil
}
type dependencyLoader struct {
@@ -399,7 +399,7 @@ func TestRunnerRegeneratesValidationIncompleteChunkPlanHit(t *testing.T) {
}},
}}
record := chunkPlanRecord(t, prepared, plan)
record.Warnings = []contracts.Warning{{Scope: "stored", ReasonCode: "old", Message: "discarded stored warning"}}
record.Diagnostics = []contracts.ProducerDiagnostic{producerDiagnostic("stored", "discarded stored diagnostic")}
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
debug := newCapturedDebugRecorder()
@@ -413,8 +413,8 @@ func TestRunnerRegeneratesValidationIncompleteChunkPlanHit(t *testing.T) {
if output.ChunkPlan == nil || output.ChunkPlan.Action != "generated" || output.ChunkPlan.LookupStatus != "invalid" || output.ChunkPlan.ValidationStatus != "approved" {
t.Fatalf("chunk plan summary = %#v", output.ChunkPlan)
}
if len(output.Warnings) != 0 {
t.Fatalf("warnings = %#v, want discarded cache warnings omitted", output.Warnings)
if len(output.Diagnostics.Groups) != 0 {
t.Fatalf("diagnostics = %#v, want discarded cache diagnostics omitted", output.Diagnostics)
}
assertAttemptEnvelopeSequence(t, debug, "chunk", 1)
}
@@ -427,7 +427,7 @@ func TestRunnerKeepsStoredPlanWhenCacheAndGeneratedValidationAreIncomplete(t *te
validator := &countingChunkValidator{err: errors.New("validator unavailable")}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
record := chunkPlanRecord(t, prepared, plan)
record.Warnings = []contracts.Warning{{Scope: "stored", ReasonCode: "old", Message: "discarded stored warning"}}
record.Diagnostics = []contracts.ProducerDiagnostic{producerDiagnostic("stored", "discarded stored diagnostic")}
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
@@ -440,8 +440,8 @@ func TestRunnerKeepsStoredPlanWhenCacheAndGeneratedValidationAreIncomplete(t *te
if output.ChunkPlan == nil || output.ChunkPlan.Action != "generated" || output.ChunkPlan.LookupStatus != "invalid" || output.ChunkPlan.ValidationStatus != "incomplete" {
t.Fatalf("chunk plan summary = %#v", output.ChunkPlan)
}
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != "validator_execution_incomplete" {
t.Fatalf("warnings = %#v, want only generated incomplete warning", output.Warnings)
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "validator_execution_incomplete" {
t.Fatalf("diagnostics = %#v, want only generated incomplete diagnostic", output.Diagnostics)
}
if !reflect.DeepEqual(store.record, record) {
t.Fatal("discarded incomplete candidates mutated the stored record")
@@ -611,8 +611,8 @@ func TestRunnerRetriesBeforePublishingAcceptedPlan(t *testing.T) {
if chunker.calls != 2 || store.saves != 1 {
t.Fatalf("module calls = %d saves = %d, want 2 and 1", chunker.calls, store.saves)
}
if len(output.Warnings) != 1 || output.Warnings[0].Scope != "accepted" || len(store.saved.Warnings) != 1 || store.saved.Warnings[0].Scope != "accepted" {
t.Fatalf("output warnings = %#v stored warnings = %#v", output.Warnings, store.saved.Warnings)
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].Samples[0].Scope != "accepted" || len(store.saved.Diagnostics) != 1 || store.saved.Diagnostics[0].Samples[0].Scope != "accepted" {
t.Fatalf("output diagnostics = %#v stored diagnostics = %#v", output.Diagnostics, store.saved.Diagnostics)
}
}
@@ -624,7 +624,7 @@ func TestRunnerAutoHitValidatesOnceWithoutRegenerationOrMutation(t *testing.T) {
wantError string
wantReject bool
}{
{name: "warning", result: contracts.ValidationResult{Approved: true, Warnings: []contracts.Warning{{Scope: "current", ReasonCode: "observed", Message: "current warning"}}}},
{name: "diagnostic", result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("current", "current diagnostic")}}},
{name: "rejection", result: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected hit", CorrectionGuidance: "return an acceptable chunk plan"}, wantReject: true},
{name: "error", validatorErr: errors.New("validator failed"), wantError: "validator failed"},
}
@@ -637,7 +637,7 @@ func TestRunnerAutoHitValidatesOnceWithoutRegenerationOrMutation(t *testing.T) {
validator := &countingChunkValidator{result: tc.result, err: tc.validatorErr}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
record := chunkPlanRecord(t, prepared, plan)
record.Warnings = []contracts.Warning{{Scope: "stored", ReasonCode: "observed", Message: "stored warning"}}
record.Diagnostics = []contracts.ProducerDiagnostic{producerDiagnostic("stored", "stored diagnostic")}
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
debug := newCapturedDebugRecorder()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store, Debug: debug})
@@ -673,8 +673,8 @@ func TestRunnerAutoHitValidatesOnceWithoutRegenerationOrMutation(t *testing.T) {
if tc.wantReject && (len(output.Rejected) != 1 || output.Rejected[0].ReasonCode != "rejected") {
t.Fatalf("rejected = %#v", output.Rejected)
}
if tc.wantError == "" && !tc.wantReject && len(output.Warnings) != 1+len(tc.result.Warnings) {
t.Fatalf("warnings = %#v, want stored warning once plus current warnings", output.Warnings)
if tc.wantError == "" && !tc.wantReject && len(output.Diagnostics.Groups) != 1+len(tc.result.Diagnostics) {
t.Fatalf("diagnostics = %#v, want stored diagnostic once plus current diagnostics", output.Diagnostics)
}
})
}
@@ -724,7 +724,7 @@ func TestRunnerPublishesOnlyAcceptedGeneratedPlans(t *testing.T) {
}
}
func TestRunnerStoresProducerProvenanceAndProducerWarnings(t *testing.T) {
func TestRunnerStoresProducerProvenanceAndDiagnostics(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.Chunk.LLMProfile = "chunk-profile"
prepared.resolved.ChunkExecutionClass = contracts.ExecutionClassLLMBacked
@@ -734,8 +734,8 @@ func TestRunnerStoresProducerProvenanceAndProducerWarnings(t *testing.T) {
"guide": {Items: []contracts.ReferenceItem{{SlotName: "guide", Digest: "sha256:guide", Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///guide.txt"}, Content: []byte("sensitive")}}},
}},
}
prepared.chunker = manifestChunker{terminalChunker: terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, warnings: []contracts.Warning{{Scope: "producer", ReasonCode: "observed", Message: "producer warning"}}}, metadata: map[string]any{"prompt_id": "chunk/prompt"}}
validator := &countingChunkValidator{result: contracts.ValidationResult{Approved: true, Warnings: []contracts.Warning{{Scope: "validator", ReasonCode: "observed", Message: "validator warning"}}}}
prepared.chunker = manifestChunker{terminalChunker: terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("producer", "producer diagnostic")}}, metadata: map[string]any{"prompt_id": "chunk/prompt"}}
validator := &countingChunkValidator{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("validator", "validator diagnostic")}}}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
store := &recordingChunkPlanStore{}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: store}); err != nil {
@@ -748,8 +748,8 @@ func TestRunnerStoresProducerProvenanceAndProducerWarnings(t *testing.T) {
if len(producer.References) != 1 || producer.References[0].Digest != "sha256:guide" {
t.Fatalf("producer references = %#v", producer.References)
}
if len(store.saved.Warnings) != 1 || store.saved.Warnings[0].Scope != "producer" {
t.Fatalf("stored warnings = %#v, want only producer warning", store.saved.Warnings)
if len(store.saved.Diagnostics) != 1 || store.saved.Diagnostics[0].Samples[0].Scope != "producer" {
t.Fatalf("stored diagnostics = %#v, want only producer diagnostic", store.saved.Diagnostics)
}
}