eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finalize structured diagnostic aggregation

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-27 16:40:17 +00:00
parent 480680b257
commit 4dbbf68051
112 changed files with 656 additions and 893 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
internal/cli/assembled_spell_pipeline_contract_test.go
View File

@@ -77,22 +77,22 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
spellnormalize.ReasonCodeDuplicateSpellCastCollapsed,
"spell_not_near_source",
}
gotWarningReasons := make([]string, len(output.Warnings))
for index, warning := range output.Warnings {
gotWarningReasons[index] = warning.ReasonCode
gotWarningReasons := make([]string, len(output.Diagnostics.Groups))
for index, group := range output.Diagnostics.Groups {
gotWarningReasons[index] = group.ReasonCode
}
if !reflect.DeepEqual(gotWarningReasons, wantWarningReasons) {
t.Fatalf("warnings = %#v, want deterministic normalize and validation warnings", output.Warnings)
t.Fatalf("diagnostics = %#v, want deterministic normalize and validation diagnostics", output.Diagnostics)
}
if output.Warnings[2].Scope != "spell_casts[0]" || !strings.Contains(output.Warnings[2].Message, "retained input index 0") || !strings.Contains(output.Warnings[2].Message, "removed input indices [1]") {
t.Fatalf("duplicate warning = %#v, want retained and removed merged indices", output.Warnings[2])
if output.Diagnostics.Groups[2].Samples[0].Scope != "spell_casts[0]" || !strings.Contains(output.Diagnostics.Groups[2].Samples[0].Message, "retained input index 0") || !strings.Contains(output.Diagnostics.Groups[2].Samples[0].Message, "removed input indices [1]") {
t.Fatalf("duplicate diagnostic = %#v, want retained and removed merged indices", output.Diagnostics.Groups[2])
}
warningsFile := decodeAssembledOutput[struct {
Warnings []contracts.Warning `json:"warnings"`
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, output.OutputFiles, "warnings.json")
if !reflect.DeepEqual(warningsFile.Warnings, output.Warnings) {
t.Fatalf("warnings file = %#v, run warnings = %#v, want manifest output path to preserve warnings", warningsFile.Warnings, output.Warnings)
if len(warningsFile.Groups) != 0 {
t.Fatalf("warnings file = %#v, want no process warnings for advisory-only diagnostics", warningsFile.Groups)
}
manifest := decodeAssembledOutput[artifacts.RunManifest](t, output.OutputFiles, "manifest.json")
if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].Normalizer != spellnormalize.Key {
@@ -155,9 +155,9 @@ func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
if err != nil || output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
t.Fatalf("Run() error = %v output = %#v, want approved override run", err, output)
}
for _, warning := range output.Warnings {
if warning.ReasonCode == "spell_not_near_source" {
t.Fatalf("warnings = %#v, want explicit validator override to replace default relatedness chain", output.Warnings)
for _, group := range output.Diagnostics.Groups {
if group.ReasonCode == "spell_not_near_source" {
t.Fatalf("diagnostics = %#v, want explicit validator override to replace default relatedness chain", output.Diagnostics)
}
}
}
@@ -190,8 +190,8 @@ func TestAssembledSpellPipelinePromotesTerminalUnknownSpellWarning(t *testing.T)
if !reflect.DeepEqual(rejectedFile.Rejected, output.Rejected) {
t.Fatalf("rejected file = %#v, run rejections = %#v, want durable rejection diagnostic", rejectedFile.Rejected, output.Rejected)
}
if len(output.Warnings) != 2 || output.Warnings[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Warnings[0].Scope != "spell_casts[0]" || output.Warnings[1].ReasonCode != "spell_not_near_source" {
t.Fatalf("warnings = %#v, want complete terminal normalize validation warnings", output.Warnings)
if len(output.Diagnostics.Groups) != 2 || output.Diagnostics.Groups[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Diagnostics.Groups[0].Samples[0].Scope != "spell_casts[0]" || output.Diagnostics.Groups[1].ReasonCode != "spell_not_near_source" {
t.Fatalf("diagnostics = %#v, want complete terminal normalize validation diagnostics", output.Diagnostics)
}
}
@@ -219,14 +219,14 @@ func TestAssembledSpellPipelinePromotesUnknownSpellWarningWhenOverrideAccepts(t
if len(normalized.SpellCasts) != 1 || normalized.SpellCasts[0].Spell != "Mysterious Burst" {
t.Fatalf("normalized casts = %#v, want unresolved name preserved", normalized.SpellCasts)
}
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Warnings[0].Scope != "spell_casts[0]" {
t.Fatalf("warnings = %#v, want promoted scoped unresolved-name warning", output.Warnings)
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Diagnostics.Groups[0].Samples[0].Scope != "spell_casts[0]" {
t.Fatalf("diagnostics = %#v, want promoted scoped unresolved-name diagnostic", output.Diagnostics)
}
warningsFile := decodeAssembledOutput[struct {
Warnings []contracts.Warning `json:"warnings"`
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, output.OutputFiles, "warnings.json")
if !reflect.DeepEqual(warningsFile.Warnings, output.Warnings) {
t.Fatalf("warnings file = %#v, run warnings = %#v, want durable unresolved-name warning", warningsFile.Warnings, output.Warnings)
if len(warningsFile.Groups) != 0 {
t.Fatalf("warnings file = %#v, want no process warnings for an advisory diagnostic", warningsFile.Groups)
}
}

4
internal/cli/dnd_scene_descriptions_contract_test.go
View File

@@ -91,8 +91,8 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
if !reflect.DeepEqual(durable, want) {
t.Fatalf("durable output payload = %#v, want %#v", durable, want)
}
if len(output.Warnings) != 0 {
t.Fatalf("warnings = %#v, want grounded descriptions without warnings", output.Warnings)
if len(output.Diagnostics.Groups) != 0 {
t.Fatalf("diagnostics = %#v, want grounded descriptions without diagnostics", output.Diagnostics)
}
}

6
internal/cli/example_contract_test.go
View File

@@ -239,10 +239,10 @@ func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
t.Fatalf("rejected = %#v, want empty rejection list", rejected.Rejected)
}
warnings := readProductionJSON[struct {
Warnings []json.RawMessage `json:"warnings"`
Groups []json.RawMessage `json:"groups"`
}](t, filepath.Join(runRoot, "warnings.json"))
if len(warnings.Warnings) != 0 {
t.Fatalf("warnings = %#v, want empty warning list", warnings.Warnings)
if len(warnings.Groups) != 0 {
t.Fatalf("warnings = %#v, want empty warning list", warnings.Groups)
}
}

6
internal/cli/production_contract_test.go
View File

@@ -852,10 +852,10 @@ func TestProductionSceneRunRecordsAnnotationFreeChunkPlanAndProvenance(t *testin
t.Fatalf("chunk map range annotations = %#v, want none", chunkMap.Chunks[0].Annotations)
}
warnings := readProductionJSON[struct {
Warnings []contracts.Warning `json:"warnings"`
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, filepath.Join(outputRoot, productionRunID, "warnings.json"))
if len(warnings.Warnings) != 0 {
t.Fatalf("warnings = %#v, want none", warnings.Warnings)
if len(warnings.Groups) != 0 {
t.Fatalf("warnings = %#v, want none", warnings.Groups)
}
if len(fake.requestsFor(scenes.PromptID)) != 1 || len(fake.requestsFor(spells.PromptID)) != 1 || len(fake.requestsFor(itemoccurrenceextract.PromptID)) != 1 {
t.Fatalf("fake prompt requests = %#v, want one scene, spell, and item-occurrence request", fake.requestPrompts())

10
internal/cli/run.go
View File

@@ -375,7 +375,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("resolve working directory: %w", err))
}
materialized, referenceWarnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
materialized, referenceDiagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
ConfigPath: loadedConfigPath,
WorkingDir: workingDir,
})
@@ -462,7 +462,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
StartedAt: startedAt,
LLMProfiles: llmProfiles,
Metadata: runMetadata(effective.Config.Output.Directory, debugPath),
Diagnostics: pipeline.ReferenceDiagnostics(referenceWarnings),
Diagnostics: referenceDiagnostics,
ChunkCacheMode: effective.Config.Cache.ChunkPlans.Mode,
ChunkPlans: chunkPlans,
Checkpoints: checkpointRecorder,
@@ -513,8 +513,8 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintf(stdout, "debug=%s\n", debugPath)
}
}
if len(output.Warnings) > 0 {
fmt.Fprintf(stderr, "notarius: run completed with %d warning(s)\n", len(output.Warnings))
if warningCount := warningGroupCount(output.Diagnostics); warningCount > 0 {
fmt.Fprintf(stderr, "notarius: run completed with %d warning(s)\n", warningCount)
}
return 0
}
@@ -538,7 +538,7 @@ func writePartialSummary(summary *debugbundle.SummaryWriter, output pipeline.Run
return err
}
}
if err := summary.WriteWarnings(output.Warnings); err != nil {
if err := summary.WriteWarnings(output.Diagnostics); err != nil {
return err
}
return summary.WriteCheckpointEvents(output.CheckpointEvents)

10
internal/cli/run_contract_test.go
View File

@@ -590,7 +590,7 @@ func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.includeWarnings = true
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "contract-warning", Message: "warning retained"}}
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, harness.options())
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning(s)") {
@@ -602,10 +602,10 @@ func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
t.Fatalf("durable output = %q, %v", output, err)
}
bundle := onlyChildDir(t, roots.debug)
var warnings []contracts.Warning
readStateTestSummaryJSON(t, bundle, "warnings.json", &warnings)
if len(warnings) != 1 || warnings[0].ReasonCode != "contract-warning" {
t.Fatalf("debug warnings = %#v", warnings)
var diagnostics contracts.DiagnosticCollection
readStateTestSummaryJSON(t, bundle, "warnings.json", &diagnostics)
if len(diagnostics.Groups) != 1 || diagnostics.Groups[0].ReasonCode != "contract-warning" {
t.Fatalf("debug diagnostics = %#v", diagnostics)
}
}

13
internal/cli/run_result.go
View File

@@ -8,6 +8,7 @@ import (
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -59,7 +60,7 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput,
OutputDirectory: absOutputDirectory,
NormalizedOutputCount: len(output.NormalizeOutputs),
RejectedOutputCount: len(output.Rejected),
WarningCount: len(output.Warnings),
WarningCount: warningGroupCount(output.Diagnostics),
ValidationStatus: output.Manifest.ValidationStatus,
ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries),
}
@@ -88,6 +89,16 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput,
return result, nil
}
func warningGroupCount(diagnostics contracts.DiagnosticCollection) int {
count := 0
for _, group := range diagnostics.Groups {
if group.Disposition == contracts.DiagnosticDispositionWarning {
count++
}
}
return count
}
func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary {
if len(summaries) == 0 {
return nil

2
internal/cli/run_result_command_test.go
View File

@@ -63,7 +63,7 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "contract-warning", Message: "warning retained"}}
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,

6
internal/cli/run_result_test.go
View File

@@ -182,8 +182,10 @@ func testRunOutput() pipeline.RunOutput {
Manifest: artifacts.RunManifest{RunID: "run-123", PipelineID: "sample", ValidationStatus: "rejected"},
NormalizeOutputs: []contracts.SerializedOutput{{}, {}},
Rejected: []contracts.RejectedOutput{{}},
Warnings: []contracts.Warning{{}},
OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
Diagnostics: contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{{
Disposition: contracts.DiagnosticDispositionWarning,
}}},
OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
}
}

2
internal/cli/run_terminal.go
View File

@@ -39,7 +39,7 @@ func (s *pipelineCommandState) observeOutput(output pipeline.RunOutput) {
}
s.report.OutputCount = len(output.NormalizeOutputs)
s.report.RejectedCount = len(output.Rejected)
s.report.WarningCount = len(output.Warnings)
s.report.WarningCount = warningGroupCount(output.Diagnostics)
s.report.ValidationStatus = output.Manifest.ValidationStatus
}

2
internal/cli/spell_catalog_identity_contract_test.go
View File

@@ -238,7 +238,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
Kind: dnd.SpellListKind, Schema: normalizeSchema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`),
},
}
if err := recorder.NormalizeSucceeded("spells", spellnormalize.Key, normalizeDependencies, normalizeArtifact, nil); err != nil {
if err := recorder.NormalizeSucceeded("spells", spellnormalize.Key, normalizeDependencies, normalizeArtifact); err != nil {
t.Fatal(err)
}

8
internal/cli/spell_catalog_retry_contract_test.go
View File

@@ -92,8 +92,8 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
if rejection.ReasonCode != "unknown_spell" || rejection.AttemptCount != retries+1 {
t.Fatalf("rejection = %#v, want exhausted unknown-spell rejection", rejection)
}
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != "spell_not_near_source" {
t.Fatalf("warnings = %#v, want complete terminal validation warnings", output.Warnings)
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "spell_not_near_source" {
t.Fatalf("diagnostics = %#v, want complete terminal validation diagnostic", output.Diagnostics)
}
return
}
@@ -108,8 +108,8 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != tt.wantSpell {
t.Fatalf("normalized spell list = %#v, want accepted overlay spell", value)
}
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != tt.wantWarningCode {
t.Fatalf("warnings = %#v, want the terminal extract compatibility projection", output.Warnings)
if len(output.Diagnostics.Groups) != 2 || output.Diagnostics.Groups[0].ReasonCode != tt.wantWarningCode {
t.Fatalf("diagnostics = %#v, want terminal extract and normalize diagnostics", output.Diagnostics)
}
if len(output.Diagnostics.Groups) != 2 {
t.Fatalf("diagnostics = %#v, want extract and normalize data-quality advisories", output.Diagnostics)

32
internal/cli/state_hardening_test.go
View File

@@ -445,7 +445,7 @@ func TestRunWritesTerminalArtifactsForResolutionPipelineAndOutputFailures(t *tes
func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "partial-warning", Message: "warning retained before failure"}}
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "partial-warning", "warning retained before failure")}
harness.extractErr = errors.New("synthetic partial pipeline failure")
result := runStateTest(t, roots, harness.options(), true, true, "bypass")
@@ -463,10 +463,10 @@ func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
if manifest.RunID != report.RunID || manifest.PipelineID != "sample" || manifest.ValidationStatus != "failed" {
t.Fatalf("partial manifest = %#v", manifest)
}
var warnings []contracts.Warning
readStateTestSummaryJSON(t, bundlePath, "warnings.json", &warnings)
if len(warnings) != 1 || warnings[0].ReasonCode != "partial-warning" {
t.Fatalf("partial warnings = %#v", warnings)
var diagnostics contracts.DiagnosticCollection
readStateTestSummaryJSON(t, bundlePath, "warnings.json", &diagnostics)
if len(diagnostics.Groups) != 1 || diagnostics.Groups[0].ReasonCode != "partial-warning" {
t.Fatalf("partial diagnostics = %#v", diagnostics)
}
var events []pipeline.CheckpointEvent
readStateTestSummaryJSON(t, bundlePath, "checkpoint-events.json", &events)
@@ -863,10 +863,10 @@ type stateTestHarness struct {
chunkCalls, extractCalls int
runIDCalls uint64
extractErr error
chunkWarnings []contracts.Warning
chunkDiagnostics []contracts.ProducerDiagnostic
moduleProfiles []string
sessionIDs []string
outputWarnings []contracts.Warning
outputDiagnostics contracts.DiagnosticCollection
includeWarnings bool
}
@@ -892,7 +892,7 @@ func (h *stateTestHarness) options() Options {
panic(err)
}
if err := registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) {
return stateTestOutput{harness: h, includeWarnings: h.includeWarnings}, nil
return stateTestOutput{harness: h, includeDiagnostics: h.includeWarnings}, nil
}); err != nil {
panic(err)
}
@@ -931,7 +931,7 @@ func (c stateTestChunker) Plan(_ context.Context, req contracts.ChunkRequest) (c
c.harness.mu.Lock()
c.harness.chunkCalls++
c.harness.mu.Unlock()
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}, Warnings: append([]contracts.Warning(nil), c.harness.chunkWarnings...)}, nil
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}, Diagnostics: contracts.CloneProducerDiagnostics(c.harness.chunkDiagnostics)}, nil
}
const stateTestArtifactKind contracts.ArtifactKind = "test/artifact"
@@ -999,22 +999,26 @@ func (n stateTestNormalizer) Normalize(_ context.Context, req contracts.TypedNor
}
type stateTestOutput struct {
harness *stateTestHarness
includeWarnings bool
harness *stateTestHarness
includeDiagnostics bool
}
func (o stateTestOutput) Key() string { return "test/output" }
func (o stateTestOutput) Encode(_ context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
o.harness.mu.Lock()
o.harness.outputWarnings = append([]contracts.Warning(nil), req.Warnings...)
o.harness.outputDiagnostics = contracts.CloneDiagnosticCollection(req.Diagnostics)
o.harness.mu.Unlock()
data := []byte("{\"ok\":true}\n")
if o.includeWarnings && len(req.Warnings) > 0 {
data = []byte(fmt.Sprintf("{\"ok\":true,\"warnings\":%q}\n", req.Warnings[0].ReasonCode))
if o.includeDiagnostics && len(req.Diagnostics.Groups) > 0 {
data = []byte(fmt.Sprintf("{\"ok\":true,\"diagnostics\":%q}\n", req.Diagnostics.Groups[0].ReasonCode))
}
return contracts.OutputResult{Files: []contracts.OutputFile{{Name: "result.json", Bytes: data}}}, nil
}
func stateTestDiagnostic(scope, reasonCode, message string) contracts.ProducerDiagnostic {
return contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryDegradation, ReasonCode: reasonCode, OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: scope, Message: message}}}
}
type failingDebugRecorder struct{}
func (failingDebugRecorder) Enabled() bool { return true }