From 54de2b816aa1f708f224f9bfaf86d5a6e9e3d0d0 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 27 Aug 2026 16:43:09 +0000 Subject: [PATCH] Publish grouped warning and diagnostic files --- docs/integrations/json-output.md | 55 ++++++++++-- docs/roadmap/implementation.md | 2 +- .../assembled_spell_pipeline_contract_test.go | 25 ++++++ .../modules/generic/output/json/encoder.go | 83 ++++++++++++++++--- .../generic/output/json/encoder_test.go | 48 ++++++++++- 5 files changed, 193 insertions(+), 20 deletions(-) diff --git a/docs/integrations/json-output.md b/docs/integrations/json-output.md index 9d83607a..c0f3aacc 100644 --- a/docs/integrations/json-output.md +++ b/docs/integrations/json-output.md @@ -9,7 +9,7 @@ Output configuration, including chunk-map and evidence-context publication, belo ## Bundle Layout All paths below are logical, relative, slash-separated bundle paths. The -encoder always emits the first four JSON files below and adds lane or +encoder always emits the first five JSON files below and adds lane or pipeline-wide artifact files when their corresponding artifacts are available: A subprocess caller first obtains the physical bundle root from the @@ -21,7 +21,8 @@ root for the logical discovery described here. | `index.json` | Entry point that names the other published files and lane payloads. | | `manifest.json` | Run provenance and result summaries. | | `rejected.json` | Rejected pipeline outputs. | -| `warnings.json` | Accepted-output and run warnings. | +| `warnings.json` | Actionable process-degradation warnings. | +| `diagnostics.json` | Accepted-artifact quality advisories and normalization observations. | | `lanes/.json` | One normalized artifact payload for each lane. | | `chunk-map.json` | Optional accepted chunk map, when its export is enabled and available. | | `evidence-context.json` | Optional selected source-unit excerpt, when evidence publication is enabled. | @@ -39,7 +40,8 @@ normalized lanes has this valid minimal index: "manifest_file": "manifest.json", "output_files": [], "rejected_file": "rejected.json", - "warnings_file": "warnings.json" + "warnings_file": "warnings.json", + "diagnostics_file": "diagnostics.json" } ``` @@ -49,6 +51,7 @@ normalized lanes has this valid minimal index: | `output_files` | Yes | Lane descriptors sorted by `lane_id`. | | `rejected_file` | Yes | Always `rejected.json`. | | `warnings_file` | Yes | Always `warnings.json`. | +| `diagnostics_file` | Yes | Always `diagnostics.json`. | | `chunk_map` | No | Descriptor for the pipeline-wide `chunk-map.json`; never a lane descriptor. | | `evidence_context` | No | Descriptor for the pipeline-wide `evidence-context.json`; never a lane descriptor. | @@ -132,7 +135,7 @@ These values describe observed execution; they are not a backend-registration interface. Entries that differ by backend or effective reasoning remain distinct even when their profile, provider, and model are otherwise equal. -## Rejections And Warnings +## Rejections, Warnings, And Diagnostics `rejected.json` is always an object with a `rejected` array. Each entry has required `stage` and `message`; `step_id`, `lane_id`, `module_key`, `chunk_id`, @@ -142,9 +145,47 @@ contain the bounded `validation` summary described above; the existing singular validator and reason fields remain the first configured rejection for compatibility. -`warnings.json` is always an object with a `warnings` array. Each warning has -`reason_code` and `message`; `scope` is optional. Both arrays are empty when -there is nothing to report. +`warnings.json` is always the `notarius.warnings.v2` envelope: + +```json +{ + "schema_version": "notarius.warnings.v2", + "group_count": 0, + "occurrence_count": 0, + "groups": [] +} +``` + +It contains only process warnings. `group_count` is exact, and +`occurrence_count` is the exact sum of its group occurrence counts. + +`diagnostics.json` is always the `notarius.diagnostics.v1` envelope: + +```json +{ + "schema_version": "notarius.diagnostics.v1", + "group_count": 0, + "occurrence_count": 0, + "truncated": false, + "unrepresented_occurrence_count": 0, + "groups": [] +} +``` + +It contains only advisory and observation groups. `group_count` counts groups +represented in `groups`; `occurrence_count` includes both represented and +unrepresented occurrences. When `truncated` is true, +`unrepresented_occurrence_count` is the exact number omitted from group +representation. + +Each group has `disposition`, `category`, `reason_code`, framework-owned +`origin`, exact `occurrence_count`, bounded `samples`, and +`omitted_sample_count`. Samples carry safe `scope` and `message`, plus a chunk +ID and zero-based chunk index when applicable. A group retains at most three +distinct samples. The framework fails rather than truncating actionable +warnings beyond 128 groups; it represents at most 256 advisory/observation +groups and records further occurrences through the diagnostic truncation +fields above. ## Compatibility diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 5f22cfd0..d57bf68a 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -517,7 +517,7 @@ assert old flat-list caps or omission prose. This stage is appropriately sized for one `gpt-5.6-terra` prompt. -## Stage 9 — Publish Versioned Warning And Diagnostic Files +## Stage 9 ✅ — Publish Versioned Warning And Diagnostic Files ### Goal diff --git a/internal/cli/assembled_spell_pipeline_contract_test.go b/internal/cli/assembled_spell_pipeline_contract_test.go index 49e4dffe..2fe9ddaf 100644 --- a/internal/cli/assembled_spell_pipeline_contract_test.go +++ b/internal/cli/assembled_spell_pipeline_contract_test.go @@ -94,6 +94,17 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) { if len(warningsFile.Groups) != 0 { t.Fatalf("warnings file = %#v, want no process warnings for advisory-only diagnostics", warningsFile.Groups) } + diagnosticsFile := decodeAssembledOutput[struct { + SchemaVersion string `json:"schema_version"` + GroupCount int `json:"group_count"` + OccurrenceCount int `json:"occurrence_count"` + Truncated bool `json:"truncated"` + UnrepresentedOccurrenceCount int `json:"unrepresented_occurrence_count"` + Groups []contracts.DiagnosticGroup `json:"groups"` + }](t, output.OutputFiles, "diagnostics.json") + if diagnosticsFile.SchemaVersion != "notarius.diagnostics.v1" || diagnosticsFile.GroupCount != len(output.Diagnostics.Groups) || !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) || diagnosticsFile.OccurrenceCount != diagnosticOccurrenceCount(output.Diagnostics.Groups)+output.Diagnostics.UnrepresentedOccurrenceCount || diagnosticsFile.Truncated != output.Diagnostics.Truncated || diagnosticsFile.UnrepresentedOccurrenceCount != output.Diagnostics.UnrepresentedOccurrenceCount { + t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile, output.Diagnostics) + } manifest := decodeAssembledOutput[artifacts.RunManifest](t, output.OutputFiles, "manifest.json") if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].Normalizer != spellnormalize.Key { t.Fatalf("manifest lanes = %#v, want assembled spell normalizer", manifest.ArtifactLanes) @@ -228,6 +239,12 @@ func TestAssembledSpellPipelinePromotesUnknownSpellWarningWhenOverrideAccepts(t if len(warningsFile.Groups) != 0 { t.Fatalf("warnings file = %#v, want no process warnings for an advisory diagnostic", warningsFile.Groups) } + diagnosticsFile := decodeAssembledOutput[struct { + Groups []contracts.DiagnosticGroup `json:"groups"` + }](t, output.OutputFiles, "diagnostics.json") + if !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) { + t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile.Groups, output.Diagnostics) + } } type assembledSpellPipelineOptions struct { @@ -429,6 +446,14 @@ func (e *assembledSpellExtractor) chunkIndexesSnapshot() []int { return append([]int(nil), e.chunkIndexes...) } +func diagnosticOccurrenceCount(groups []contracts.DiagnosticGroup) int { + count := 0 + for _, group := range groups { + count += group.OccurrenceCount + } + return count +} + func decodeAssembledOutput[T any](t *testing.T, files []contracts.OutputFile, name string) T { t.Helper() for _, file := range files { diff --git a/internal/modules/generic/output/json/encoder.go b/internal/modules/generic/output/json/encoder.go index 7cbe84e3..00b36235 100644 --- a/internal/modules/generic/output/json/encoder.go +++ b/internal/modules/generic/output/json/encoder.go @@ -25,6 +25,11 @@ const chunkMapFileName = "chunk-map.json" const evidenceContextFileName = "evidence-context.json" +const ( + warningsSchemaVersion = "notarius.warnings.v2" + diagnosticsSchemaVersion = "notarius.diagnostics.v1" +) + var safeOutputFileChar = regexp.MustCompile(`[^A-Za-z0-9._-]`) var _ contracts.OutputEncoder = (*Encoder)(nil) @@ -231,6 +236,7 @@ type indexFile struct { OutputFiles []outputFileIndex `json:"output_files"` RejectedFile string `json:"rejected_file"` WarningsFile string `json:"warnings_file"` + DiagnosticsFile string `json:"diagnostics_file"` ChunkMap *artifactIndex `json:"chunk_map,omitempty"` EvidenceContext *artifactIndex `json:"evidence_context,omitempty"` } @@ -259,7 +265,19 @@ type rejectedFile struct { } type warningsFile struct { - Groups []contracts.DiagnosticGroup `json:"groups"` + SchemaVersion string `json:"schema_version"` + GroupCount int `json:"group_count"` + OccurrenceCount int `json:"occurrence_count"` + Groups []contracts.DiagnosticGroup `json:"groups"` +} + +type diagnosticsFile struct { + SchemaVersion string `json:"schema_version"` + GroupCount int `json:"group_count"` + OccurrenceCount int `json:"occurrence_count"` + Truncated bool `json:"truncated"` + UnrepresentedOccurrenceCount int `json:"unrepresented_occurrence_count"` + Groups []contracts.DiagnosticGroup `json:"groups"` } func logicalFiles(req contracts.OutputRequest, options Options) ([]contracts.OutputFile, error) { @@ -269,7 +287,7 @@ func logicalFiles(req contracts.OutputRequest, options Options) ([]contracts.Out }) outputIndexes := make([]outputFileIndex, 0, len(outputs)) - files := make([]contracts.OutputFile, 0, len(outputs)+5) + files := make([]contracts.OutputFile, 0, len(outputs)+6) manifestFile, err := jsonFile("manifest.json", req.Manifest) if err != nil { return nil, err @@ -303,10 +321,11 @@ func logicalFiles(req contracts.OutputRequest, options Options) ([]contracts.Out } index := indexFile{ - ManifestFile: "manifest.json", - OutputFiles: outputIndexes, - RejectedFile: "rejected.json", - WarningsFile: "warnings.json", + ManifestFile: "manifest.json", + OutputFiles: outputIndexes, + RejectedFile: "rejected.json", + WarningsFile: "warnings.json", + DiagnosticsFile: "diagnostics.json", } if options.IncludeChunkMap && req.ChunkMap != nil { chunkMapOutput, chunkMapDescriptor, err := serializedChunkMapFile(*req.ChunkMap) @@ -332,11 +351,15 @@ func logicalFiles(req contracts.OutputRequest, options Options) ([]contracts.Out if err != nil { return nil, err } - warningsOutput, err := jsonFile("warnings.json", warningsFile{Groups: warningGroups(req.Diagnostics)}) + warningsOutput, err := jsonFile("warnings.json", newWarningsFile(req.Diagnostics)) if err != nil { return nil, err } - files = append(files, indexOutput, rejectedOutput, warningsOutput) + diagnosticsOutput, err := jsonFile("diagnostics.json", newDiagnosticsFile(req.Diagnostics)) + if err != nil { + return nil, err + } + files = append(files, indexOutput, rejectedOutput, warningsOutput, diagnosticsOutput) sort.Slice(files, func(i, j int) bool { return files[i].Name < files[j].Name }) @@ -487,16 +510,56 @@ func cloneRejected(rejected []contracts.RejectedOutput) []contracts.RejectedOutp return append([]contracts.RejectedOutput(nil), rejected...) } -func warningGroups(collection contracts.DiagnosticCollection) []contracts.DiagnosticGroup { +func newWarningsFile(collection contracts.DiagnosticCollection) warningsFile { + groups := diagnosticGroupsByDisposition(collection, contracts.DiagnosticDispositionWarning) + return warningsFile{ + SchemaVersion: warningsSchemaVersion, + GroupCount: len(groups), + OccurrenceCount: diagnosticOccurrenceCount(groups), + Groups: groups, + } +} + +func newDiagnosticsFile(collection contracts.DiagnosticCollection) diagnosticsFile { + groups := diagnosticGroupsExceptDisposition(collection, contracts.DiagnosticDispositionWarning) + return diagnosticsFile{ + SchemaVersion: diagnosticsSchemaVersion, + GroupCount: len(groups), + OccurrenceCount: diagnosticOccurrenceCount(groups) + collection.UnrepresentedOccurrenceCount, + Truncated: collection.Truncated, + UnrepresentedOccurrenceCount: collection.UnrepresentedOccurrenceCount, + Groups: groups, + } +} + +func diagnosticGroupsByDisposition(collection contracts.DiagnosticCollection, disposition contracts.DiagnosticDisposition) []contracts.DiagnosticGroup { groups := make([]contracts.DiagnosticGroup, 0) for _, group := range collection.Groups { - if group.Disposition == contracts.DiagnosticDispositionWarning { + if group.Disposition == disposition { groups = append(groups, group) } } return contracts.CloneDiagnosticCollection(contracts.DiagnosticCollection{Groups: groups}).Groups } +func diagnosticGroupsExceptDisposition(collection contracts.DiagnosticCollection, disposition contracts.DiagnosticDisposition) []contracts.DiagnosticGroup { + groups := make([]contracts.DiagnosticGroup, 0) + for _, group := range collection.Groups { + if group.Disposition != disposition { + groups = append(groups, group) + } + } + return contracts.CloneDiagnosticCollection(contracts.DiagnosticCollection{Groups: groups}).Groups +} + +func diagnosticOccurrenceCount(groups []contracts.DiagnosticGroup) int { + count := 0 + for _, group := range groups { + count += group.OccurrenceCount + } + return count +} + func encoderErrorf(format string, args ...any) error { return fmt.Errorf("json output encoder: "+format, args...) } diff --git a/internal/modules/generic/output/json/encoder_test.go b/internal/modules/generic/output/json/encoder_test.go index e742ecad..e4823c86 100644 --- a/internal/modules/generic/output/json/encoder_test.go +++ b/internal/modules/generic/output/json/encoder_test.go @@ -155,6 +155,7 @@ func TestEncodeReturnsLogicalFilesForNormalizedOutputs(t *testing.T) { } wantNames := []string{ + "diagnostics.json", "index.json", "lanes/notes_items.json", "lanes/spells.json", @@ -216,6 +217,49 @@ func TestEncodeIncludesRejectedAndWarningsWhenEmpty(t *testing.T) { } } +func TestEncodePublishesVersionedDiagnosticPartitions(t *testing.T) { + diagnostics := contracts.DiagnosticCollection{ + Groups: []contracts.DiagnosticGroup{ + { + Disposition: contracts.DiagnosticDispositionWarning, + Category: contracts.DiagnosticCategoryDegradation, + ReasonCode: "degraded", + Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk}, + OccurrenceCount: 2, + Samples: []contracts.DiagnosticSample{{Scope: "chunk", Message: "degraded"}}, + OmittedSampleCount: 1, + }, + { + Disposition: contracts.DiagnosticDispositionAdvisory, + Category: contracts.DiagnosticCategoryDataQuality, + ReasonCode: "unrelated", + Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract}, + OccurrenceCount: 3, + Samples: []contracts.DiagnosticSample{{Scope: "items[0]", Message: "unrelated"}}, + OmittedSampleCount: 2, + }, + }, + Truncated: true, + UnrepresentedOccurrenceCount: 5, + } + result, err := New().Encode(context.Background(), contracts.OutputRequest{Diagnostics: diagnostics}) + if err != nil { + t.Fatalf("Encode() error = %v", err) + } + warnings := decodeObject(t, fileBytes(t, result.Files, "warnings.json")) + if warnings["schema_version"] != warningsSchemaVersion || warnings["group_count"] != float64(1) || warnings["occurrence_count"] != float64(2) || len(warnings["groups"].([]any)) != 1 { + t.Fatalf("warnings file = %#v", warnings) + } + publishedDiagnostics := decodeObject(t, fileBytes(t, result.Files, "diagnostics.json")) + if publishedDiagnostics["schema_version"] != diagnosticsSchemaVersion || publishedDiagnostics["group_count"] != float64(1) || publishedDiagnostics["occurrence_count"] != float64(8) || publishedDiagnostics["truncated"] != true || publishedDiagnostics["unrepresented_occurrence_count"] != float64(5) || len(publishedDiagnostics["groups"].([]any)) != 1 { + t.Fatalf("diagnostics file = %#v", publishedDiagnostics) + } + index := decodeObject(t, fileBytes(t, result.Files, "index.json")) + if index["warnings_file"] != "warnings.json" || index["diagnostics_file"] != "diagnostics.json" { + t.Fatalf("index = %#v", index) + } +} + func TestEncodeChunkMapExportIsOptIn(t *testing.T) { artifact := acceptedChunkMapArtifact(t) request := contracts.OutputRequest{ @@ -234,7 +278,7 @@ func TestEncodeChunkMapExportIsOptIn(t *testing.T) { if err != nil { t.Fatalf("Encode() error = %v, want nil", err) } - if got, want := outputFileNames(result.Files), []string{"index.json", "manifest.json", "rejected.json", "warnings.json"}; !reflect.DeepEqual(got, want) { + if got, want := outputFileNames(result.Files), []string{"diagnostics.json", "index.json", "manifest.json", "rejected.json", "warnings.json"}; !reflect.DeepEqual(got, want) { t.Fatalf("file names = %#v, want %#v", got, want) } index := decodeObject(t, fileBytes(t, result.Files, "index.json")) @@ -257,7 +301,7 @@ func TestEncodeIncludesValidatedChunkMap(t *testing.T) { if err != nil { t.Fatalf("Encode() error = %v, want nil", err) } - if got, want := outputFileNames(result.Files), []string{"chunk-map.json", "index.json", "lanes/spells.json", "manifest.json", "rejected.json", "warnings.json"}; !reflect.DeepEqual(got, want) { + if got, want := outputFileNames(result.Files), []string{"chunk-map.json", "diagnostics.json", "index.json", "lanes/spells.json", "manifest.json", "rejected.json", "warnings.json"}; !reflect.DeepEqual(got, want) { t.Fatalf("file names = %#v, want %#v", got, want) } chunkMapFile := fileBytes(t, result.Files, chunkMapFileName)