eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Publish grouped warning and diagnostic files

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-27 16:43:09 +00:00
parent 4dbbf68051
commit 54de2b816a
5 changed files with 193 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
docs/integrations/json-output.md
View File

@@ -9,7 +9,7 @@ Output configuration, including chunk-map and evidence-context publication, belo
## Bundle Layout
All paths below are logical, relative, slash-separated bundle paths. The
encoder always emits the first four JSON files below and adds lane or
encoder always emits the first five JSON files below and adds lane or
pipeline-wide artifact files when their corresponding artifacts are available:
A subprocess caller first obtains the physical bundle root from the
@@ -21,7 +21,8 @@ root for the logical discovery described here.
| `index.json` | Entry point that names the other published files and lane payloads. |
| `manifest.json` | Run provenance and result summaries. |
| `rejected.json` | Rejected pipeline outputs. |
| `warnings.json` | Accepted-output and run warnings. |
| `warnings.json` | Actionable process-degradation warnings. |
| `diagnostics.json` | Accepted-artifact quality advisories and normalization observations. |
| `lanes/<safe-lane-id>.json` | One normalized artifact payload for each lane. |
| `chunk-map.json` | Optional accepted chunk map, when its export is enabled and available. |
| `evidence-context.json` | Optional selected source-unit excerpt, when evidence publication is enabled. |
@@ -39,7 +40,8 @@ normalized lanes has this valid minimal index:
"manifest_file": "manifest.json",
"output_files": [],
"rejected_file": "rejected.json",
"warnings_file": "warnings.json"
"warnings_file": "warnings.json",
"diagnostics_file": "diagnostics.json"
}
```
@@ -49,6 +51,7 @@ normalized lanes has this valid minimal index:
| `output_files` | Yes | Lane descriptors sorted by `lane_id`. |
| `rejected_file` | Yes | Always `rejected.json`. |
| `warnings_file` | Yes | Always `warnings.json`. |
| `diagnostics_file` | Yes | Always `diagnostics.json`. |
| `chunk_map` | No | Descriptor for the pipeline-wide `chunk-map.json`; never a lane descriptor. |
| `evidence_context` | No | Descriptor for the pipeline-wide `evidence-context.json`; never a lane descriptor. |
@@ -132,7 +135,7 @@ These values describe observed execution; they are not a backend-registration
interface. Entries that differ by backend or effective reasoning remain
distinct even when their profile, provider, and model are otherwise equal.
## Rejections And Warnings
## Rejections, Warnings, And Diagnostics
`rejected.json` is always an object with a `rejected` array. Each entry has
required `stage` and `message`; `step_id`, `lane_id`, `module_key`, `chunk_id`,
@@ -142,9 +145,47 @@ contain the bounded `validation` summary described above; the existing singular
validator and reason fields remain the first configured rejection for
compatibility.
`warnings.json` is always an object with a `warnings` array. Each warning has
`reason_code` and `message`; `scope` is optional. Both arrays are empty when
there is nothing to report.
`warnings.json` is always the `notarius.warnings.v2` envelope:
```json
{
"schema_version": "notarius.warnings.v2",
"group_count": 0,
"occurrence_count": 0,
"groups": []
}
```
It contains only process warnings. `group_count` is exact, and
`occurrence_count` is the exact sum of its group occurrence counts.
`diagnostics.json` is always the `notarius.diagnostics.v1` envelope:
```json
{
"schema_version": "notarius.diagnostics.v1",
"group_count": 0,
"occurrence_count": 0,
"truncated": false,
"unrepresented_occurrence_count": 0,
"groups": []
}
```
It contains only advisory and observation groups. `group_count` counts groups
represented in `groups`; `occurrence_count` includes both represented and
unrepresented occurrences. When `truncated` is true,
`unrepresented_occurrence_count` is the exact number omitted from group
representation.
Each group has `disposition`, `category`, `reason_code`, framework-owned
`origin`, exact `occurrence_count`, bounded `samples`, and
`omitted_sample_count`. Samples carry safe `scope` and `message`, plus a chunk
ID and zero-based chunk index when applicable. A group retains at most three
distinct samples. The framework fails rather than truncating actionable
warnings beyond 128 groups; it represents at most 256 advisory/observation
groups and records further occurrences through the diagnostic truncation
fields above.
## Compatibility

2
docs/roadmap/implementation.md
View File

@@ -517,7 +517,7 @@ assert old flat-list caps or omission prose.
This stage is appropriately sized for one `gpt-5.6-terra` prompt.
## Stage 9 — Publish Versioned Warning And Diagnostic Files
## Stage 9 ✅ — Publish Versioned Warning And Diagnostic Files
### Goal

25
internal/cli/assembled_spell_pipeline_contract_test.go
View File

@@ -94,6 +94,17 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
if len(warningsFile.Groups) != 0 {
t.Fatalf("warnings file = %#v, want no process warnings for advisory-only diagnostics", warningsFile.Groups)
}
diagnosticsFile := decodeAssembledOutput[struct {
SchemaVersion string `json:"schema_version"`
GroupCount int `json:"group_count"`
OccurrenceCount int `json:"occurrence_count"`
Truncated bool `json:"truncated"`
UnrepresentedOccurrenceCount int `json:"unrepresented_occurrence_count"`
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, output.OutputFiles, "diagnostics.json")
if diagnosticsFile.SchemaVersion != "notarius.diagnostics.v1" || diagnosticsFile.GroupCount != len(output.Diagnostics.Groups) || !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) || diagnosticsFile.OccurrenceCount != diagnosticOccurrenceCount(output.Diagnostics.Groups)+output.Diagnostics.UnrepresentedOccurrenceCount || diagnosticsFile.Truncated != output.Diagnostics.Truncated || diagnosticsFile.UnrepresentedOccurrenceCount != output.Diagnostics.UnrepresentedOccurrenceCount {
t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile, output.Diagnostics)
}
manifest := decodeAssembledOutput[artifacts.RunManifest](t, output.OutputFiles, "manifest.json")
if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].Normalizer != spellnormalize.Key {
t.Fatalf("manifest lanes = %#v, want assembled spell normalizer", manifest.ArtifactLanes)
@@ -228,6 +239,12 @@ func TestAssembledSpellPipelinePromotesUnknownSpellWarningWhenOverrideAccepts(t
if len(warningsFile.Groups) != 0 {
t.Fatalf("warnings file = %#v, want no process warnings for an advisory diagnostic", warningsFile.Groups)
}
diagnosticsFile := decodeAssembledOutput[struct {
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, output.OutputFiles, "diagnostics.json")
if !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) {
t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile.Groups, output.Diagnostics)
}
}
type assembledSpellPipelineOptions struct {
@@ -429,6 +446,14 @@ func (e *assembledSpellExtractor) chunkIndexesSnapshot() []int {
return append([]int(nil), e.chunkIndexes...)
}
func diagnosticOccurrenceCount(groups []contracts.DiagnosticGroup) int {
count := 0
for _, group := range groups {
count += group.OccurrenceCount
}
return count
}
func decodeAssembledOutput[T any](t *testing.T, files []contracts.OutputFile, name string) T {
t.Helper()
for _, file := range files {

73
internal/modules/generic/output/json/encoder.go
View File

@@ -25,6 +25,11 @@ const chunkMapFileName = "chunk-map.json"
const evidenceContextFileName = "evidence-context.json"
const (
warningsSchemaVersion = "notarius.warnings.v2"
diagnosticsSchemaVersion = "notarius.diagnostics.v1"
)
var safeOutputFileChar = regexp.MustCompile(`[^A-Za-z0-9._-]`)
var _ contracts.OutputEncoder = (*Encoder)(nil)
@@ -231,6 +236,7 @@ type indexFile struct {
OutputFiles []outputFileIndex `json:"output_files"`
RejectedFile string `json:"rejected_file"`
WarningsFile string `json:"warnings_file"`
DiagnosticsFile string `json:"diagnostics_file"`
ChunkMap *artifactIndex `json:"chunk_map,omitempty"`
EvidenceContext *artifactIndex `json:"evidence_context,omitempty"`
}
@@ -259,6 +265,18 @@ type rejectedFile struct {
}
type warningsFile struct {
SchemaVersion string `json:"schema_version"`
GroupCount int `json:"group_count"`
OccurrenceCount int `json:"occurrence_count"`
Groups []contracts.DiagnosticGroup `json:"groups"`
}
type diagnosticsFile struct {
SchemaVersion string `json:"schema_version"`
GroupCount int `json:"group_count"`
OccurrenceCount int `json:"occurrence_count"`
Truncated bool `json:"truncated"`
UnrepresentedOccurrenceCount int `json:"unrepresented_occurrence_count"`
Groups []contracts.DiagnosticGroup `json:"groups"`
}
@@ -269,7 +287,7 @@ func logicalFiles(req contracts.OutputRequest, options Options) ([]contracts.Out
})
outputIndexes := make([]outputFileIndex, 0, len(outputs))
files := make([]contracts.OutputFile, 0, len(outputs)+5)
files := make([]contracts.OutputFile, 0, len(outputs)+6)
manifestFile, err := jsonFile("manifest.json", req.Manifest)
if err != nil {
return nil, err
@@ -307,6 +325,7 @@ func logicalFiles(req contracts.OutputRequest, options Options) ([]contracts.Out
OutputFiles: outputIndexes,
RejectedFile: "rejected.json",
WarningsFile: "warnings.json",
DiagnosticsFile: "diagnostics.json",
}
if options.IncludeChunkMap && req.ChunkMap != nil {
chunkMapOutput, chunkMapDescriptor, err := serializedChunkMapFile(*req.ChunkMap)
@@ -332,11 +351,15 @@ func logicalFiles(req contracts.OutputRequest, options Options) ([]contracts.Out
if err != nil {
return nil, err
}
warningsOutput, err := jsonFile("warnings.json", warningsFile{Groups: warningGroups(req.Diagnostics)})
warningsOutput, err := jsonFile("warnings.json", newWarningsFile(req.Diagnostics))
if err != nil {
return nil, err
}
files = append(files, indexOutput, rejectedOutput, warningsOutput)
diagnosticsOutput, err := jsonFile("diagnostics.json", newDiagnosticsFile(req.Diagnostics))
if err != nil {
return nil, err
}
files = append(files, indexOutput, rejectedOutput, warningsOutput, diagnosticsOutput)
sort.Slice(files, func(i, j int) bool {
return files[i].Name < files[j].Name
})
@@ -487,16 +510,56 @@ func cloneRejected(rejected []contracts.RejectedOutput) []contracts.RejectedOutp
return append([]contracts.RejectedOutput(nil), rejected...)
}
func warningGroups(collection contracts.DiagnosticCollection) []contracts.DiagnosticGroup {
func newWarningsFile(collection contracts.DiagnosticCollection) warningsFile {
groups := diagnosticGroupsByDisposition(collection, contracts.DiagnosticDispositionWarning)
return warningsFile{
SchemaVersion: warningsSchemaVersion,
GroupCount: len(groups),
OccurrenceCount: diagnosticOccurrenceCount(groups),
Groups: groups,
}
}
func newDiagnosticsFile(collection contracts.DiagnosticCollection) diagnosticsFile {
groups := diagnosticGroupsExceptDisposition(collection, contracts.DiagnosticDispositionWarning)
return diagnosticsFile{
SchemaVersion: diagnosticsSchemaVersion,
GroupCount: len(groups),
OccurrenceCount: diagnosticOccurrenceCount(groups) + collection.UnrepresentedOccurrenceCount,
Truncated: collection.Truncated,
UnrepresentedOccurrenceCount: collection.UnrepresentedOccurrenceCount,
Groups: groups,
}
}
func diagnosticGroupsByDisposition(collection contracts.DiagnosticCollection, disposition contracts.DiagnosticDisposition) []contracts.DiagnosticGroup {
groups := make([]contracts.DiagnosticGroup, 0)
for _, group := range collection.Groups {
if group.Disposition == contracts.DiagnosticDispositionWarning {
if group.Disposition == disposition {
groups = append(groups, group)
}
}
return contracts.CloneDiagnosticCollection(contracts.DiagnosticCollection{Groups: groups}).Groups
}
func diagnosticGroupsExceptDisposition(collection contracts.DiagnosticCollection, disposition contracts.DiagnosticDisposition) []contracts.DiagnosticGroup {
groups := make([]contracts.DiagnosticGroup, 0)
for _, group := range collection.Groups {
if group.Disposition != disposition {
groups = append(groups, group)
}
}
return contracts.CloneDiagnosticCollection(contracts.DiagnosticCollection{Groups: groups}).Groups
}
func diagnosticOccurrenceCount(groups []contracts.DiagnosticGroup) int {
count := 0
for _, group := range groups {
count += group.OccurrenceCount
}
return count
}
func encoderErrorf(format string, args ...any) error {
return fmt.Errorf("json output encoder: "+format, args...)
}

48
internal/modules/generic/output/json/encoder_test.go
View File

@@ -155,6 +155,7 @@ func TestEncodeReturnsLogicalFilesForNormalizedOutputs(t *testing.T) {
}
wantNames := []string{
"diagnostics.json",
"index.json",
"lanes/notes_items.json",
"lanes/spells.json",
@@ -216,6 +217,49 @@ func TestEncodeIncludesRejectedAndWarningsWhenEmpty(t *testing.T) {
}
}
func TestEncodePublishesVersionedDiagnosticPartitions(t *testing.T) {
diagnostics := contracts.DiagnosticCollection{
Groups: []contracts.DiagnosticGroup{
{
Disposition: contracts.DiagnosticDispositionWarning,
Category: contracts.DiagnosticCategoryDegradation,
ReasonCode: "degraded",
Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk},
OccurrenceCount: 2,
Samples: []contracts.DiagnosticSample{{Scope: "chunk", Message: "degraded"}},
OmittedSampleCount: 1,
},
{
Disposition: contracts.DiagnosticDispositionAdvisory,
Category: contracts.DiagnosticCategoryDataQuality,
ReasonCode: "unrelated",
Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract},
OccurrenceCount: 3,
Samples: []contracts.DiagnosticSample{{Scope: "items[0]", Message: "unrelated"}},
OmittedSampleCount: 2,
},
},
Truncated: true,
UnrepresentedOccurrenceCount: 5,
}
result, err := New().Encode(context.Background(), contracts.OutputRequest{Diagnostics: diagnostics})
if err != nil {
t.Fatalf("Encode() error = %v", err)
}
warnings := decodeObject(t, fileBytes(t, result.Files, "warnings.json"))
if warnings["schema_version"] != warningsSchemaVersion || warnings["group_count"] != float64(1) || warnings["occurrence_count"] != float64(2) || len(warnings["groups"].([]any)) != 1 {
t.Fatalf("warnings file = %#v", warnings)
}
publishedDiagnostics := decodeObject(t, fileBytes(t, result.Files, "diagnostics.json"))
if publishedDiagnostics["schema_version"] != diagnosticsSchemaVersion || publishedDiagnostics["group_count"] != float64(1) || publishedDiagnostics["occurrence_count"] != float64(8) || publishedDiagnostics["truncated"] != true || publishedDiagnostics["unrepresented_occurrence_count"] != float64(5) || len(publishedDiagnostics["groups"].([]any)) != 1 {
t.Fatalf("diagnostics file = %#v", publishedDiagnostics)
}
index := decodeObject(t, fileBytes(t, result.Files, "index.json"))
if index["warnings_file"] != "warnings.json" || index["diagnostics_file"] != "diagnostics.json" {
t.Fatalf("index = %#v", index)
}
}
func TestEncodeChunkMapExportIsOptIn(t *testing.T) {
artifact := acceptedChunkMapArtifact(t)
request := contracts.OutputRequest{
@@ -234,7 +278,7 @@ func TestEncodeChunkMapExportIsOptIn(t *testing.T) {
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
if got, want := outputFileNames(result.Files), []string{"index.json", "manifest.json", "rejected.json", "warnings.json"}; !reflect.DeepEqual(got, want) {
if got, want := outputFileNames(result.Files), []string{"diagnostics.json", "index.json", "manifest.json", "rejected.json", "warnings.json"}; !reflect.DeepEqual(got, want) {
t.Fatalf("file names = %#v, want %#v", got, want)
}
index := decodeObject(t, fileBytes(t, result.Files, "index.json"))
@@ -257,7 +301,7 @@ func TestEncodeIncludesValidatedChunkMap(t *testing.T) {
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
if got, want := outputFileNames(result.Files), []string{"chunk-map.json", "index.json", "lanes/spells.json", "manifest.json", "rejected.json", "warnings.json"}; !reflect.DeepEqual(got, want) {
if got, want := outputFileNames(result.Files), []string{"chunk-map.json", "diagnostics.json", "index.json", "lanes/spells.json", "manifest.json", "rejected.json", "warnings.json"}; !reflect.DeepEqual(got, want) {
t.Fatalf("file names = %#v, want %#v", got, want)
}
chunkMapFile := fileBytes(t, result.Files, chunkMapFileName)