From af0119cc1d88f214f80d5378de080396146e79af Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 27 Aug 2026 16:50:32 +0000 Subject: [PATCH] Add diagnostic counts to run receipts --- docs/cli.md | 8 +- docs/consumers/dnd-pipeline.md | 5 +- docs/consumers/subprocess.md | 2 +- docs/integrations/run-result.md | 18 +++-- docs/roadmap/implementation.md | 2 +- .../assembled_spell_pipeline_contract_test.go | 4 +- internal/cli/run.go | 4 +- internal/cli/run_contract_test.go | 2 +- internal/cli/run_result.go | 78 ++++++++++++++----- internal/cli/run_result_command_test.go | 12 +-- internal/cli/run_result_test.go | 7 +- 11 files changed, 95 insertions(+), 47 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index e0f0903a..d96248e4 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -103,11 +103,13 @@ names, requiredness, and configured bindings are part of the Without **--json**, standard output contains the completed pipeline ID, counts of normalized and rejected outputs, and the output directory. A debug-enabled run also prints its debug-bundle path to standard output. A successful run with -warnings reports the warning count to standard error. The published JSON bundle +actionable process warnings report their group and occurrence counts plus the +durable warning-file path to standard error. Advisory and observation findings +do not produce a warning line. The published JSON bundle is defined by the [JSON output contract](integrations/json-output.md). With **--json**, successful standard output is exactly one -`notarius.run-result.v1` JSON document followed by a newline, with no +`notarius.run-result.v2` JSON document followed by a newline, with no human-oriented status or debug-path line. Its fields and compatibility policy are defined by the [run-result contract](integrations/run-result.md). A caller must check for exit status 0 before decoding this output; a failed write can @@ -171,7 +173,7 @@ go run ./cmd/notarius pipelines list \ Successful commands write their primary result to standard output. Warnings and errors are written to standard error. -For **run --json**, warnings remain on standard error and standard output is a +For **run --json**, actionable process warnings remain on standard error and standard output is a machine-readable success result only. Syntax and runtime diagnostics remain on standard error. Parse the result only after the process exits with status 0. diff --git a/docs/consumers/dnd-pipeline.md b/docs/consumers/dnd-pipeline.md index 57b1173c..86beedf0 100644 --- a/docs/consumers/dnd-pipeline.md +++ b/docs/consumers/dnd-pipeline.md @@ -92,7 +92,7 @@ stream and exit-status contract. ## Discover The Published Bundle Decode the successful stdout document as a supported run-result schema. For -the current contract, `schema_version` is `notarius.run-result.v1`. Tolerate +the current contract, `schema_version` is `notarius.run-result.v2`. Tolerate unknown fields allowed by that version, but reject an unsupported schema version. @@ -145,7 +145,8 @@ The JSON encoder always publishes these bundle-management files: | `index.json` | Discovery document for lane and pipeline-wide artifacts. | | `manifest.json` | Run provenance and result summaries. | | `rejected.json` | Rejected pipeline outputs. | -| `warnings.json` | Accepted-output and run warnings. | +| `warnings.json` | Actionable process-degradation warnings. | +| `diagnostics.json` | Advisory and observation findings for accepted artifacts. | The complete configuration also requests two pipeline-wide artifacts: diff --git a/docs/consumers/subprocess.md b/docs/consumers/subprocess.md index e5dc94be..70353ec1 100644 --- a/docs/consumers/subprocess.md +++ b/docs/consumers/subprocess.md @@ -79,7 +79,7 @@ silently treated as fully reviewed by the caller. ## Preserve Provenance And Handle Data Carefully Keep the receipt with the published `manifest.json`, and retain -`rejected.json` and `warnings.json` when review or later provenance requires +`rejected.json`, `warnings.json`, and `diagnostics.json` when review or later provenance requires them. Treat the input, output bundle, cache, debug bundle, and captured process logs as potentially sensitive data. Apply the caller's access controls and retention policy, and avoid copying secrets into arguments, logs, or diff --git a/docs/integrations/run-result.md b/docs/integrations/run-result.md index 2d2c531b..7790867a 100644 --- a/docs/integrations/run-result.md +++ b/docs/integrations/run-result.md @@ -9,18 +9,22 @@ Command syntax, streams, and exit statuses are defined in the ## Schema -The current schema version is `notarius.run-result.v1`. +The current schema version is `notarius.run-result.v2`. | Field | Required | Meaning | | --- | --- | --- | -| `schema_version` | Yes | Exactly `notarius.run-result.v1`. | +| `schema_version` | Yes | Exactly `notarius.run-result.v2`. | | `run_id` | Yes | The finalized Notarius run identifier. | | `pipeline_id` | Yes | The effective pipeline identifier. | | `output_directory` | Yes | Absolute path to the published, run-specific output bundle. | | `index_file` | For the production JSON output | Logical path `index.json`; omitted for other output modules. | | `normalized_output_count` | Yes | Number of final normalized outputs. | | `rejected_output_count` | Yes | Number of recorded rejected outputs. | -| `warning_count` | Yes | Number of final run warnings. | +| `warning_group_count` | Yes | Exact number of actionable warning groups. | +| `warning_occurrence_count` | Yes | Exact occurrences represented by actionable warning groups. | +| `diagnostic_group_count` | Yes | Number of represented advisory and observation groups. | +| `diagnostic_occurrence_count` | Yes | Advisory and observation occurrences, including unrepresented occurrences. | +| `diagnostics_truncated` | Yes | Whether advisory/observation group representation was truncated. | | `validation_status` | Yes | The final run manifest validation status. | | `validation_summaries` | No | Bounded per-producer validation outcomes; present when producer work ran. | | `debug_directory` | No | Absolute path to the run-specific debug bundle when requested debug capture completed. | @@ -34,14 +38,18 @@ means one or more otherwise accepted results advanced under validator-failure ```json { - "schema_version": "notarius.run-result.v1", + "schema_version": "notarius.run-result.v2", "run_id": "run-1770000000000000000-0123456789abcdef0123456789abcdef", "pipeline_id": "dnd-session", "output_directory": "/work/results/run-1770000000000000000-0123456789abcdef0123456789abcdef", "index_file": "index.json", "normalized_output_count": 6, "rejected_output_count": 2, - "warning_count": 1, + "warning_group_count": 1, + "warning_occurrence_count": 2, + "diagnostic_group_count": 3, + "diagnostic_occurrence_count": 5, + "diagnostics_truncated": false, "validation_status": "incomplete", "validation_summaries": [ { diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index d57bf68a..3ca5d1d9 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -554,7 +554,7 @@ durable files with the finalized in-memory collections. This stage is appropriately sized for one `gpt-5.6-terra` prompt. -## Stage 10 — Introduce Run Result V2 And Quiet CLI Presentation +## Stage 10 ✅ — Introduce Run Result V2 And Quiet CLI Presentation ### Goal diff --git a/internal/cli/assembled_spell_pipeline_contract_test.go b/internal/cli/assembled_spell_pipeline_contract_test.go index 2fe9ddaf..72691804 100644 --- a/internal/cli/assembled_spell_pipeline_contract_test.go +++ b/internal/cli/assembled_spell_pipeline_contract_test.go @@ -102,7 +102,7 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) { UnrepresentedOccurrenceCount int `json:"unrepresented_occurrence_count"` Groups []contracts.DiagnosticGroup `json:"groups"` }](t, output.OutputFiles, "diagnostics.json") - if diagnosticsFile.SchemaVersion != "notarius.diagnostics.v1" || diagnosticsFile.GroupCount != len(output.Diagnostics.Groups) || !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) || diagnosticsFile.OccurrenceCount != diagnosticOccurrenceCount(output.Diagnostics.Groups)+output.Diagnostics.UnrepresentedOccurrenceCount || diagnosticsFile.Truncated != output.Diagnostics.Truncated || diagnosticsFile.UnrepresentedOccurrenceCount != output.Diagnostics.UnrepresentedOccurrenceCount { + if diagnosticsFile.SchemaVersion != "notarius.diagnostics.v1" || diagnosticsFile.GroupCount != len(output.Diagnostics.Groups) || !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) || diagnosticsFile.OccurrenceCount != diagnosticGroupOccurrences(output.Diagnostics.Groups)+output.Diagnostics.UnrepresentedOccurrenceCount || diagnosticsFile.Truncated != output.Diagnostics.Truncated || diagnosticsFile.UnrepresentedOccurrenceCount != output.Diagnostics.UnrepresentedOccurrenceCount { t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile, output.Diagnostics) } manifest := decodeAssembledOutput[artifacts.RunManifest](t, output.OutputFiles, "manifest.json") @@ -446,7 +446,7 @@ func (e *assembledSpellExtractor) chunkIndexesSnapshot() []int { return append([]int(nil), e.chunkIndexes...) } -func diagnosticOccurrenceCount(groups []contracts.DiagnosticGroup) int { +func diagnosticGroupOccurrences(groups []contracts.DiagnosticGroup) int { count := 0 for _, group := range groups { count += group.OccurrenceCount diff --git a/internal/cli/run.go b/internal/cli/run.go index 51de4d5c..bd071975 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -513,8 +513,8 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i fmt.Fprintf(stdout, "debug=%s\n", debugPath) } } - if warningCount := warningGroupCount(output.Diagnostics); warningCount > 0 { - fmt.Fprintf(stderr, "notarius: run completed with %d warning(s)\n", warningCount) + if warningGroups := warningGroupCount(output.Diagnostics); warningGroups > 0 { + fmt.Fprintf(stderr, "notarius: run completed with %d warning group(s), %d occurrence(s); details=%s\n", warningGroups, warningOccurrenceCount(output.Diagnostics), filepath.Join(runOutputDir, "warnings.json")) } return 0 } diff --git a/internal/cli/run_contract_test.go b/internal/cli/run_contract_test.go index d9b9998b..15ede91c 100644 --- a/internal/cli/run_contract_test.go +++ b/internal/cli/run_contract_test.go @@ -593,7 +593,7 @@ func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) { harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")} var stdout, stderr bytes.Buffer code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, harness.options()) - if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning(s)") { + if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || !strings.Contains(stderr.String(), "warnings.json") { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } outputPath := filepath.Join(onlyChildDir(t, roots.output), "result.json") diff --git a/internal/cli/run_result.go b/internal/cli/run_result.go index b0c5c693..4e4d1e9b 100644 --- a/internal/cli/run_result.go +++ b/internal/cli/run_result.go @@ -12,20 +12,24 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" ) -const runResultSchemaVersion = "notarius.run-result.v1" +const runResultSchemaVersion = "notarius.run-result.v2" type runResult struct { - SchemaVersion string `json:"schema_version"` - RunID string `json:"run_id"` - PipelineID string `json:"pipeline_id"` - OutputDirectory string `json:"output_directory"` - IndexFile string `json:"index_file,omitempty"` - NormalizedOutputCount int `json:"normalized_output_count"` - RejectedOutputCount int `json:"rejected_output_count"` - WarningCount int `json:"warning_count"` - ValidationStatus string `json:"validation_status"` - ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"` - DebugDirectory string `json:"debug_directory,omitempty"` + SchemaVersion string `json:"schema_version"` + RunID string `json:"run_id"` + PipelineID string `json:"pipeline_id"` + OutputDirectory string `json:"output_directory"` + IndexFile string `json:"index_file,omitempty"` + NormalizedOutputCount int `json:"normalized_output_count"` + RejectedOutputCount int `json:"rejected_output_count"` + WarningGroupCount int `json:"warning_group_count"` + WarningOccurrenceCount int `json:"warning_occurrence_count"` + DiagnosticGroupCount int `json:"diagnostic_group_count"` + DiagnosticOccurrenceCount int `json:"diagnostic_occurrence_count"` + DiagnosticsTruncated bool `json:"diagnostics_truncated"` + ValidationStatus string `json:"validation_status"` + ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"` + DebugDirectory string `json:"debug_directory,omitempty"` } func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string) (runResult, error) { @@ -54,15 +58,19 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, } result := runResult{ - SchemaVersion: runResultSchemaVersion, - RunID: output.Manifest.RunID, - PipelineID: output.Manifest.PipelineID, - OutputDirectory: absOutputDirectory, - NormalizedOutputCount: len(output.NormalizeOutputs), - RejectedOutputCount: len(output.Rejected), - WarningCount: warningGroupCount(output.Diagnostics), - ValidationStatus: output.Manifest.ValidationStatus, - ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries), + SchemaVersion: runResultSchemaVersion, + RunID: output.Manifest.RunID, + PipelineID: output.Manifest.PipelineID, + OutputDirectory: absOutputDirectory, + NormalizedOutputCount: len(output.NormalizeOutputs), + RejectedOutputCount: len(output.Rejected), + WarningGroupCount: warningGroupCount(output.Diagnostics), + WarningOccurrenceCount: warningOccurrenceCount(output.Diagnostics), + DiagnosticGroupCount: diagnosticGroupCount(output.Diagnostics), + DiagnosticOccurrenceCount: diagnosticOccurrenceCount(output.Diagnostics), + DiagnosticsTruncated: output.Diagnostics.Truncated, + ValidationStatus: output.Manifest.ValidationStatus, + ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries), } if strings.TrimSpace(debugDirectory) != "" { @@ -99,6 +107,34 @@ func warningGroupCount(diagnostics contracts.DiagnosticCollection) int { return count } +func warningOccurrenceCount(diagnostics contracts.DiagnosticCollection) int { + return occurrenceCountByDisposition(diagnostics, contracts.DiagnosticDispositionWarning, true) +} + +func diagnosticGroupCount(diagnostics contracts.DiagnosticCollection) int { + count := 0 + for _, group := range diagnostics.Groups { + if group.Disposition != contracts.DiagnosticDispositionWarning { + count++ + } + } + return count +} + +func diagnosticOccurrenceCount(diagnostics contracts.DiagnosticCollection) int { + return occurrenceCountByDisposition(diagnostics, contracts.DiagnosticDispositionWarning, false) + diagnostics.UnrepresentedOccurrenceCount +} + +func occurrenceCountByDisposition(diagnostics contracts.DiagnosticCollection, disposition contracts.DiagnosticDisposition, include bool) int { + count := 0 + for _, group := range diagnostics.Groups { + if (group.Disposition == disposition) == include { + count += group.OccurrenceCount + } + } + return count +} + func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary { if len(summaries) == 0 { return nil diff --git a/internal/cli/run_result_command_test.go b/internal/cli/run_result_command_test.go index 477c0f65..ee79e018 100644 --- a/internal/cli/run_result_command_test.go +++ b/internal/cli/run_result_command_test.go @@ -27,7 +27,7 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) { } receipt := decodeRunResultDocument(t, stdout.String()) - if got := receipt["schema_version"]; got != "notarius.run-result.v1" { + if got := receipt["schema_version"]; got != "notarius.run-result.v2" { t.Fatalf("schema_version = %q", got) } if got := receipt["run_id"]; got != productionRunID { @@ -45,8 +45,8 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) { if got := receipt["rejected_output_count"]; got != float64(0) { t.Fatalf("rejected_output_count = %v", got) } - if got := receipt["warning_count"]; got != float64(0) { - t.Fatalf("warning_count = %v", got) + if got := receipt["warning_group_count"]; got != float64(0) || receipt["warning_occurrence_count"] != float64(0) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false { + t.Fatalf("diagnostic counts = %#v", receipt) } if got := receipt["validation_status"]; got != "approved" { t.Fatalf("validation_status = %q", got) @@ -69,13 +69,13 @@ func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) { "run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug", "--json", }, &stdout, &stderr, harness.options()) - if code != 0 || !strings.Contains(stderr.String(), "1 warning(s)") { + if code != 0 || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || !strings.Contains(stderr.String(), "warnings.json") { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } receipt := decodeRunResultDocument(t, stdout.String()) - if got := receipt["warning_count"]; got != float64(1) { - t.Fatalf("warning_count = %v", got) + if got := receipt["warning_group_count"]; got != float64(1) || receipt["warning_occurrence_count"] != float64(1) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false { + t.Fatalf("diagnostic counts = %#v", receipt) } debugDirectory, ok := receipt["debug_directory"].(string) if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) { diff --git a/internal/cli/run_result_test.go b/internal/cli/run_result_test.go index 3245e762..cffe4251 100644 --- a/internal/cli/run_result_test.go +++ b/internal/cli/run_result_test.go @@ -52,8 +52,8 @@ func TestRunResultEncodesRequiredFieldsAndCounts(t *testing.T) { if got := decoded["rejected_output_count"]; got != float64(1) { t.Fatalf("rejected_output_count = %v", got) } - if got := decoded["warning_count"]; got != float64(1) { - t.Fatalf("warning_count = %v", got) + if got := decoded["warning_group_count"]; got != float64(1) || decoded["warning_occurrence_count"] != float64(1) || decoded["diagnostic_group_count"] != float64(0) || decoded["diagnostic_occurrence_count"] != float64(0) || decoded["diagnostics_truncated"] != false { + t.Fatalf("diagnostic counts = %#v", decoded) } if got := decoded["validation_summaries"]; got != nil { t.Fatalf("validation_summaries = %#v, want omitted when empty", got) @@ -183,7 +183,8 @@ func testRunOutput() pipeline.RunOutput { NormalizeOutputs: []contracts.SerializedOutput{{}, {}}, Rejected: []contracts.RejectedOutput{{}}, Diagnostics: contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{{ - Disposition: contracts.DiagnosticDispositionWarning, + Disposition: contracts.DiagnosticDispositionWarning, + OccurrenceCount: 1, }}}, OutputFiles: []contracts.OutputFile{{Name: "index.json"}}, }