Add diagnostic counts to run receipts

This commit is contained in:
2026-08-27 16:50:32 +00:00
parent 54de2b816a
commit af0119cc1d
11 changed files with 95 additions and 47 deletions

View File

@@ -103,11 +103,13 @@ names, requiredness, and configured bindings are part of the
Without **--json**, standard output contains the completed pipeline ID, counts Without **--json**, standard output contains the completed pipeline ID, counts
of normalized and rejected outputs, and the output directory. A debug-enabled of normalized and rejected outputs, and the output directory. A debug-enabled
run also prints its debug-bundle path to standard output. A successful run with run also prints its debug-bundle path to standard output. A successful run with
warnings reports the warning count to standard error. The published JSON bundle actionable process warnings report their group and occurrence counts plus the
durable warning-file path to standard error. Advisory and observation findings
do not produce a warning line. The published JSON bundle
is defined by the [JSON output contract](integrations/json-output.md). is defined by the [JSON output contract](integrations/json-output.md).
With **--json**, successful standard output is exactly one With **--json**, successful standard output is exactly one
`notarius.run-result.v1` JSON document followed by a newline, with no `notarius.run-result.v2` JSON document followed by a newline, with no
human-oriented status or debug-path line. Its fields and compatibility policy human-oriented status or debug-path line. Its fields and compatibility policy
are defined by the [run-result contract](integrations/run-result.md). A caller are defined by the [run-result contract](integrations/run-result.md). A caller
must check for exit status 0 before decoding this output; a failed write can must check for exit status 0 before decoding this output; a failed write can
@@ -171,7 +173,7 @@ go run ./cmd/notarius pipelines list \
Successful commands write their primary result to standard output. Warnings and Successful commands write their primary result to standard output. Warnings and
errors are written to standard error. errors are written to standard error.
For **run --json**, warnings remain on standard error and standard output is a For **run --json**, actionable process warnings remain on standard error and standard output is a
machine-readable success result only. Syntax and runtime diagnostics remain on machine-readable success result only. Syntax and runtime diagnostics remain on
standard error. Parse the result only after the process exits with status 0. standard error. Parse the result only after the process exits with status 0.

View File

@@ -92,7 +92,7 @@ stream and exit-status contract.
## Discover The Published Bundle ## Discover The Published Bundle
Decode the successful stdout document as a supported run-result schema. For Decode the successful stdout document as a supported run-result schema. For
the current contract, `schema_version` is `notarius.run-result.v1`. Tolerate the current contract, `schema_version` is `notarius.run-result.v2`. Tolerate
unknown fields allowed by that version, but reject an unsupported schema unknown fields allowed by that version, but reject an unsupported schema
version. version.
@@ -145,7 +145,8 @@ The JSON encoder always publishes these bundle-management files:
| `index.json` | Discovery document for lane and pipeline-wide artifacts. | | `index.json` | Discovery document for lane and pipeline-wide artifacts. |
| `manifest.json` | Run provenance and result summaries. | | `manifest.json` | Run provenance and result summaries. |
| `rejected.json` | Rejected pipeline outputs. | | `rejected.json` | Rejected pipeline outputs. |
| `warnings.json` | Accepted-output and run warnings. | | `warnings.json` | Actionable process-degradation warnings. |
| `diagnostics.json` | Advisory and observation findings for accepted artifacts. |
The complete configuration also requests two pipeline-wide artifacts: The complete configuration also requests two pipeline-wide artifacts:

View File

@@ -79,7 +79,7 @@ silently treated as fully reviewed by the caller.
## Preserve Provenance And Handle Data Carefully ## Preserve Provenance And Handle Data Carefully
Keep the receipt with the published `manifest.json`, and retain Keep the receipt with the published `manifest.json`, and retain
`rejected.json` and `warnings.json` when review or later provenance requires `rejected.json`, `warnings.json`, and `diagnostics.json` when review or later provenance requires
them. Treat the input, output bundle, cache, debug bundle, and captured process them. Treat the input, output bundle, cache, debug bundle, and captured process
logs as potentially sensitive data. Apply the caller's access controls and logs as potentially sensitive data. Apply the caller's access controls and
retention policy, and avoid copying secrets into arguments, logs, or retention policy, and avoid copying secrets into arguments, logs, or

View File

@@ -9,18 +9,22 @@ Command syntax, streams, and exit statuses are defined in the
## Schema ## Schema
The current schema version is `notarius.run-result.v1`. The current schema version is `notarius.run-result.v2`.
| Field | Required | Meaning | | Field | Required | Meaning |
| --- | --- | --- | | --- | --- | --- |
| `schema_version` | Yes | Exactly `notarius.run-result.v1`. | | `schema_version` | Yes | Exactly `notarius.run-result.v2`. |
| `run_id` | Yes | The finalized Notarius run identifier. | | `run_id` | Yes | The finalized Notarius run identifier. |
| `pipeline_id` | Yes | The effective pipeline identifier. | | `pipeline_id` | Yes | The effective pipeline identifier. |
| `output_directory` | Yes | Absolute path to the published, run-specific output bundle. | | `output_directory` | Yes | Absolute path to the published, run-specific output bundle. |
| `index_file` | For the production JSON output | Logical path `index.json`; omitted for other output modules. | | `index_file` | For the production JSON output | Logical path `index.json`; omitted for other output modules. |
| `normalized_output_count` | Yes | Number of final normalized outputs. | | `normalized_output_count` | Yes | Number of final normalized outputs. |
| `rejected_output_count` | Yes | Number of recorded rejected outputs. | | `rejected_output_count` | Yes | Number of recorded rejected outputs. |
| `warning_count` | Yes | Number of final run warnings. | | `warning_group_count` | Yes | Exact number of actionable warning groups. |
| `warning_occurrence_count` | Yes | Exact occurrences represented by actionable warning groups. |
| `diagnostic_group_count` | Yes | Number of represented advisory and observation groups. |
| `diagnostic_occurrence_count` | Yes | Advisory and observation occurrences, including unrepresented occurrences. |
| `diagnostics_truncated` | Yes | Whether advisory/observation group representation was truncated. |
| `validation_status` | Yes | The final run manifest validation status. | | `validation_status` | Yes | The final run manifest validation status. |
| `validation_summaries` | No | Bounded per-producer validation outcomes; present when producer work ran. | | `validation_summaries` | No | Bounded per-producer validation outcomes; present when producer work ran. |
| `debug_directory` | No | Absolute path to the run-specific debug bundle when requested debug capture completed. | | `debug_directory` | No | Absolute path to the run-specific debug bundle when requested debug capture completed. |
@@ -34,14 +38,18 @@ means one or more otherwise accepted results advanced under validator-failure
```json ```json
{ {
"schema_version": "notarius.run-result.v1", "schema_version": "notarius.run-result.v2",
"run_id": "run-1770000000000000000-0123456789abcdef0123456789abcdef", "run_id": "run-1770000000000000000-0123456789abcdef0123456789abcdef",
"pipeline_id": "dnd-session", "pipeline_id": "dnd-session",
"output_directory": "/work/results/run-1770000000000000000-0123456789abcdef0123456789abcdef", "output_directory": "/work/results/run-1770000000000000000-0123456789abcdef0123456789abcdef",
"index_file": "index.json", "index_file": "index.json",
"normalized_output_count": 6, "normalized_output_count": 6,
"rejected_output_count": 2, "rejected_output_count": 2,
"warning_count": 1, "warning_group_count": 1,
"warning_occurrence_count": 2,
"diagnostic_group_count": 3,
"diagnostic_occurrence_count": 5,
"diagnostics_truncated": false,
"validation_status": "incomplete", "validation_status": "incomplete",
"validation_summaries": [ "validation_summaries": [
{ {

View File

@@ -554,7 +554,7 @@ durable files with the finalized in-memory collections.
This stage is appropriately sized for one `gpt-5.6-terra` prompt. This stage is appropriately sized for one `gpt-5.6-terra` prompt.
## Stage 10 — Introduce Run Result V2 And Quiet CLI Presentation ## Stage 10 — Introduce Run Result V2 And Quiet CLI Presentation
### Goal ### Goal

View File

@@ -102,7 +102,7 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
UnrepresentedOccurrenceCount int `json:"unrepresented_occurrence_count"` UnrepresentedOccurrenceCount int `json:"unrepresented_occurrence_count"`
Groups []contracts.DiagnosticGroup `json:"groups"` Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, output.OutputFiles, "diagnostics.json") }](t, output.OutputFiles, "diagnostics.json")
if diagnosticsFile.SchemaVersion != "notarius.diagnostics.v1" || diagnosticsFile.GroupCount != len(output.Diagnostics.Groups) || !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) || diagnosticsFile.OccurrenceCount != diagnosticOccurrenceCount(output.Diagnostics.Groups)+output.Diagnostics.UnrepresentedOccurrenceCount || diagnosticsFile.Truncated != output.Diagnostics.Truncated || diagnosticsFile.UnrepresentedOccurrenceCount != output.Diagnostics.UnrepresentedOccurrenceCount { if diagnosticsFile.SchemaVersion != "notarius.diagnostics.v1" || diagnosticsFile.GroupCount != len(output.Diagnostics.Groups) || !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) || diagnosticsFile.OccurrenceCount != diagnosticGroupOccurrences(output.Diagnostics.Groups)+output.Diagnostics.UnrepresentedOccurrenceCount || diagnosticsFile.Truncated != output.Diagnostics.Truncated || diagnosticsFile.UnrepresentedOccurrenceCount != output.Diagnostics.UnrepresentedOccurrenceCount {
t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile, output.Diagnostics) t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile, output.Diagnostics)
} }
manifest := decodeAssembledOutput[artifacts.RunManifest](t, output.OutputFiles, "manifest.json") manifest := decodeAssembledOutput[artifacts.RunManifest](t, output.OutputFiles, "manifest.json")
@@ -446,7 +446,7 @@ func (e *assembledSpellExtractor) chunkIndexesSnapshot() []int {
return append([]int(nil), e.chunkIndexes...) return append([]int(nil), e.chunkIndexes...)
} }
func diagnosticOccurrenceCount(groups []contracts.DiagnosticGroup) int { func diagnosticGroupOccurrences(groups []contracts.DiagnosticGroup) int {
count := 0 count := 0
for _, group := range groups { for _, group := range groups {
count += group.OccurrenceCount count += group.OccurrenceCount

View File

@@ -513,8 +513,8 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintf(stdout, "debug=%s\n", debugPath) fmt.Fprintf(stdout, "debug=%s\n", debugPath)
} }
} }
if warningCount := warningGroupCount(output.Diagnostics); warningCount > 0 { if warningGroups := warningGroupCount(output.Diagnostics); warningGroups > 0 {
fmt.Fprintf(stderr, "notarius: run completed with %d warning(s)\n", warningCount) fmt.Fprintf(stderr, "notarius: run completed with %d warning group(s), %d occurrence(s); details=%s\n", warningGroups, warningOccurrenceCount(output.Diagnostics), filepath.Join(runOutputDir, "warnings.json"))
} }
return 0 return 0
} }

View File

@@ -593,7 +593,7 @@ func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")} harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
var stdout, stderr bytes.Buffer var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, harness.options()) code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, harness.options())
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning(s)") { if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || !strings.Contains(stderr.String(), "warnings.json") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
} }
outputPath := filepath.Join(onlyChildDir(t, roots.output), "result.json") outputPath := filepath.Join(onlyChildDir(t, roots.output), "result.json")

View File

@@ -12,20 +12,24 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
) )
const runResultSchemaVersion = "notarius.run-result.v1" const runResultSchemaVersion = "notarius.run-result.v2"
type runResult struct { type runResult struct {
SchemaVersion string `json:"schema_version"` SchemaVersion string `json:"schema_version"`
RunID string `json:"run_id"` RunID string `json:"run_id"`
PipelineID string `json:"pipeline_id"` PipelineID string `json:"pipeline_id"`
OutputDirectory string `json:"output_directory"` OutputDirectory string `json:"output_directory"`
IndexFile string `json:"index_file,omitempty"` IndexFile string `json:"index_file,omitempty"`
NormalizedOutputCount int `json:"normalized_output_count"` NormalizedOutputCount int `json:"normalized_output_count"`
RejectedOutputCount int `json:"rejected_output_count"` RejectedOutputCount int `json:"rejected_output_count"`
WarningCount int `json:"warning_count"` WarningGroupCount int `json:"warning_group_count"`
ValidationStatus string `json:"validation_status"` WarningOccurrenceCount int `json:"warning_occurrence_count"`
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"` DiagnosticGroupCount int `json:"diagnostic_group_count"`
DebugDirectory string `json:"debug_directory,omitempty"` DiagnosticOccurrenceCount int `json:"diagnostic_occurrence_count"`
DiagnosticsTruncated bool `json:"diagnostics_truncated"`
ValidationStatus string `json:"validation_status"`
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
DebugDirectory string `json:"debug_directory,omitempty"`
} }
func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string) (runResult, error) { func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string) (runResult, error) {
@@ -54,15 +58,19 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput,
} }
result := runResult{ result := runResult{
SchemaVersion: runResultSchemaVersion, SchemaVersion: runResultSchemaVersion,
RunID: output.Manifest.RunID, RunID: output.Manifest.RunID,
PipelineID: output.Manifest.PipelineID, PipelineID: output.Manifest.PipelineID,
OutputDirectory: absOutputDirectory, OutputDirectory: absOutputDirectory,
NormalizedOutputCount: len(output.NormalizeOutputs), NormalizedOutputCount: len(output.NormalizeOutputs),
RejectedOutputCount: len(output.Rejected), RejectedOutputCount: len(output.Rejected),
WarningCount: warningGroupCount(output.Diagnostics), WarningGroupCount: warningGroupCount(output.Diagnostics),
ValidationStatus: output.Manifest.ValidationStatus, WarningOccurrenceCount: warningOccurrenceCount(output.Diagnostics),
ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries), DiagnosticGroupCount: diagnosticGroupCount(output.Diagnostics),
DiagnosticOccurrenceCount: diagnosticOccurrenceCount(output.Diagnostics),
DiagnosticsTruncated: output.Diagnostics.Truncated,
ValidationStatus: output.Manifest.ValidationStatus,
ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries),
} }
if strings.TrimSpace(debugDirectory) != "" { if strings.TrimSpace(debugDirectory) != "" {
@@ -99,6 +107,34 @@ func warningGroupCount(diagnostics contracts.DiagnosticCollection) int {
return count return count
} }
func warningOccurrenceCount(diagnostics contracts.DiagnosticCollection) int {
return occurrenceCountByDisposition(diagnostics, contracts.DiagnosticDispositionWarning, true)
}
func diagnosticGroupCount(diagnostics contracts.DiagnosticCollection) int {
count := 0
for _, group := range diagnostics.Groups {
if group.Disposition != contracts.DiagnosticDispositionWarning {
count++
}
}
return count
}
func diagnosticOccurrenceCount(diagnostics contracts.DiagnosticCollection) int {
return occurrenceCountByDisposition(diagnostics, contracts.DiagnosticDispositionWarning, false) + diagnostics.UnrepresentedOccurrenceCount
}
func occurrenceCountByDisposition(diagnostics contracts.DiagnosticCollection, disposition contracts.DiagnosticDisposition, include bool) int {
count := 0
for _, group := range diagnostics.Groups {
if (group.Disposition == disposition) == include {
count += group.OccurrenceCount
}
}
return count
}
func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary { func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary {
if len(summaries) == 0 { if len(summaries) == 0 {
return nil return nil

View File

@@ -27,7 +27,7 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
} }
receipt := decodeRunResultDocument(t, stdout.String()) receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["schema_version"]; got != "notarius.run-result.v1" { if got := receipt["schema_version"]; got != "notarius.run-result.v2" {
t.Fatalf("schema_version = %q", got) t.Fatalf("schema_version = %q", got)
} }
if got := receipt["run_id"]; got != productionRunID { if got := receipt["run_id"]; got != productionRunID {
@@ -45,8 +45,8 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
if got := receipt["rejected_output_count"]; got != float64(0) { if got := receipt["rejected_output_count"]; got != float64(0) {
t.Fatalf("rejected_output_count = %v", got) t.Fatalf("rejected_output_count = %v", got)
} }
if got := receipt["warning_count"]; got != float64(0) { if got := receipt["warning_group_count"]; got != float64(0) || receipt["warning_occurrence_count"] != float64(0) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
t.Fatalf("warning_count = %v", got) t.Fatalf("diagnostic counts = %#v", receipt)
} }
if got := receipt["validation_status"]; got != "approved" { if got := receipt["validation_status"]; got != "approved" {
t.Fatalf("validation_status = %q", got) t.Fatalf("validation_status = %q", got)
@@ -69,13 +69,13 @@ func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
"run", "sample", "--config", roots.config, "--input", roots.input, "run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--debug", "--json", "--chunk_cache", "bypass", "--debug", "--json",
}, &stdout, &stderr, harness.options()) }, &stdout, &stderr, harness.options())
if code != 0 || !strings.Contains(stderr.String(), "1 warning(s)") { if code != 0 || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || !strings.Contains(stderr.String(), "warnings.json") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
} }
receipt := decodeRunResultDocument(t, stdout.String()) receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["warning_count"]; got != float64(1) { if got := receipt["warning_group_count"]; got != float64(1) || receipt["warning_occurrence_count"] != float64(1) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
t.Fatalf("warning_count = %v", got) t.Fatalf("diagnostic counts = %#v", receipt)
} }
debugDirectory, ok := receipt["debug_directory"].(string) debugDirectory, ok := receipt["debug_directory"].(string)
if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) { if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) {

View File

@@ -52,8 +52,8 @@ func TestRunResultEncodesRequiredFieldsAndCounts(t *testing.T) {
if got := decoded["rejected_output_count"]; got != float64(1) { if got := decoded["rejected_output_count"]; got != float64(1) {
t.Fatalf("rejected_output_count = %v", got) t.Fatalf("rejected_output_count = %v", got)
} }
if got := decoded["warning_count"]; got != float64(1) { if got := decoded["warning_group_count"]; got != float64(1) || decoded["warning_occurrence_count"] != float64(1) || decoded["diagnostic_group_count"] != float64(0) || decoded["diagnostic_occurrence_count"] != float64(0) || decoded["diagnostics_truncated"] != false {
t.Fatalf("warning_count = %v", got) t.Fatalf("diagnostic counts = %#v", decoded)
} }
if got := decoded["validation_summaries"]; got != nil { if got := decoded["validation_summaries"]; got != nil {
t.Fatalf("validation_summaries = %#v, want omitted when empty", got) t.Fatalf("validation_summaries = %#v, want omitted when empty", got)
@@ -183,7 +183,8 @@ func testRunOutput() pipeline.RunOutput {
NormalizeOutputs: []contracts.SerializedOutput{{}, {}}, NormalizeOutputs: []contracts.SerializedOutput{{}, {}},
Rejected: []contracts.RejectedOutput{{}}, Rejected: []contracts.RejectedOutput{{}},
Diagnostics: contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{{ Diagnostics: contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{{
Disposition: contracts.DiagnosticDispositionWarning, Disposition: contracts.DiagnosticDispositionWarning,
OccurrenceCount: 1,
}}}, }}},
OutputFiles: []contracts.OutputFile{{Name: "index.json"}}, OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
} }