eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add diagnostic counts to run receipts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-27 16:50:32 +00:00
parent 54de2b816a
commit af0119cc1d
11 changed files with 95 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
internal/cli/assembled_spell_pipeline_contract_test.go
View File

@@ -102,7 +102,7 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
UnrepresentedOccurrenceCount int `json:"unrepresented_occurrence_count"`
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, output.OutputFiles, "diagnostics.json")
if diagnosticsFile.SchemaVersion != "notarius.diagnostics.v1" || diagnosticsFile.GroupCount != len(output.Diagnostics.Groups) || !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) || diagnosticsFile.OccurrenceCount != diagnosticOccurrenceCount(output.Diagnostics.Groups)+output.Diagnostics.UnrepresentedOccurrenceCount || diagnosticsFile.Truncated != output.Diagnostics.Truncated || diagnosticsFile.UnrepresentedOccurrenceCount != output.Diagnostics.UnrepresentedOccurrenceCount {
if diagnosticsFile.SchemaVersion != "notarius.diagnostics.v1" || diagnosticsFile.GroupCount != len(output.Diagnostics.Groups) || !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) || diagnosticsFile.OccurrenceCount != diagnosticGroupOccurrences(output.Diagnostics.Groups)+output.Diagnostics.UnrepresentedOccurrenceCount || diagnosticsFile.Truncated != output.Diagnostics.Truncated || diagnosticsFile.UnrepresentedOccurrenceCount != output.Diagnostics.UnrepresentedOccurrenceCount {
t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile, output.Diagnostics)
}
manifest := decodeAssembledOutput[artifacts.RunManifest](t, output.OutputFiles, "manifest.json")
@@ -446,7 +446,7 @@ func (e *assembledSpellExtractor) chunkIndexesSnapshot() []int {
return append([]int(nil), e.chunkIndexes...)
}
func diagnosticOccurrenceCount(groups []contracts.DiagnosticGroup) int {
func diagnosticGroupOccurrences(groups []contracts.DiagnosticGroup) int {
count := 0
for _, group := range groups {
count += group.OccurrenceCount

4
internal/cli/run.go
View File

@@ -513,8 +513,8 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintf(stdout, "debug=%s\n", debugPath)
}
}
if warningCount := warningGroupCount(output.Diagnostics); warningCount > 0 {
fmt.Fprintf(stderr, "notarius: run completed with %d warning(s)\n", warningCount)
if warningGroups := warningGroupCount(output.Diagnostics); warningGroups > 0 {
fmt.Fprintf(stderr, "notarius: run completed with %d warning group(s), %d occurrence(s); details=%s\n", warningGroups, warningOccurrenceCount(output.Diagnostics), filepath.Join(runOutputDir, "warnings.json"))
}
return 0
}

2
internal/cli/run_contract_test.go
View File

@@ -593,7 +593,7 @@ func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, harness.options())
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning(s)") {
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || !strings.Contains(stderr.String(), "warnings.json") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
outputPath := filepath.Join(onlyChildDir(t, roots.output), "result.json")

78
internal/cli/run_result.go
View File

@@ -12,20 +12,24 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const runResultSchemaVersion = "notarius.run-result.v1"
const runResultSchemaVersion = "notarius.run-result.v2"
type runResult struct {
SchemaVersion string `json:"schema_version"`
RunID string `json:"run_id"`
PipelineID string `json:"pipeline_id"`
OutputDirectory string `json:"output_directory"`
IndexFile string `json:"index_file,omitempty"`
NormalizedOutputCount int `json:"normalized_output_count"`
RejectedOutputCount int `json:"rejected_output_count"`
WarningCount int `json:"warning_count"`
ValidationStatus string `json:"validation_status"`
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
DebugDirectory string `json:"debug_directory,omitempty"`
SchemaVersion string `json:"schema_version"`
RunID string `json:"run_id"`
PipelineID string `json:"pipeline_id"`
OutputDirectory string `json:"output_directory"`
IndexFile string `json:"index_file,omitempty"`
NormalizedOutputCount int `json:"normalized_output_count"`
RejectedOutputCount int `json:"rejected_output_count"`
WarningGroupCount int `json:"warning_group_count"`
WarningOccurrenceCount int `json:"warning_occurrence_count"`
DiagnosticGroupCount int `json:"diagnostic_group_count"`
DiagnosticOccurrenceCount int `json:"diagnostic_occurrence_count"`
DiagnosticsTruncated bool `json:"diagnostics_truncated"`
ValidationStatus string `json:"validation_status"`
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
DebugDirectory string `json:"debug_directory,omitempty"`
}
func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string) (runResult, error) {
@@ -54,15 +58,19 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput,
}
result := runResult{
SchemaVersion: runResultSchemaVersion,
RunID: output.Manifest.RunID,
PipelineID: output.Manifest.PipelineID,
OutputDirectory: absOutputDirectory,
NormalizedOutputCount: len(output.NormalizeOutputs),
RejectedOutputCount: len(output.Rejected),
WarningCount: warningGroupCount(output.Diagnostics),
ValidationStatus: output.Manifest.ValidationStatus,
ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries),
SchemaVersion: runResultSchemaVersion,
RunID: output.Manifest.RunID,
PipelineID: output.Manifest.PipelineID,
OutputDirectory: absOutputDirectory,
NormalizedOutputCount: len(output.NormalizeOutputs),
RejectedOutputCount: len(output.Rejected),
WarningGroupCount: warningGroupCount(output.Diagnostics),
WarningOccurrenceCount: warningOccurrenceCount(output.Diagnostics),
DiagnosticGroupCount: diagnosticGroupCount(output.Diagnostics),
DiagnosticOccurrenceCount: diagnosticOccurrenceCount(output.Diagnostics),
DiagnosticsTruncated: output.Diagnostics.Truncated,
ValidationStatus: output.Manifest.ValidationStatus,
ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries),
}
if strings.TrimSpace(debugDirectory) != "" {
@@ -99,6 +107,34 @@ func warningGroupCount(diagnostics contracts.DiagnosticCollection) int {
return count
}
func warningOccurrenceCount(diagnostics contracts.DiagnosticCollection) int {
return occurrenceCountByDisposition(diagnostics, contracts.DiagnosticDispositionWarning, true)
}
func diagnosticGroupCount(diagnostics contracts.DiagnosticCollection) int {
count := 0
for _, group := range diagnostics.Groups {
if group.Disposition != contracts.DiagnosticDispositionWarning {
count++
}
}
return count
}
func diagnosticOccurrenceCount(diagnostics contracts.DiagnosticCollection) int {
return occurrenceCountByDisposition(diagnostics, contracts.DiagnosticDispositionWarning, false) + diagnostics.UnrepresentedOccurrenceCount
}
func occurrenceCountByDisposition(diagnostics contracts.DiagnosticCollection, disposition contracts.DiagnosticDisposition, include bool) int {
count := 0
for _, group := range diagnostics.Groups {
if (group.Disposition == disposition) == include {
count += group.OccurrenceCount
}
}
return count
}
func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary {
if len(summaries) == 0 {
return nil

12
internal/cli/run_result_command_test.go
View File

@@ -27,7 +27,7 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
}
receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["schema_version"]; got != "notarius.run-result.v1" {
if got := receipt["schema_version"]; got != "notarius.run-result.v2" {
t.Fatalf("schema_version = %q", got)
}
if got := receipt["run_id"]; got != productionRunID {
@@ -45,8 +45,8 @@ func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
if got := receipt["rejected_output_count"]; got != float64(0) {
t.Fatalf("rejected_output_count = %v", got)
}
if got := receipt["warning_count"]; got != float64(0) {
t.Fatalf("warning_count = %v", got)
if got := receipt["warning_group_count"]; got != float64(0) || receipt["warning_occurrence_count"] != float64(0) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
t.Fatalf("diagnostic counts = %#v", receipt)
}
if got := receipt["validation_status"]; got != "approved" {
t.Fatalf("validation_status = %q", got)
@@ -69,13 +69,13 @@ func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--debug", "--json",
}, &stdout, &stderr, harness.options())
if code != 0 || !strings.Contains(stderr.String(), "1 warning(s)") {
if code != 0 || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || !strings.Contains(stderr.String(), "warnings.json") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["warning_count"]; got != float64(1) {
t.Fatalf("warning_count = %v", got)
if got := receipt["warning_group_count"]; got != float64(1) || receipt["warning_occurrence_count"] != float64(1) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
t.Fatalf("diagnostic counts = %#v", receipt)
}
debugDirectory, ok := receipt["debug_directory"].(string)
if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) {

7
internal/cli/run_result_test.go
View File

@@ -52,8 +52,8 @@ func TestRunResultEncodesRequiredFieldsAndCounts(t *testing.T) {
if got := decoded["rejected_output_count"]; got != float64(1) {
t.Fatalf("rejected_output_count = %v", got)
}
if got := decoded["warning_count"]; got != float64(1) {
t.Fatalf("warning_count = %v", got)
if got := decoded["warning_group_count"]; got != float64(1) || decoded["warning_occurrence_count"] != float64(1) || decoded["diagnostic_group_count"] != float64(0) || decoded["diagnostic_occurrence_count"] != float64(0) || decoded["diagnostics_truncated"] != false {
t.Fatalf("diagnostic counts = %#v", decoded)
}
if got := decoded["validation_summaries"]; got != nil {
t.Fatalf("validation_summaries = %#v, want omitted when empty", got)
@@ -183,7 +183,8 @@ func testRunOutput() pipeline.RunOutput {
NormalizeOutputs: []contracts.SerializedOutput{{}, {}},
Rejected: []contracts.RejectedOutput{{}},
Diagnostics: contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{{
Disposition: contracts.DiagnosticDispositionWarning,
Disposition: contracts.DiagnosticDispositionWarning,
OccurrenceCount: 1,
}}},
OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
}