eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden diagnostic handling and warning presentation

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-27 18:41:59 +00:00
parent 1025001f20
commit 079d5af337
67 changed files with 518 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
internal/cli/assembled_spell_pipeline_contract_test.go
View File

@@ -71,17 +71,17 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
t.Fatalf("distinct cast = %#v, want separate evidence event", distinct)
}
wantWarningReasons := []string{
wantDiagnosticReasons := []string{
spellnormalize.ReasonCodeSpellNameCanonicalized,
spellnormalize.ReasonCodeSourceReferencesNormalized,
spellnormalize.ReasonCodeDuplicateSpellCastCollapsed,
"spell_not_near_source",
}
gotWarningReasons := make([]string, len(output.Diagnostics.Groups))
gotDiagnosticReasons := make([]string, len(output.Diagnostics.Groups))
for index, group := range output.Diagnostics.Groups {
gotWarningReasons[index] = group.ReasonCode
gotDiagnosticReasons[index] = group.ReasonCode
}
if !reflect.DeepEqual(gotWarningReasons, wantWarningReasons) {
if !reflect.DeepEqual(gotDiagnosticReasons, wantDiagnosticReasons) {
t.Fatalf("diagnostics = %#v, want deterministic normalize and validation diagnostics", output.Diagnostics)
}
if output.Diagnostics.Groups[2].Samples[0].Scope != "spell_casts[0]" || !strings.Contains(output.Diagnostics.Groups[2].Samples[0].Message, "retained input index 0") || !strings.Contains(output.Diagnostics.Groups[2].Samples[0].Message, "removed input indices [1]") {
@@ -173,7 +173,7 @@ func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
}
}
func TestAssembledSpellPipelinePromotesTerminalUnknownSpellWarning(t *testing.T) {
func TestAssembledSpellPipelinePromotesTerminalUnknownSpellDiagnostics(t *testing.T) {
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{unknownSpell: true})
resolved.Steps[0].ArtifactLanes[0].NormalizeValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
@@ -206,7 +206,7 @@ func TestAssembledSpellPipelinePromotesTerminalUnknownSpellWarning(t *testing.T)
}
}
func TestAssembledSpellPipelinePromotesUnknownSpellWarningWhenOverrideAccepts(t *testing.T) {
func TestAssembledSpellPipelinePromotesUnknownSpellAdvisoryWhenOverrideAccepts(t *testing.T) {
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true, unknownSpell: true})
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
if err != nil {

30
internal/cli/run.go
View File

@@ -471,7 +471,10 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
Debug: debugRecorder,
ExtractWorkers: cfg.Concurrency.StageWorkers["extract"],
})
commandState.observeOutput(output)
diagnosticProjection, diagnosticErr := contracts.ProjectDiagnosticCollection(output.Diagnostics)
if diagnosticErr == nil {
commandState.observeOutput(output, diagnosticProjection)
}
if err != nil {
primaryErr := fmt.Errorf("run pipeline %q: %w", pipelineID, err)
if output.Manifest.PipelineID != "" {
@@ -479,15 +482,21 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("write debug summary: %w", summaryErr))
}
}
if diagnosticErr != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr))
}
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr)
}
if diagnosticErr != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr))
}
if err := writePartialSummary(summary, output); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug summary: %w", err))
}
var encodedResult []byte
if *machineOutput {
result, err := newRunResult(effective.ResolvedPipeline, output, runOutputDir, debugPath)
result, err := newRunResultWithDiagnostics(effective.ResolvedPipeline, output, runOutputDir, debugPath, diagnosticProjection)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
@@ -513,12 +522,25 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintf(stdout, "debug=%s\n", debugPath)
}
}
if warningGroups := warningGroupCount(output.Diagnostics); warningGroups > 0 {
fmt.Fprintf(stderr, "notarius: run completed with %d warning group(s), %d occurrence(s); details=%s\n", warningGroups, warningOccurrenceCount(output.Diagnostics), filepath.Join(runOutputDir, "warnings.json"))
if warningGroups := len(diagnosticProjection.Warnings); warningGroups > 0 {
fmt.Fprintf(stderr, "notarius: run completed with %d warning group(s), %d occurrence(s)", warningGroups, diagnosticProjection.WarningOccurrenceCount)
if warningFile, ok := logicalOutputFile(output.OutputFiles, "warnings.json"); ok {
fmt.Fprintf(stderr, "; details=%s", filepath.Join(runOutputDir, warningFile))
}
fmt.Fprintln(stderr)
}
return 0
}
func logicalOutputFile(files []contracts.OutputFile, name string) (string, bool) {
for _, file := range files {
if file.Name == name {
return file.Name, true
}
}
return "", false
}
func writeSummary(summary *debugbundle.SummaryWriter, write func() error) error {
if summary == nil {
return nil

2
internal/cli/run_contract_test.go
View File

@@ -590,6 +590,7 @@ func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.includeWarnings = true
harness.includeWarningFile = true
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, harness.options())
@@ -601,6 +602,7 @@ func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
if err != nil || !strings.Contains(string(output), "contract-warning") {
t.Fatalf("durable output = %q, %v", output, err)
}
assertFile(t, filepath.Join(filepath.Dir(outputPath), "warnings.json"))
bundle := onlyChildDir(t, roots.debug)
var diagnostics contracts.DiagnosticCollection
readStateTestSummaryJSON(t, bundle, "final-diagnostics.json", &diagnostics)

54
internal/cli/run_result.go
View File

@@ -33,6 +33,14 @@ type runResult struct {
}
func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string) (runResult, error) {
diagnosticProjection, err := contracts.ProjectDiagnosticCollection(output.Diagnostics)
if err != nil {
return runResult{}, fmt.Errorf("summarize run diagnostics: %w", err)
}
return newRunResultWithDiagnostics(resolved, output, outputDirectory, debugDirectory, diagnosticProjection)
}
func newRunResultWithDiagnostics(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string, diagnosticProjection contracts.DiagnosticProjection) (runResult, error) {
if strings.TrimSpace(output.Manifest.RunID) == "" {
return runResult{}, fmt.Errorf("run result requires a run ID")
}
@@ -64,10 +72,10 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput,
OutputDirectory: absOutputDirectory,
NormalizedOutputCount: len(output.NormalizeOutputs),
RejectedOutputCount: len(output.Rejected),
WarningGroupCount: warningGroupCount(output.Diagnostics),
WarningOccurrenceCount: warningOccurrenceCount(output.Diagnostics),
DiagnosticGroupCount: diagnosticGroupCount(output.Diagnostics),
DiagnosticOccurrenceCount: diagnosticOccurrenceCount(output.Diagnostics),
WarningGroupCount: len(diagnosticProjection.Warnings),
WarningOccurrenceCount: diagnosticProjection.WarningOccurrenceCount,
DiagnosticGroupCount: len(diagnosticProjection.Diagnostics),
DiagnosticOccurrenceCount: diagnosticProjection.DiagnosticOccurrenceCount,
DiagnosticsTruncated: output.Diagnostics.Truncated,
ValidationStatus: output.Manifest.ValidationStatus,
ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries),
@@ -97,44 +105,6 @@ func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput,
return result, nil
}
func warningGroupCount(diagnostics contracts.DiagnosticCollection) int {
count := 0
for _, group := range diagnostics.Groups {
if group.Disposition == contracts.DiagnosticDispositionWarning {
count++
}
}
return count
}
func warningOccurrenceCount(diagnostics contracts.DiagnosticCollection) int {
return occurrenceCountByDisposition(diagnostics, contracts.DiagnosticDispositionWarning, true)
}
func diagnosticGroupCount(diagnostics contracts.DiagnosticCollection) int {
count := 0
for _, group := range diagnostics.Groups {
if group.Disposition != contracts.DiagnosticDispositionWarning {
count++
}
}
return count
}
func diagnosticOccurrenceCount(diagnostics contracts.DiagnosticCollection) int {
return occurrenceCountByDisposition(diagnostics, contracts.DiagnosticDispositionWarning, false) + diagnostics.UnrepresentedOccurrenceCount
}
func occurrenceCountByDisposition(diagnostics contracts.DiagnosticCollection, disposition contracts.DiagnosticDisposition, include bool) int {
count := 0
for _, group := range diagnostics.Groups {
if (group.Disposition == disposition) == include {
count += group.OccurrenceCount
}
}
return count
}
func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary {
if len(summaries) == 0 {
return nil

2
internal/cli/run_result_command_test.go
View File

@@ -69,7 +69,7 @@ func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--debug", "--json",
}, &stdout, &stderr, harness.options())
if code != 0 || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || !strings.Contains(stderr.String(), "warnings.json") {
if code != 0 || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || strings.Contains(stderr.String(), "warnings.json") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}

4
internal/cli/run_result_test.go
View File

@@ -184,7 +184,11 @@ func testRunOutput() pipeline.RunOutput {
Rejected: []contracts.RejectedOutput{{}},
Diagnostics: contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{{
Disposition: contracts.DiagnosticDispositionWarning,
Category: contracts.DiagnosticCategoryFallback,
ReasonCode: "fallback",
Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: "step", LaneID: "lane", ModuleKey: "module"},
OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{Scope: "scope", Message: "message"}},
}}},
OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
}

11
internal/cli/run_terminal.go
View File

@@ -6,6 +6,7 @@ import (
"io"
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -33,16 +34,16 @@ func (s *pipelineCommandState) setDebugPath(debugPath string) {
}
}
func (s *pipelineCommandState) observeOutput(output pipeline.RunOutput) {
func (s *pipelineCommandState) observeOutput(output pipeline.RunOutput, diagnostics contracts.DiagnosticProjection) {
if s == nil {
return
}
s.report.OutputCount = len(output.NormalizeOutputs)
s.report.RejectedCount = len(output.Rejected)
s.report.WarningGroupCount = warningGroupCount(output.Diagnostics)
s.report.WarningOccurrenceCount = warningOccurrenceCount(output.Diagnostics)
s.report.DiagnosticGroupCount = diagnosticGroupCount(output.Diagnostics)
s.report.DiagnosticOccurrenceCount = diagnosticOccurrenceCount(output.Diagnostics)
s.report.WarningGroupCount = len(diagnostics.Warnings)
s.report.WarningOccurrenceCount = diagnostics.WarningOccurrenceCount
s.report.DiagnosticGroupCount = len(diagnostics.Diagnostics)
s.report.DiagnosticOccurrenceCount = diagnostics.DiagnosticOccurrenceCount
s.report.DiagnosticsTruncated = output.Diagnostics.Truncated
s.report.ValidationStatus = output.Manifest.ValidationStatus
}

22
internal/cli/spell_catalog_retry_contract_test.go
View File

@@ -19,12 +19,12 @@ import (
func TestProductionSpellCatalogValidationRetries(t *testing.T) {
const retries = 2
tests := []struct {
name string
responses []string
wantCalls int
wantRejected bool
wantSpell string
wantWarningCode string
name string
responses []string
wantCalls int
wantRejected bool
wantSpell string
wantAdvisoryCode string
}{
{
name: "unknown spell remains rejected after exhaustion",
@@ -42,9 +42,9 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
productionSpellResponse("Unknown Spell"),
productionSpellResponse("Aegis of Emberfall"),
},
wantCalls: 2,
wantSpell: "Aegis of Emberfall",
wantWarningCode: "spell_not_near_source",
wantCalls: 2,
wantSpell: "Aegis of Emberfall",
wantAdvisoryCode: "spell_not_near_source",
},
}
@@ -108,14 +108,14 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != tt.wantSpell {
t.Fatalf("normalized spell list = %#v, want accepted overlay spell", value)
}
if len(output.Diagnostics.Groups) != 2 || output.Diagnostics.Groups[0].ReasonCode != tt.wantWarningCode {
if len(output.Diagnostics.Groups) != 2 || output.Diagnostics.Groups[0].ReasonCode != tt.wantAdvisoryCode {
t.Fatalf("diagnostics = %#v, want terminal extract and normalize diagnostics", output.Diagnostics)
}
if len(output.Diagnostics.Groups) != 2 {
t.Fatalf("diagnostics = %#v, want extract and normalize data-quality advisories", output.Diagnostics)
}
for _, diagnostic := range output.Diagnostics.Groups {
if diagnostic.Disposition != contracts.DiagnosticDispositionAdvisory || diagnostic.ReasonCode != tt.wantWarningCode {
if diagnostic.Disposition != contracts.DiagnosticDispositionAdvisory || diagnostic.ReasonCode != tt.wantAdvisoryCode {
t.Fatalf("diagnostics = %#v, want only data-quality advisories", output.Diagnostics)
}
}

7
internal/cli/state_hardening_test.go
View File

@@ -868,6 +868,7 @@ type stateTestHarness struct {
sessionIDs []string
outputDiagnostics contracts.DiagnosticCollection
includeWarnings bool
includeWarningFile bool
}
func newStateTestHarness() *stateTestHarness { return &stateTestHarness{} }
@@ -1012,7 +1013,11 @@ func (o stateTestOutput) Encode(_ context.Context, req contracts.OutputRequest)
if o.includeDiagnostics && len(req.Diagnostics.Groups) > 0 {
data = []byte(fmt.Sprintf("{\"ok\":true,\"diagnostics\":%q}\n", req.Diagnostics.Groups[0].ReasonCode))
}
return contracts.OutputResult{Files: []contracts.OutputFile{{Name: "result.json", Bytes: data}}}, nil
files := []contracts.OutputFile{{Name: "result.json", Bytes: data}}
if o.harness.includeWarningFile {
files = append(files, contracts.OutputFile{Name: "warnings.json", Bytes: []byte("{\"warnings\":true}\n")})
}
return contracts.OutputResult{Files: files}, nil
}
func stateTestDiagnostic(scope, reasonCode, message string) contracts.ProducerDiagnostic {