761 lines
29 KiB
Go
761 lines
29 KiB
Go
package notarius
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
|
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
|
)
|
|
|
|
const (
|
|
maxReceiptBytes = 1 << 20
|
|
maxIndexBytes = 4 << 20
|
|
maxSummaryBytes = 4 << 20
|
|
canonicalIndexFile = "index.json"
|
|
canonicalManifestFile = "manifest.json"
|
|
canonicalRejectedFile = "rejected.json"
|
|
canonicalWarningsFile = "warnings.json"
|
|
canonicalDiagnosticsFile = "diagnostics.json"
|
|
warningsSchemaVersion = "notarius.warnings.v2"
|
|
diagnosticsSchemaVersion = "notarius.diagnostics.v1"
|
|
maxWarningGroups = 128
|
|
maxDiagnosticGroups = 256
|
|
maxFindingSamples = 3
|
|
)
|
|
|
|
type subprocessRun func(context.Context, subprocess.RunRequest) (subprocess.RunResult, error)
|
|
|
|
// SubprocessRunner invokes Notarius through its public CLI.
|
|
type SubprocessRunner struct {
|
|
run subprocessRun
|
|
}
|
|
|
|
// NewSubprocessRunner constructs a production Notarius subprocess runner.
|
|
func NewSubprocessRunner() *SubprocessRunner {
|
|
return &SubprocessRunner{run: subprocess.Run}
|
|
}
|
|
|
|
// Run executes a complete Notarius pipeline and discovers its published bundle.
|
|
func (r *SubprocessRunner) Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
|
if r == nil || r.run == nil {
|
|
return RunResult{}, fmt.Errorf("notarius subprocess runner is nil")
|
|
}
|
|
if err := validateRunRequest(req); err != nil {
|
|
return RunResult{}, err
|
|
}
|
|
|
|
args := []string{
|
|
"run", req.PipelineID,
|
|
"--config", req.ConfigPath,
|
|
"--input", req.InputPath,
|
|
"--output-dir", req.OutputRoot,
|
|
"--json",
|
|
}
|
|
processResult, err := r.run(ctx, subprocess.RunRequest{
|
|
Executable: req.Binary,
|
|
Args: args,
|
|
WorkingDir: req.WorkingDirectory,
|
|
Timeout: req.Timeout,
|
|
DiagnosticOwner: "notarius",
|
|
StdoutLogPath: req.ReceiptPath,
|
|
StderrLogPath: req.LogPath,
|
|
})
|
|
baseResult := RunResult{
|
|
ReceiptPath: req.ReceiptPath,
|
|
LogPath: req.LogPath,
|
|
ExitCode: processResult.ExitCode,
|
|
Duration: processResult.Duration,
|
|
}
|
|
if err != nil {
|
|
return baseResult, fmt.Errorf("run notarius pipeline %q: %w", req.PipelineID, err)
|
|
}
|
|
|
|
receipt, err := loadReceipt(req.ReceiptPath, req.PipelineID)
|
|
if err != nil {
|
|
return baseResult, err
|
|
}
|
|
bundleRoot, err := validateBundleRoot(req.OutputRoot, receipt.OutputDirectory)
|
|
if err != nil {
|
|
return baseResult, err
|
|
}
|
|
indexPath, err := resolveRegularFile(bundleRoot, receipt.IndexFile)
|
|
if err != nil {
|
|
return baseResult, fmt.Errorf("resolve receipt index file: %w", err)
|
|
}
|
|
index, err := loadIndex(bundleRoot, indexPath)
|
|
if err != nil {
|
|
return baseResult, err
|
|
}
|
|
rejections, err := loadRejections(index.RejectedPath)
|
|
if err != nil {
|
|
return baseResult, err
|
|
}
|
|
warnings, err := loadWarnings(index.WarningsPath)
|
|
if err != nil {
|
|
return baseResult, err
|
|
}
|
|
diagnostics, diagnosticOccurrences, diagnosticsTruncated, err := loadDiagnostics(index.DiagnosticsPath)
|
|
if err != nil {
|
|
return baseResult, err
|
|
}
|
|
if receipt.NormalizedOutputCount != len(index.Lanes) || receipt.RejectedOutputCount != len(rejections) ||
|
|
receipt.WarningGroupCount != len(warnings) || receipt.DiagnosticGroupCount != len(diagnostics) {
|
|
return baseResult, fmt.Errorf("notarius receipt counts do not match published bundle")
|
|
}
|
|
warningOccurrences, err := sumWarningOccurrences(warnings)
|
|
if err != nil {
|
|
return baseResult, err
|
|
}
|
|
if receipt.WarningOccurrenceCount != warningOccurrences ||
|
|
receipt.DiagnosticOccurrenceCount != diagnosticOccurrences ||
|
|
receipt.DiagnosticsTruncated != diagnosticsTruncated {
|
|
return baseResult, fmt.Errorf("notarius receipt occurrence counts do not match published bundle")
|
|
}
|
|
|
|
baseResult.Receipt = receipt
|
|
baseResult.Index = index
|
|
baseResult.BundleRoot = bundleRoot
|
|
baseResult.Rejections = rejections
|
|
baseResult.Warnings = warnings
|
|
baseResult.Diagnostics = diagnostics
|
|
return baseResult, nil
|
|
}
|
|
|
|
func validateRunRequest(req RunRequest) error {
|
|
if strings.TrimSpace(req.Binary) == "" {
|
|
return fmt.Errorf("notarius binary is required")
|
|
}
|
|
if strings.TrimSpace(req.PipelineID) == "" {
|
|
return fmt.Errorf("notarius pipeline id is required")
|
|
}
|
|
if req.Timeout <= 0 {
|
|
return fmt.Errorf("notarius timeout must be positive")
|
|
}
|
|
for label, path := range map[string]string{
|
|
"config": req.ConfigPath,
|
|
"input": req.InputPath,
|
|
"output root": req.OutputRoot,
|
|
"working directory": req.WorkingDirectory,
|
|
"receipt": req.ReceiptPath,
|
|
"log": req.LogPath,
|
|
} {
|
|
if strings.TrimSpace(path) == "" {
|
|
return fmt.Errorf("notarius %s path is required", label)
|
|
}
|
|
if !filepath.IsAbs(path) {
|
|
return fmt.Errorf("notarius %s path must be absolute", label)
|
|
}
|
|
}
|
|
if filepath.Clean(req.ReceiptPath) == filepath.Clean(req.LogPath) {
|
|
return fmt.Errorf("notarius receipt and log paths must be different")
|
|
}
|
|
if err := requireRegularFile(req.ConfigPath); err != nil {
|
|
return fmt.Errorf("validate notarius config path: %w", err)
|
|
}
|
|
if err := requireRegularFile(req.InputPath); err != nil {
|
|
return fmt.Errorf("validate notarius input path: %w", err)
|
|
}
|
|
if err := requireDirectory(req.OutputRoot); err != nil {
|
|
return fmt.Errorf("validate notarius output root: %w", err)
|
|
}
|
|
if err := requireDirectory(req.WorkingDirectory); err != nil {
|
|
return fmt.Errorf("validate notarius working directory: %w", err)
|
|
}
|
|
if err := validateLogDestination(req.ReceiptPath); err != nil {
|
|
return fmt.Errorf("validate notarius receipt path: %w", err)
|
|
}
|
|
if err := validateLogDestination(req.LogPath); err != nil {
|
|
return fmt.Errorf("validate notarius log path: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type receiptDocument struct {
|
|
SchemaVersion string `json:"schema_version"`
|
|
RunID string `json:"run_id"`
|
|
PipelineID string `json:"pipeline_id"`
|
|
OutputDirectory string `json:"output_directory"`
|
|
IndexFile string `json:"index_file"`
|
|
NormalizedOutputCount *int `json:"normalized_output_count"`
|
|
RejectedOutputCount *int `json:"rejected_output_count"`
|
|
WarningGroupCount *int `json:"warning_group_count"`
|
|
WarningOccurrenceCount *int `json:"warning_occurrence_count"`
|
|
DiagnosticGroupCount *int `json:"diagnostic_group_count"`
|
|
DiagnosticOccurrenceCount *int `json:"diagnostic_occurrence_count"`
|
|
DiagnosticsTruncated *bool `json:"diagnostics_truncated"`
|
|
ValidationStatus string `json:"validation_status"`
|
|
ValidationSummaries []validationSummaryDocument `json:"validation_summaries"`
|
|
DebugDirectory string `json:"debug_directory"`
|
|
}
|
|
|
|
type validationSummaryDocument struct {
|
|
Stage string `json:"stage"`
|
|
StepID string `json:"step_id"`
|
|
LaneID string `json:"lane_id"`
|
|
ModuleKey string `json:"module_key"`
|
|
ChunkID string `json:"chunk_id"`
|
|
Status string `json:"status"`
|
|
RejectingValidators []string `json:"rejecting_validators"`
|
|
ReasonCodes []string `json:"reason_codes"`
|
|
IncompleteValidators []string `json:"incomplete_validators"`
|
|
ProducerAttemptCount *int `json:"producer_attempt_count"`
|
|
TerminalAction string `json:"terminal_action"`
|
|
}
|
|
|
|
func validValidationStatus(value string) bool {
|
|
switch value {
|
|
case "approved", "rejected", "incomplete":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func validateValidationSummaries(documents []validationSummaryDocument) ([]ValidationSummary, error) {
|
|
summaries := make([]ValidationSummary, 0, len(documents))
|
|
for _, document := range documents {
|
|
if document.Status != "complete" && document.Status != "rejected" && document.Status != "incomplete" {
|
|
return nil, fmt.Errorf("notarius validation summary status %q is invalid", document.Status)
|
|
}
|
|
if document.ProducerAttemptCount == nil || *document.ProducerAttemptCount <= 0 || !validTerminalAction(document.TerminalAction) {
|
|
return nil, fmt.Errorf("notarius validation summary is missing required fields")
|
|
}
|
|
summaries = append(summaries, ValidationSummary{
|
|
Stage: document.Stage, StepID: document.StepID, LaneID: document.LaneID,
|
|
ModuleKey: document.ModuleKey, ChunkID: document.ChunkID, Status: document.Status,
|
|
RejectingValidators: append([]string(nil), document.RejectingValidators...),
|
|
ReasonCodes: append([]string(nil), document.ReasonCodes...),
|
|
IncompleteValidators: append([]string(nil), document.IncompleteValidators...),
|
|
ProducerAttemptCount: *document.ProducerAttemptCount, TerminalAction: document.TerminalAction,
|
|
})
|
|
}
|
|
return summaries, nil
|
|
}
|
|
|
|
func validTerminalAction(value string) bool {
|
|
switch value {
|
|
case "accepted", "reject_output", "warn_continue", "fail_run":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func loadReceipt(path, pipelineID string) (Receipt, error) {
|
|
var document receiptDocument
|
|
if err := decodeBoundedJSON(path, maxReceiptBytes, &document); err != nil {
|
|
return Receipt{}, fmt.Errorf("decode notarius receipt: %w", err)
|
|
}
|
|
if document.SchemaVersion != ReceiptSchemaVersion {
|
|
return Receipt{}, fmt.Errorf("unsupported notarius receipt schema version %q", document.SchemaVersion)
|
|
}
|
|
if strings.TrimSpace(document.RunID) == "" || strings.TrimSpace(document.PipelineID) == "" ||
|
|
strings.TrimSpace(document.OutputDirectory) == "" || strings.TrimSpace(document.ValidationStatus) == "" ||
|
|
document.NormalizedOutputCount == nil ||
|
|
document.RejectedOutputCount == nil || document.WarningGroupCount == nil ||
|
|
document.WarningOccurrenceCount == nil || document.DiagnosticGroupCount == nil ||
|
|
document.DiagnosticOccurrenceCount == nil || document.DiagnosticsTruncated == nil {
|
|
return Receipt{}, fmt.Errorf("notarius receipt is missing required fields")
|
|
}
|
|
if document.IndexFile != canonicalIndexFile {
|
|
return Receipt{}, fmt.Errorf("notarius receipt index_file %q is incompatible; want %q", document.IndexFile, canonicalIndexFile)
|
|
}
|
|
if document.PipelineID != pipelineID {
|
|
return Receipt{}, fmt.Errorf("notarius receipt pipeline id %q does not match requested pipeline %q", document.PipelineID, pipelineID)
|
|
}
|
|
if *document.NormalizedOutputCount < 0 || *document.RejectedOutputCount < 0 ||
|
|
*document.WarningGroupCount < 0 || *document.WarningOccurrenceCount < 0 ||
|
|
*document.DiagnosticGroupCount < 0 || *document.DiagnosticOccurrenceCount < 0 {
|
|
return Receipt{}, fmt.Errorf("notarius receipt counts must be non-negative")
|
|
}
|
|
if !validValidationStatus(document.ValidationStatus) {
|
|
return Receipt{}, fmt.Errorf("notarius receipt validation_status %q is invalid", document.ValidationStatus)
|
|
}
|
|
validationSummaries, err := validateValidationSummaries(document.ValidationSummaries)
|
|
if err != nil {
|
|
return Receipt{}, err
|
|
}
|
|
if !filepath.IsAbs(document.OutputDirectory) {
|
|
return Receipt{}, fmt.Errorf("notarius receipt output directory must be absolute")
|
|
}
|
|
if document.DebugDirectory != "" && !filepath.IsAbs(document.DebugDirectory) {
|
|
return Receipt{}, fmt.Errorf("notarius receipt debug directory must be absolute when present")
|
|
}
|
|
return Receipt{
|
|
SchemaVersion: document.SchemaVersion,
|
|
RunID: document.RunID,
|
|
PipelineID: document.PipelineID,
|
|
OutputDirectory: filepath.Clean(document.OutputDirectory),
|
|
IndexFile: document.IndexFile,
|
|
NormalizedOutputCount: *document.NormalizedOutputCount,
|
|
RejectedOutputCount: *document.RejectedOutputCount,
|
|
WarningGroupCount: *document.WarningGroupCount,
|
|
WarningOccurrenceCount: *document.WarningOccurrenceCount,
|
|
DiagnosticGroupCount: *document.DiagnosticGroupCount,
|
|
DiagnosticOccurrenceCount: *document.DiagnosticOccurrenceCount,
|
|
DiagnosticsTruncated: *document.DiagnosticsTruncated,
|
|
ValidationStatus: document.ValidationStatus,
|
|
ValidationSummaries: validationSummaries,
|
|
DebugDirectory: document.DebugDirectory,
|
|
}, nil
|
|
}
|
|
|
|
type indexDocument struct {
|
|
ManifestFile string `json:"manifest_file"`
|
|
OutputFiles *[]laneDocument `json:"output_files"`
|
|
RejectedFile string `json:"rejected_file"`
|
|
WarningsFile string `json:"warnings_file"`
|
|
DiagnosticsFile string `json:"diagnostics_file"`
|
|
ChunkMap *pipelineDocument `json:"chunk_map"`
|
|
EvidenceContext *pipelineDocument `json:"evidence_context"`
|
|
}
|
|
|
|
type laneDocument struct {
|
|
LaneID string `json:"lane_id"`
|
|
File string `json:"file"`
|
|
MediaType string `json:"media_type"`
|
|
ModuleKey string `json:"module_key"`
|
|
SchemaID string `json:"schema_id"`
|
|
SchemaName string `json:"schema_name"`
|
|
SchemaVersion string `json:"schema_version"`
|
|
}
|
|
|
|
type pipelineDocument struct {
|
|
ArtifactKind string `json:"artifact_kind"`
|
|
File string `json:"file"`
|
|
MediaType string `json:"media_type"`
|
|
SchemaID string `json:"schema_id"`
|
|
SchemaName string `json:"schema_name"`
|
|
SchemaVersion string `json:"schema_version"`
|
|
}
|
|
|
|
func loadIndex(bundleRoot, indexPath string) (Index, error) {
|
|
var document indexDocument
|
|
if err := decodeBoundedJSON(indexPath, maxIndexBytes, &document); err != nil {
|
|
return Index{}, fmt.Errorf("decode notarius index: %w", err)
|
|
}
|
|
for _, field := range []struct {
|
|
name string
|
|
got string
|
|
want string
|
|
}{
|
|
{name: "manifest_file", got: document.ManifestFile, want: canonicalManifestFile},
|
|
{name: "rejected_file", got: document.RejectedFile, want: canonicalRejectedFile},
|
|
{name: "warnings_file", got: document.WarningsFile, want: canonicalWarningsFile},
|
|
{name: "diagnostics_file", got: document.DiagnosticsFile, want: canonicalDiagnosticsFile},
|
|
} {
|
|
if field.got != field.want {
|
|
return Index{}, fmt.Errorf("notarius index %s %q is incompatible; want %q", field.name, field.got, field.want)
|
|
}
|
|
}
|
|
if document.OutputFiles == nil {
|
|
return Index{}, fmt.Errorf("notarius index is missing required output_files")
|
|
}
|
|
|
|
index := Index{
|
|
Path: indexPath,
|
|
ManifestFile: document.ManifestFile,
|
|
RejectedFile: document.RejectedFile,
|
|
WarningsFile: document.WarningsFile,
|
|
DiagnosticsFile: document.DiagnosticsFile,
|
|
}
|
|
var err error
|
|
if index.ManifestPath, err = resolveRegularFile(bundleRoot, index.ManifestFile); err != nil {
|
|
return Index{}, fmt.Errorf("resolve notarius manifest file: %w", err)
|
|
}
|
|
if index.RejectedPath, err = resolveRegularFile(bundleRoot, index.RejectedFile); err != nil {
|
|
return Index{}, fmt.Errorf("resolve notarius rejection file: %w", err)
|
|
}
|
|
if index.WarningsPath, err = resolveRegularFile(bundleRoot, index.WarningsFile); err != nil {
|
|
return Index{}, fmt.Errorf("resolve notarius warning file: %w", err)
|
|
}
|
|
if index.DiagnosticsPath, err = resolveRegularFile(bundleRoot, index.DiagnosticsFile); err != nil {
|
|
return Index{}, fmt.Errorf("resolve notarius diagnostics file: %w", err)
|
|
}
|
|
|
|
seenLanes := make(map[string]struct{}, len(*document.OutputFiles))
|
|
for _, lane := range *document.OutputFiles {
|
|
if strings.TrimSpace(lane.LaneID) == "" || strings.TrimSpace(lane.File) == "" {
|
|
return Index{}, fmt.Errorf("notarius lane descriptors require lane_id and file")
|
|
}
|
|
if _, exists := seenLanes[lane.LaneID]; exists {
|
|
return Index{}, fmt.Errorf("notarius index contains duplicate lane id %q", lane.LaneID)
|
|
}
|
|
seenLanes[lane.LaneID] = struct{}{}
|
|
path, err := resolveRegularFile(bundleRoot, lane.File)
|
|
if err != nil {
|
|
return Index{}, fmt.Errorf("resolve notarius lane %q file: %w", lane.LaneID, err)
|
|
}
|
|
index.Lanes = append(index.Lanes, LaneDescriptor{
|
|
LaneID: lane.LaneID, File: lane.File, Path: path, MediaType: lane.MediaType,
|
|
ModuleKey: lane.ModuleKey, SchemaID: lane.SchemaID, SchemaName: lane.SchemaName,
|
|
SchemaVersion: lane.SchemaVersion,
|
|
})
|
|
}
|
|
if document.ChunkMap != nil {
|
|
index.ChunkMap, err = resolvePipelineDescriptor(bundleRoot, "chunk_map", *document.ChunkMap)
|
|
if err != nil {
|
|
return Index{}, err
|
|
}
|
|
}
|
|
if document.EvidenceContext != nil {
|
|
index.EvidenceContext, err = resolvePipelineDescriptor(bundleRoot, "evidence_context", *document.EvidenceContext)
|
|
if err != nil {
|
|
return Index{}, err
|
|
}
|
|
}
|
|
return index, nil
|
|
}
|
|
|
|
func resolvePipelineDescriptor(bundleRoot, label string, document pipelineDocument) (*PipelineDescriptor, error) {
|
|
if strings.TrimSpace(document.ArtifactKind) == "" || strings.TrimSpace(document.File) == "" ||
|
|
strings.TrimSpace(document.MediaType) == "" || strings.TrimSpace(document.SchemaID) == "" ||
|
|
strings.TrimSpace(document.SchemaName) == "" || strings.TrimSpace(document.SchemaVersion) == "" {
|
|
return nil, fmt.Errorf("notarius %s descriptor is missing required fields", label)
|
|
}
|
|
path, err := resolveRegularFile(bundleRoot, document.File)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("resolve notarius %s file: %w", label, err)
|
|
}
|
|
return &PipelineDescriptor{
|
|
ArtifactKind: document.ArtifactKind, File: document.File, Path: path,
|
|
MediaType: document.MediaType, SchemaID: document.SchemaID,
|
|
SchemaName: document.SchemaName, SchemaVersion: document.SchemaVersion,
|
|
}, nil
|
|
}
|
|
|
|
type rejectionDocument struct {
|
|
Rejected *[]struct {
|
|
Stage string `json:"stage"`
|
|
StepID string `json:"step_id"`
|
|
LaneID string `json:"lane_id"`
|
|
ModuleKey string `json:"module_key"`
|
|
ChunkID string `json:"chunk_id"`
|
|
ValidatorName string `json:"validator_name"`
|
|
ReasonCode string `json:"reason_code"`
|
|
Message string `json:"message"`
|
|
} `json:"rejected"`
|
|
}
|
|
|
|
func loadRejections(path string) ([]RejectionSummary, error) {
|
|
var document rejectionDocument
|
|
if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil {
|
|
return nil, fmt.Errorf("decode notarius rejections: %w", err)
|
|
}
|
|
if document.Rejected == nil {
|
|
return nil, fmt.Errorf("notarius rejection document is missing rejected array")
|
|
}
|
|
summaries := make([]RejectionSummary, 0, len(*document.Rejected))
|
|
for _, item := range *document.Rejected {
|
|
if strings.TrimSpace(item.Stage) == "" || strings.TrimSpace(item.Message) == "" {
|
|
return nil, fmt.Errorf("notarius rejection entries require stage and message")
|
|
}
|
|
summaries = append(summaries, RejectionSummary{
|
|
Stage: item.Stage, StepID: item.StepID, LaneID: item.LaneID,
|
|
ModuleKey: item.ModuleKey, ChunkID: item.ChunkID,
|
|
ValidatorName: item.ValidatorName, ReasonCode: item.ReasonCode,
|
|
})
|
|
}
|
|
return summaries, nil
|
|
}
|
|
|
|
type findingGroupDocument struct {
|
|
Disposition string `json:"disposition"`
|
|
Category string `json:"category"`
|
|
ReasonCode string `json:"reason_code"`
|
|
Origin diagnosticOriginDocument `json:"origin"`
|
|
OccurrenceCount *int `json:"occurrence_count"`
|
|
Samples *[]struct {
|
|
Scope string `json:"scope"`
|
|
Message string `json:"message"`
|
|
ChunkID string `json:"chunk_id"`
|
|
ChunkIndex *int `json:"chunk_index"`
|
|
} `json:"samples"`
|
|
OmittedSampleCount *int `json:"omitted_sample_count"`
|
|
}
|
|
|
|
type diagnosticOriginDocument struct {
|
|
Stage string `json:"stage"`
|
|
StepID string `json:"step_id"`
|
|
LaneID string `json:"lane_id"`
|
|
ModuleKey string `json:"module_key"`
|
|
ValidatorKey string `json:"validator_key"`
|
|
}
|
|
|
|
type warningDocument struct {
|
|
SchemaVersion string `json:"schema_version"`
|
|
GroupCount *int `json:"group_count"`
|
|
OccurrenceCount *int `json:"occurrence_count"`
|
|
Groups *[]findingGroupDocument `json:"groups"`
|
|
}
|
|
|
|
func loadWarnings(path string) ([]WarningSummary, error) {
|
|
var document warningDocument
|
|
if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil {
|
|
return nil, fmt.Errorf("decode notarius warnings: %w", err)
|
|
}
|
|
if document.SchemaVersion != warningsSchemaVersion || document.GroupCount == nil ||
|
|
document.OccurrenceCount == nil || document.Groups == nil {
|
|
return nil, fmt.Errorf("notarius warning document is missing or incompatible required fields")
|
|
}
|
|
if *document.GroupCount < 0 || *document.GroupCount > maxWarningGroups || *document.OccurrenceCount < 0 ||
|
|
*document.GroupCount != len(*document.Groups) {
|
|
return nil, fmt.Errorf("notarius warning document counts are inconsistent")
|
|
}
|
|
summaries := make([]WarningSummary, 0, len(*document.Groups))
|
|
occurrences := 0
|
|
for _, group := range *document.Groups {
|
|
if err := validateFindingGroup(group); err != nil {
|
|
return nil, fmt.Errorf("notarius warning group: %w", err)
|
|
}
|
|
if group.Disposition != "warning" {
|
|
return nil, fmt.Errorf("notarius warning group disposition %q is invalid", group.Disposition)
|
|
}
|
|
if *group.OccurrenceCount > int(^uint(0)>>1)-occurrences {
|
|
return nil, fmt.Errorf("notarius warning occurrence count overflows")
|
|
}
|
|
occurrences += *group.OccurrenceCount
|
|
summaries = append(summaries, WarningSummary{
|
|
Disposition: group.Disposition, Category: group.Category, ReasonCode: group.ReasonCode,
|
|
Origin: diagnosticOrigin(group.Origin), OccurrenceCount: *group.OccurrenceCount,
|
|
})
|
|
}
|
|
if occurrences != *document.OccurrenceCount {
|
|
return nil, fmt.Errorf("notarius warning document occurrence count is inconsistent")
|
|
}
|
|
return summaries, nil
|
|
}
|
|
|
|
type diagnosticDocument struct {
|
|
SchemaVersion string `json:"schema_version"`
|
|
GroupCount *int `json:"group_count"`
|
|
OccurrenceCount *int `json:"occurrence_count"`
|
|
Truncated *bool `json:"truncated"`
|
|
UnrepresentedOccurrenceCount *int `json:"unrepresented_occurrence_count"`
|
|
Groups *[]findingGroupDocument `json:"groups"`
|
|
}
|
|
|
|
func loadDiagnostics(path string) ([]DiagnosticSummary, int, bool, error) {
|
|
var document diagnosticDocument
|
|
if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil {
|
|
return nil, 0, false, fmt.Errorf("decode notarius diagnostics: %w", err)
|
|
}
|
|
if document.SchemaVersion != diagnosticsSchemaVersion || document.GroupCount == nil ||
|
|
document.OccurrenceCount == nil || document.Truncated == nil ||
|
|
document.UnrepresentedOccurrenceCount == nil || document.Groups == nil {
|
|
return nil, 0, false, fmt.Errorf("notarius diagnostics document is missing or incompatible required fields")
|
|
}
|
|
if *document.GroupCount < 0 || *document.GroupCount > maxDiagnosticGroups || *document.OccurrenceCount < 0 ||
|
|
*document.UnrepresentedOccurrenceCount < 0 || *document.GroupCount != len(*document.Groups) {
|
|
return nil, 0, false, fmt.Errorf("notarius diagnostics document counts are inconsistent")
|
|
}
|
|
if !*document.Truncated && *document.UnrepresentedOccurrenceCount != 0 {
|
|
return nil, 0, false, fmt.Errorf("notarius diagnostics document has unrepresented occurrences without truncation")
|
|
}
|
|
summaries := make([]DiagnosticSummary, 0, len(*document.Groups))
|
|
representedOccurrences := 0
|
|
for _, group := range *document.Groups {
|
|
if err := validateFindingGroup(group); err != nil {
|
|
return nil, 0, false, fmt.Errorf("notarius diagnostic group: %w", err)
|
|
}
|
|
if group.Disposition != "advisory" && group.Disposition != "observation" {
|
|
return nil, 0, false, fmt.Errorf("notarius diagnostic group disposition %q is invalid", group.Disposition)
|
|
}
|
|
if *group.OccurrenceCount > int(^uint(0)>>1)-representedOccurrences {
|
|
return nil, 0, false, fmt.Errorf("notarius diagnostic occurrence count overflows")
|
|
}
|
|
representedOccurrences += *group.OccurrenceCount
|
|
summaries = append(summaries, DiagnosticSummary{
|
|
Disposition: group.Disposition, Category: group.Category, ReasonCode: group.ReasonCode,
|
|
Origin: diagnosticOrigin(group.Origin), OccurrenceCount: *group.OccurrenceCount,
|
|
})
|
|
}
|
|
if *document.UnrepresentedOccurrenceCount > int(^uint(0)>>1)-representedOccurrences ||
|
|
representedOccurrences+*document.UnrepresentedOccurrenceCount != *document.OccurrenceCount {
|
|
return nil, 0, false, fmt.Errorf("notarius diagnostics document occurrence count is inconsistent")
|
|
}
|
|
return summaries, *document.OccurrenceCount, *document.Truncated, nil
|
|
}
|
|
|
|
func validateFindingGroup(group findingGroupDocument) error {
|
|
if strings.TrimSpace(group.Disposition) == "" || strings.TrimSpace(group.Category) == "" ||
|
|
strings.TrimSpace(group.ReasonCode) == "" || !validDiagnosticOriginStage(group.Origin.Stage) ||
|
|
!validDiagnosticCategory(group.Disposition, group.Category) ||
|
|
group.OccurrenceCount == nil || *group.OccurrenceCount <= 0 || group.Samples == nil ||
|
|
group.OmittedSampleCount == nil || *group.OmittedSampleCount < 0 {
|
|
return fmt.Errorf("missing required fields")
|
|
}
|
|
if len(*group.Samples) == 0 || len(*group.Samples) > maxFindingSamples ||
|
|
*group.OmittedSampleCount != *group.OccurrenceCount-len(*group.Samples) {
|
|
return fmt.Errorf("sample counts are inconsistent")
|
|
}
|
|
for _, sample := range *group.Samples {
|
|
if strings.TrimSpace(sample.Scope) == "" || strings.TrimSpace(sample.Message) == "" ||
|
|
(sample.ChunkIndex != nil && *sample.ChunkIndex < 0) {
|
|
return fmt.Errorf("samples require scope and message")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validDiagnosticCategory(disposition, category string) bool {
|
|
switch disposition {
|
|
case "warning":
|
|
return category == "configuration" || category == "degradation" ||
|
|
category == "validation_incomplete" || category == "fallback"
|
|
case "advisory":
|
|
return category == "data_quality"
|
|
case "observation":
|
|
return category == "normalization"
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func validDiagnosticOriginStage(stage string) bool {
|
|
switch stage {
|
|
case "references", "chunk", "extract", "merge", "normalize":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func diagnosticOrigin(document diagnosticOriginDocument) DiagnosticOrigin {
|
|
return DiagnosticOrigin{
|
|
Stage: document.Stage, StepID: document.StepID, LaneID: document.LaneID,
|
|
ModuleKey: document.ModuleKey, ValidatorKey: document.ValidatorKey,
|
|
}
|
|
}
|
|
|
|
func sumWarningOccurrences(values []WarningSummary) (int, error) {
|
|
total := 0
|
|
for _, value := range values {
|
|
if value.OccurrenceCount > int(^uint(0)>>1)-total {
|
|
return 0, fmt.Errorf("notarius warning occurrence count overflows")
|
|
}
|
|
total += value.OccurrenceCount
|
|
}
|
|
return total, nil
|
|
}
|
|
|
|
func decodeBoundedJSON(path string, limit int64, destination any) error {
|
|
data, err := fileops.ReadRegularFile(path, limit)
|
|
if err != nil {
|
|
return fmt.Errorf("notarius JSON result exceeds or cannot be read within %d-byte limit: %w", limit, err)
|
|
}
|
|
if err := json.Unmarshal(data, destination); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateBundleRoot(outputRoot, bundleRoot string) (string, error) {
|
|
root := filepath.Clean(outputRoot)
|
|
bundle := filepath.Clean(bundleRoot)
|
|
relative, err := filepath.Rel(root, bundle)
|
|
if err != nil {
|
|
return "", fmt.Errorf("compare notarius output paths: %w", err)
|
|
}
|
|
if relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
|
return "", fmt.Errorf("notarius output directory %q is not beneath output root %q", bundleRoot, outputRoot)
|
|
}
|
|
if err := requireDirectoryTree(root, relative); err != nil {
|
|
return "", fmt.Errorf("validate notarius output directory: %w", err)
|
|
}
|
|
return bundle, nil
|
|
}
|
|
|
|
func resolveRegularFile(root, logicalPath string) (string, error) {
|
|
resolved, err := pathsafe.JoinSlashRelativeUnderRoot(root, logicalPath)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
relative, err := filepath.Rel(root, resolved)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if err := requireRegularFileTree(root, relative); err != nil {
|
|
return "", err
|
|
}
|
|
return resolved, nil
|
|
}
|
|
|
|
func requireDirectoryTree(root, relative string) error {
|
|
if err := requireDirectory(root); err != nil {
|
|
return err
|
|
}
|
|
current := root
|
|
for _, component := range strings.Split(relative, string(filepath.Separator)) {
|
|
current = filepath.Join(current, component)
|
|
if err := requireDirectory(current); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func requireRegularFileTree(root, relative string) error {
|
|
components := strings.Split(relative, string(filepath.Separator))
|
|
if len(components) == 0 {
|
|
return fmt.Errorf("regular file path is required")
|
|
}
|
|
if err := requireDirectory(root); err != nil {
|
|
return err
|
|
}
|
|
current := root
|
|
for _, component := range components[:len(components)-1] {
|
|
current = filepath.Join(current, component)
|
|
if err := requireDirectory(current); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return requireRegularFile(filepath.Join(current, components[len(components)-1]))
|
|
}
|
|
|
|
func requireDirectory(path string) error {
|
|
info, err := os.Lstat(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
|
return fmt.Errorf("path %q must be a directory without symlinks", path)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func requireRegularFile(path string) error {
|
|
info, err := os.Lstat(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
|
return fmt.Errorf("path %q must be a regular file without symlinks", path)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateLogDestination(path string) error {
|
|
if err := requireDirectory(filepath.Dir(path)); err != nil {
|
|
return err
|
|
}
|
|
info, err := os.Lstat(path)
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
|
return fmt.Errorf("path %q must be absent or a regular file without symlinks", path)
|
|
}
|
|
return nil
|
|
}
|