eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add stable checkpoint decision diagnostics

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-22 02:13:50 +00:00
parent 5bdd56cfb1
commit 9de399432e
10 changed files with 391 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
internal/framework/checkpoint/loader.go
View File

@@ -52,13 +52,13 @@ func (l *FilesystemLoader) Source(moduleKey string) (pipeline.SourceCheckpoint,
}
doc := cloneSourceDocument(payload.Document)
if err := source.ValidateDocument(&doc); err != nil {
return pipeline.SourceCheckpoint{}, invalidDecision("source checkpoint document is invalid: %v", err)
return pipeline.SourceCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactPayloadInvalid, "source checkpoint document is invalid")
}
if strings.TrimSpace(manifest.SourceID) != "" && manifest.SourceID != doc.ID {
return pipeline.SourceCheckpoint{}, invalidDecision("source checkpoint source id does not match payload")
return pipeline.SourceCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactPayloadInvalid, "source checkpoint identity does not match its payload")
}
if !fingerprintsEqual(checkpointToPipelineFingerprints(manifest.OutputDigests), digestFingerprints("source_document", doc.Digest)) {
return pipeline.SourceCheckpoint{}, invalidDecision("source checkpoint output digest does not match payload")
return pipeline.SourceCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactDigestMismatch, "source checkpoint output digest does not match its payload")
}
return pipeline.SourceCheckpoint{Document: &doc}, reusedDecision()
}
@@ -81,10 +81,10 @@ func (l *FilesystemLoader) ExtractForStep(stepID, laneID, moduleKey string, depe
}
outputs, err := artifactCheckpointOutputs(payload.Outputs)
if err != nil {
return pipeline.ExtractCheckpoint{}, invalidDecision("extract artifact checkpoint payload is invalid: %v", err)
return pipeline.ExtractCheckpoint{}, artifactDecision(err, "extract checkpoint artifact is invalid")
}
if !fingerprintsEqual(checkpointToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(outputs)) {
return pipeline.ExtractCheckpoint{}, invalidDecision("extract artifact checkpoint output digests do not match payload")
return pipeline.ExtractCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactDigestMismatch, "extract checkpoint output digest does not match its payload")
}
return pipeline.ExtractCheckpoint{Outputs: outputs, Rejected: cloneRejectedOutputs(payload.Rejected), Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
@@ -107,10 +107,10 @@ func (l *FilesystemLoader) MergeForStep(stepID, laneID, moduleKey string, depend
}
values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output})
if err != nil {
return pipeline.MergeCheckpoint{}, invalidDecision("merge artifact checkpoint payload is invalid: %v", err)
return pipeline.MergeCheckpoint{}, artifactDecision(err, "merge checkpoint artifact is invalid")
}
if !fingerprintsEqual(checkpointToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
return pipeline.MergeCheckpoint{}, invalidDecision("merge artifact checkpoint output digest does not match payload")
return pipeline.MergeCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactDigestMismatch, "merge checkpoint output digest does not match its payload")
}
return pipeline.MergeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
@@ -133,10 +133,10 @@ func (l *FilesystemLoader) NormalizeForStep(stepID, laneID, moduleKey string, de
}
values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output})
if err != nil {
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint payload is invalid: %v", err)
return pipeline.NormalizeCheckpoint{}, artifactDecision(err, "normalize checkpoint artifact is invalid")
}
if !fingerprintsEqual(checkpointToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint output digest does not match payload")
return pipeline.NormalizeCheckpoint{}, decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonArtifactDigestMismatch, "normalize checkpoint output digest does not match its payload")
}
return pipeline.NormalizeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
@@ -152,7 +152,7 @@ func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.
return nil, err
}
if strings.TrimSpace(string(v.Kind)) == "" || strings.TrimSpace(v.Schema.ID) == "" || strings.TrimSpace(v.Schema.Version) == "" || strings.TrimSpace(v.SchemaDigest) == "" {
return nil, fmt.Errorf("artifact codec identity is incomplete")
return nil, &artifactPayloadError{code: pipeline.CheckpointReasonArtifactCodecIncompatible, err: fmt.Errorf("artifact codec identity is incomplete")}
}
out = append(out, pipeline.CheckpointArtifact{LaneID: v.LaneID, ModuleKey: v.ModuleKey, SourceID: v.SourceID, ChunkID: v.ChunkID, ChunkIndex: v.ChunkIndex, ChunkRef: v.ChunkRef, SchemaDigest: v.SchemaDigest, Artifact: contracts.SerializedArtifact{Kind: v.Kind, Schema: v.Schema, MediaType: v.Content.MediaType, Content: content, Metadata: cloneMetadata(v.Content.Metadata)}})
}
@@ -161,21 +161,21 @@ func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.
func (l *FilesystemLoader) readJSON(name string, out any) pipeline.CheckpointDecision {
if !l.Enabled() {
return pipeline.CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"}
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonLoadingDisabled, "checkpoint loading disabled")
}
target, err := fileio.SafePath(l.root, name)
if err != nil {
return invalidDecision("checkpoint path is invalid: %v", err)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonPathInvalid, "checkpoint path is invalid")
}
data, err := os.ReadFile(target)
if err != nil {
if os.IsNotExist(err) {
return pipeline.CheckpointDecision{Category: "executed", ReasonCode: "checkpoint_missing", Reason: "checkpoint artifact is missing"}
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonMissing, "checkpoint artifact is missing")
}
return invalidDecision("read checkpoint artifact: %v", err)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonReadFailed, "checkpoint artifact could not be read")
}
if err := json.Unmarshal(data, out); err != nil {
return invalidDecision("decode checkpoint artifact: %v", err)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonDecodeFailed, "checkpoint artifact could not be decoded")
}
return reusedDecision()
}
@@ -186,28 +186,28 @@ func (l *FilesystemLoader) validateManifest(manifest StageManifest, stage StageN
func (l *FilesystemLoader) validateLaneManifest(manifest StageManifest, stage StageName, stepID string, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, statuses ...StageStatus) pipeline.CheckpointDecision {
if manifest.WorkspaceSchemaVersion == WorkspaceSchemaVersionV1 {
return invalidDecision("checkpoint workspace schema version %q is incompatible with %q and must be recomputed", manifest.WorkspaceSchemaVersion, WorkspaceSchemaVersion)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonWorkspaceSchemaIncompatible, "checkpoint workspace schema is incompatible")
}
if manifest.WorkspaceSchemaVersion == WorkspaceSchemaVersionV2 {
return invalidDecision("checkpoint workspace schema version %q is incompatible with %q and must be recomputed", manifest.WorkspaceSchemaVersion, WorkspaceSchemaVersion)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonWorkspaceSchemaIncompatible, "checkpoint workspace schema is incompatible")
}
if manifest.WorkspaceSchemaVersion != WorkspaceSchemaVersion {
return invalidDecision("checkpoint workspace schema version %q is not supported", manifest.WorkspaceSchemaVersion)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonWorkspaceSchemaIncompatible, "checkpoint workspace schema is incompatible")
}
if strings.TrimSpace(l.identityDigest) != "" && manifest.Metadata["checkpoint_identity_digest"] != l.identityDigest {
return invalidDecision("checkpoint identity digest does not match current invocation")
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonIdentityMismatch, "checkpoint identity does not match the current invocation")
}
if manifest.Stage != stage {
return invalidDecision("checkpoint stage %q does not match %q", manifest.Stage, stage)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonStageMismatch, "checkpoint stage does not match the requested stage")
}
if strings.TrimSpace(stepID) != "" && manifest.StepID != stepID {
return invalidDecision("checkpoint step does not match requested step")
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonStepMismatch, "checkpoint step does not match the requested step")
}
if strings.TrimSpace(laneID) != "" && manifest.LaneID != laneID {
return invalidDecision("checkpoint lane %q does not match %q", manifest.LaneID, laneID)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonLaneMismatch, "checkpoint lane does not match the requested lane")
}
if strings.TrimSpace(moduleKey) != "" && manifest.ModuleKey != moduleKey {
return invalidDecision("checkpoint module %q does not match %q", manifest.ModuleKey, moduleKey)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonModuleMismatch, "checkpoint module does not match the requested module")
}
statusOK := false
for _, status := range statuses {
@@ -217,10 +217,10 @@ func (l *FilesystemLoader) validateLaneManifest(manifest StageManifest, stage St
}
}
if !statusOK {
return invalidDecision("checkpoint status %q cannot be reused", manifest.Status)
return decision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonStatusNotReusable, "checkpoint status cannot be reused")
}
if !fingerprintsEqual(checkpointToPipelineFingerprints(manifest.DependencyFingerprints), dependencies) {
return invalidDecision("checkpoint dependency fingerprints do not match")
return decision(pipeline.CheckpointDecisionDependencyInvalidated, pipeline.CheckpointReasonDependencyMismatch, "checkpoint dependencies do not match")
}
return reusedDecision()
}
@@ -228,10 +228,10 @@ func (l *FilesystemLoader) validateLaneManifest(manifest StageManifest, stage St
func contentFromEnvelope(value binaryEnvelope) ([]byte, error) {
content, err := base64.StdEncoding.DecodeString(value.ContentBase64)
if err != nil {
return nil, fmt.Errorf("decode content_base64: %w", err)
return nil, &artifactPayloadError{code: pipeline.CheckpointReasonArtifactPayloadInvalid, err: fmt.Errorf("decode content_base64: %w", err)}
}
if digest := strings.TrimSpace(value.ContentDigest); digest != "" && digest != contentDigest(content) {
return nil, fmt.Errorf("content digest mismatch")
return nil, &artifactPayloadError{code: pipeline.CheckpointReasonArtifactDigestMismatch, err: fmt.Errorf("content digest mismatch")}
}
return content, nil
}
@@ -262,65 +262,24 @@ func fingerprintsEqual(a []pipeline.CheckpointFingerprint, b []pipeline.Checkpoi
}
func reusedDecision() pipeline.CheckpointDecision {
return pipeline.CheckpointDecision{Reused: true, Category: "reused", ReasonCode: "checkpoint_valid", Reason: "checkpoint is valid"}
return decision(pipeline.CheckpointDecisionReused, pipeline.CheckpointReasonReused, "checkpoint is reusable")
}
func invalidDecision(format string, args ...any) pipeline.CheckpointDecision {
reason := fmt.Sprintf(format, args...)
code := "checkpoint_invalid"
category := "executed"
switch {
case strings.Contains(reason, "dependency fingerprints"):
code, category = "dependency_mismatch", "dependency_invalidated"
case strings.Contains(reason, "missing"):
code = "checkpoint_missing"
case strings.Contains(reason, "schema"):
code = "workspace_schema_incompatible"
case strings.Contains(reason, "codec"):
code = "artifact_codec_incompatible"
case strings.Contains(reason, "content"):
code = "artifact_content_invalid"
case strings.Contains(reason, "identity"):
code = "identity_mismatch"
}
safe := safeReasonText(reason, code)
return pipeline.CheckpointDecision{Category: category, ReasonCode: code, Detail: safe, Reason: safe}
func decision(category pipeline.CheckpointDecisionCategory, code pipeline.CheckpointReasonCode, detail string) pipeline.CheckpointDecision {
return pipeline.NewCheckpointDecision(category, code, detail)
}
func safeReasonText(reason, code string) string {
lower := strings.ToLower(reason)
switch {
case strings.Contains(lower, "workspace schema"):
return "checkpoint workspace schema is incompatible"
case strings.Contains(lower, "source checkpoint document"):
return "source checkpoint document is invalid"
case strings.Contains(lower, "artifact codec identity"):
return "artifact codec identity is incomplete"
case strings.Contains(lower, "base64"):
return "checkpoint content base64 is invalid"
case strings.Contains(lower, "content digest"):
return "checkpoint content digest is invalid"
case strings.Contains(lower, "output digest"):
return "checkpoint output digest does not match payload"
case strings.Contains(lower, "identity"):
return "checkpoint identity does not match"
case strings.Contains(lower, "dependency"):
return "checkpoint dependency fingerprints do not match"
case strings.Contains(lower, "stage"):
return "checkpoint stage does not match"
case strings.Contains(lower, "step"):
return "checkpoint step does not match"
case strings.Contains(lower, "lane"):
return "checkpoint lane does not match"
case strings.Contains(lower, "module"):
return "checkpoint module does not match"
case strings.Contains(lower, "status"):
return "checkpoint status cannot be reused"
case strings.Contains(lower, "payload"):
return "checkpoint artifact payload is invalid"
case strings.Contains(lower, "decode"):
return "checkpoint artifact decode failed"
default:
return "checkpoint is not reusable (" + code + ")"
}
type artifactPayloadError struct {
code pipeline.CheckpointReasonCode
err error
}
func (e *artifactPayloadError) Error() string { return e.err.Error() }
func artifactDecision(err error, detail string) pipeline.CheckpointDecision {
code := pipeline.CheckpointReasonArtifactPayloadInvalid
if payloadErr, ok := err.(*artifactPayloadError); ok {
code = payloadErr.code
}
return decision(pipeline.CheckpointDecisionExecuted, code, detail)
}