eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement selective checkpoint recomputation

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-21 21:59:48 +00:00
parent 22d4f29670
commit c437682407
17 changed files with 902 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
internal/framework/checkpoint/loader.go
View File

@@ -64,15 +64,19 @@ func (l *FilesystemLoader) Source(moduleKey string) (pipeline.SourceCheckpoint,
}
func (l *FilesystemLoader) Extract(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) {
return l.ExtractForStep("", laneID, moduleKey, dependencies)
}
func (l *FilesystemLoader) ExtractForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) {
var manifest ExtractLaneManifest
if d := l.readJSON(laneManifestPath("extract", laneID), &manifest); !d.Reused {
if d := l.readJSON(laneManifestPath("extract", stepID, laneID), &manifest); !d.Reused {
return pipeline.ExtractCheckpoint{}, d
}
if d := l.validateLaneManifest(manifest.StageManifest, StageExtract, laneID, moduleKey, dependencies, StatusSucceeded, StatusSucceededWithRejections); !d.Reused {
if d := l.validateLaneManifest(manifest.StageManifest, StageExtract, stepID, laneID, moduleKey, dependencies, StatusSucceeded, StatusSucceededWithRejections); !d.Reused {
return pipeline.ExtractCheckpoint{}, d
}
var payload artifactExtractEnvelope
if d := l.readJSON(lanePayloadPath("extract", laneID, "outputs.json"), &payload); !d.Reused {
if d := l.readJSON(lanePayloadPath("extract", stepID, laneID, "outputs.json"), &payload); !d.Reused {
return pipeline.ExtractCheckpoint{}, d
}
outputs, err := artifactCheckpointOutputs(payload.Outputs)
@@ -86,15 +90,19 @@ func (l *FilesystemLoader) Extract(laneID, moduleKey string, dependencies []pipe
}
func (l *FilesystemLoader) Merge(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.MergeCheckpoint, pipeline.CheckpointDecision) {
return l.MergeForStep("", laneID, moduleKey, dependencies)
}
func (l *FilesystemLoader) MergeForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.MergeCheckpoint, pipeline.CheckpointDecision) {
var manifest MergeLaneManifest
if d := l.readJSON(laneManifestPath("merge", laneID), &manifest); !d.Reused {
if d := l.readJSON(laneManifestPath("merge", stepID, laneID), &manifest); !d.Reused {
return pipeline.MergeCheckpoint{}, d
}
if d := l.validateLaneManifest(manifest.StageManifest, StageMerge, laneID, moduleKey, dependencies, StatusSucceeded); !d.Reused {
if d := l.validateLaneManifest(manifest.StageManifest, StageMerge, stepID, laneID, moduleKey, dependencies, StatusSucceeded); !d.Reused {
return pipeline.MergeCheckpoint{}, d
}
var payload artifactSingleEnvelope
if d := l.readJSON(lanePayloadPath("merge", laneID, "output.json"), &payload); !d.Reused {
if d := l.readJSON(lanePayloadPath("merge", stepID, laneID, "output.json"), &payload); !d.Reused {
return pipeline.MergeCheckpoint{}, d
}
values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output})
@@ -108,15 +116,19 @@ func (l *FilesystemLoader) Merge(laneID, moduleKey string, dependencies []pipeli
}
func (l *FilesystemLoader) Normalize(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.NormalizeCheckpoint, pipeline.CheckpointDecision) {
return l.NormalizeForStep("", laneID, moduleKey, dependencies)
}
func (l *FilesystemLoader) NormalizeForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.NormalizeCheckpoint, pipeline.CheckpointDecision) {
var manifest NormalizeLaneManifest
if d := l.readJSON(laneManifestPath("normalize", laneID), &manifest); !d.Reused {
if d := l.readJSON(laneManifestPath("normalize", stepID, laneID), &manifest); !d.Reused {
return pipeline.NormalizeCheckpoint{}, d
}
if d := l.validateLaneManifest(manifest.StageManifest, StageNormalize, laneID, moduleKey, dependencies, StatusSucceeded); !d.Reused {
if d := l.validateLaneManifest(manifest.StageManifest, StageNormalize, stepID, laneID, moduleKey, dependencies, StatusSucceeded); !d.Reused {
return pipeline.NormalizeCheckpoint{}, d
}
var payload artifactSingleEnvelope
if d := l.readJSON(lanePayloadPath("normalize", laneID, "output.json"), &payload); !d.Reused {
if d := l.readJSON(lanePayloadPath("normalize", stepID, laneID, "output.json"), &payload); !d.Reused {
return pipeline.NormalizeCheckpoint{}, d
}
values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output})
@@ -149,7 +161,7 @@ func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.
func (l *FilesystemLoader) readJSON(name string, out any) pipeline.CheckpointDecision {
if !l.Enabled() {
return pipeline.CheckpointDecision{Reason: "checkpoint loading disabled"}
return pipeline.CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"}
}
target, err := fileio.SafePath(l.root, name)
if err != nil {
@@ -158,7 +170,7 @@ func (l *FilesystemLoader) readJSON(name string, out any) pipeline.CheckpointDec
data, err := os.ReadFile(target)
if err != nil {
if os.IsNotExist(err) {
return pipeline.CheckpointDecision{Reason: "checkpoint artifact is missing"}
return pipeline.CheckpointDecision{Category: "executed", ReasonCode: "checkpoint_missing", Reason: "checkpoint artifact is missing"}
}
return invalidDecision("read checkpoint artifact: %v", err)
}
@@ -169,13 +181,16 @@ func (l *FilesystemLoader) readJSON(name string, out any) pipeline.CheckpointDec
}
func (l *FilesystemLoader) validateManifest(manifest StageManifest, stage StageName, laneID string, moduleKey string, status StageStatus, dependencies []pipeline.CheckpointFingerprint) pipeline.CheckpointDecision {
return l.validateLaneManifest(manifest, stage, laneID, moduleKey, dependencies, status)
return l.validateLaneManifest(manifest, stage, "", laneID, moduleKey, dependencies, status)
}
func (l *FilesystemLoader) validateLaneManifest(manifest StageManifest, stage StageName, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, statuses ...StageStatus) pipeline.CheckpointDecision {
func (l *FilesystemLoader) validateLaneManifest(manifest StageManifest, stage StageName, stepID string, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, statuses ...StageStatus) pipeline.CheckpointDecision {
if manifest.WorkspaceSchemaVersion == WorkspaceSchemaVersionV1 {
return invalidDecision("checkpoint workspace schema version %q is incompatible with %q and must be recomputed", manifest.WorkspaceSchemaVersion, WorkspaceSchemaVersion)
}
if manifest.WorkspaceSchemaVersion == WorkspaceSchemaVersionV2 {
return invalidDecision("checkpoint workspace schema version %q is incompatible with %q and must be recomputed", manifest.WorkspaceSchemaVersion, WorkspaceSchemaVersion)
}
if manifest.WorkspaceSchemaVersion != WorkspaceSchemaVersion {
return invalidDecision("checkpoint workspace schema version %q is not supported", manifest.WorkspaceSchemaVersion)
}
@@ -185,6 +200,9 @@ func (l *FilesystemLoader) validateLaneManifest(manifest StageManifest, stage St
if manifest.Stage != stage {
return invalidDecision("checkpoint stage %q does not match %q", manifest.Stage, stage)
}
if strings.TrimSpace(stepID) != "" && manifest.StepID != stepID {
return invalidDecision("checkpoint step does not match requested step")
}
if strings.TrimSpace(laneID) != "" && manifest.LaneID != laneID {
return invalidDecision("checkpoint lane %q does not match %q", manifest.LaneID, laneID)
}
@@ -244,9 +262,65 @@ func fingerprintsEqual(a []pipeline.CheckpointFingerprint, b []pipeline.Checkpoi
}
func reusedDecision() pipeline.CheckpointDecision {
return pipeline.CheckpointDecision{Reused: true, Reason: "checkpoint is valid"}
return pipeline.CheckpointDecision{Reused: true, Category: "reused", ReasonCode: "checkpoint_valid", Reason: "checkpoint is valid"}
}
func invalidDecision(format string, args ...any) pipeline.CheckpointDecision {
return pipeline.CheckpointDecision{Reason: fmt.Sprintf(format, args...)}
reason := fmt.Sprintf(format, args...)
code := "checkpoint_invalid"
category := "executed"
switch {
case strings.Contains(reason, "dependency fingerprints"):
code, category = "dependency_mismatch", "dependency_invalidated"
case strings.Contains(reason, "missing"):
code = "checkpoint_missing"
case strings.Contains(reason, "schema"):
code = "workspace_schema_incompatible"
case strings.Contains(reason, "codec"):
code = "artifact_codec_incompatible"
case strings.Contains(reason, "content"):
code = "artifact_content_invalid"
case strings.Contains(reason, "identity"):
code = "identity_mismatch"
}
safe := safeReasonText(reason, code)
return pipeline.CheckpointDecision{Category: category, ReasonCode: code, Detail: safe, Reason: safe}
}
func safeReasonText(reason, code string) string {
lower := strings.ToLower(reason)
switch {
case strings.Contains(lower, "workspace schema"):
return "checkpoint workspace schema is incompatible"
case strings.Contains(lower, "source checkpoint document"):
return "source checkpoint document is invalid"
case strings.Contains(lower, "artifact codec identity"):
return "artifact codec identity is incomplete"
case strings.Contains(lower, "base64"):
return "checkpoint content base64 is invalid"
case strings.Contains(lower, "content digest"):
return "checkpoint content digest is invalid"
case strings.Contains(lower, "output digest"):
return "checkpoint output digest does not match payload"
case strings.Contains(lower, "identity"):
return "checkpoint identity does not match"
case strings.Contains(lower, "dependency"):
return "checkpoint dependency fingerprints do not match"
case strings.Contains(lower, "stage"):
return "checkpoint stage does not match"
case strings.Contains(lower, "step"):
return "checkpoint step does not match"
case strings.Contains(lower, "lane"):
return "checkpoint lane does not match"
case strings.Contains(lower, "module"):
return "checkpoint module does not match"
case strings.Contains(lower, "status"):
return "checkpoint status cannot be reused"
case strings.Contains(lower, "payload"):
return "checkpoint artifact payload is invalid"
case strings.Contains(lower, "decode"):
return "checkpoint artifact decode failed"
default:
return "checkpoint is not reusable (" + code + ")"
}
}