eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement selective checkpoint recomputation

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-21 21:59:48 +00:00
parent 22d4f29670
commit c437682407
17 changed files with 902 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
internal/framework/checkpoint/filesystem_test.go
View File

@@ -157,6 +157,7 @@ func TestFilesystemCheckpointRejectsIncompatibleManifests(t *testing.T) {
want string
}{
{"v1 schema", func(m map[string]any) { m["workspace_schema_version"] = WorkspaceSchemaVersionV1 }, "workspace schema"},
{"v2 schema", func(m map[string]any) { m["workspace_schema_version"] = WorkspaceSchemaVersionV2 }, "workspace schema"},
{"unknown schema", func(m map[string]any) { m["workspace_schema_version"] = "notarius.workspace.future" }, "workspace schema"},
{"identity", func(m map[string]any) { m["metadata"].(map[string]any)["checkpoint_identity_digest"] = "sha256:other" }, "identity"},
{"stage", func(m map[string]any) { m["stage"] = string(StageMerge) }, "stage"},
@@ -247,6 +248,17 @@ func TestFilesystemCheckpointReusesExtractWithRejections(t *testing.T) {
}
}
func TestFilesystemCheckpointDependencyDecisionIsBoundedAndCategorized(t *testing.T) {
fixture := seedFilesystemCheckpoints(t)
_, decision := fixture.loader.Extract("lane-a", "extract-module", []pipeline.CheckpointFingerprint{{Name: "source", Value: "sha256:changed"}})
if decision.Reused || decision.Category != "dependency_invalidated" || decision.ReasonCode != "dependency_mismatch" {
t.Fatalf("dependency decision = %#v", decision)
}
if strings.Contains(decision.Reason, fixture.root) || strings.Contains(decision.Detail, fixture.root) {
t.Fatalf("dependency decision leaked checkpoint path: %#v", decision)
}
}
type checkpointStage struct {
name string
manifest func(filesystemCheckpointFixture) string

18
internal/framework/checkpoint/identity.go
View File

@@ -31,6 +31,7 @@ type Identity struct {
Digest string `json:"digest"`
PipelineID string `json:"pipeline_id"`
PipelineDigest string `json:"pipeline_digest"`
PipelineTopology []string `json:"pipeline_topology,omitempty"`
InputKey string `json:"input_key"`
RawInputDigest string `json:"raw_input_digest,omitempty"`
SourceDigest string `json:"source_digest,omitempty"`
@@ -57,8 +58,8 @@ func NewIdentity(input IdentityInput) (Identity, error) {
if strings.TrimSpace(input.RawInputDigest) == "" && strings.TrimSpace(input.SourceDigest) == "" {
return Identity{}, fmt.Errorf("checkpoint identity raw input digest or source digest must be set")
}
v := Identity{PipelineID: pipelineID, PipelineDigest: pipelineDigest, InputKey: inputKey, RawInputDigest: strings.TrimSpace(input.RawInputDigest), SourceDigest: strings.TrimSpace(input.SourceDigest), SelectedLanes: normalizedLanes(input.SelectedLanes, input.Pipeline.AllArtifactLanes()), RuntimeOverrides: normalizeIdentityFingerprints(input.RuntimeOverrides), ReferenceDigests: referenceFingerprints(input.References), ProvenanceFingerprints: normalizeIdentityFingerprints(input.ProvenanceFingerprints)}
data, err := json.Marshal(Identity{PipelineID: v.PipelineID, PipelineDigest: v.PipelineDigest, InputKey: v.InputKey, RawInputDigest: v.RawInputDigest, SourceDigest: v.SourceDigest, SelectedLanes: v.SelectedLanes, RuntimeOverrides: v.RuntimeOverrides, ReferenceDigests: v.ReferenceDigests, ProvenanceFingerprints: v.ProvenanceFingerprints})
v := Identity{PipelineID: pipelineID, PipelineDigest: pipelineDigest, PipelineTopology: pipelineTopology(input.Pipeline), InputKey: inputKey, RawInputDigest: strings.TrimSpace(input.RawInputDigest), SourceDigest: strings.TrimSpace(input.SourceDigest), SelectedLanes: normalizedLanes(input.SelectedLanes, input.Pipeline.AllArtifactLanes()), RuntimeOverrides: normalizeIdentityFingerprints(input.RuntimeOverrides), ReferenceDigests: referenceFingerprints(input.References), ProvenanceFingerprints: normalizeIdentityFingerprints(input.ProvenanceFingerprints)}
data, err := json.Marshal(Identity{PipelineID: v.PipelineID, PipelineDigest: v.PipelineDigest, PipelineTopology: v.PipelineTopology, InputKey: v.InputKey, RawInputDigest: v.RawInputDigest, SourceDigest: v.SourceDigest, SelectedLanes: v.SelectedLanes, RuntimeOverrides: v.RuntimeOverrides, ReferenceDigests: v.ReferenceDigests, ProvenanceFingerprints: v.ProvenanceFingerprints})
if err != nil {
return Identity{}, fmt.Errorf("marshal checkpoint identity: %w", err)
}
@@ -66,6 +67,19 @@ func NewIdentity(input IdentityInput) (Identity, error) {
v.Digest = "sha256:" + hex.EncodeToString(sum[:])
return v, nil
}
func pipelineTopology(resolved pipeline.ResolvedPipeline) []string {
var topology []string
for _, step := range resolved.Steps {
for _, lane := range step.ArtifactLanes {
topology = append(topology, strings.TrimSpace(step.ID)+"\x00"+strings.TrimSpace(lane.ID))
}
}
if len(topology) == 0 {
return nil
}
return topology
}
func (i Identity) RelativePath() (string, error) {
p, err := safeComponent(i.PipelineID)
if err != nil {

10
internal/framework/checkpoint/identity_test.go
View File

@@ -32,7 +32,9 @@ func TestNewIdentityNormalizesOrderAndEmptyValues(t *testing.T) {
v.ProvenanceFingerprints = []Fingerprint{{Name: "source", Value: "v2"}, {Name: "runner", Value: "v1"}}
}},
{"resolved lanes", func(v *IdentityInput) {
v.Pipeline.Steps[0].ArtifactLanes = []pipeline.ResolvedArtifactLane{v.Pipeline.Steps[0].ArtifactLanes[1], v.Pipeline.Steps[0].ArtifactLanes[0]}
steps := append([]pipeline.ResolvedPipelineStep(nil), v.Pipeline.Steps...)
steps[0].ArtifactLanes = []pipeline.ResolvedArtifactLane{steps[0].ArtifactLanes[1], steps[0].ArtifactLanes[0]}
v.Pipeline.Steps = steps
}},
} {
t.Run(tt.name, func(t *testing.T) {
@@ -42,6 +44,12 @@ func TestNewIdentityNormalizesOrderAndEmptyValues(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if tt.name == "resolved lanes" {
if reflect.DeepEqual(identity, got) {
t.Fatal("reordered topology did not change checkpoint identity")
}
return
}
if !reflect.DeepEqual(identity, got) {
t.Fatalf("reordered identity differs:\nbase=%#v\ngot=%#v", identity, got)
}

104
internal/framework/checkpoint/loader.go
View File

@@ -64,15 +64,19 @@ func (l *FilesystemLoader) Source(moduleKey string) (pipeline.SourceCheckpoint,
}
func (l *FilesystemLoader) Extract(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) {
return l.ExtractForStep("", laneID, moduleKey, dependencies)
}
func (l *FilesystemLoader) ExtractForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) {
var manifest ExtractLaneManifest
if d := l.readJSON(laneManifestPath("extract", laneID), &manifest); !d.Reused {
if d := l.readJSON(laneManifestPath("extract", stepID, laneID), &manifest); !d.Reused {
return pipeline.ExtractCheckpoint{}, d
}
if d := l.validateLaneManifest(manifest.StageManifest, StageExtract, laneID, moduleKey, dependencies, StatusSucceeded, StatusSucceededWithRejections); !d.Reused {
if d := l.validateLaneManifest(manifest.StageManifest, StageExtract, stepID, laneID, moduleKey, dependencies, StatusSucceeded, StatusSucceededWithRejections); !d.Reused {
return pipeline.ExtractCheckpoint{}, d
}
var payload artifactExtractEnvelope
if d := l.readJSON(lanePayloadPath("extract", laneID, "outputs.json"), &payload); !d.Reused {
if d := l.readJSON(lanePayloadPath("extract", stepID, laneID, "outputs.json"), &payload); !d.Reused {
return pipeline.ExtractCheckpoint{}, d
}
outputs, err := artifactCheckpointOutputs(payload.Outputs)
@@ -86,15 +90,19 @@ func (l *FilesystemLoader) Extract(laneID, moduleKey string, dependencies []pipe
}
func (l *FilesystemLoader) Merge(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.MergeCheckpoint, pipeline.CheckpointDecision) {
return l.MergeForStep("", laneID, moduleKey, dependencies)
}
func (l *FilesystemLoader) MergeForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.MergeCheckpoint, pipeline.CheckpointDecision) {
var manifest MergeLaneManifest
if d := l.readJSON(laneManifestPath("merge", laneID), &manifest); !d.Reused {
if d := l.readJSON(laneManifestPath("merge", stepID, laneID), &manifest); !d.Reused {
return pipeline.MergeCheckpoint{}, d
}
if d := l.validateLaneManifest(manifest.StageManifest, StageMerge, laneID, moduleKey, dependencies, StatusSucceeded); !d.Reused {
if d := l.validateLaneManifest(manifest.StageManifest, StageMerge, stepID, laneID, moduleKey, dependencies, StatusSucceeded); !d.Reused {
return pipeline.MergeCheckpoint{}, d
}
var payload artifactSingleEnvelope
if d := l.readJSON(lanePayloadPath("merge", laneID, "output.json"), &payload); !d.Reused {
if d := l.readJSON(lanePayloadPath("merge", stepID, laneID, "output.json"), &payload); !d.Reused {
return pipeline.MergeCheckpoint{}, d
}
values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output})
@@ -108,15 +116,19 @@ func (l *FilesystemLoader) Merge(laneID, moduleKey string, dependencies []pipeli
}
func (l *FilesystemLoader) Normalize(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.NormalizeCheckpoint, pipeline.CheckpointDecision) {
return l.NormalizeForStep("", laneID, moduleKey, dependencies)
}
func (l *FilesystemLoader) NormalizeForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.NormalizeCheckpoint, pipeline.CheckpointDecision) {
var manifest NormalizeLaneManifest
if d := l.readJSON(laneManifestPath("normalize", laneID), &manifest); !d.Reused {
if d := l.readJSON(laneManifestPath("normalize", stepID, laneID), &manifest); !d.Reused {
return pipeline.NormalizeCheckpoint{}, d
}
if d := l.validateLaneManifest(manifest.StageManifest, StageNormalize, laneID, moduleKey, dependencies, StatusSucceeded); !d.Reused {
if d := l.validateLaneManifest(manifest.StageManifest, StageNormalize, stepID, laneID, moduleKey, dependencies, StatusSucceeded); !d.Reused {
return pipeline.NormalizeCheckpoint{}, d
}
var payload artifactSingleEnvelope
if d := l.readJSON(lanePayloadPath("normalize", laneID, "output.json"), &payload); !d.Reused {
if d := l.readJSON(lanePayloadPath("normalize", stepID, laneID, "output.json"), &payload); !d.Reused {
return pipeline.NormalizeCheckpoint{}, d
}
values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output})
@@ -149,7 +161,7 @@ func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.
func (l *FilesystemLoader) readJSON(name string, out any) pipeline.CheckpointDecision {
if !l.Enabled() {
return pipeline.CheckpointDecision{Reason: "checkpoint loading disabled"}
return pipeline.CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"}
}
target, err := fileio.SafePath(l.root, name)
if err != nil {
@@ -158,7 +170,7 @@ func (l *FilesystemLoader) readJSON(name string, out any) pipeline.CheckpointDec
data, err := os.ReadFile(target)
if err != nil {
if os.IsNotExist(err) {
return pipeline.CheckpointDecision{Reason: "checkpoint artifact is missing"}
return pipeline.CheckpointDecision{Category: "executed", ReasonCode: "checkpoint_missing", Reason: "checkpoint artifact is missing"}
}
return invalidDecision("read checkpoint artifact: %v", err)
}
@@ -169,13 +181,16 @@ func (l *FilesystemLoader) readJSON(name string, out any) pipeline.CheckpointDec
}
func (l *FilesystemLoader) validateManifest(manifest StageManifest, stage StageName, laneID string, moduleKey string, status StageStatus, dependencies []pipeline.CheckpointFingerprint) pipeline.CheckpointDecision {
return l.validateLaneManifest(manifest, stage, laneID, moduleKey, dependencies, status)
return l.validateLaneManifest(manifest, stage, "", laneID, moduleKey, dependencies, status)
}
func (l *FilesystemLoader) validateLaneManifest(manifest StageManifest, stage StageName, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, statuses ...StageStatus) pipeline.CheckpointDecision {
func (l *FilesystemLoader) validateLaneManifest(manifest StageManifest, stage StageName, stepID string, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, statuses ...StageStatus) pipeline.CheckpointDecision {
if manifest.WorkspaceSchemaVersion == WorkspaceSchemaVersionV1 {
return invalidDecision("checkpoint workspace schema version %q is incompatible with %q and must be recomputed", manifest.WorkspaceSchemaVersion, WorkspaceSchemaVersion)
}
if manifest.WorkspaceSchemaVersion == WorkspaceSchemaVersionV2 {
return invalidDecision("checkpoint workspace schema version %q is incompatible with %q and must be recomputed", manifest.WorkspaceSchemaVersion, WorkspaceSchemaVersion)
}
if manifest.WorkspaceSchemaVersion != WorkspaceSchemaVersion {
return invalidDecision("checkpoint workspace schema version %q is not supported", manifest.WorkspaceSchemaVersion)
}
@@ -185,6 +200,9 @@ func (l *FilesystemLoader) validateLaneManifest(manifest StageManifest, stage St
if manifest.Stage != stage {
return invalidDecision("checkpoint stage %q does not match %q", manifest.Stage, stage)
}
if strings.TrimSpace(stepID) != "" && manifest.StepID != stepID {
return invalidDecision("checkpoint step does not match requested step")
}
if strings.TrimSpace(laneID) != "" && manifest.LaneID != laneID {
return invalidDecision("checkpoint lane %q does not match %q", manifest.LaneID, laneID)
}
@@ -244,9 +262,65 @@ func fingerprintsEqual(a []pipeline.CheckpointFingerprint, b []pipeline.Checkpoi
}
func reusedDecision() pipeline.CheckpointDecision {
return pipeline.CheckpointDecision{Reused: true, Reason: "checkpoint is valid"}
return pipeline.CheckpointDecision{Reused: true, Category: "reused", ReasonCode: "checkpoint_valid", Reason: "checkpoint is valid"}
}
func invalidDecision(format string, args ...any) pipeline.CheckpointDecision {
return pipeline.CheckpointDecision{Reason: fmt.Sprintf(format, args...)}
reason := fmt.Sprintf(format, args...)
code := "checkpoint_invalid"
category := "executed"
switch {
case strings.Contains(reason, "dependency fingerprints"):
code, category = "dependency_mismatch", "dependency_invalidated"
case strings.Contains(reason, "missing"):
code = "checkpoint_missing"
case strings.Contains(reason, "schema"):
code = "workspace_schema_incompatible"
case strings.Contains(reason, "codec"):
code = "artifact_codec_incompatible"
case strings.Contains(reason, "content"):
code = "artifact_content_invalid"
case strings.Contains(reason, "identity"):
code = "identity_mismatch"
}
safe := safeReasonText(reason, code)
return pipeline.CheckpointDecision{Category: category, ReasonCode: code, Detail: safe, Reason: safe}
}
func safeReasonText(reason, code string) string {
lower := strings.ToLower(reason)
switch {
case strings.Contains(lower, "workspace schema"):
return "checkpoint workspace schema is incompatible"
case strings.Contains(lower, "source checkpoint document"):
return "source checkpoint document is invalid"
case strings.Contains(lower, "artifact codec identity"):
return "artifact codec identity is incomplete"
case strings.Contains(lower, "base64"):
return "checkpoint content base64 is invalid"
case strings.Contains(lower, "content digest"):
return "checkpoint content digest is invalid"
case strings.Contains(lower, "output digest"):
return "checkpoint output digest does not match payload"
case strings.Contains(lower, "identity"):
return "checkpoint identity does not match"
case strings.Contains(lower, "dependency"):
return "checkpoint dependency fingerprints do not match"
case strings.Contains(lower, "stage"):
return "checkpoint stage does not match"
case strings.Contains(lower, "step"):
return "checkpoint step does not match"
case strings.Contains(lower, "lane"):
return "checkpoint lane does not match"
case strings.Contains(lower, "module"):
return "checkpoint module does not match"
case strings.Contains(lower, "status"):
return "checkpoint status cannot be reused"
case strings.Contains(lower, "payload"):
return "checkpoint artifact payload is invalid"
case strings.Contains(lower, "decode"):
return "checkpoint artifact decode failed"
default:
return "checkpoint is not reusable (" + code + ")"
}
}

6
internal/framework/checkpoint/manifest.go
View File

@@ -3,9 +3,8 @@ package checkpoint
import "time"
const (
// These names and values are frozen checkpoint wire-compatibility
// identifiers. They intentionally retain the former terminology.
WorkspaceSchemaVersion = "notarius.workspace.v2"
WorkspaceSchemaVersion = "notarius.workspace.v3"
WorkspaceSchemaVersionV2 = "notarius.workspace.v2"
WorkspaceSchemaVersionV1 = "notarius.workspace.v1"
)
@@ -32,6 +31,7 @@ const (
type StageManifest struct {
WorkspaceSchemaVersion string `json:"workspace_schema_version"`
Stage StageName `json:"stage"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
DependencyFingerprints []Fingerprint `json:"dependency_fingerprints,omitempty"`

108
internal/framework/checkpoint/recorder.go
View File

@@ -71,93 +71,137 @@ func (r *FilesystemRecorder) SourceFailed(moduleKey string, err error) error {
}
func (r *FilesystemRecorder) ExtractRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
manifest := r.laneManifest(StageExtract, StatusRunning, laneID, moduleKey, dependencies)
return r.ExtractRunningForStep("", laneID, moduleKey, dependencies)
}
func (r *FilesystemRecorder) ExtractRunningForStep(stepID, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
manifest := r.laneManifest(StageExtract, StatusRunning, stepID, laneID, moduleKey, dependencies)
manifest.StartedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("extract", laneID), ExtractLaneManifest{StageManifest: manifest})
return r.writeManifest(laneManifestPath("extract", stepID, laneID), ExtractLaneManifest{StageManifest: manifest})
}
func (r *FilesystemRecorder) ExtractSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
return r.ExtractSucceededForStep("", laneID, moduleKey, dependencies, outputs, rejected, warnings)
}
func (r *FilesystemRecorder) ExtractSucceededForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
payload := artifactExtractEnvelope{Outputs: artifactCheckpointEnvelopes(outputs), Rejected: cloneRejectedOutputs(rejected), Warnings: cloneWarnings(warnings)}
if err := r.writePayload(lanePayloadPath("extract", laneID, "outputs.json"), payload); err != nil {
if err := r.writePayload(lanePayloadPath("extract", stepID, laneID, "outputs.json"), payload); err != nil {
return err
}
manifest := r.laneManifest(StageExtract, statusForRejected(rejected), laneID, moduleKey, dependencies)
manifest := r.laneManifest(StageExtract, statusForRejected(rejected), stepID, laneID, moduleKey, dependencies)
manifest.OutputDigests = checkpointFingerprints(artifactOutputDigests(outputs))
manifest.ValidationStatus = validationStatusString(warnings, rejected)
manifest.Rejections = rejectionSummaries(rejected)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("extract", laneID), ExtractLaneManifest{StageManifest: manifest, ChunkCount: len(outputs) + len(rejected), OutputCount: len(outputs)})
return r.writeManifest(laneManifestPath("extract", stepID, laneID), ExtractLaneManifest{StageManifest: manifest, ChunkCount: len(outputs) + len(rejected), OutputCount: len(outputs)})
}
func (r *FilesystemRecorder) ExtractFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
manifest := r.laneManifest(StageExtract, StatusFailed, laneID, moduleKey, dependencies)
return r.ExtractFailedForStep("", laneID, moduleKey, dependencies, err)
}
func (r *FilesystemRecorder) ExtractFailedForStep(stepID, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
manifest := r.laneManifest(StageExtract, StatusFailed, stepID, laneID, moduleKey, dependencies)
manifest.CompletedAt = timePtr(r.timestamp())
manifest.Metadata = errorMetadata(err)
return r.writeManifest(laneManifestPath("extract", laneID), ExtractLaneManifest{StageManifest: manifest})
return r.writeManifest(laneManifestPath("extract", stepID, laneID), ExtractLaneManifest{StageManifest: manifest})
}
func (r *FilesystemRecorder) MergeRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
manifest := r.laneManifest(StageMerge, StatusRunning, laneID, moduleKey, dependencies)
return r.MergeRunningForStep("", laneID, moduleKey, dependencies)
}
func (r *FilesystemRecorder) MergeRunningForStep(stepID, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
manifest := r.laneManifest(StageMerge, StatusRunning, stepID, laneID, moduleKey, dependencies)
manifest.StartedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("merge", laneID), MergeLaneManifest{StageManifest: manifest})
return r.writeManifest(laneManifestPath("merge", stepID, laneID), MergeLaneManifest{StageManifest: manifest})
}
func (r *FilesystemRecorder) MergeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
if err := r.writePayload(lanePayloadPath("merge", laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
return r.MergeSucceededForStep("", laneID, moduleKey, dependencies, output, warnings)
}
func (r *FilesystemRecorder) MergeSucceededForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
if err := r.writePayload(lanePayloadPath("merge", stepID, laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
return err
}
manifest := r.laneManifest(StageMerge, StatusSucceeded, laneID, moduleKey, dependencies)
manifest := r.laneManifest(StageMerge, StatusSucceeded, stepID, laneID, moduleKey, dependencies)
manifest.OutputDigests = checkpointFingerprints(artifactOutputDigests([]pipeline.CheckpointArtifact{output}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("merge", laneID), MergeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
return r.writeManifest(laneManifestPath("merge", stepID, laneID), MergeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
}
func (r *FilesystemRecorder) MergeRejected(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, rejected contracts.RejectedOutput) error {
manifest := r.laneManifest(StageMerge, StatusSucceededWithRejections, laneID, moduleKey, dependencies)
return r.MergeRejectedForStep("", laneID, moduleKey, dependencies, rejected)
}
func (r *FilesystemRecorder) MergeRejectedForStep(stepID, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, rejected contracts.RejectedOutput) error {
manifest := r.laneManifest(StageMerge, StatusSucceededWithRejections, stepID, laneID, moduleKey, dependencies)
manifest.ValidationStatus = "rejected"
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("merge", laneID), MergeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
return r.writeManifest(laneManifestPath("merge", stepID, laneID), MergeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
}
func (r *FilesystemRecorder) MergeFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
manifest := r.laneManifest(StageMerge, StatusFailed, laneID, moduleKey, dependencies)
return r.MergeFailedForStep("", laneID, moduleKey, dependencies, err)
}
func (r *FilesystemRecorder) MergeFailedForStep(stepID, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
manifest := r.laneManifest(StageMerge, StatusFailed, stepID, laneID, moduleKey, dependencies)
manifest.CompletedAt = timePtr(r.timestamp())
manifest.Metadata = errorMetadata(err)
return r.writeManifest(laneManifestPath("merge", laneID), MergeLaneManifest{StageManifest: manifest})
return r.writeManifest(laneManifestPath("merge", stepID, laneID), MergeLaneManifest{StageManifest: manifest})
}
func (r *FilesystemRecorder) NormalizeRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
manifest := r.laneManifest(StageNormalize, StatusRunning, laneID, moduleKey, dependencies)
return r.NormalizeRunningForStep("", laneID, moduleKey, dependencies)
}
func (r *FilesystemRecorder) NormalizeRunningForStep(stepID, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
manifest := r.laneManifest(StageNormalize, StatusRunning, stepID, laneID, moduleKey, dependencies)
manifest.StartedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("normalize", laneID), NormalizeLaneManifest{StageManifest: manifest})
return r.writeManifest(laneManifestPath("normalize", stepID, laneID), NormalizeLaneManifest{StageManifest: manifest})
}
func (r *FilesystemRecorder) NormalizeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
if err := r.writePayload(lanePayloadPath("normalize", laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
return r.NormalizeSucceededForStep("", laneID, moduleKey, dependencies, output, warnings)
}
func (r *FilesystemRecorder) NormalizeSucceededForStep(stepID, laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
if err := r.writePayload(lanePayloadPath("normalize", stepID, laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
return err
}
manifest := r.laneManifest(StageNormalize, StatusSucceeded, laneID, moduleKey, dependencies)
manifest := r.laneManifest(StageNormalize, StatusSucceeded, stepID, laneID, moduleKey, dependencies)
manifest.OutputDigests = checkpointFingerprints(artifactOutputDigests([]pipeline.CheckpointArtifact{output}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("normalize", laneID), NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
return r.writeManifest(laneManifestPath("normalize", stepID, laneID), NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
}
func (r *FilesystemRecorder) NormalizeRejected(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, rejected contracts.RejectedOutput) error {
manifest := r.laneManifest(StageNormalize, StatusSucceededWithRejections, laneID, moduleKey, dependencies)
return r.NormalizeRejectedForStep("", laneID, moduleKey, dependencies, rejected)
}
func (r *FilesystemRecorder) NormalizeRejectedForStep(stepID, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, rejected contracts.RejectedOutput) error {
manifest := r.laneManifest(StageNormalize, StatusSucceededWithRejections, stepID, laneID, moduleKey, dependencies)
manifest.ValidationStatus = "rejected"
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("normalize", laneID), NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
return r.writeManifest(laneManifestPath("normalize", stepID, laneID), NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
}
func (r *FilesystemRecorder) NormalizeFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
manifest := r.laneManifest(StageNormalize, StatusFailed, laneID, moduleKey, dependencies)
return r.NormalizeFailedForStep("", laneID, moduleKey, dependencies, err)
}
func (r *FilesystemRecorder) NormalizeFailedForStep(stepID, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
manifest := r.laneManifest(StageNormalize, StatusFailed, stepID, laneID, moduleKey, dependencies)
manifest.CompletedAt = timePtr(r.timestamp())
manifest.Metadata = errorMetadata(err)
return r.writeManifest(laneManifestPath("normalize", laneID), NormalizeLaneManifest{StageManifest: manifest})
return r.writeManifest(laneManifestPath("normalize", stepID, laneID), NormalizeLaneManifest{StageManifest: manifest})
}
func (r *FilesystemRecorder) writeManifest(name string, payload any) error {
@@ -190,8 +234,9 @@ func (r *FilesystemRecorder) newStageManifest(stage StageName, status StageStatu
return manifest
}
func (r *FilesystemRecorder) laneManifest(stage StageName, status StageStatus, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) StageManifest {
func (r *FilesystemRecorder) laneManifest(stage StageName, status StageStatus, stepID string, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) StageManifest {
manifest := r.newStageManifest(stage, status)
manifest.StepID = stepID
manifest.LaneID = laneID
manifest.ModuleKey = moduleKey
manifest.DependencyFingerprints = checkpointFingerprints(dependencies)
@@ -424,12 +469,15 @@ func errorMetadata(err error) map[string]string {
return map[string]string{"error": err.Error()}
}
func laneManifestPath(stage string, laneID string) string {
return lanePayloadPath(stage, laneID, "manifest.json")
func laneManifestPath(stage string, stepID string, laneID string) string {
return lanePayloadPath(stage, stepID, laneID, "manifest.json")
}
func lanePayloadPath(stage string, laneID string, file string) string {
return path.Join(stage, checkpointPathComponent(laneID), file)
func lanePayloadPath(stage string, stepID string, laneID string, file string) string {
if strings.TrimSpace(stepID) == "" {
return path.Join(stage, checkpointPathComponent(laneID), file)
}
return path.Join(stage, checkpointPathComponent(stepID), checkpointPathComponent(laneID), file)
}
func checkpointPathComponent(value string) string {

37
internal/framework/checkpoint/recorder_test.go
View File

@@ -35,9 +35,40 @@ func TestRootBasedRecorderOutputIsReusable(t *testing.T) {
}
}
func TestCheckpointSchemaCompatibilityIsUnchanged(t *testing.T) {
if WorkspaceSchemaVersion != "notarius.workspace.v2" || WorkspaceSchemaVersionV1 != "notarius.workspace.v1" {
t.Fatal("checkpoint schema identifiers changed")
func TestStepAwareRecorderAndLoaderIsolateLaneState(t *testing.T) {
root := t.TempDir()
identity := testIdentity(t)
recorder, err := NewFilesystemRecorder(root, identity)
if err != nil {
t.Fatal(err)
}
if err := recorder.(pipeline.StepCheckpointRecorder).ExtractSucceededForStep("step-a", "lane", "module", nil, nil, nil, nil); err != nil {
t.Fatal(err)
}
loader, err := NewFilesystemLoader(root, identity)
if err != nil {
t.Fatal(err)
}
stepLoader := loader.(pipeline.StepCheckpointLoader)
if _, decision := stepLoader.ExtractForStep("step-b", "lane", "module", nil); decision.Reused {
t.Fatal("checkpoint from another step was reused")
}
loaded, decision := stepLoader.ExtractForStep("step-a", "lane", "module", nil)
if !decision.Reused || len(loaded.Outputs) != 0 {
t.Fatalf("step-aware load = %#v, decision=%#v", loaded, decision)
}
relative, err := identity.RelativePath()
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(root, relative, "extract", "step-a", "lane", "manifest.json")); err != nil {
t.Fatal(err)
}
}
func TestCheckpointSchemaCompatibilityIdentifiers(t *testing.T) {
if WorkspaceSchemaVersion != "notarius.workspace.v3" || WorkspaceSchemaVersionV2 != "notarius.workspace.v2" || WorkspaceSchemaVersionV1 != "notarius.workspace.v1" {
t.Fatal("checkpoint schema identifiers are incorrect")
}
}