eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reuse valid workspace checkpoints on request

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-08 03:02:50 +00:00
parent 1d3a444df8
commit ae9c2e1d5e
14 changed files with 1406 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
internal/framework/checkpoint/recorder.go
View File

@@ -17,8 +17,9 @@ import (
)
type WorkspaceRecorder struct {
root string
now func() time.Time
root string
identityDigest string
now func() time.Time
}
func NewWorkspaceRecorder(settings coreworkspace.Settings, identity coreworkspace.CheckpointIdentity) (pipeline.CheckpointRecorder, error) {
@@ -29,11 +30,11 @@ func NewWorkspaceRecorder(settings coreworkspace.Settings, identity coreworkspac
if strings.TrimSpace(root) == "" {
return pipeline.NoopCheckpointRecorder(), nil
}
return &WorkspaceRecorder{root: root, now: time.Now}, nil
return &WorkspaceRecorder{root: root, identityDigest: identity.Digest, now: time.Now}, nil
}
func (r *WorkspaceRecorder) SourceRunning(moduleKey string) error {
manifest := coreworkspace.NewStageManifest(coreworkspace.StageSource, coreworkspace.StatusRunning)
manifest := r.newStageManifest(coreworkspace.StageSource, coreworkspace.StatusRunning)
manifest.ModuleKey = moduleKey
manifest.StartedAt = timePtr(r.timestamp())
return r.writeManifest("source/manifest.json", coreworkspace.SourceManifest{StageManifest: manifest})
@@ -46,7 +47,7 @@ func (r *WorkspaceRecorder) SourceSucceeded(moduleKey string, doc *source.Source
if err := r.writePayload("source/source-document.json", sourceDocumentEnvelope{Document: cloneSourceDocument(*doc)}); err != nil {
return err
}
manifest := coreworkspace.NewStageManifest(coreworkspace.StageSource, coreworkspace.StatusSucceeded)
manifest := r.newStageManifest(coreworkspace.StageSource, coreworkspace.StatusSucceeded)
manifest.ModuleKey = moduleKey
manifest.OutputDigests = workspaceFingerprints(digestFingerprints("source_document", doc.Digest))
manifest.CompletedAt = timePtr(r.timestamp())
@@ -57,7 +58,7 @@ func (r *WorkspaceRecorder) SourceSucceeded(moduleKey string, doc *source.Source
}
func (r *WorkspaceRecorder) SourceFailed(moduleKey string, err error) error {
manifest := coreworkspace.NewStageManifest(coreworkspace.StageSource, coreworkspace.StatusFailed)
manifest := r.newStageManifest(coreworkspace.StageSource, coreworkspace.StatusFailed)
manifest.ModuleKey = moduleKey
manifest.CompletedAt = timePtr(r.timestamp())
manifest.Metadata = errorMetadata(err)
@@ -65,7 +66,7 @@ func (r *WorkspaceRecorder) SourceFailed(moduleKey string, err error) error {
}
func (r *WorkspaceRecorder) ChunkRunning(moduleKey string, sourceDigest string) error {
manifest := coreworkspace.NewStageManifest(coreworkspace.StageChunk, coreworkspace.StatusRunning)
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusRunning)
manifest.ModuleKey = moduleKey
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
manifest.StartedAt = timePtr(r.timestamp())
@@ -77,7 +78,7 @@ func (r *WorkspaceRecorder) ChunkSucceeded(moduleKey string, sourceDigest string
if err := r.writePayload("chunk/chunks.json", payload); err != nil {
return err
}
manifest := coreworkspace.NewStageManifest(coreworkspace.StageChunk, coreworkspace.StatusSucceeded)
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusSucceeded)
manifest.ModuleKey = moduleKey
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
manifest.OutputDigests = workspaceFingerprints(chunkOutputDigests(chunks))
@@ -90,7 +91,7 @@ func (r *WorkspaceRecorder) ChunkSucceeded(moduleKey string, sourceDigest string
}
func (r *WorkspaceRecorder) ChunkRejected(moduleKey string, sourceDigest string, rejected contracts.RejectedOutput) error {
manifest := coreworkspace.NewStageManifest(coreworkspace.StageChunk, coreworkspace.StatusSucceededWithRejections)
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusSucceededWithRejections)
manifest.ModuleKey = moduleKey
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
manifest.ValidationStatus = "rejected"
@@ -100,7 +101,7 @@ func (r *WorkspaceRecorder) ChunkRejected(moduleKey string, sourceDigest string,
}
func (r *WorkspaceRecorder) ChunkFailed(moduleKey string, sourceDigest string, err error) error {
manifest := coreworkspace.NewStageManifest(coreworkspace.StageChunk, coreworkspace.StatusFailed)
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusFailed)
manifest.ModuleKey = moduleKey
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
manifest.CompletedAt = timePtr(r.timestamp())
@@ -109,7 +110,7 @@ func (r *WorkspaceRecorder) ChunkFailed(moduleKey string, sourceDigest string, e
}
func (r *WorkspaceRecorder) ExtractRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
manifest := laneManifest(coreworkspace.StageExtract, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
manifest := r.laneManifest(coreworkspace.StageExtract, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
manifest.StartedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest})
}
@@ -123,7 +124,7 @@ func (r *WorkspaceRecorder) ExtractSucceeded(laneID string, moduleKey string, de
if err := r.writePayload(lanePayloadPath("extract", laneID, "outputs.json"), payload); err != nil {
return err
}
manifest := laneManifest(coreworkspace.StageExtract, statusForRejected(rejected), laneID, moduleKey, dependencies)
manifest := r.laneManifest(coreworkspace.StageExtract, statusForRejected(rejected), laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests(extractPayloads(outputs)))
manifest.ValidationStatus = validationStatusString(warnings, rejected)
manifest.Rejections = rejectionSummaries(rejected)
@@ -136,14 +137,14 @@ func (r *WorkspaceRecorder) ExtractSucceeded(laneID string, moduleKey string, de
}
func (r *WorkspaceRecorder) ExtractFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
manifest := laneManifest(coreworkspace.StageExtract, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
manifest := r.laneManifest(coreworkspace.StageExtract, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
manifest.CompletedAt = timePtr(r.timestamp())
manifest.Metadata = errorMetadata(err)
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) MergeRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
manifest := laneManifest(coreworkspace.StageMerge, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
manifest.StartedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest})
}
@@ -153,7 +154,7 @@ func (r *WorkspaceRecorder) MergeSucceeded(laneID string, moduleKey string, depe
if err := r.writePayload(lanePayloadPath("merge", laneID, "output.json"), payload); err != nil {
return err
}
manifest := laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests([]contracts.RawPayload{output.Payload}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
@@ -164,7 +165,7 @@ func (r *WorkspaceRecorder) MergeSucceeded(laneID string, moduleKey string, depe
}
func (r *WorkspaceRecorder) MergeRejected(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, rejected contracts.RejectedOutput) error {
manifest := laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceededWithRejections, laneID, moduleKey, dependencies)
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceededWithRejections, laneID, moduleKey, dependencies)
manifest.ValidationStatus = "rejected"
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
manifest.CompletedAt = timePtr(r.timestamp())
@@ -172,14 +173,14 @@ func (r *WorkspaceRecorder) MergeRejected(laneID string, moduleKey string, depen
}
func (r *WorkspaceRecorder) MergeFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
manifest := laneManifest(coreworkspace.StageMerge, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
manifest.CompletedAt = timePtr(r.timestamp())
manifest.Metadata = errorMetadata(err)
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) NormalizeRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
manifest := laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
manifest.StartedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest})
}
@@ -189,7 +190,7 @@ func (r *WorkspaceRecorder) NormalizeSucceeded(laneID string, moduleKey string,
if err := r.writePayload(lanePayloadPath("normalize", laneID, "output.json"), payload); err != nil {
return err
}
manifest := laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests([]contracts.RawPayload{output.Payload}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
@@ -197,7 +198,7 @@ func (r *WorkspaceRecorder) NormalizeSucceeded(laneID string, moduleKey string,
}
func (r *WorkspaceRecorder) NormalizeRejected(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, rejected contracts.RejectedOutput) error {
manifest := laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceededWithRejections, laneID, moduleKey, dependencies)
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceededWithRejections, laneID, moduleKey, dependencies)
manifest.ValidationStatus = "rejected"
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
manifest.CompletedAt = timePtr(r.timestamp())
@@ -205,7 +206,7 @@ func (r *WorkspaceRecorder) NormalizeRejected(laneID string, moduleKey string, d
}
func (r *WorkspaceRecorder) NormalizeFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
manifest := laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
manifest.CompletedAt = timePtr(r.timestamp())
manifest.Metadata = errorMetadata(err)
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest})
@@ -233,8 +234,16 @@ func (r *WorkspaceRecorder) timestamp() time.Time {
return r.now().UTC()
}
func laneManifest(stage coreworkspace.StageName, status coreworkspace.StageStatus, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) coreworkspace.StageManifest {
func (r *WorkspaceRecorder) newStageManifest(stage coreworkspace.StageName, status coreworkspace.StageStatus) coreworkspace.StageManifest {
manifest := coreworkspace.NewStageManifest(stage, status)
if strings.TrimSpace(r.identityDigest) != "" {
manifest.Metadata = map[string]string{"checkpoint_identity_digest": r.identityDigest}
}
return manifest
}
func (r *WorkspaceRecorder) laneManifest(stage coreworkspace.StageName, status coreworkspace.StageStatus, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) coreworkspace.StageManifest {
manifest := r.newStageManifest(stage, status)
manifest.LaneID = laneID
manifest.ModuleKey = moduleKey
manifest.DependencyFingerprints = workspaceFingerprints(dependencies)