eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add stable checkpoint decision diagnostics

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-22 02:13:50 +00:00
parent 5bdd56cfb1
commit 9de399432e
10 changed files with 391 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
internal/framework/pipeline/checkpoint.go
View File

@@ -57,26 +57,96 @@ type StepCheckpointRecorder interface {
NormalizeFailedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
}
type CheckpointDecisionCategory string
const (
CheckpointDecisionExecuted CheckpointDecisionCategory = "executed"
CheckpointDecisionReused CheckpointDecisionCategory = "reused"
CheckpointDecisionForcedRecompute CheckpointDecisionCategory = "forced_recompute"
CheckpointDecisionDependencyInvalidated CheckpointDecisionCategory = "dependency_invalidated"
)
type CheckpointReasonCode string
const (
CheckpointReasonLoadingDisabled CheckpointReasonCode = "loading_disabled"
CheckpointReasonMissing CheckpointReasonCode = "checkpoint_missing"
CheckpointReasonPathInvalid CheckpointReasonCode = "checkpoint_path_invalid"
CheckpointReasonReadFailed CheckpointReasonCode = "checkpoint_read_failed"
CheckpointReasonDecodeFailed CheckpointReasonCode = "checkpoint_decode_failed"
CheckpointReasonWorkspaceSchemaIncompatible CheckpointReasonCode = "workspace_schema_incompatible"
CheckpointReasonIdentityMismatch CheckpointReasonCode = "identity_mismatch"
CheckpointReasonStageMismatch CheckpointReasonCode = "stage_mismatch"
CheckpointReasonStepMismatch CheckpointReasonCode = "step_mismatch"
CheckpointReasonLaneMismatch CheckpointReasonCode = "lane_mismatch"
CheckpointReasonModuleMismatch CheckpointReasonCode = "module_mismatch"
CheckpointReasonStatusNotReusable CheckpointReasonCode = "status_not_reusable"
CheckpointReasonDependencyMismatch CheckpointReasonCode = "dependency_mismatch"
CheckpointReasonArtifactPayloadInvalid CheckpointReasonCode = "artifact_payload_invalid"
CheckpointReasonArtifactDigestMismatch CheckpointReasonCode = "artifact_digest_mismatch"
CheckpointReasonArtifactCodecIncompatible CheckpointReasonCode = "artifact_codec_incompatible"
CheckpointReasonArtifactNotCanonical CheckpointReasonCode = "artifact_not_canonical"
CheckpointReasonReused CheckpointReasonCode = "checkpoint_reused"
CheckpointReasonAcceptedArtifactReused CheckpointReasonCode = "accepted_artifact_reused"
CheckpointReasonRecomputeStep CheckpointReasonCode = "recompute_step"
)
type CheckpointDecision struct {
Reused bool `json:"reused"`
Category string `json:"category,omitempty"`
ReasonCode string `json:"reason_code,omitempty"`
Detail string `json:"detail,omitempty"`
Reused bool `json:"reused"`
Category CheckpointDecisionCategory `json:"category,omitempty"`
ReasonCode CheckpointReasonCode `json:"reason_code,omitempty"`
Detail string `json:"detail,omitempty"`
// Reason is retained as a compatibility/debug field for existing callers.
// New checkpoint stores should put bounded, non-sensitive text in Detail.
Reason string `json:"reason,omitempty"`
}
const checkpointDecisionDetailLimit = 512
func NewCheckpointDecision(category CheckpointDecisionCategory, reasonCode CheckpointReasonCode, detail string) CheckpointDecision {
return checkpointDecision(category, reasonCode, detail)
}
func checkpointDecision(category CheckpointDecisionCategory, reasonCode CheckpointReasonCode, detail string) CheckpointDecision {
detail = sanitizeCheckpointDecisionDetail(detail)
return CheckpointDecision{
Reused: category == CheckpointDecisionReused,
Category: category,
ReasonCode: reasonCode,
Detail: detail,
Reason: detail,
}
}
func sanitizeCheckpointDecisionDetail(detail string) string {
detail = strings.TrimSpace(strings.ToValidUTF8(detail, "?"))
lower := strings.ToLower(detail)
if strings.ContainsAny(detail, `/\\`) || strings.Contains(lower, "secret") || strings.Contains(lower, "token") || strings.Contains(lower, "password") || strings.Contains(lower, "credential") || strings.Contains(lower, "environment") {
return "checkpoint decision detail redacted"
}
var b strings.Builder
for _, r := range detail {
if r < 0x20 || r == 0x7f {
r = ' '
}
if b.Len()+len(string(r)) > checkpointDecisionDetailLimit {
break
}
b.WriteRune(r)
}
return strings.Join(strings.Fields(b.String()), " ")
}
type CheckpointEvent struct {
Stage string `json:"stage"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
Action string `json:"action"`
Category string `json:"category,omitempty"`
ReasonCode string `json:"reason_code,omitempty"`
Detail string `json:"detail,omitempty"`
Reason string `json:"reason,omitempty"`
Stage string `json:"stage"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
Action CheckpointDecisionCategory `json:"action"`
Category CheckpointDecisionCategory `json:"category,omitempty"`
ReasonCode CheckpointReasonCode `json:"reason_code,omitempty"`
Detail string `json:"detail,omitempty"`
Reason string `json:"reason,omitempty"`
}
type CheckpointExecutionPolicy struct {
@@ -100,14 +170,14 @@ func (policy CheckpointExecutionPolicy) requiresReusable(stepID, laneID string)
func forceCheckpointDecision(policy CheckpointExecutionPolicy, stepID, laneID string, decision CheckpointDecision) CheckpointDecision {
if policy.forced(stepID, laneID) {
return CheckpointDecision{Category: "forced_recompute", ReasonCode: "recompute_step", Detail: "selected step requires execution"}
return checkpointDecision(CheckpointDecisionForcedRecompute, CheckpointReasonRecomputeStep, "selected step requires execution")
}
return decision
}
func requireReusableCheckpoint(policy CheckpointExecutionPolicy, stepID, laneID string, decision CheckpointDecision) error {
if policy.requiresReusable(stepID, laneID) && !decision.Reused {
return fmt.Errorf("required reusable checkpoint unavailable for step %q lane %q", strings.TrimSpace(stepID), strings.TrimSpace(laneID))
return fmt.Errorf("required reusable checkpoint unavailable for step %q lane %q (%s)", strings.TrimSpace(stepID), strings.TrimSpace(laneID), decision.ReasonCode)
}
return nil
}
@@ -116,23 +186,20 @@ func requireReusableCheckpoint(policy CheckpointExecutionPolicy, stepID, laneID
// validation at the single point where a stage's observable decision is made.
func resolveCheckpointDecision(output *RunOutput, loader CheckpointLoader, policy CheckpointExecutionPolicy, stage ModuleStage, stepID, laneID, moduleKey string, decision CheckpointDecision, codec artifactCodecEntry, artifacts []CheckpointArtifact) (CheckpointDecision, error) {
decision = forceCheckpointDecision(policy, stepID, laneID, decision)
if err := requireReusableCheckpoint(policy, stepID, laneID, decision); err != nil {
return decision, err
}
if decision.Reused {
for _, artifact := range artifacts {
if _, _, err := decodeCanonicalCheckpointArtifact(codec, artifact); err != nil {
decision = CheckpointDecision{Category: "executed", ReasonCode: "artifact_not_canonical", Detail: "stored " + string(stage) + " artifact failed canonical codec validation", Reason: string(stage) + " artifact checkpoint is not canonical"}
decision = checkpointDecision(CheckpointDecisionExecuted, checkpointArtifactReasonCode(err), "stored "+string(stage)+" artifact failed canonical codec validation")
break
}
}
}
if err := requireReusableCheckpoint(policy, stepID, laneID, decision); err != nil {
return decision, err
}
if output != nil {
recordCheckpointEvent(output, loader, string(stage), stepID, laneID, moduleKey, decision)
}
if err := requireReusableCheckpoint(policy, stepID, laneID, decision); err != nil {
return decision, err
}
return decision, nil
}
@@ -228,16 +295,16 @@ func (noopCheckpointRecorder) NormalizeFailed(string, string, []CheckpointFinger
func (noopCheckpointLoader) Enabled() bool { return false }
func (noopCheckpointLoader) Source(string) (SourceCheckpoint, CheckpointDecision) {
return SourceCheckpoint{}, CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"}
return SourceCheckpoint{}, checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonLoadingDisabled, "checkpoint loading disabled")
}
func (noopCheckpointLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
return ExtractCheckpoint{}, CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"}
return ExtractCheckpoint{}, checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonLoadingDisabled, "checkpoint loading disabled")
}
func (noopCheckpointLoader) Merge(string, string, []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
return MergeCheckpoint{}, CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"}
return MergeCheckpoint{}, checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonLoadingDisabled, "checkpoint loading disabled")
}
func (noopCheckpointLoader) Normalize(string, string, []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
return NormalizeCheckpoint{}, CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"}
return NormalizeCheckpoint{}, checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonLoadingDisabled, "checkpoint loading disabled")
}
func checkpointExtractRunning(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) error {