package pipeline import ( "crypto/sha256" "encoding/hex" "fmt" "sort" "strings" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) type CheckpointFingerprint struct { Name string `json:"name"` Value string `json:"value"` } // CheckpointFingerprintProvider supplies stable, non-secret semantic identity // for a prepared module or validator. Values must not contain source content, // credentials, local paths, timestamps, or other invocation-specific data. type CheckpointFingerprintProvider interface { CheckpointFingerprints() []CheckpointFingerprint } type CheckpointRecorder interface { SourceRunning(moduleKey string) error SourceSucceeded(moduleKey string, doc *source.SourceDocument) error SourceFailed(moduleKey string, err error) error ExtractRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error ExtractSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error ExtractFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error MergeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error MergeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error MergeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error MergeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error NormalizeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error NormalizeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error NormalizeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error NormalizeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error } // StepCheckpointRecorder is implemented by checkpoint stores that isolate // lane artifacts by their ordered pipeline step. The legacy recorder methods // remain available for callers that do not have step context. type StepCheckpointRecorder interface { ExtractRunningForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) error ExtractSucceededForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error ExtractFailedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error MergeRunningForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) error MergeSucceededForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error MergeRejectedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error MergeFailedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error NormalizeRunningForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) error NormalizeSucceededForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error NormalizeRejectedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error NormalizeFailedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error } type CheckpointDecision struct { Reused bool `json:"reused"` Category string `json:"category,omitempty"` ReasonCode string `json:"reason_code,omitempty"` Detail string `json:"detail,omitempty"` // Reason is retained as a compatibility/debug field for existing callers. // New checkpoint stores should put bounded, non-sensitive text in Detail. Reason string `json:"reason,omitempty"` } type CheckpointEvent struct { Stage string `json:"stage"` StepID string `json:"step_id,omitempty"` LaneID string `json:"lane_id,omitempty"` ModuleKey string `json:"module_key,omitempty"` Action string `json:"action"` Category string `json:"category,omitempty"` ReasonCode string `json:"reason_code,omitempty"` Detail string `json:"detail,omitempty"` Reason string `json:"reason,omitempty"` } type CheckpointExecutionPolicy struct { ForcedLanes map[string]struct{} RequireReusableLanes map[string]struct{} } func CheckpointLaneKey(stepID, laneID string) string { return strings.TrimSpace(stepID) + "\x00" + strings.TrimSpace(laneID) } func (policy CheckpointExecutionPolicy) forced(stepID, laneID string) bool { _, ok := policy.ForcedLanes[CheckpointLaneKey(stepID, laneID)] return ok } func (policy CheckpointExecutionPolicy) requiresReusable(stepID, laneID string) bool { _, ok := policy.RequireReusableLanes[CheckpointLaneKey(stepID, laneID)] return ok } func forceCheckpointDecision(policy CheckpointExecutionPolicy, stepID, laneID string, decision CheckpointDecision) CheckpointDecision { if policy.forced(stepID, laneID) { return CheckpointDecision{Category: "forced_recompute", ReasonCode: "recompute_step", Detail: "selected step requires execution"} } return decision } func requireReusableCheckpoint(policy CheckpointExecutionPolicy, stepID, laneID string, decision CheckpointDecision) error { if policy.requiresReusable(stepID, laneID) && !decision.Reused { return fmt.Errorf("required reusable checkpoint unavailable for step %q lane %q", strings.TrimSpace(stepID), strings.TrimSpace(laneID)) } return nil } // resolveCheckpointDecision applies runner policy and canonical payload // validation at the single point where a stage's observable decision is made. func resolveCheckpointDecision(output *RunOutput, loader CheckpointLoader, policy CheckpointExecutionPolicy, stage ModuleStage, stepID, laneID, moduleKey string, decision CheckpointDecision, codec artifactCodecEntry, artifacts []CheckpointArtifact) (CheckpointDecision, error) { decision = forceCheckpointDecision(policy, stepID, laneID, decision) if err := requireReusableCheckpoint(policy, stepID, laneID, decision); err != nil { return decision, err } if decision.Reused { for _, artifact := range artifacts { if _, _, err := decodeCanonicalCheckpointArtifact(codec, artifact); err != nil { decision = CheckpointDecision{Category: "executed", ReasonCode: "artifact_not_canonical", Detail: "stored " + string(stage) + " artifact failed canonical codec validation", Reason: string(stage) + " artifact checkpoint is not canonical"} break } } } if err := requireReusableCheckpoint(policy, stepID, laneID, decision); err != nil { return decision, err } if output != nil { recordCheckpointEvent(output, loader, string(stage), stepID, laneID, moduleKey, decision) } return decision, nil } type SourceCheckpoint struct { Document *source.SourceDocument } // CheckpointArtifact is the durable, domain-neutral value stored at a lane // checkpoint boundary. type CheckpointArtifact struct { LaneID string ModuleKey string SourceID string ChunkID string ChunkIndex int ChunkRef source.SourceRef Artifact contracts.SerializedArtifact SchemaDigest string } type ExtractCheckpoint struct { Outputs []CheckpointArtifact Rejected []contracts.RejectedOutput Warnings []contracts.Warning } type MergeCheckpoint struct { Output CheckpointArtifact Warnings []contracts.Warning } type NormalizeCheckpoint struct { Output CheckpointArtifact Warnings []contracts.Warning } type CheckpointLoader interface { Enabled() bool Source(moduleKey string) (SourceCheckpoint, CheckpointDecision) Extract(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) Merge(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) Normalize(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) } // StepCheckpointLoader is the step-aware counterpart used by the persistent // checkpoint implementation. Loaders without this optional interface remain // usable by framework callers and test doubles through the legacy methods. type StepCheckpointLoader interface { ExtractForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) MergeForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) NormalizeForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) } type noopCheckpointRecorder struct{} type noopCheckpointLoader struct{} func NoopCheckpointRecorder() CheckpointRecorder { return noopCheckpointRecorder{} } func NoopCheckpointLoader() CheckpointLoader { return noopCheckpointLoader{} } func (noopCheckpointRecorder) SourceRunning(string) error { return nil } func (noopCheckpointRecorder) SourceSucceeded(string, *source.SourceDocument) error { return nil } func (noopCheckpointRecorder) SourceFailed(string, error) error { return nil } func (noopCheckpointRecorder) ExtractRunning(string, string, []CheckpointFingerprint) error { return nil } func (noopCheckpointRecorder) ExtractSucceeded(string, string, []CheckpointFingerprint, []CheckpointArtifact, []contracts.RejectedOutput, []contracts.Warning) error { return nil } func (noopCheckpointRecorder) ExtractFailed(string, string, []CheckpointFingerprint, error) error { return nil } func (noopCheckpointRecorder) MergeRunning(string, string, []CheckpointFingerprint) error { return nil } func (noopCheckpointRecorder) MergeSucceeded(string, string, []CheckpointFingerprint, CheckpointArtifact, []contracts.Warning) error { return nil } func (noopCheckpointRecorder) MergeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error { return nil } func (noopCheckpointRecorder) MergeFailed(string, string, []CheckpointFingerprint, error) error { return nil } func (noopCheckpointRecorder) NormalizeRunning(string, string, []CheckpointFingerprint) error { return nil } func (noopCheckpointRecorder) NormalizeSucceeded(string, string, []CheckpointFingerprint, CheckpointArtifact, []contracts.Warning) error { return nil } func (noopCheckpointRecorder) NormalizeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error { return nil } func (noopCheckpointRecorder) NormalizeFailed(string, string, []CheckpointFingerprint, error) error { return nil } func (noopCheckpointLoader) Enabled() bool { return false } func (noopCheckpointLoader) Source(string) (SourceCheckpoint, CheckpointDecision) { return SourceCheckpoint{}, CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"} } func (noopCheckpointLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) { return ExtractCheckpoint{}, CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"} } func (noopCheckpointLoader) Merge(string, string, []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) { return MergeCheckpoint{}, CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"} } func (noopCheckpointLoader) Normalize(string, string, []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) { return NormalizeCheckpoint{}, CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"} } func checkpointExtractRunning(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) error { if stepAware, ok := recorder.(StepCheckpointRecorder); ok { return stepAware.ExtractRunningForStep(stepID, laneID, moduleKey, deps) } return recorder.ExtractRunning(laneID, moduleKey, deps) } func checkpointExtractSucceeded(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error { if stepAware, ok := recorder.(StepCheckpointRecorder); ok { return stepAware.ExtractSucceededForStep(stepID, laneID, moduleKey, deps, outputs, rejected, warnings) } return recorder.ExtractSucceeded(laneID, moduleKey, deps, outputs, rejected, warnings) } func checkpointExtractFailed(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, err error) error { if stepAware, ok := recorder.(StepCheckpointRecorder); ok { return stepAware.ExtractFailedForStep(stepID, laneID, moduleKey, deps, err) } return recorder.ExtractFailed(laneID, moduleKey, deps, err) } func checkpointMergeRunning(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) error { if stepAware, ok := recorder.(StepCheckpointRecorder); ok { return stepAware.MergeRunningForStep(stepID, laneID, moduleKey, deps) } return recorder.MergeRunning(laneID, moduleKey, deps) } func checkpointMergeSucceeded(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error { if stepAware, ok := recorder.(StepCheckpointRecorder); ok { return stepAware.MergeSucceededForStep(stepID, laneID, moduleKey, deps, output, warnings) } return recorder.MergeSucceeded(laneID, moduleKey, deps, output, warnings) } func checkpointMergeRejected(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error { if stepAware, ok := recorder.(StepCheckpointRecorder); ok { return stepAware.MergeRejectedForStep(stepID, laneID, moduleKey, deps, rejected) } return recorder.MergeRejected(laneID, moduleKey, deps, rejected) } func checkpointMergeFailed(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, err error) error { if stepAware, ok := recorder.(StepCheckpointRecorder); ok { return stepAware.MergeFailedForStep(stepID, laneID, moduleKey, deps, err) } return recorder.MergeFailed(laneID, moduleKey, deps, err) } func checkpointNormalizeRunning(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) error { if stepAware, ok := recorder.(StepCheckpointRecorder); ok { return stepAware.NormalizeRunningForStep(stepID, laneID, moduleKey, deps) } return recorder.NormalizeRunning(laneID, moduleKey, deps) } func checkpointNormalizeSucceeded(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error { if stepAware, ok := recorder.(StepCheckpointRecorder); ok { return stepAware.NormalizeSucceededForStep(stepID, laneID, moduleKey, deps, output, warnings) } return recorder.NormalizeSucceeded(laneID, moduleKey, deps, output, warnings) } func checkpointNormalizeRejected(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error { if stepAware, ok := recorder.(StepCheckpointRecorder); ok { return stepAware.NormalizeRejectedForStep(stepID, laneID, moduleKey, deps, rejected) } return recorder.NormalizeRejected(laneID, moduleKey, deps, rejected) } func checkpointNormalizeFailed(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, err error) error { if stepAware, ok := recorder.(StepCheckpointRecorder); ok { return stepAware.NormalizeFailedForStep(stepID, laneID, moduleKey, deps, err) } return recorder.NormalizeFailed(laneID, moduleKey, deps, err) } func digestFingerprints(name string, digest string) []CheckpointFingerprint { digest = strings.TrimSpace(digest) if digest == "" { return nil } return []CheckpointFingerprint{{Name: name, Value: digest}} } func joinedChunkDigest(chunks []source.Chunk) (string, error) { if len(chunks) == 0 { return "", nil } values := make([]string, 0, len(chunks)) for _, chunk := range chunks { digest, err := source.DigestChunk(chunk) if err != nil { return "", fmt.Errorf("digest chunk %q: %w", chunk.ID, err) } values = append(values, chunk.ID+"="+digest) } sort.Strings(values) sum := sha256.Sum256([]byte(strings.Join(values, "\n"))) return "sha256:" + hex.EncodeToString(sum[:]), nil } func normalizeCheckpointFingerprints(values []CheckpointFingerprint) []CheckpointFingerprint { if len(values) == 0 { return nil } byName := make(map[string]string, len(values)) for _, value := range values { name := strings.TrimSpace(value.Name) fingerprint := strings.TrimSpace(value.Value) if name == "" || fingerprint == "" { continue } byName[name] = fingerprint } if len(byName) == 0 { return nil } names := make([]string, 0, len(byName)) for name := range byName { names = append(names, name) } sort.Strings(names) out := make([]CheckpointFingerprint, 0, len(names)) for _, name := range names { out = append(out, CheckpointFingerprint{Name: name, Value: byName[name]}) } return out } func checkpointContentDigest(content []byte) string { sum := sha256.Sum256(content) return "sha256:" + hex.EncodeToString(sum[:]) }