Implement selective checkpoint recomputation

This commit is contained in:
2026-07-21 21:59:48 +00:00
parent 22d4f29670
commit c437682407
17 changed files with 902 additions and 180 deletions

View File

@@ -40,17 +40,76 @@ type CheckpointRecorder interface {
NormalizeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
}
// StepCheckpointRecorder is implemented by checkpoint stores that isolate
// lane artifacts by their ordered pipeline step. The legacy recorder methods
// remain available for callers that do not have step context.
type StepCheckpointRecorder interface {
ExtractRunningForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
ExtractSucceededForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error
ExtractFailedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
MergeRunningForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
MergeSucceededForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error
MergeRejectedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
MergeFailedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
NormalizeRunningForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
NormalizeSucceededForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error
NormalizeRejectedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
NormalizeFailedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
}
type CheckpointDecision struct {
Reused bool `json:"reused"`
Reused bool `json:"reused"`
Category string `json:"category,omitempty"`
ReasonCode string `json:"reason_code,omitempty"`
Detail string `json:"detail,omitempty"`
// Reason is retained as a compatibility/debug field for existing callers.
// New checkpoint stores should put bounded, non-sensitive text in Detail.
Reason string `json:"reason,omitempty"`
}
type CheckpointEvent struct {
Stage string `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
Action string `json:"action"`
Reason string `json:"reason,omitempty"`
Stage string `json:"stage"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
Action string `json:"action"`
Category string `json:"category,omitempty"`
ReasonCode string `json:"reason_code,omitempty"`
Detail string `json:"detail,omitempty"`
Reason string `json:"reason,omitempty"`
}
type CheckpointExecutionPolicy struct {
ForcedLanes map[string]struct{}
RequireReusableLanes map[string]struct{}
}
func CheckpointLaneKey(stepID, laneID string) string {
return strings.TrimSpace(stepID) + "\x00" + strings.TrimSpace(laneID)
}
func (policy CheckpointExecutionPolicy) forced(stepID, laneID string) bool {
_, ok := policy.ForcedLanes[CheckpointLaneKey(stepID, laneID)]
return ok
}
func (policy CheckpointExecutionPolicy) requiresReusable(stepID, laneID string) bool {
_, ok := policy.RequireReusableLanes[CheckpointLaneKey(stepID, laneID)]
return ok
}
func forceCheckpointDecision(policy CheckpointExecutionPolicy, stepID, laneID string, decision CheckpointDecision) CheckpointDecision {
if policy.forced(stepID, laneID) {
return CheckpointDecision{Category: "forced_recompute", ReasonCode: "recompute_step", Detail: "selected step requires execution"}
}
return decision
}
func requireReusableCheckpoint(policy CheckpointExecutionPolicy, stepID, laneID string, decision CheckpointDecision) error {
if policy.requiresReusable(stepID, laneID) && !decision.Reused {
return fmt.Errorf("required reusable checkpoint unavailable for step %q lane %q", strings.TrimSpace(stepID), strings.TrimSpace(laneID))
}
return nil
}
type SourceCheckpoint struct {
@@ -93,6 +152,15 @@ type CheckpointLoader interface {
Normalize(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision)
}
// StepCheckpointLoader is the step-aware counterpart used by the persistent
// checkpoint implementation. Loaders without this optional interface remain
// usable by framework callers and test doubles through the legacy methods.
type StepCheckpointLoader interface {
ExtractForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision)
MergeForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision)
NormalizeForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision)
}
type noopCheckpointRecorder struct{}
type noopCheckpointLoader struct{}
@@ -136,16 +204,83 @@ func (noopCheckpointRecorder) NormalizeFailed(string, string, []CheckpointFinger
func (noopCheckpointLoader) Enabled() bool { return false }
func (noopCheckpointLoader) Source(string) (SourceCheckpoint, CheckpointDecision) {
return SourceCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
return SourceCheckpoint{}, CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"}
}
func (noopCheckpointLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
return ExtractCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
return ExtractCheckpoint{}, CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"}
}
func (noopCheckpointLoader) Merge(string, string, []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
return MergeCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
return MergeCheckpoint{}, CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"}
}
func (noopCheckpointLoader) Normalize(string, string, []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
return NormalizeCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
return NormalizeCheckpoint{}, CheckpointDecision{Category: "executed", ReasonCode: "loading_disabled", Reason: "checkpoint loading disabled"}
}
func checkpointExtractRunning(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.ExtractRunningForStep(stepID, laneID, moduleKey, deps)
}
return recorder.ExtractRunning(laneID, moduleKey, deps)
}
func checkpointExtractSucceeded(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.ExtractSucceededForStep(stepID, laneID, moduleKey, deps, outputs, rejected, warnings)
}
return recorder.ExtractSucceeded(laneID, moduleKey, deps, outputs, rejected, warnings)
}
func checkpointExtractFailed(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, err error) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.ExtractFailedForStep(stepID, laneID, moduleKey, deps, err)
}
return recorder.ExtractFailed(laneID, moduleKey, deps, err)
}
func checkpointMergeRunning(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.MergeRunningForStep(stepID, laneID, moduleKey, deps)
}
return recorder.MergeRunning(laneID, moduleKey, deps)
}
func checkpointMergeSucceeded(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.MergeSucceededForStep(stepID, laneID, moduleKey, deps, output, warnings)
}
return recorder.MergeSucceeded(laneID, moduleKey, deps, output, warnings)
}
func checkpointMergeRejected(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.MergeRejectedForStep(stepID, laneID, moduleKey, deps, rejected)
}
return recorder.MergeRejected(laneID, moduleKey, deps, rejected)
}
func checkpointMergeFailed(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, err error) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.MergeFailedForStep(stepID, laneID, moduleKey, deps, err)
}
return recorder.MergeFailed(laneID, moduleKey, deps, err)
}
func checkpointNormalizeRunning(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.NormalizeRunningForStep(stepID, laneID, moduleKey, deps)
}
return recorder.NormalizeRunning(laneID, moduleKey, deps)
}
func checkpointNormalizeSucceeded(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.NormalizeSucceededForStep(stepID, laneID, moduleKey, deps, output, warnings)
}
return recorder.NormalizeSucceeded(laneID, moduleKey, deps, output, warnings)
}
func checkpointNormalizeRejected(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.NormalizeRejectedForStep(stepID, laneID, moduleKey, deps, rejected)
}
return recorder.NormalizeRejected(laneID, moduleKey, deps, rejected)
}
func checkpointNormalizeFailed(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, err error) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.NormalizeFailedForStep(stepID, laneID, moduleKey, deps, err)
}
return recorder.NormalizeFailed(laneID, moduleKey, deps, err)
}
func digestFingerprints(name string, digest string) []CheckpointFingerprint {

View File

@@ -38,21 +38,22 @@ func New() *Runner {
}
type RunInput struct {
Prepared *PreparedPipeline
SourceID string
Path string
RawInput []byte
SessionID string
RunID string
StartedAt time.Time
LLMProfiles []artifacts.LLMProfileManifest
Metadata map[string]any
Warnings []contracts.Warning
ChunkCacheMode ChunkCacheMode
ChunkPlans ChunkPlanStore
Checkpoints CheckpointRecorder
Checkpoint CheckpointLoader
Debug DebugRecorder
Prepared *PreparedPipeline
SourceID string
Path string
RawInput []byte
SessionID string
RunID string
StartedAt time.Time
LLMProfiles []artifacts.LLMProfileManifest
Metadata map[string]any
Warnings []contracts.Warning
ChunkCacheMode ChunkCacheMode
ChunkPlans ChunkPlanStore
Checkpoints CheckpointRecorder
Checkpoint CheckpointLoader
CheckpointPolicy CheckpointExecutionPolicy
Debug DebugRecorder
// ExtractWorkers bounds run-wide extract jobs. Values less than one use a
// single worker so direct framework callers retain deterministic behavior.
ExtractWorkers int
@@ -131,7 +132,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return failOutput(output), err
}
sourceCheckpoint, sourceDecision := checkpointLoader.Source(adapter.Key())
recordCheckpointEvent(&output, checkpointLoader, "source", "", adapter.Key(), sourceDecision)
recordCheckpointEvent(&output, checkpointLoader, "source", "", "", adapter.Key(), sourceDecision)
doc := sourceCheckpoint.Document
sourceStarted := time.Now().UTC()
if err := writeDebugTimed(debugRecorder, "source/input.json", debugTimedEnvelope{
@@ -597,29 +598,47 @@ func failOutput(output RunOutput) RunOutput {
return output
}
func recordCheckpointEvent(output *RunOutput, loader CheckpointLoader, stage string, laneID string, moduleKey string, decision CheckpointDecision) {
func recordCheckpointEvent(output *RunOutput, loader CheckpointLoader, stage string, stepID string, laneID string, moduleKey string, decision CheckpointDecision) {
if output == nil || loader == nil || !loader.Enabled() {
return
}
action := "executed"
if decision.Reused {
action = "reused"
}
action := checkpointDecisionCategory(decision)
output.CheckpointEvents = append(output.CheckpointEvents, CheckpointEvent{
Stage: stage,
LaneID: laneID,
ModuleKey: moduleKey,
Action: action,
Reason: decision.Reason,
Stage: stage,
StepID: stepID,
LaneID: laneID,
ModuleKey: moduleKey,
Action: action,
Category: checkpointDecisionCategory(decision),
ReasonCode: decision.ReasonCode,
Detail: decision.Detail,
Reason: decision.Reason,
})
}
func checkpointDecisionCategory(decision CheckpointDecision) string {
if decision.Category != "" {
return decision.Category
}
if decision.Reused {
return "reused"
}
return "executed"
}
func populateOutputManifest(output *RunOutput) {
if output == nil {
return
}
output.Manifest.NormalizedOutputs = normalizedOutputManifests(output.NormalizeOutputs)
output.Manifest.RejectedOutputs = rejectedOutputManifests(output.Rejected)
if len(output.CheckpointEvents) > 0 {
decisions := make([]artifacts.CheckpointDecisionManifest, 0, len(output.CheckpointEvents))
for _, event := range output.CheckpointEvents {
decisions = append(decisions, artifacts.CheckpointDecisionManifest{Stage: event.Stage, StepID: event.StepID, LaneID: event.LaneID, ModuleKey: event.ModuleKey, Category: event.Category, ReasonCode: event.ReasonCode, Detail: event.Detail})
}
output.Manifest.CheckpointDecisions = decisions
}
}
func normalizedOutputManifests(outputs []contracts.SerializedOutput) []artifacts.NormalizedOutputManifest {

View File

@@ -36,12 +36,15 @@ type finalizedExtractResults struct {
decision CheckpointDecision
}
func loadExtract(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
func loadExtract(loader CheckpointLoader, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
if stepAware, ok := loader.(StepCheckpointLoader); ok {
return stepAware.ExtractForStep(stepID, laneID, moduleKey, deps)
}
return loader.Extract(laneID, moduleKey, deps)
}
func recordExtract(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
return recorder.ExtractSucceeded(laneID, moduleKey, deps, outputs, rejected, warnings)
func recordExtract(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
return checkpointExtractSucceeded(recorder, stepID, laneID, moduleKey, deps, outputs, rejected, warnings)
}
type extractJob struct {
@@ -87,10 +90,10 @@ func (r *Runner) runLanes(parent context.Context, input RunInput, step PreparedP
return output, err
}
if !state.decision.Reused {
if err := checkpoints.ExtractRunning(prepared.resolved.ID, prepared.resolved.Extract.Module, state.deps); err != nil {
if err := checkpointExtractRunning(checkpoints, input.stepID, prepared.resolved.ID, prepared.resolved.Extract.Module, state.deps); err != nil {
return output, fmt.Errorf("write extract checkpoint for lane %q: %w", prepared.resolved.ID, err)
}
} else if err := finalizeLaneExtract(checkpoints, state); err != nil {
} else if err := finalizeLaneExtract(checkpoints, input.stepID, state); err != nil {
return output, err
}
states[i] = state
@@ -188,13 +191,13 @@ func (r *Runner) runLanes(parent context.Context, input RunInput, step PreparedP
if result.err != nil {
state.failed = true
runErrors = append(runErrors, orderedRunError{stage: 0, lane: state.index, chunk: result.chunkIndex, err: result.err})
_ = checkpoints.ExtractFailed(state.prepared.resolved.ID, state.prepared.resolved.Extract.Module, state.deps, result.err)
_ = checkpointExtractFailed(checkpoints, input.stepID, state.prepared.resolved.ID, state.prepared.resolved.Extract.Module, state.deps, result.err)
cancel()
} else {
state.results[result.chunkIndex] = result
}
if state.remaining == 0 && !state.failed && ctx.Err() == nil {
if err := finalizeLaneExtract(checkpoints, state); err != nil {
if err := finalizeLaneExtract(checkpoints, input.stepID, state); err != nil {
state.failed = true
runErrors = append(runErrors, orderedRunError{stage: 0, lane: state.index, chunk: len(chunks), err: err})
cancel()
@@ -233,27 +236,31 @@ func prepareLaneExtract(input RunInput, loader CheckpointLoader, doc *source.Sou
extractReferences := operationReferenceSet(input, lane.ExtractReferences)
deps := append(digestFingerprints("chunks", digest), generatedReferenceDependencies(extractReferences)...)
state := &laneExtractState{index: index, prepared: prepared, deps: normalizeCheckpointFingerprints(deps), remaining: len(chunks), results: make(map[int]extractJobResult, len(chunks))}
cp, decision := loadExtract(loader, lane.ID, lane.Extract.Module, state.deps)
cp, decision := loadExtract(loader, input.stepID, lane.ID, lane.Extract.Module, state.deps)
decision = forceCheckpointDecision(input.CheckpointPolicy, input.stepID, lane.ID, decision)
if err := requireReusableCheckpoint(input.CheckpointPolicy, input.stepID, lane.ID, decision); err != nil {
return nil, err
}
if decision.Reused {
for _, stored := range cp.Outputs {
if _, decodeErr := decodeCheckpointArtifact(typed.codec, stored); decodeErr != nil {
decision = CheckpointDecision{Reason: "extract artifact checkpoint codec is incompatible: " + decodeErr.Error()}
if _, _, decodeErr := decodeCanonicalCheckpointArtifact(typed.codec, stored); decodeErr != nil {
decision = CheckpointDecision{Category: "executed", ReasonCode: "artifact_not_canonical", Detail: "stored extract artifact failed canonical codec validation", Reason: "extract artifact checkpoint is not canonical"}
break
}
}
}
if err := requireReusableCheckpoint(input.CheckpointPolicy, input.stepID, lane.ID, decision); err != nil {
return nil, err
}
state.decision = decision
if decision.Reused {
state.remaining = 0
for _, stored := range cp.Outputs {
value, decodeErr := decodeCheckpointArtifact(typed.codec, stored)
value, hydrated, decodeErr := decodeCanonicalCheckpointArtifact(typed.codec, stored)
if decodeErr != nil {
return nil, fmt.Errorf("decode extract checkpoint for lane %q: %w", lane.ID, decodeErr)
}
stored, decodeErr = hydrateCheckpointArtifact(typed.codec, stored, value)
if decodeErr != nil {
return nil, fmt.Errorf("hydrate extract checkpoint for lane %q: %w", lane.ID, decodeErr)
}
stored = hydrated
artifact := erasedExtractArtifact{LaneID: lane.ID, ExtractorKey: lane.Extract.Module, SourceID: doc.ID, ChunkID: stored.ChunkID, ChunkIndex: stored.ChunkIndex, ChunkRef: stored.ChunkRef, Value: value}
if stored.ChunkIndex >= 0 && stored.ChunkIndex < len(chunks) && artifact.ChunkRef == (source.SourceRef{}) {
artifact.ChunkRef = chunks[stored.ChunkIndex].Ref
@@ -330,7 +337,7 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.
return result
}
func finalizeLaneExtract(checkpoints CheckpointRecorder, state *laneExtractState) error {
func finalizeLaneExtract(checkpoints CheckpointRecorder, stepID string, state *laneExtractState) error {
lane := state.prepared.resolved
indexes := make([]int, 0, len(state.results))
for index := range state.results {
@@ -351,7 +358,7 @@ func finalizeLaneExtract(checkpoints CheckpointRecorder, state *laneExtractState
sort.SliceStable(state.serialized, func(i, j int) bool { return state.serialized[i].ChunkIndex < state.serialized[j].ChunkIndex })
sort.SliceStable(state.rejected, func(i, j int) bool { return state.rejected[i].ChunkIndex < state.rejected[j].ChunkIndex })
if !state.decision.Reused {
if err := recordExtract(checkpoints, lane.ID, lane.Extract.Module, state.deps, state.serialized, state.rejected, state.warnings); err != nil {
if err := recordExtract(checkpoints, stepID, lane.ID, lane.Extract.Module, state.deps, state.serialized, state.rejected, state.warnings); err != nil {
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
}
}
@@ -370,7 +377,7 @@ func (r *Runner) continueLane(ctx context.Context, input RunInput, checkpoints C
}
local.Warnings = append(local.Warnings, cloneWarnings(results.warnings)...)
local.Rejected = append(local.Rejected, cloneRejectedOutputs(results.rejected)...)
recordCheckpointEvent(&local, loader, string(StageExtract), lane.ID, lane.Extract.Module, results.decision)
recordCheckpointEvent(&local, loader, string(StageExtract), input.stepID, lane.ID, lane.Extract.Module, results.decision)
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": results.decision.Reused, "decision": results.decision, "source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "options": redactSensitiveMap(lane.Extract.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
return local, &laneRunError{stage: StageExtract, err: err}
}

View File

@@ -13,17 +13,23 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func loadMerge(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
func loadMerge(loader CheckpointLoader, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
if stepAware, ok := loader.(StepCheckpointLoader); ok {
return stepAware.MergeForStep(stepID, laneID, moduleKey, deps)
}
return loader.Merge(laneID, moduleKey, deps)
}
func loadNormalize(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
func loadNormalize(loader CheckpointLoader, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
if stepAware, ok := loader.(StepCheckpointLoader); ok {
return stepAware.NormalizeForStep(stepID, laneID, moduleKey, deps)
}
return loader.Normalize(laneID, moduleKey, deps)
}
func recordMerge(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error {
return recorder.MergeSucceeded(laneID, moduleKey, deps, output, warnings)
func recordMerge(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error {
return checkpointMergeSucceeded(recorder, stepID, laneID, moduleKey, deps, output, warnings)
}
func recordNormalize(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error {
return recorder.NormalizeSucceeded(laneID, moduleKey, deps, output, warnings)
func recordNormalize(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error {
return checkpointNormalizeSucceeded(recorder, stepID, laneID, moduleKey, deps, output, warnings)
}
func cloneCheckpointArtifact(output CheckpointArtifact) CheckpointArtifact {
output.Artifact = contracts.CloneSerializedArtifact(output.Artifact)
@@ -111,6 +117,25 @@ func decodeCheckpointArtifact(codec artifactCodecEntry, artifact CheckpointArtif
return codec.decode(append([]byte(nil), artifact.Artifact.Content...))
}
func decodeCanonicalCheckpointArtifact(codec artifactCodecEntry, artifact CheckpointArtifact) (any, CheckpointArtifact, error) {
value, err := decodeCheckpointArtifact(codec, artifact)
if err != nil {
return nil, CheckpointArtifact{}, err
}
canonical, err := serializeArtifact(codec, value, false)
if err != nil {
return nil, CheckpointArtifact{}, err
}
if canonical.Kind != artifact.Artifact.Kind || canonical.MediaType != artifact.Artifact.MediaType || checkpointContentDigest(canonical.Content) != checkpointContentDigest(artifact.Artifact.Content) {
return nil, CheckpointArtifact{}, fmt.Errorf("stored artifact is not canonical")
}
hydrated, err := hydrateCheckpointArtifact(codec, cloneCheckpointArtifact(artifact), value)
if err != nil {
return nil, CheckpointArtifact{}, err
}
return value, hydrated, nil
}
func checkpointArtifact(codec artifactCodecEntry, laneID, moduleKey, sourceID string, value any) (CheckpointArtifact, error) {
serialized, err := serializeArtifact(codec, value, false)
if err != nil {
@@ -157,13 +182,20 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoi
mergeReferences := operationReferenceSet(input, lane.MergeReferences)
mergeDeps := append(artifactCheckpointDigests(extracts.serialized), generatedReferenceDependencies(mergeReferences)...)
mergeDeps = normalizeCheckpointFingerprints(mergeDeps)
mergeCP, mergeDecision := loadMerge(loader, lane.ID, lane.Merge.Module, mergeDeps)
mergeCP, mergeDecision := loadMerge(loader, input.stepID, lane.ID, lane.Merge.Module, mergeDeps)
mergeDecision = forceCheckpointDecision(input.CheckpointPolicy, input.stepID, lane.ID, mergeDecision)
if err := requireReusableCheckpoint(input.CheckpointPolicy, input.stepID, lane.ID, mergeDecision); err != nil {
return err
}
if mergeDecision.Reused {
if _, decodeErr := decodeCheckpointArtifact(typed.codec, mergeCP.Output); decodeErr != nil {
mergeDecision = CheckpointDecision{Reason: "merge artifact checkpoint codec is incompatible: " + decodeErr.Error()}
if _, _, decodeErr := decodeCanonicalCheckpointArtifact(typed.codec, mergeCP.Output); decodeErr != nil {
mergeDecision = CheckpointDecision{Category: "executed", ReasonCode: "artifact_not_canonical", Detail: "stored merge artifact failed canonical codec validation", Reason: "merge artifact checkpoint is not canonical"}
}
}
recordCheckpointEvent(output, loader, string(StageMerge), lane.ID, lane.Merge.Module, mergeDecision)
if err := requireReusableCheckpoint(input.CheckpointPolicy, input.stepID, lane.ID, mergeDecision); err != nil {
return err
}
recordCheckpointEvent(output, loader, string(StageMerge), input.stepID, lane.ID, lane.Merge.Module, mergeDecision)
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageMerge), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": mergeDecision.Reused, "decision": mergeDecision, "source": debugSourceDocumentEnvelope(doc), "extract_outputs": debugCheckpointArtifacts(extracts.serialized), "options": redactSensitiveMap(lane.Merge.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
return err
}
@@ -171,19 +203,16 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoi
var serializedMerge CheckpointArtifact
var mergeWarnings []contracts.Warning
if mergeDecision.Reused {
value, decodeErr := decodeCheckpointArtifact(typed.codec, mergeCP.Output)
value, hydrated, decodeErr := decodeCanonicalCheckpointArtifact(typed.codec, mergeCP.Output)
if decodeErr != nil {
return fmt.Errorf("decode merge checkpoint for lane %q: %w", lane.ID, decodeErr)
}
merged = erasedMergeArtifact{LaneID: lane.ID, MergerKey: lane.Merge.Module, SourceID: doc.ID, Value: value}
serializedMerge, decodeErr = hydrateCheckpointArtifact(typed.codec, cloneCheckpointArtifact(mergeCP.Output), value)
if decodeErr != nil {
return fmt.Errorf("hydrate merge checkpoint for lane %q: %w", lane.ID, decodeErr)
}
serializedMerge = hydrated
mergeWarnings = cloneWarnings(mergeCP.Warnings)
output.Warnings = append(output.Warnings, mergeWarnings...)
} else {
if err := checkpoints.MergeRunning(lane.ID, lane.Merge.Module, mergeDeps); err != nil {
if err := checkpointMergeRunning(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps); err != nil {
return err
}
ok, rejection, runErr := runWithRetry(ctx, lane.Merge.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
@@ -226,18 +255,18 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoi
return true, nil, nil
})
if runErr != nil {
_ = checkpoints.MergeFailed(lane.ID, lane.Merge.Module, mergeDeps, runErr)
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, runErr)
return runErr
}
if !ok {
output.Rejected = append(output.Rejected, *rejection)
if err := checkpoints.MergeRejected(lane.ID, lane.Merge.Module, mergeDeps, *rejection); err != nil {
if err := checkpointMergeRejected(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, *rejection); err != nil {
return err
}
return nil
}
output.Warnings = append(output.Warnings, mergeWarnings...)
if err := recordMerge(checkpoints, lane.ID, lane.Merge.Module, mergeDeps, serializedMerge, mergeWarnings); err != nil {
if err := recordMerge(checkpoints, input.stepID, lane.ID, lane.Merge.Module, mergeDeps, serializedMerge, mergeWarnings); err != nil {
return err
}
}
@@ -249,31 +278,35 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoi
normalizeReferences := operationReferenceSet(input, lane.NormalizeReferences)
normalizeDeps := append(artifactCheckpointDigests([]CheckpointArtifact{serializedMerge}), generatedReferenceDependencies(normalizeReferences)...)
normalizeDeps = normalizeCheckpointFingerprints(normalizeDeps)
normalizeCP, normalizeDecision := loadNormalize(loader, lane.ID, lane.Normalize.Module, normalizeDeps)
normalizeCP, normalizeDecision := loadNormalize(loader, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps)
normalizeDecision = forceCheckpointDecision(input.CheckpointPolicy, input.stepID, lane.ID, normalizeDecision)
if err := requireReusableCheckpoint(input.CheckpointPolicy, input.stepID, lane.ID, normalizeDecision); err != nil {
return err
}
if normalizeDecision.Reused {
if _, decodeErr := decodeCheckpointArtifact(typed.codec, normalizeCP.Output); decodeErr != nil {
normalizeDecision = CheckpointDecision{Reason: "normalize artifact checkpoint codec is incompatible: " + decodeErr.Error()}
if _, _, decodeErr := decodeCanonicalCheckpointArtifact(typed.codec, normalizeCP.Output); decodeErr != nil {
normalizeDecision = CheckpointDecision{Category: "executed", ReasonCode: "artifact_not_canonical", Detail: "stored normalize artifact failed canonical codec validation", Reason: "normalize artifact checkpoint is not canonical"}
}
}
recordCheckpointEvent(output, loader, string(StageNormalize), lane.ID, lane.Normalize.Module, normalizeDecision)
if err := requireReusableCheckpoint(input.CheckpointPolicy, input.stepID, lane.ID, normalizeDecision); err != nil {
return err
}
recordCheckpointEvent(output, loader, string(StageNormalize), input.stepID, lane.ID, lane.Normalize.Module, normalizeDecision)
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageNormalize), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": normalizeDecision.Reused, "decision": normalizeDecision, "source": debugSourceDocumentEnvelope(doc), "merge_output": debugCheckpointArtifact(serializedMerge), "options": redactSensitiveMap(lane.Normalize.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
return err
}
var serializedNormalize CheckpointArtifact
var normalizeWarnings []contracts.Warning
if normalizeDecision.Reused {
value, decodeErr := decodeCheckpointArtifact(typed.codec, normalizeCP.Output)
_, hydrated, decodeErr := decodeCanonicalCheckpointArtifact(typed.codec, normalizeCP.Output)
if decodeErr != nil {
return fmt.Errorf("decode normalize checkpoint for lane %q: %w", lane.ID, decodeErr)
}
serializedNormalize, decodeErr = hydrateCheckpointArtifact(typed.codec, cloneCheckpointArtifact(normalizeCP.Output), value)
if decodeErr != nil {
return fmt.Errorf("hydrate normalize checkpoint for lane %q: %w", lane.ID, decodeErr)
}
serializedNormalize = hydrated
normalizeWarnings = cloneWarnings(normalizeCP.Warnings)
output.Warnings = append(output.Warnings, normalizeWarnings...)
} else {
if err := checkpoints.NormalizeRunning(lane.ID, lane.Normalize.Module, normalizeDeps); err != nil {
if err := checkpointNormalizeRunning(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps); err != nil {
return err
}
ok, rejection, runErr := runWithRetry(ctx, lane.Normalize.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
@@ -315,18 +348,18 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoi
return true, nil, nil
})
if runErr != nil {
_ = checkpoints.NormalizeFailed(lane.ID, lane.Normalize.Module, normalizeDeps, runErr)
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, runErr)
return runErr
}
if !ok {
output.Rejected = append(output.Rejected, *rejection)
if err := checkpoints.NormalizeRejected(lane.ID, lane.Normalize.Module, normalizeDeps, *rejection); err != nil {
if err := checkpointNormalizeRejected(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, *rejection); err != nil {
return err
}
return nil
}
output.Warnings = append(output.Warnings, normalizeWarnings...)
if err := recordNormalize(checkpoints, lane.ID, lane.Normalize.Module, normalizeDeps, serializedNormalize, normalizeWarnings); err != nil {
if err := recordNormalize(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, normalizeDeps, serializedNormalize, normalizeWarnings); err != nil {
return err
}
}

View File

@@ -43,3 +43,26 @@ func TestDecodeCheckpointArtifactRejectsIncompatibleCodecIdentityAndBytes(t *tes
})
}
}
func TestDecodeCanonicalCheckpointArtifactRejectsNonCanonicalBytes(t *testing.T) {
registry := NewArtifactCodecRegistry()
if err := RegisterArtifactCodec(registry, notesCodec()); err != nil {
t.Fatalf("RegisterArtifactCodec: %v", err)
}
codec, _, err := registry.entry("test/notes")
if err != nil {
t.Fatalf("entry: %v", err)
}
artifact, err := serializeArtifact(codec, codecNotes{Items: []string{"one"}}, false)
if err != nil {
t.Fatalf("serializeArtifact: %v", err)
}
stored := CheckpointArtifact{Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(artifact.Schema)}
if _, _, err := decodeCanonicalCheckpointArtifact(codec, stored); err != nil {
t.Fatalf("canonical artifact rejected: %v", err)
}
stored.Artifact.Content = []byte(`{"items": ["one"]}`)
if _, _, err := decodeCanonicalCheckpointArtifact(codec, stored); err == nil {
t.Fatal("non-canonical artifact was accepted")
}
}

View File

@@ -76,6 +76,30 @@ func (l *lockedCheckpointLoader) Normalize(lane, key string, deps []CheckpointFi
defer l.mu.Unlock()
return l.inner.Normalize(lane, key, deps)
}
func (l *lockedCheckpointLoader) ExtractForStep(step, lane, key string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
if stepAware, ok := l.inner.(StepCheckpointLoader); ok {
return stepAware.ExtractForStep(step, lane, key, deps)
}
return l.inner.Extract(lane, key, deps)
}
func (l *lockedCheckpointLoader) MergeForStep(step, lane, key string, deps []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
if stepAware, ok := l.inner.(StepCheckpointLoader); ok {
return stepAware.MergeForStep(step, lane, key, deps)
}
return l.inner.Merge(lane, key, deps)
}
func (l *lockedCheckpointLoader) NormalizeForStep(step, lane, key string, deps []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
if stepAware, ok := l.inner.(StepCheckpointLoader); ok {
return stepAware.NormalizeForStep(step, lane, key, deps)
}
return l.inner.Normalize(lane, key, deps)
}
type lockedCheckpointRecorder struct {
inner CheckpointRecorder
@@ -135,3 +159,92 @@ func (r *lockedCheckpointRecorder) NormalizeRejected(lane, key string, deps []Ch
func (r *lockedCheckpointRecorder) NormalizeFailed(lane, key string, deps []CheckpointFingerprint, err error) error {
return r.call(func() error { return r.inner.NormalizeFailed(lane, key, deps, err) })
}
func (r *lockedCheckpointRecorder) ExtractRunningForStep(step, lane, key string, deps []CheckpointFingerprint) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.ExtractRunningForStep(step, lane, key, deps)
}
return r.inner.ExtractRunning(lane, key, deps)
})
}
func (r *lockedCheckpointRecorder) ExtractSucceededForStep(step, lane, key string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.ExtractSucceededForStep(step, lane, key, deps, outputs, rejected, warnings)
}
return r.inner.ExtractSucceeded(lane, key, deps, outputs, rejected, warnings)
})
}
func (r *lockedCheckpointRecorder) ExtractFailedForStep(step, lane, key string, deps []CheckpointFingerprint, err error) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.ExtractFailedForStep(step, lane, key, deps, err)
}
return r.inner.ExtractFailed(lane, key, deps, err)
})
}
func (r *lockedCheckpointRecorder) MergeRunningForStep(step, lane, key string, deps []CheckpointFingerprint) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.MergeRunningForStep(step, lane, key, deps)
}
return r.inner.MergeRunning(lane, key, deps)
})
}
func (r *lockedCheckpointRecorder) MergeSucceededForStep(step, lane, key string, deps []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.MergeSucceededForStep(step, lane, key, deps, output, warnings)
}
return r.inner.MergeSucceeded(lane, key, deps, output, warnings)
})
}
func (r *lockedCheckpointRecorder) MergeRejectedForStep(step, lane, key string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.MergeRejectedForStep(step, lane, key, deps, rejected)
}
return r.inner.MergeRejected(lane, key, deps, rejected)
})
}
func (r *lockedCheckpointRecorder) MergeFailedForStep(step, lane, key string, deps []CheckpointFingerprint, err error) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.MergeFailedForStep(step, lane, key, deps, err)
}
return r.inner.MergeFailed(lane, key, deps, err)
})
}
func (r *lockedCheckpointRecorder) NormalizeRunningForStep(step, lane, key string, deps []CheckpointFingerprint) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.NormalizeRunningForStep(step, lane, key, deps)
}
return r.inner.NormalizeRunning(lane, key, deps)
})
}
func (r *lockedCheckpointRecorder) NormalizeSucceededForStep(step, lane, key string, deps []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.NormalizeSucceededForStep(step, lane, key, deps, output, warnings)
}
return r.inner.NormalizeSucceeded(lane, key, deps, output, warnings)
})
}
func (r *lockedCheckpointRecorder) NormalizeRejectedForStep(step, lane, key string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.NormalizeRejectedForStep(step, lane, key, deps, rejected)
}
return r.inner.NormalizeRejected(lane, key, deps, rejected)
})
}
func (r *lockedCheckpointRecorder) NormalizeFailedForStep(step, lane, key string, deps []CheckpointFingerprint, err error) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.NormalizeFailedForStep(step, lane, key, deps, err)
}
return r.inner.NormalizeFailed(lane, key, deps, err)
})
}