Implement selective checkpoint recomputation

This commit is contained in:
2026-07-21 21:59:48 +00:00
parent 22d4f29670
commit c437682407
17 changed files with 902 additions and 180 deletions

View File

@@ -38,21 +38,22 @@ func New() *Runner {
}
type RunInput struct {
Prepared *PreparedPipeline
SourceID string
Path string
RawInput []byte
SessionID string
RunID string
StartedAt time.Time
LLMProfiles []artifacts.LLMProfileManifest
Metadata map[string]any
Warnings []contracts.Warning
ChunkCacheMode ChunkCacheMode
ChunkPlans ChunkPlanStore
Checkpoints CheckpointRecorder
Checkpoint CheckpointLoader
Debug DebugRecorder
Prepared *PreparedPipeline
SourceID string
Path string
RawInput []byte
SessionID string
RunID string
StartedAt time.Time
LLMProfiles []artifacts.LLMProfileManifest
Metadata map[string]any
Warnings []contracts.Warning
ChunkCacheMode ChunkCacheMode
ChunkPlans ChunkPlanStore
Checkpoints CheckpointRecorder
Checkpoint CheckpointLoader
CheckpointPolicy CheckpointExecutionPolicy
Debug DebugRecorder
// ExtractWorkers bounds run-wide extract jobs. Values less than one use a
// single worker so direct framework callers retain deterministic behavior.
ExtractWorkers int
@@ -131,7 +132,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return failOutput(output), err
}
sourceCheckpoint, sourceDecision := checkpointLoader.Source(adapter.Key())
recordCheckpointEvent(&output, checkpointLoader, "source", "", adapter.Key(), sourceDecision)
recordCheckpointEvent(&output, checkpointLoader, "source", "", "", adapter.Key(), sourceDecision)
doc := sourceCheckpoint.Document
sourceStarted := time.Now().UTC()
if err := writeDebugTimed(debugRecorder, "source/input.json", debugTimedEnvelope{
@@ -597,29 +598,47 @@ func failOutput(output RunOutput) RunOutput {
return output
}
func recordCheckpointEvent(output *RunOutput, loader CheckpointLoader, stage string, laneID string, moduleKey string, decision CheckpointDecision) {
func recordCheckpointEvent(output *RunOutput, loader CheckpointLoader, stage string, stepID string, laneID string, moduleKey string, decision CheckpointDecision) {
if output == nil || loader == nil || !loader.Enabled() {
return
}
action := "executed"
if decision.Reused {
action = "reused"
}
action := checkpointDecisionCategory(decision)
output.CheckpointEvents = append(output.CheckpointEvents, CheckpointEvent{
Stage: stage,
LaneID: laneID,
ModuleKey: moduleKey,
Action: action,
Reason: decision.Reason,
Stage: stage,
StepID: stepID,
LaneID: laneID,
ModuleKey: moduleKey,
Action: action,
Category: checkpointDecisionCategory(decision),
ReasonCode: decision.ReasonCode,
Detail: decision.Detail,
Reason: decision.Reason,
})
}
func checkpointDecisionCategory(decision CheckpointDecision) string {
if decision.Category != "" {
return decision.Category
}
if decision.Reused {
return "reused"
}
return "executed"
}
func populateOutputManifest(output *RunOutput) {
if output == nil {
return
}
output.Manifest.NormalizedOutputs = normalizedOutputManifests(output.NormalizeOutputs)
output.Manifest.RejectedOutputs = rejectedOutputManifests(output.Rejected)
if len(output.CheckpointEvents) > 0 {
decisions := make([]artifacts.CheckpointDecisionManifest, 0, len(output.CheckpointEvents))
for _, event := range output.CheckpointEvents {
decisions = append(decisions, artifacts.CheckpointDecisionManifest{Stage: event.Stage, StepID: event.StepID, LaneID: event.LaneID, ModuleKey: event.ModuleKey, Category: event.Category, ReasonCode: event.ReasonCode, Detail: event.Detail})
}
output.Manifest.CheckpointDecisions = decisions
}
}
func normalizedOutputManifests(outputs []contracts.SerializedOutput) []artifacts.NormalizedOutputManifest {