eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Write workspace checkpoints during runs

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-08 02:46:38 +00:00
parent f044c00a7c
commit 1d3a444df8
11 changed files with 1258 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
internal/framework/pipeline/runner.go
View File

@@ -48,6 +48,7 @@ type RunInput struct {
LLMProfiles []artifacts.LLMProfileManifest
Metadata map[string]any
Warnings []contracts.Warning
Checkpoints CheckpointRecorder
}
type RunOutput struct {
@@ -70,6 +71,10 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}
output.Manifest = manifestFromPipeline(input)
checkpoints := input.Checkpoints
if checkpoints == nil {
checkpoints = NoopCheckpointRecorder()
}
defer func() {
output.Manifest.LLMProfiles = mergeLLMProfileManifests(input.LLMProfiles, llmProfileManifests(input.LLMClient))
}()
@@ -80,6 +85,9 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return failOutput(output), fmt.Errorf("build input adapter %q: %w", input.Pipeline.Input.Module, err)
}
attachModuleManifestMetadata(&output, "input", adapter)
if err := checkpoints.SourceRunning(adapter.Key()); err != nil {
return failOutput(output), fmt.Errorf("write source checkpoint: %w", err)
}
doc, err := adapter.Parse(ctx, contracts.ParseRequest{
SourceID: input.SourceID,
Path: input.Path,
@@ -89,11 +97,16 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
Metadata: input.Metadata,
})
if err != nil {
_ = checkpoints.SourceFailed(adapter.Key(), err)
return failOutput(output), fmt.Errorf("parse input with adapter %q: %w", adapter.Key(), err)
}
if err := source.ValidateDocument(doc); err != nil {
_ = checkpoints.SourceFailed(adapter.Key(), err)
return failOutput(output), fmt.Errorf("validate source document: %w", err)
}
if err := checkpoints.SourceSucceeded(adapter.Key(), doc); err != nil {
return failOutput(output), fmt.Errorf("write source checkpoint: %w", err)
}
sourceInput := sourceInputMaterial(input.Path, input.RawInput)
sessionID := resolvedSessionID(input.SessionID, doc.ID)
output.Manifest.Metadata = manifestMetadataWithSessionID(output.Manifest.Metadata, sessionID)
@@ -104,6 +117,9 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return failOutput(output), fmt.Errorf("build chunker %q: %w", input.Pipeline.Chunk.Module, err)
}
attachModuleManifestMetadata(&output, "chunker", chunker)
if err := checkpoints.ChunkRunning(chunker.Key(), doc.Digest); err != nil {
return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
}
var canonicalChunks []contracts.SourceChunk
var chunkWarnings []contracts.Warning
chunksAccepted, chunkRejection, err := runWithRetry(ctx, input.Pipeline.Chunk.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
@@ -136,17 +152,24 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return true, nil, nil
})
if err != nil {
_ = checkpoints.ChunkFailed(chunker.Key(), doc.Digest, err)
return failOutput(output), err
}
if !chunksAccepted {
output.Rejected = append(output.Rejected, *chunkRejection)
if err := checkpoints.ChunkRejected(chunker.Key(), doc.Digest, *chunkRejection); err != nil {
return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
}
} else {
output.Warnings = append(output.Warnings, chunkWarnings...)
if err := checkpoints.ChunkSucceeded(chunker.Key(), doc.Digest, canonicalChunks, chunkWarnings); err != nil {
return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
}
}
if chunksAccepted {
for _, lane := range input.Pipeline.ArtifactLanes {
if err := r.runLane(ctx, input, doc, sourceInput, sessionID, canonicalChunks, lane, &output); err != nil {
if err := r.runLane(ctx, input, checkpoints, doc, sourceInput, sessionID, canonicalChunks, lane, &output); err != nil {
return failOutput(output), err
}
}
@@ -187,7 +210,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return output, nil
}
func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []contracts.SourceChunk, lane ResolvedArtifactLane, output *RunOutput) error {
func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []contracts.SourceChunk, lane ResolvedArtifactLane, output *RunOutput) error {
extractor, err := r.registries.Extractors.Build(lane.Extract.Module)
if err != nil {
return fmt.Errorf("build extractor %q for lane %q: %w", lane.Extract.Module, lane.ID, err)
@@ -203,6 +226,12 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
setLaneManifestMetadata(output, lane.ID, extractor, merger, normalizer)
extractOutputs := make([]contracts.ExtractOutput, 0, len(chunks))
extractWarnings := []contracts.Warning{}
extractRejectedStart := len(output.Rejected)
extractDependencies := digestFingerprints("chunks", joinedChunkDigest(chunks))
if err := checkpoints.ExtractRunning(lane.ID, extractor.Key(), extractDependencies); err != nil {
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
}
for index := range chunks {
chunk := chunks[index]
var acceptedOutput contracts.ExtractOutput
@@ -256,6 +285,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
return true, nil, nil
})
if err != nil {
_ = checkpoints.ExtractFailed(lane.ID, extractor.Key(), extractDependencies, err)
return err
}
if !accepted {
@@ -263,8 +293,13 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
continue
}
output.Warnings = append(output.Warnings, acceptedWarnings...)
extractWarnings = append(extractWarnings, acceptedWarnings...)
extractOutputs = append(extractOutputs, acceptedOutput)
}
extractRejected := cloneRejectedOutputs(output.Rejected[extractRejectedStart:])
if err := checkpoints.ExtractSucceeded(lane.ID, extractor.Key(), extractDependencies, extractOutputs, extractRejected, extractWarnings); err != nil {
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
}
if len(extractOutputs) == 0 {
return nil
@@ -272,6 +307,10 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
var acceptedMerge contracts.MergeOutput
var mergeWarnings []contracts.Warning
mergeDependencies := rawOutputDigests(extractPayloads(extractOutputs))
if err := checkpoints.MergeRunning(lane.ID, merger.Key(), mergeDependencies); err != nil {
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
}
mergeAccepted, mergeRejection, err := runWithRetry(ctx, lane.Merge.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
mergeResult, err := merger.Merge(ctx, contracts.MergeRequest{
Source: doc,
@@ -318,16 +357,27 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
return true, nil, nil
})
if err != nil {
_ = checkpoints.MergeFailed(lane.ID, merger.Key(), mergeDependencies, err)
return err
}
if !mergeAccepted {
output.Rejected = append(output.Rejected, *mergeRejection)
if err := checkpoints.MergeRejected(lane.ID, merger.Key(), mergeDependencies, *mergeRejection); err != nil {
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
}
return nil
}
output.Warnings = append(output.Warnings, mergeWarnings...)
if err := checkpoints.MergeSucceeded(lane.ID, merger.Key(), mergeDependencies, acceptedMerge, mergeWarnings); err != nil {
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
}
var acceptedNormalize contracts.NormalizeOutput
var normalizeWarnings []contracts.Warning
normalizeDependencies := rawOutputDigests([]contracts.RawPayload{acceptedMerge.Payload})
if err := checkpoints.NormalizeRunning(lane.ID, normalizer.Key(), normalizeDependencies); err != nil {
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
}
normalizeAccepted, normalizeRejection, err := runWithRetry(ctx, lane.Normalize.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
normalizeResult, err := normalizer.Normalize(ctx, contracts.NormalizeRequest{
Source: doc,
@@ -374,13 +424,20 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
return true, nil, nil
})
if err != nil {
_ = checkpoints.NormalizeFailed(lane.ID, normalizer.Key(), normalizeDependencies, err)
return err
}
if !normalizeAccepted {
output.Rejected = append(output.Rejected, *normalizeRejection)
if err := checkpoints.NormalizeRejected(lane.ID, normalizer.Key(), normalizeDependencies, *normalizeRejection); err != nil {
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
}
return nil
}
output.Warnings = append(output.Warnings, normalizeWarnings...)
if err := checkpoints.NormalizeSucceeded(lane.ID, normalizer.Key(), normalizeDependencies, acceptedNormalize, normalizeWarnings); err != nil {
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
}
output.NormalizeOutputs = append(output.NormalizeOutputs, acceptedNormalize)
return nil
}