Reuse valid workspace checkpoints on request
This commit is contained in:
@@ -49,6 +49,7 @@ type RunInput struct {
|
||||
Metadata map[string]any
|
||||
Warnings []contracts.Warning
|
||||
Checkpoints CheckpointRecorder
|
||||
Checkpoint CheckpointLoader
|
||||
}
|
||||
|
||||
type RunOutput struct {
|
||||
@@ -57,6 +58,7 @@ type RunOutput struct {
|
||||
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
OutputFiles []contracts.OutputFile `json:"-"`
|
||||
CheckpointEvents []CheckpointEvent `json:"checkpoint_events,omitempty"`
|
||||
}
|
||||
|
||||
func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err error) {
|
||||
@@ -75,6 +77,10 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
||||
if checkpoints == nil {
|
||||
checkpoints = NoopCheckpointRecorder()
|
||||
}
|
||||
checkpointLoader := input.Checkpoint
|
||||
if checkpointLoader == nil {
|
||||
checkpointLoader = NoopCheckpointLoader()
|
||||
}
|
||||
defer func() {
|
||||
output.Manifest.LLMProfiles = mergeLLMProfileManifests(input.LLMProfiles, llmProfileManifests(input.LLMClient))
|
||||
}()
|
||||
@@ -85,27 +91,32 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
||||
return failOutput(output), fmt.Errorf("build input adapter %q: %w", input.Pipeline.Input.Module, err)
|
||||
}
|
||||
attachModuleManifestMetadata(&output, "input", adapter)
|
||||
if err := checkpoints.SourceRunning(adapter.Key()); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write source checkpoint: %w", err)
|
||||
}
|
||||
doc, err := adapter.Parse(ctx, contracts.ParseRequest{
|
||||
SourceID: input.SourceID,
|
||||
Path: input.Path,
|
||||
Raw: input.RawInput,
|
||||
LLMProfile: input.Pipeline.Input.LLMProfile,
|
||||
Options: cloneOptions(input.Pipeline.Input.Options),
|
||||
Metadata: input.Metadata,
|
||||
})
|
||||
if err != nil {
|
||||
_ = checkpoints.SourceFailed(adapter.Key(), err)
|
||||
return failOutput(output), fmt.Errorf("parse input with adapter %q: %w", adapter.Key(), err)
|
||||
}
|
||||
if err := source.ValidateDocument(doc); err != nil {
|
||||
_ = checkpoints.SourceFailed(adapter.Key(), err)
|
||||
return failOutput(output), fmt.Errorf("validate source document: %w", err)
|
||||
}
|
||||
if err := checkpoints.SourceSucceeded(adapter.Key(), doc); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write source checkpoint: %w", err)
|
||||
sourceCheckpoint, sourceDecision := checkpointLoader.Source(adapter.Key())
|
||||
recordCheckpointEvent(&output, checkpointLoader, "source", "", adapter.Key(), sourceDecision)
|
||||
doc := sourceCheckpoint.Document
|
||||
if !sourceDecision.Reused {
|
||||
if err := checkpoints.SourceRunning(adapter.Key()); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write source checkpoint: %w", err)
|
||||
}
|
||||
doc, err = adapter.Parse(ctx, contracts.ParseRequest{
|
||||
SourceID: input.SourceID,
|
||||
Path: input.Path,
|
||||
Raw: input.RawInput,
|
||||
LLMProfile: input.Pipeline.Input.LLMProfile,
|
||||
Options: cloneOptions(input.Pipeline.Input.Options),
|
||||
Metadata: input.Metadata,
|
||||
})
|
||||
if err != nil {
|
||||
_ = checkpoints.SourceFailed(adapter.Key(), err)
|
||||
return failOutput(output), fmt.Errorf("parse input with adapter %q: %w", adapter.Key(), err)
|
||||
}
|
||||
if err := source.ValidateDocument(doc); err != nil {
|
||||
_ = checkpoints.SourceFailed(adapter.Key(), err)
|
||||
return failOutput(output), fmt.Errorf("validate source document: %w", err)
|
||||
}
|
||||
if err := checkpoints.SourceSucceeded(adapter.Key(), doc); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write source checkpoint: %w", err)
|
||||
}
|
||||
}
|
||||
sourceInput := sourceInputMaterial(input.Path, input.RawInput)
|
||||
sessionID := resolvedSessionID(input.SessionID, doc.ID)
|
||||
@@ -117,59 +128,69 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
||||
return failOutput(output), fmt.Errorf("build chunker %q: %w", input.Pipeline.Chunk.Module, err)
|
||||
}
|
||||
attachModuleManifestMetadata(&output, "chunker", chunker)
|
||||
if err := checkpoints.ChunkRunning(chunker.Key(), doc.Digest); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
|
||||
}
|
||||
var canonicalChunks []contracts.SourceChunk
|
||||
var chunkWarnings []contracts.Warning
|
||||
chunksAccepted, chunkRejection, err := runWithRetry(ctx, input.Pipeline.Chunk.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
|
||||
Source: doc,
|
||||
SourceInput: sourceInput.Clone(),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(input.Pipeline.ChunkReferences.ReferenceSet),
|
||||
LLMClient: input.LLMClient,
|
||||
LLMProfile: input.Pipeline.Chunk.LLMProfile,
|
||||
Options: cloneOptions(input.Pipeline.Chunk.Options),
|
||||
Metadata: input.Metadata,
|
||||
chunkCheckpoint, chunkDecision := checkpointLoader.Chunk(chunker.Key(), doc.Digest)
|
||||
recordCheckpointEvent(&output, checkpointLoader, string(StageChunk), "", chunker.Key(), chunkDecision)
|
||||
chunksAccepted := chunkDecision.Reused
|
||||
var chunkRejection *contracts.RejectedOutput
|
||||
if chunkDecision.Reused {
|
||||
canonicalChunks = cloneSourceChunks(chunkCheckpoint.Chunks)
|
||||
chunkWarnings = cloneWarnings(chunkCheckpoint.Warnings)
|
||||
output.Warnings = append(output.Warnings, chunkWarnings...)
|
||||
} else {
|
||||
if err := checkpoints.ChunkRunning(chunker.Key(), doc.Digest); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
|
||||
}
|
||||
chunksAccepted, chunkRejection, err = runWithRetry(ctx, input.Pipeline.Chunk.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
|
||||
Source: doc,
|
||||
SourceInput: sourceInput.Clone(),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(input.Pipeline.ChunkReferences.ReferenceSet),
|
||||
LLMClient: input.LLMClient,
|
||||
LLMProfile: input.Pipeline.Chunk.LLMProfile,
|
||||
Options: cloneOptions(input.Pipeline.Chunk.Options),
|
||||
Metadata: input.Metadata,
|
||||
})
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("chunk source with chunker %q: %w", chunker.Key(), err)
|
||||
}
|
||||
if len(chunkResult.Chunks) == 0 {
|
||||
return false, nil, fmt.Errorf("chunker %q returned no chunks", chunker.Key())
|
||||
}
|
||||
chunks, err := validateAndCanonicalizeChunkResult(doc, chunkResult.Chunks)
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
|
||||
}
|
||||
validationWarnings, rejection, err := r.validateChunksRaw(ctx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.Pipeline.ChunkReferences.ReferenceSet, input.LLMClient, input.Metadata, input.Pipeline.ValidatorChains, attempt)
|
||||
if err != nil || rejection != nil {
|
||||
return false, rejection, err
|
||||
}
|
||||
canonicalChunks = chunks
|
||||
chunkWarnings = append(cloneWarnings(chunkResult.Warnings), validationWarnings...)
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("chunk source with chunker %q: %w", chunker.Key(), err)
|
||||
_ = checkpoints.ChunkFailed(chunker.Key(), doc.Digest, err)
|
||||
return failOutput(output), err
|
||||
}
|
||||
if len(chunkResult.Chunks) == 0 {
|
||||
return false, nil, fmt.Errorf("chunker %q returned no chunks", chunker.Key())
|
||||
}
|
||||
chunks, err := validateAndCanonicalizeChunkResult(doc, chunkResult.Chunks)
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
|
||||
}
|
||||
validationWarnings, rejection, err := r.validateChunksRaw(ctx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.Pipeline.ChunkReferences.ReferenceSet, input.LLMClient, input.Metadata, input.Pipeline.ValidatorChains, attempt)
|
||||
if err != nil || rejection != nil {
|
||||
return false, rejection, err
|
||||
}
|
||||
canonicalChunks = chunks
|
||||
chunkWarnings = append(cloneWarnings(chunkResult.Warnings), validationWarnings...)
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
_ = checkpoints.ChunkFailed(chunker.Key(), doc.Digest, err)
|
||||
return failOutput(output), err
|
||||
}
|
||||
if !chunksAccepted {
|
||||
output.Rejected = append(output.Rejected, *chunkRejection)
|
||||
if err := checkpoints.ChunkRejected(chunker.Key(), doc.Digest, *chunkRejection); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
|
||||
}
|
||||
} else {
|
||||
output.Warnings = append(output.Warnings, chunkWarnings...)
|
||||
if err := checkpoints.ChunkSucceeded(chunker.Key(), doc.Digest, canonicalChunks, chunkWarnings); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
|
||||
if !chunksAccepted {
|
||||
output.Rejected = append(output.Rejected, *chunkRejection)
|
||||
if err := checkpoints.ChunkRejected(chunker.Key(), doc.Digest, *chunkRejection); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
|
||||
}
|
||||
} else {
|
||||
output.Warnings = append(output.Warnings, chunkWarnings...)
|
||||
if err := checkpoints.ChunkSucceeded(chunker.Key(), doc.Digest, canonicalChunks, chunkWarnings); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if chunksAccepted {
|
||||
for _, lane := range input.Pipeline.ArtifactLanes {
|
||||
if err := r.runLane(ctx, input, checkpoints, doc, sourceInput, sessionID, canonicalChunks, lane, &output); err != nil {
|
||||
if err := r.runLane(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, canonicalChunks, lane, &output); err != nil {
|
||||
return failOutput(output), err
|
||||
}
|
||||
}
|
||||
@@ -210,7 +231,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
||||
return output, nil
|
||||
}
|
||||
|
||||
func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []contracts.SourceChunk, lane ResolvedArtifactLane, output *RunOutput) error {
|
||||
func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, checkpointLoader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []contracts.SourceChunk, lane ResolvedArtifactLane, output *RunOutput) error {
|
||||
extractor, err := r.registries.Extractors.Build(lane.Extract.Module)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build extractor %q for lane %q: %w", lane.Extract.Module, lane.ID, err)
|
||||
@@ -229,76 +250,85 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
|
||||
extractWarnings := []contracts.Warning{}
|
||||
extractRejectedStart := len(output.Rejected)
|
||||
extractDependencies := digestFingerprints("chunks", joinedChunkDigest(chunks))
|
||||
if err := checkpoints.ExtractRunning(lane.ID, extractor.Key(), extractDependencies); err != nil {
|
||||
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
for index := range chunks {
|
||||
chunk := chunks[index]
|
||||
var acceptedOutput contracts.ExtractOutput
|
||||
var acceptedWarnings []contracts.Warning
|
||||
accepted, rejection, err := runWithRetry(ctx, lane.Extract.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
result, err := extractor.Extract(ctx, contracts.ExtractionRequest{
|
||||
Source: doc,
|
||||
Chunk: &chunk,
|
||||
SourceInput: chunkInputMaterial(sourceInput, chunk),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet),
|
||||
LLMClient: input.LLMClient,
|
||||
LLMProfile: lane.Extract.LLMProfile,
|
||||
Options: cloneOptions(lane.Extract.Options),
|
||||
Metadata: input.Metadata,
|
||||
extractCheckpoint, extractDecision := checkpointLoader.Extract(lane.ID, extractor.Key(), extractDependencies)
|
||||
recordCheckpointEvent(output, checkpointLoader, string(StageExtract), lane.ID, extractor.Key(), extractDecision)
|
||||
if extractDecision.Reused {
|
||||
extractOutputs = cloneExtractOutputs(extractCheckpoint.Outputs)
|
||||
extractWarnings = cloneWarnings(extractCheckpoint.Warnings)
|
||||
output.Rejected = append(output.Rejected, cloneRejectedOutputs(extractCheckpoint.Rejected)...)
|
||||
output.Warnings = append(output.Warnings, extractWarnings...)
|
||||
} else {
|
||||
if err := checkpoints.ExtractRunning(lane.ID, extractor.Key(), extractDependencies); err != nil {
|
||||
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
for index := range chunks {
|
||||
chunk := chunks[index]
|
||||
var acceptedOutput contracts.ExtractOutput
|
||||
var acceptedWarnings []contracts.Warning
|
||||
accepted, rejection, err := runWithRetry(ctx, lane.Extract.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
result, err := extractor.Extract(ctx, contracts.ExtractionRequest{
|
||||
Source: doc,
|
||||
Chunk: &chunk,
|
||||
SourceInput: chunkInputMaterial(sourceInput, chunk),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet),
|
||||
LLMClient: input.LLMClient,
|
||||
LLMProfile: lane.Extract.LLMProfile,
|
||||
Options: cloneOptions(lane.Extract.Options),
|
||||
Metadata: input.Metadata,
|
||||
})
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("extract lane %q chunk %q with extractor %q: %w", lane.ID, chunk.ID, extractor.Key(), err)
|
||||
}
|
||||
extractOutput := result.Output
|
||||
extractOutput.LaneID = lane.ID
|
||||
extractOutput.ExtractorKey = extractor.Key()
|
||||
extractOutput.SourceID = doc.ID
|
||||
extractOutput.ChunkID = chunk.ID
|
||||
extractOutput.ChunkIndex = chunk.Index
|
||||
extractOutput.Payload.Warnings = append(extractOutput.Payload.Warnings, result.Warnings...)
|
||||
validationWarnings, rejection, err := r.validateRaw(ctx, rawValidationTarget{
|
||||
stage: StageExtract,
|
||||
laneID: lane.ID,
|
||||
moduleKey: extractor.Key(),
|
||||
source: doc,
|
||||
sourceID: doc.ID,
|
||||
chunkID: chunk.ID,
|
||||
chunkIndex: chunk.Index,
|
||||
chunk: &chunk,
|
||||
sourceInput: chunkInputMaterial(sourceInput, chunk),
|
||||
sessionID: sessionID,
|
||||
references: lane.ExtractReferences.ReferenceSet,
|
||||
llmClient: input.LLMClient,
|
||||
schema: extractOutput.Schema,
|
||||
payload: extractOutput.Payload,
|
||||
metadata: input.Metadata,
|
||||
chains: input.Pipeline.ValidatorChains,
|
||||
attempt: attempt,
|
||||
})
|
||||
if err != nil || rejection != nil {
|
||||
return false, rejection, err
|
||||
}
|
||||
acceptedOutput = cloneExtractOutput(extractOutput)
|
||||
acceptedWarnings = append(cloneWarnings(result.Warnings), validationWarnings...)
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("extract lane %q chunk %q with extractor %q: %w", lane.ID, chunk.ID, extractor.Key(), err)
|
||||
_ = checkpoints.ExtractFailed(lane.ID, extractor.Key(), extractDependencies, err)
|
||||
return err
|
||||
}
|
||||
extractOutput := result.Output
|
||||
extractOutput.LaneID = lane.ID
|
||||
extractOutput.ExtractorKey = extractor.Key()
|
||||
extractOutput.SourceID = doc.ID
|
||||
extractOutput.ChunkID = chunk.ID
|
||||
extractOutput.ChunkIndex = chunk.Index
|
||||
extractOutput.Payload.Warnings = append(extractOutput.Payload.Warnings, result.Warnings...)
|
||||
validationWarnings, rejection, err := r.validateRaw(ctx, rawValidationTarget{
|
||||
stage: StageExtract,
|
||||
laneID: lane.ID,
|
||||
moduleKey: extractor.Key(),
|
||||
source: doc,
|
||||
sourceID: doc.ID,
|
||||
chunkID: chunk.ID,
|
||||
chunkIndex: chunk.Index,
|
||||
chunk: &chunk,
|
||||
sourceInput: chunkInputMaterial(sourceInput, chunk),
|
||||
sessionID: sessionID,
|
||||
references: lane.ExtractReferences.ReferenceSet,
|
||||
llmClient: input.LLMClient,
|
||||
schema: extractOutput.Schema,
|
||||
payload: extractOutput.Payload,
|
||||
metadata: input.Metadata,
|
||||
chains: input.Pipeline.ValidatorChains,
|
||||
attempt: attempt,
|
||||
})
|
||||
if err != nil || rejection != nil {
|
||||
return false, rejection, err
|
||||
if !accepted {
|
||||
output.Rejected = append(output.Rejected, *rejection)
|
||||
continue
|
||||
}
|
||||
acceptedOutput = cloneExtractOutput(extractOutput)
|
||||
acceptedWarnings = append(cloneWarnings(result.Warnings), validationWarnings...)
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
_ = checkpoints.ExtractFailed(lane.ID, extractor.Key(), extractDependencies, err)
|
||||
return err
|
||||
output.Warnings = append(output.Warnings, acceptedWarnings...)
|
||||
extractWarnings = append(extractWarnings, acceptedWarnings...)
|
||||
extractOutputs = append(extractOutputs, acceptedOutput)
|
||||
}
|
||||
if !accepted {
|
||||
output.Rejected = append(output.Rejected, *rejection)
|
||||
continue
|
||||
extractRejected := cloneRejectedOutputs(output.Rejected[extractRejectedStart:])
|
||||
if err := checkpoints.ExtractSucceeded(lane.ID, extractor.Key(), extractDependencies, extractOutputs, extractRejected, extractWarnings); err != nil {
|
||||
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
output.Warnings = append(output.Warnings, acceptedWarnings...)
|
||||
extractWarnings = append(extractWarnings, acceptedWarnings...)
|
||||
extractOutputs = append(extractOutputs, acceptedOutput)
|
||||
}
|
||||
extractRejected := cloneRejectedOutputs(output.Rejected[extractRejectedStart:])
|
||||
if err := checkpoints.ExtractSucceeded(lane.ID, extractor.Key(), extractDependencies, extractOutputs, extractRejected, extractWarnings); err != nil {
|
||||
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
|
||||
if len(extractOutputs) == 0 {
|
||||
@@ -308,135 +338,151 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
|
||||
var acceptedMerge contracts.MergeOutput
|
||||
var mergeWarnings []contracts.Warning
|
||||
mergeDependencies := rawOutputDigests(extractPayloads(extractOutputs))
|
||||
if err := checkpoints.MergeRunning(lane.ID, merger.Key(), mergeDependencies); err != nil {
|
||||
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
mergeAccepted, mergeRejection, err := runWithRetry(ctx, lane.Merge.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
mergeResult, err := merger.Merge(ctx, contracts.MergeRequest{
|
||||
Source: doc,
|
||||
LaneID: lane.ID,
|
||||
ExtractOutputs: cloneExtractOutputs(extractOutputs),
|
||||
SourceInput: sourceInput.Clone(),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(lane.MergeReferences.ReferenceSet),
|
||||
LLMClient: input.LLMClient,
|
||||
LLMProfile: lane.Merge.LLMProfile,
|
||||
Options: cloneOptions(lane.Merge.Options),
|
||||
Metadata: input.Metadata,
|
||||
})
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("merge lane %q with merger %q: %w", lane.ID, merger.Key(), err)
|
||||
}
|
||||
mergeOutput := mergeResult.Output
|
||||
mergeOutput.LaneID = lane.ID
|
||||
mergeOutput.MergerKey = merger.Key()
|
||||
mergeOutput.SourceID = doc.ID
|
||||
mergeOutput.Payload.Warnings = append(mergeOutput.Payload.Warnings, mergeResult.Warnings...)
|
||||
validationWarnings, rejection, err := r.validateRaw(ctx, rawValidationTarget{
|
||||
stage: StageMerge,
|
||||
laneID: lane.ID,
|
||||
moduleKey: merger.Key(),
|
||||
source: doc,
|
||||
sourceID: doc.ID,
|
||||
sourceInput: sourceInput.Clone(),
|
||||
sessionID: sessionID,
|
||||
references: lane.MergeReferences.ReferenceSet,
|
||||
llmClient: input.LLMClient,
|
||||
schema: mergeOutput.Schema,
|
||||
payload: mergeOutput.Payload,
|
||||
extractOutputs: extractOutputs,
|
||||
metadata: input.Metadata,
|
||||
chains: input.Pipeline.ValidatorChains,
|
||||
attempt: attempt,
|
||||
})
|
||||
if err != nil || rejection != nil {
|
||||
return false, rejection, err
|
||||
}
|
||||
acceptedMerge = cloneMergeOutput(mergeOutput)
|
||||
mergeWarnings = append(cloneWarnings(mergeResult.Warnings), validationWarnings...)
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
_ = checkpoints.MergeFailed(lane.ID, merger.Key(), mergeDependencies, err)
|
||||
return err
|
||||
}
|
||||
if !mergeAccepted {
|
||||
output.Rejected = append(output.Rejected, *mergeRejection)
|
||||
if err := checkpoints.MergeRejected(lane.ID, merger.Key(), mergeDependencies, *mergeRejection); err != nil {
|
||||
mergeCheckpoint, mergeDecision := checkpointLoader.Merge(lane.ID, merger.Key(), mergeDependencies)
|
||||
recordCheckpointEvent(output, checkpointLoader, string(StageMerge), lane.ID, merger.Key(), mergeDecision)
|
||||
if mergeDecision.Reused {
|
||||
acceptedMerge = cloneMergeOutput(mergeCheckpoint.Output)
|
||||
mergeWarnings = cloneWarnings(mergeCheckpoint.Warnings)
|
||||
output.Warnings = append(output.Warnings, mergeWarnings...)
|
||||
} else {
|
||||
if err := checkpoints.MergeRunning(lane.ID, merger.Key(), mergeDependencies); err != nil {
|
||||
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
mergeAccepted, mergeRejection, err := runWithRetry(ctx, lane.Merge.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
mergeResult, err := merger.Merge(ctx, contracts.MergeRequest{
|
||||
Source: doc,
|
||||
LaneID: lane.ID,
|
||||
ExtractOutputs: cloneExtractOutputs(extractOutputs),
|
||||
SourceInput: sourceInput.Clone(),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(lane.MergeReferences.ReferenceSet),
|
||||
LLMClient: input.LLMClient,
|
||||
LLMProfile: lane.Merge.LLMProfile,
|
||||
Options: cloneOptions(lane.Merge.Options),
|
||||
Metadata: input.Metadata,
|
||||
})
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("merge lane %q with merger %q: %w", lane.ID, merger.Key(), err)
|
||||
}
|
||||
mergeOutput := mergeResult.Output
|
||||
mergeOutput.LaneID = lane.ID
|
||||
mergeOutput.MergerKey = merger.Key()
|
||||
mergeOutput.SourceID = doc.ID
|
||||
mergeOutput.Payload.Warnings = append(mergeOutput.Payload.Warnings, mergeResult.Warnings...)
|
||||
validationWarnings, rejection, err := r.validateRaw(ctx, rawValidationTarget{
|
||||
stage: StageMerge,
|
||||
laneID: lane.ID,
|
||||
moduleKey: merger.Key(),
|
||||
source: doc,
|
||||
sourceID: doc.ID,
|
||||
sourceInput: sourceInput.Clone(),
|
||||
sessionID: sessionID,
|
||||
references: lane.MergeReferences.ReferenceSet,
|
||||
llmClient: input.LLMClient,
|
||||
schema: mergeOutput.Schema,
|
||||
payload: mergeOutput.Payload,
|
||||
extractOutputs: extractOutputs,
|
||||
metadata: input.Metadata,
|
||||
chains: input.Pipeline.ValidatorChains,
|
||||
attempt: attempt,
|
||||
})
|
||||
if err != nil || rejection != nil {
|
||||
return false, rejection, err
|
||||
}
|
||||
acceptedMerge = cloneMergeOutput(mergeOutput)
|
||||
mergeWarnings = append(cloneWarnings(mergeResult.Warnings), validationWarnings...)
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
_ = checkpoints.MergeFailed(lane.ID, merger.Key(), mergeDependencies, err)
|
||||
return err
|
||||
}
|
||||
if !mergeAccepted {
|
||||
output.Rejected = append(output.Rejected, *mergeRejection)
|
||||
if err := checkpoints.MergeRejected(lane.ID, merger.Key(), mergeDependencies, *mergeRejection); err != nil {
|
||||
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
output.Warnings = append(output.Warnings, mergeWarnings...)
|
||||
if err := checkpoints.MergeSucceeded(lane.ID, merger.Key(), mergeDependencies, acceptedMerge, mergeWarnings); err != nil {
|
||||
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
output.Warnings = append(output.Warnings, mergeWarnings...)
|
||||
if err := checkpoints.MergeSucceeded(lane.ID, merger.Key(), mergeDependencies, acceptedMerge, mergeWarnings); err != nil {
|
||||
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
|
||||
var acceptedNormalize contracts.NormalizeOutput
|
||||
var normalizeWarnings []contracts.Warning
|
||||
normalizeDependencies := rawOutputDigests([]contracts.RawPayload{acceptedMerge.Payload})
|
||||
if err := checkpoints.NormalizeRunning(lane.ID, normalizer.Key(), normalizeDependencies); err != nil {
|
||||
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
normalizeAccepted, normalizeRejection, err := runWithRetry(ctx, lane.Normalize.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
normalizeResult, err := normalizer.Normalize(ctx, contracts.NormalizeRequest{
|
||||
Source: doc,
|
||||
LaneID: lane.ID,
|
||||
MergeOutput: cloneMergeOutput(acceptedMerge),
|
||||
SourceInput: sourceInput.Clone(),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(lane.NormalizeReferences.ReferenceSet),
|
||||
LLMClient: input.LLMClient,
|
||||
LLMProfile: lane.Normalize.LLMProfile,
|
||||
Options: cloneOptions(lane.Normalize.Options),
|
||||
Metadata: input.Metadata,
|
||||
})
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("normalize lane %q with normalizer %q: %w", lane.ID, normalizer.Key(), err)
|
||||
}
|
||||
normalizeOutput := normalizeResult.Output
|
||||
normalizeOutput.LaneID = lane.ID
|
||||
normalizeOutput.NormalizerKey = normalizer.Key()
|
||||
normalizeOutput.SourceID = doc.ID
|
||||
normalizeOutput.Payload.Warnings = append(normalizeOutput.Payload.Warnings, normalizeResult.Warnings...)
|
||||
validationWarnings, rejection, err := r.validateRaw(ctx, rawValidationTarget{
|
||||
stage: StageNormalize,
|
||||
laneID: lane.ID,
|
||||
moduleKey: normalizer.Key(),
|
||||
source: doc,
|
||||
sourceID: doc.ID,
|
||||
sourceInput: sourceInput.Clone(),
|
||||
sessionID: sessionID,
|
||||
references: lane.NormalizeReferences.ReferenceSet,
|
||||
llmClient: input.LLMClient,
|
||||
schema: normalizeOutput.Schema,
|
||||
payload: normalizeOutput.Payload,
|
||||
mergeOutput: acceptedMerge,
|
||||
metadata: input.Metadata,
|
||||
chains: input.Pipeline.ValidatorChains,
|
||||
attempt: attempt,
|
||||
})
|
||||
if err != nil || rejection != nil {
|
||||
return false, rejection, err
|
||||
}
|
||||
acceptedNormalize = cloneNormalizeOutput(normalizeOutput)
|
||||
normalizeWarnings = append(cloneWarnings(normalizeResult.Warnings), validationWarnings...)
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
_ = checkpoints.NormalizeFailed(lane.ID, normalizer.Key(), normalizeDependencies, err)
|
||||
return err
|
||||
}
|
||||
if !normalizeAccepted {
|
||||
output.Rejected = append(output.Rejected, *normalizeRejection)
|
||||
if err := checkpoints.NormalizeRejected(lane.ID, normalizer.Key(), normalizeDependencies, *normalizeRejection); err != nil {
|
||||
normalizeCheckpoint, normalizeDecision := checkpointLoader.Normalize(lane.ID, normalizer.Key(), normalizeDependencies)
|
||||
recordCheckpointEvent(output, checkpointLoader, string(StageNormalize), lane.ID, normalizer.Key(), normalizeDecision)
|
||||
if normalizeDecision.Reused {
|
||||
acceptedNormalize = cloneNormalizeOutput(normalizeCheckpoint.Output)
|
||||
normalizeWarnings = cloneWarnings(normalizeCheckpoint.Warnings)
|
||||
output.Warnings = append(output.Warnings, normalizeWarnings...)
|
||||
} else {
|
||||
if err := checkpoints.NormalizeRunning(lane.ID, normalizer.Key(), normalizeDependencies); err != nil {
|
||||
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
normalizeAccepted, normalizeRejection, err := runWithRetry(ctx, lane.Normalize.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
normalizeResult, err := normalizer.Normalize(ctx, contracts.NormalizeRequest{
|
||||
Source: doc,
|
||||
LaneID: lane.ID,
|
||||
MergeOutput: cloneMergeOutput(acceptedMerge),
|
||||
SourceInput: sourceInput.Clone(),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(lane.NormalizeReferences.ReferenceSet),
|
||||
LLMClient: input.LLMClient,
|
||||
LLMProfile: lane.Normalize.LLMProfile,
|
||||
Options: cloneOptions(lane.Normalize.Options),
|
||||
Metadata: input.Metadata,
|
||||
})
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("normalize lane %q with normalizer %q: %w", lane.ID, normalizer.Key(), err)
|
||||
}
|
||||
normalizeOutput := normalizeResult.Output
|
||||
normalizeOutput.LaneID = lane.ID
|
||||
normalizeOutput.NormalizerKey = normalizer.Key()
|
||||
normalizeOutput.SourceID = doc.ID
|
||||
normalizeOutput.Payload.Warnings = append(normalizeOutput.Payload.Warnings, normalizeResult.Warnings...)
|
||||
validationWarnings, rejection, err := r.validateRaw(ctx, rawValidationTarget{
|
||||
stage: StageNormalize,
|
||||
laneID: lane.ID,
|
||||
moduleKey: normalizer.Key(),
|
||||
source: doc,
|
||||
sourceID: doc.ID,
|
||||
sourceInput: sourceInput.Clone(),
|
||||
sessionID: sessionID,
|
||||
references: lane.NormalizeReferences.ReferenceSet,
|
||||
llmClient: input.LLMClient,
|
||||
schema: normalizeOutput.Schema,
|
||||
payload: normalizeOutput.Payload,
|
||||
mergeOutput: acceptedMerge,
|
||||
metadata: input.Metadata,
|
||||
chains: input.Pipeline.ValidatorChains,
|
||||
attempt: attempt,
|
||||
})
|
||||
if err != nil || rejection != nil {
|
||||
return false, rejection, err
|
||||
}
|
||||
acceptedNormalize = cloneNormalizeOutput(normalizeOutput)
|
||||
normalizeWarnings = append(cloneWarnings(normalizeResult.Warnings), validationWarnings...)
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
_ = checkpoints.NormalizeFailed(lane.ID, normalizer.Key(), normalizeDependencies, err)
|
||||
return err
|
||||
}
|
||||
if !normalizeAccepted {
|
||||
output.Rejected = append(output.Rejected, *normalizeRejection)
|
||||
if err := checkpoints.NormalizeRejected(lane.ID, normalizer.Key(), normalizeDependencies, *normalizeRejection); err != nil {
|
||||
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
output.Warnings = append(output.Warnings, normalizeWarnings...)
|
||||
if err := checkpoints.NormalizeSucceeded(lane.ID, normalizer.Key(), normalizeDependencies, acceptedNormalize, normalizeWarnings); err != nil {
|
||||
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
output.Warnings = append(output.Warnings, normalizeWarnings...)
|
||||
if err := checkpoints.NormalizeSucceeded(lane.ID, normalizer.Key(), normalizeDependencies, acceptedNormalize, normalizeWarnings); err != nil {
|
||||
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
output.NormalizeOutputs = append(output.NormalizeOutputs, acceptedNormalize)
|
||||
return nil
|
||||
@@ -759,6 +805,23 @@ func failOutput(output RunOutput) RunOutput {
|
||||
return output
|
||||
}
|
||||
|
||||
func recordCheckpointEvent(output *RunOutput, loader CheckpointLoader, stage string, laneID string, moduleKey string, decision CheckpointDecision) {
|
||||
if output == nil || loader == nil || !loader.Enabled() {
|
||||
return
|
||||
}
|
||||
action := "executed"
|
||||
if decision.Reused {
|
||||
action = "reused"
|
||||
}
|
||||
output.CheckpointEvents = append(output.CheckpointEvents, CheckpointEvent{
|
||||
Stage: stage,
|
||||
LaneID: laneID,
|
||||
ModuleKey: moduleKey,
|
||||
Action: action,
|
||||
Reason: decision.Reason,
|
||||
})
|
||||
}
|
||||
|
||||
func populateRawOutputManifest(output *RunOutput) {
|
||||
if output == nil {
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user