eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove obsolete sequential extraction path

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 14:05:21 +00:00
parent 2df7084d5d
commit ef6926322d
4 changed files with 226 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
internal/framework/pipeline/runner_concurrent.go
View File

@@ -28,6 +28,22 @@ type laneExtractState struct {
failed bool
}
type finalizedExtractResults struct {
accepted []erasedExtractArtifact
serialized []CheckpointArtifact
warnings []contracts.Warning
rejected []contracts.RejectedOutput
decision CheckpointDecision
}
func loadExtract(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
return loader.Extract(laneID, moduleKey, deps)
}
func recordExtract(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
return recorder.ExtractSucceeded(laneID, moduleKey, deps, outputs, rejected, warnings)
}
type extractJob struct {
lane *laneExtractState
chunk source.Chunk
@@ -56,19 +72,6 @@ type orderedRunError struct {
err error
}
type completedExtractLoader struct {
CheckpointLoader
laneID string
checkpoint ExtractCheckpoint
}
func (l completedExtractLoader) Extract(laneID, _ string, _ []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
if laneID == l.laneID {
return l.checkpoint, CheckpointDecision{Reused: true, Reason: "coordinated extract result"}
}
return ExtractCheckpoint{}, CheckpointDecision{Reason: "extract result unavailable"}
}
func (r *Runner) runLanes(parent context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk) (RunOutput, error) {
output := RunOutput{Manifest: manifestFromPipeline(input)}
states := make([]*laneExtractState, len(input.Prepared.lanes))
@@ -85,6 +88,8 @@ func (r *Runner) runLanes(parent context.Context, input RunInput, checkpoints Ch
if err := checkpoints.ExtractRunning(prepared.resolved.ID, prepared.resolved.Extract.Module, state.deps); err != nil {
return output, fmt.Errorf("write extract checkpoint for lane %q: %w", prepared.resolved.ID, err)
}
} else if err := finalizeLaneExtract(checkpoints, state); err != nil {
return output, err
}
states[i] = state
}
@@ -325,10 +330,26 @@ func finalizeLaneExtract(checkpoints CheckpointRecorder, state *laneExtractState
func (r *Runner) continueLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, state *laneExtractState) (RunOutput, error) {
lane := state.prepared.resolved
local := RunOutput{Manifest: manifestFromPipeline(input)}
checkpoint := ExtractCheckpoint{Outputs: state.serialized, Rejected: state.rejected, Warnings: state.warnings}
coordinatedLoader := completedExtractLoader{CheckpointLoader: loader, laneID: lane.ID, checkpoint: checkpoint}
input.extractDecision = &state.decision
err := r.runTypedLane(ctx, input, checkpoints, coordinatedLoader, doc, sourceInput, sessionID, chunks, state.prepared, &local)
results := finalizedExtractResults{
accepted: state.values,
serialized: state.serialized,
warnings: state.warnings,
rejected: state.rejected,
decision: state.decision,
}
local.Warnings = append(local.Warnings, cloneWarnings(results.warnings)...)
local.Rejected = append(local.Rejected, cloneRejectedOutputs(results.rejected)...)
recordCheckpointEvent(&local, loader, string(StageExtract), lane.ID, lane.Extract.Module, results.decision)
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": results.decision.Reused, "decision": results.decision, "source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "options": redactSensitiveMap(lane.Extract.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
return local, &laneRunError{stage: StageExtract, err: err}
}
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": results.decision.Reused, "outputs": debugCheckpointArtifacts(results.serialized), "rejected": debugRejectedOutputEnvelopes(results.rejected), "warnings": debugWarningEnvelopes(results.warnings)}}); err != nil {
return local, &laneRunError{stage: StageExtract, err: err}
}
if len(results.accepted) == 0 {
return local, nil
}
err := r.continueTypedLane(ctx, input, checkpoints, loader, doc, sourceInput, sessionID, state.prepared, results, &local)
return local, err
}