eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement final fixes and close out the implemetation roadmap

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 23:16:16 -05:00
parent 604c7a7945
commit 9fa9154dda
14 changed files with 463 additions and 879 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
internal/framework/pipeline/runner_concurrent.go
View File

@@ -79,7 +79,9 @@ func (r *Runner) runLanes(parent context.Context, input RunInput, checkpoints Ch
if prepared.typed == nil {
return output, fmt.Errorf("typed lane %q executor is not prepared", prepared.resolved.ID)
}
setTypedLaneManifestMetadata(&output, prepared.resolved.ID, prepared.typed.extractor, prepared.typed.merger, prepared.typed.normalizer)
if err := setTypedLaneManifestMetadata(&output, prepared.resolved.ID, prepared.typed.extractor, prepared.typed.merger, prepared.typed.normalizer); err != nil {
return output, err
}
state, err := prepareLaneExtract(input, loader, doc, chunks, i, prepared)
if err != nil {
return output, err
@@ -212,7 +214,9 @@ func (r *Runner) runLanes(parent context.Context, input RunInput, checkpoints Ch
close(continuations)
continuationWorkers.Wait()
for i := range completedOutputs {
mergeLaneOutput(&output, completedOutputs[i])
if err := mergeLaneOutput(&output, completedOutputs[i]); err != nil {
return output, err
}
}
if err := selectRunError(parent, runErrors); err != nil {
return output, err
@@ -244,7 +248,10 @@ func prepareLaneExtract(input RunInput, loader CheckpointLoader, doc *source.Sou
if decodeErr != nil {
return nil, fmt.Errorf("decode extract checkpoint for lane %q: %w", lane.ID, decodeErr)
}
stored = hydrateCheckpointArtifact(typed.codec, stored, value)
stored, decodeErr = hydrateCheckpointArtifact(typed.codec, stored, value)
if decodeErr != nil {
return nil, fmt.Errorf("hydrate extract checkpoint for lane %q: %w", lane.ID, decodeErr)
}
artifact := erasedExtractArtifact{LaneID: lane.ID, ExtractorKey: lane.Extract.Module, SourceID: doc.ID, ChunkID: stored.ChunkID, ChunkIndex: stored.ChunkIndex, ChunkRef: stored.ChunkRef, Value: value}
if stored.ChunkIndex >= 0 && stored.ChunkIndex < len(chunks) && artifact.ChunkRef == (source.SourceRef{}) {
artifact.ChunkRef = chunks[stored.ChunkIndex].Ref
@@ -258,9 +265,14 @@ func prepareLaneExtract(input RunInput, loader CheckpointLoader, doc *source.Sou
}
func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, job extractJob) extractJobResult {
state, chunk := job.lane, cloneSourceChunk(job.chunk)
state := job.lane
chunk, cloneErr := cloneSourceChunk(job.chunk)
lane, typed := state.prepared.resolved, state.prepared.typed
result := extractJobResult{laneIndex: state.index, chunkIndex: chunk.Index}
result := extractJobResult{laneIndex: state.index, chunkIndex: job.chunk.Index}
if cloneErr != nil {
result.err = fmt.Errorf("clone chunk %q for extraction: %w", job.chunk.ID, cloneErr)
return result
}
var accepted erasedExtractArtifact
var serialized CheckpointArtifact
var acceptedWarnings []contracts.Warning
@@ -269,7 +281,11 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.
attemptPath := path.Join("extract", debugPathComponent(lane.ID), fmt.Sprintf("chunk-%06d", chunk.Index+1), fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
terminal := newAttemptTerminalRecorder(input.Debug, attemptPath, "extract", llmScope, debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, Attempt: attempt, StartedAt: started})
extracted, callErr := typed.extract(attemptCtx, typed.extractor, contracts.TypedExtractionRequest{Source: doc, Chunk: &chunk, SourceInput: chunkInputMaterial(sourceInput, chunk), SessionID: sessionID, References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet), LLMProfile: lane.Extract.LLMProfile, Metadata: cloneMetadata(input.Metadata)})
requestMetadata, metadataErr := cloneMetadata(input.Metadata)
if metadataErr != nil {
return false, nil, terminal.record(nil, fmt.Errorf("clone extract request metadata: %w", metadataErr))
}
extracted, callErr := typed.extract(attemptCtx, typed.extractor, contracts.TypedExtractionRequest{Source: doc, Chunk: &chunk, SourceInput: chunkInputMaterial(sourceInput, chunk), SessionID: sessionID, References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet), LLMProfile: lane.Extract.LLMProfile, Metadata: requestMetadata})
if callErr != nil {
attemptErr := fmt.Errorf("extract lane %q chunk %q with extractor %q: %w", lane.ID, chunk.ID, lane.Extract.Module, callErr)
return false, nil, terminal.record(nil, attemptErr)
@@ -414,9 +430,9 @@ func selectRunError(parent context.Context, values []orderedRunError) error {
return filtered[0].err
}
func mergeLaneOutput(dst *RunOutput, src RunOutput) {
func mergeLaneOutput(dst *RunOutput, src RunOutput) error {
if dst == nil {
return
return nil
}
dst.NormalizeOutputs = append(dst.NormalizeOutputs, cloneSerializedOutputs(src.NormalizeOutputs)...)
dst.Rejected = append(dst.Rejected, cloneRejectedOutputs(src.Rejected)...)
@@ -425,8 +441,13 @@ func mergeLaneOutput(dst *RunOutput, src RunOutput) {
for i := range dst.Manifest.ArtifactLanes {
for j := range src.Manifest.ArtifactLanes {
if dst.Manifest.ArtifactLanes[i].ID == src.Manifest.ArtifactLanes[j].ID && src.Manifest.ArtifactLanes[j].Metadata != nil {
dst.Manifest.ArtifactLanes[i].Metadata = cloneMetadata(src.Manifest.ArtifactLanes[j].Metadata)
metadata, err := cloneMetadata(src.Manifest.ArtifactLanes[j].Metadata)
if err != nil {
return fmt.Errorf("clone lane %q manifest metadata: %w", dst.Manifest.ArtifactLanes[i].ID, err)
}
dst.Manifest.ArtifactLanes[i].Metadata = metadata
}
}
}
return nil
}