eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prepare pipelines before source execution

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 06:18:46 +00:00
parent 1c84d19e5f
commit ce3a07512f
26 changed files with 1562 additions and 404 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
internal/framework/pipeline/runner.go
View File

@@ -29,20 +29,17 @@ type Registries struct {
Outputs *OutputEncoderRegistry
}
type Runner struct {
registries Registries
}
type Runner struct{}
func New(registries Registries) *Runner {
return &Runner{registries: registries}
func New() *Runner {
return &Runner{}
}
type RunInput struct {
Pipeline ResolvedPipeline
Prepared *PreparedPipeline
SourceID string
Path string
RawInput []byte
LLMClient contracts.StructuredLLMClient
SessionID string
RunID string
StartedAt time.Time
@@ -52,6 +49,9 @@ type RunInput struct {
Checkpoints CheckpointRecorder
Checkpoint CheckpointLoader
Debug DebugRecorder
pipeline ResolvedPipeline
llmClient contracts.StructuredLLMClient
}
type RunOutput struct {
@@ -70,9 +70,8 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
if err := validateRunInput(input); err != nil {
return output, err
}
if err := r.validateRegistries(input.Pipeline); err != nil {
return output, err
}
input.pipeline = input.Prepared.resolved
input.llmClient = input.Prepared.dependencies.LLM
output.Manifest = manifestFromPipeline(input)
checkpoints := input.Checkpoints
@@ -88,27 +87,24 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
debugRecorder = NoopDebugRecorder()
}
input.Debug = debugRecorder
input.LLMClient = wrapDebugLLMClient(input.LLMClient, debugRecorder)
input.llmClient = wrapDebugLLMClient(input.llmClient, debugRecorder)
defer func() {
output.Manifest.LLMProfiles = mergeLLMProfileManifests(input.LLMProfiles, llmProfileManifests(input.LLMClient))
output.Manifest.LLMProfiles = mergeLLMProfileManifests(input.LLMProfiles, llmProfileManifests(input.llmClient))
}()
output.Warnings = append(output.Warnings, cloneWarnings(input.Warnings)...)
if err := writeDebugTimed(debugRecorder, "run.json", debugTimedEnvelope{
Stage: "run",
StartedAt: startedTime(input.StartedAt),
Payload: map[string]any{
"pipeline_id": input.Pipeline.ID,
"pipeline_digest": input.Pipeline.Digest,
"pipeline_id": input.pipeline.ID,
"pipeline_digest": input.pipeline.Digest,
"run_id": output.Manifest.RunID,
},
}); err != nil {
return failOutput(output), fmt.Errorf("write debug run artifact: %w", err)
}
adapter, err := r.registries.Inputs.Build(input.Pipeline.Input.Module)
if err != nil {
return failOutput(output), fmt.Errorf("build input adapter %q: %w", input.Pipeline.Input.Module, err)
}
adapter := input.Prepared.input
attachModuleManifestMetadata(&output, "input", adapter)
sourceCheckpoint, sourceDecision := checkpointLoader.Source(adapter.Key())
recordCheckpointEvent(&output, checkpointLoader, "source", "", adapter.Key(), sourceDecision)
@@ -122,7 +118,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
SourceID: input.SourceID,
Path: input.Path,
Raw: debugContentEnvelope(input.RawInput, sourceInputMediaType(input.Path), nil, nil),
Options: redactSensitiveMap(input.Pipeline.Input.Options),
Options: redactSensitiveMap(input.pipeline.Input.Options),
Metadata: redactSensitiveMap(input.Metadata),
},
}); err != nil {
@@ -136,8 +132,8 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
SourceID: input.SourceID,
Path: input.Path,
Raw: input.RawInput,
LLMProfile: input.Pipeline.Input.LLMProfile,
Options: cloneOptions(input.Pipeline.Input.Options),
LLMProfile: input.pipeline.Input.LLMProfile,
Options: cloneOptions(input.pipeline.Input.Options),
Metadata: input.Metadata,
})
if err != nil {
@@ -169,10 +165,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
output.Manifest.Metadata = manifestMetadataWithSessionID(output.Manifest.Metadata, sessionID)
output.Manifest.SourceDigests = []string{doc.Digest}
chunker, err := r.registries.Chunkers.Build(input.Pipeline.Chunk.Module)
if err != nil {
return failOutput(output), fmt.Errorf("build chunker %q: %w", input.Pipeline.Chunk.Module, err)
}
chunker := input.Prepared.chunker
attachModuleManifestMetadata(&output, "chunker", chunker)
var canonicalChunks []source.Chunk
var chunkWarnings []contracts.Warning
@@ -188,7 +181,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
"decision": chunkDecision,
"source": debugSourceDocumentEnvelope(doc),
"source_input": debugContentEnvelope(sourceInput.Content, sourceInput.MediaType, nil, nil),
"options": redactSensitiveMap(input.Pipeline.Chunk.Options),
"options": redactSensitiveMap(input.pipeline.Chunk.Options),
"metadata": redactSensitiveMap(input.Metadata),
},
}); err != nil {
@@ -204,7 +197,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
if err := checkpoints.ChunkRunning(chunker.Key(), doc.Digest); err != nil {
return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
}
chunksAccepted, chunkRejection, err = runWithRetry(ctx, input.Pipeline.Chunk.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
chunksAccepted, chunkRejection, err = runWithRetry(ctx, input.pipeline.Chunk.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
attemptStarted := time.Now().UTC()
attemptPath := path.Join("chunk", fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
@@ -212,10 +205,10 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
Source: doc,
SourceInput: sourceInput.Clone(),
SessionID: sessionID,
References: CloneReferenceSet(input.Pipeline.ChunkReferences.ReferenceSet),
LLMClient: input.LLMClient,
LLMProfile: input.Pipeline.Chunk.LLMProfile,
Options: cloneOptions(input.Pipeline.Chunk.Options),
References: CloneReferenceSet(input.pipeline.ChunkReferences.ReferenceSet),
LLMClient: input.llmClient,
LLMProfile: input.pipeline.Chunk.LLMProfile,
Options: cloneOptions(input.pipeline.Chunk.Options),
Metadata: input.Metadata,
})
if err != nil {
@@ -254,7 +247,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}, llmScope))
return false, nil, err
}
validationWarnings, rejection, err := r.validateChunksRaw(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.Pipeline.ChunkReferences.ReferenceSet, input.LLMClient, input.Metadata, input.Pipeline.ValidatorChains, attempt, input.Debug)
validationWarnings, rejection, err := r.validateChunksRaw(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.llmClient, input.Metadata, input.Prepared.chunkValidators, attempt, input.Debug)
if err != nil || rejection != nil {
_ = writeDebugTimed(debugRecorder, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageChunk),
@@ -320,7 +313,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}
if chunksAccepted {
for _, lane := range input.Pipeline.ArtifactLanes {
for _, lane := range input.Prepared.lanes {
if err := r.runLane(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, canonicalChunks, lane, &output); err != nil {
return failOutput(output), err
}
@@ -335,10 +328,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
populateRawOutputManifest(&output)
output.Manifest.CompletedAt = timePtr(time.Now().UTC())
encoder, err := r.registries.Outputs.Build(input.Pipeline.Output.Module)
if err != nil {
return failOutput(output), fmt.Errorf("build output encoder %q: %w", input.Pipeline.Output.Module, err)
}
encoder := input.Prepared.output
attachModuleManifestMetadata(&output, "output", encoder)
outputStarted := time.Now().UTC()
if err := writeDebugTimed(debugRecorder, "output/input.json", debugTimedEnvelope{
@@ -350,7 +340,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
"normalize_outputs": debugNormalizeOutputEnvelopes(output.NormalizeOutputs),
"rejected": debugRejectedOutputEnvelopes(output.Rejected),
"warnings": output.Warnings,
"options": redactSensitiveMap(input.Pipeline.Output.Options),
"options": redactSensitiveMap(input.pipeline.Output.Options),
"metadata": redactSensitiveMap(input.Metadata),
},
}); err != nil {
@@ -361,8 +351,8 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
NormalizeOutputs: cloneNormalizeOutputs(output.NormalizeOutputs),
Rejected: cloneRejectedOutputs(output.Rejected),
Warnings: output.Warnings,
LLMProfile: input.Pipeline.Output.LLMProfile,
Options: cloneOptions(input.Pipeline.Output.Options),
LLMProfile: input.pipeline.Output.LLMProfile,
Options: cloneOptions(input.pipeline.Output.Options),
Metadata: input.Metadata,
})
output.Warnings = append(output.Warnings, encoded.Warnings...)
@@ -389,19 +379,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return output, nil
}
func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, checkpointLoader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, lane ResolvedArtifactLane, output *RunOutput) error {
extractor, err := r.registries.Extractors.BuildLegacyRaw(lane.Extract.Module)
if err != nil {
return fmt.Errorf("build extractor %q for lane %q: %w", lane.Extract.Module, lane.ID, err)
}
merger, err := r.registries.Mergers.BuildLegacyRaw(lane.Merge.Module)
if err != nil {
return fmt.Errorf("build merger %q for lane %q: %w", lane.Merge.Module, lane.ID, err)
}
normalizer, err := r.registries.Normalizers.BuildLegacyRaw(lane.Normalize.Module)
if err != nil {
return fmt.Errorf("build normalizer %q for lane %q: %w", lane.Normalize.Module, lane.ID, err)
func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, checkpointLoader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, prepared preparedLaneExecutor, output *RunOutput) error {
lane := prepared.resolved
if prepared.legacy == nil {
return fmt.Errorf("resolved pipeline lane %q uses typed artifact kind %q, which the legacy raw runner cannot execute", lane.ID, lane.ArtifactKind)
}
extractor := prepared.legacy.extractor
merger := prepared.legacy.merger
normalizer := prepared.legacy.normalizer
setLaneManifestMetadata(output, lane.ID, extractor, merger, normalizer)
extractOutputs := make([]contracts.ExtractOutput, 0, len(chunks))
@@ -454,7 +439,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
SourceInput: chunkInputMaterial(sourceInput, chunk),
SessionID: sessionID,
References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet),
LLMClient: input.LLMClient,
LLMClient: input.llmClient,
LLMProfile: lane.Extract.LLMProfile,
Options: cloneOptions(lane.Extract.Options),
Metadata: input.Metadata,
@@ -489,11 +474,11 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
sourceInput: chunkInputMaterial(sourceInput, chunk),
sessionID: sessionID,
references: lane.ExtractReferences.ReferenceSet,
llmClient: input.LLMClient,
llmClient: input.llmClient,
schema: extractOutput.Schema,
payload: extractOutput.Payload,
metadata: input.Metadata,
chains: input.Pipeline.ValidatorChains,
prepared: prepared.extractValidators,
attempt: attempt,
debug: input.Debug,
})
@@ -606,7 +591,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
SourceInput: sourceInput.Clone(),
SessionID: sessionID,
References: CloneReferenceSet(lane.MergeReferences.ReferenceSet),
LLMClient: input.LLMClient,
LLMClient: input.llmClient,
LLMProfile: lane.Merge.LLMProfile,
Options: cloneOptions(lane.Merge.Options),
Metadata: input.Metadata,
@@ -636,12 +621,12 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
sourceInput: sourceInput.Clone(),
sessionID: sessionID,
references: lane.MergeReferences.ReferenceSet,
llmClient: input.LLMClient,
llmClient: input.llmClient,
schema: mergeOutput.Schema,
payload: mergeOutput.Payload,
extractOutputs: extractOutputs,
metadata: input.Metadata,
chains: input.Pipeline.ValidatorChains,
prepared: prepared.mergeValidators,
attempt: attempt,
debug: input.Debug,
})
@@ -762,7 +747,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
SourceInput: sourceInput.Clone(),
SessionID: sessionID,
References: CloneReferenceSet(lane.NormalizeReferences.ReferenceSet),
LLMClient: input.LLMClient,
LLMClient: input.llmClient,
LLMProfile: lane.Normalize.LLMProfile,
Options: cloneOptions(lane.Normalize.Options),
Metadata: input.Metadata,
@@ -792,12 +777,12 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
sourceInput: sourceInput.Clone(),
sessionID: sessionID,
references: lane.NormalizeReferences.ReferenceSet,
llmClient: input.LLMClient,
llmClient: input.llmClient,
schema: normalizeOutput.Schema,
payload: normalizeOutput.Payload,
mergeOutput: acceptedMerge,
metadata: input.Metadata,
chains: input.Pipeline.ValidatorChains,
prepared: prepared.normalizeValidators,
attempt: attempt,
debug: input.Debug,
})
@@ -899,7 +884,7 @@ type rawValidationTarget struct {
extractOutputs []contracts.ExtractOutput
mergeOutput contracts.MergeOutput
metadata map[string]any
chains []ResolvedValidatorChain
prepared preparedValidatorChain
attempt int
debug DebugRecorder
}
@@ -951,7 +936,7 @@ func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool,
return false, lastRejection, nil
}
func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, llmClient contracts.StructuredLLMClient, metadata map[string]any, chains []ResolvedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, llmClient contracts.StructuredLLMClient, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
return r.validateRaw(ctx, rawValidationTarget{
stage: StageChunk,
moduleKey: moduleKey,
@@ -963,26 +948,23 @@ func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocume
llmClient: llmClient,
chunks: chunks,
metadata: metadata,
chains: chains,
prepared: prepared,
attempt: attempt,
debug: debug,
})
}
func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([]contracts.Warning, *contracts.RejectedOutput, error) {
chain := resolvedValidatorChain(target.stage, target.laneID, target.moduleKey, target.chains)
if len(chain.Validators) == 0 {
if len(target.prepared.validators) == 0 {
return nil, nil, nil
}
if r.registries.Validators == nil {
return nil, nil, fmt.Errorf("validator registry must not be nil")
}
var warnings []contracts.Warning
for index, validatorBinding := range chain.Validators {
validator, err := r.registries.Validators.BuildLegacyRaw(validatorBinding.Binding.Module)
if err != nil {
return nil, nil, fmt.Errorf("build validator %q: %w", validatorBinding.Binding.Module, err)
for index, preparedValidator := range target.prepared.validators {
validator := preparedValidator.legacy
validatorBinding := preparedValidator.resolved
if validator == nil {
return nil, nil, fmt.Errorf("validator %q is not available on the legacy raw path", validatorBinding.Binding.Module)
}
request := target.validationRequest(validatorBinding.Binding)
started := time.Now().UTC()
@@ -1088,52 +1070,37 @@ func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string,
}
}
func (r *Runner) validateRegistries(pipeline ResolvedPipeline) error {
if r.registries.Inputs == nil {
return fmt.Errorf("input registry must not be nil")
func validateRunInput(input RunInput) error {
if input.Prepared == nil {
return fmt.Errorf("prepared pipeline must not be nil")
}
if r.registries.Chunkers == nil {
return fmt.Errorf("chunker registry must not be nil")
}
if r.registries.Extractors == nil {
return fmt.Errorf("extractor registry must not be nil")
}
if r.registries.Mergers == nil {
return fmt.Errorf("merger registry must not be nil")
}
if r.registries.Normalizers == nil {
return fmt.Errorf("normalizer registry must not be nil")
}
if r.registries.Outputs == nil {
return fmt.Errorf("output encoder registry must not be nil")
}
return nil
return validateResolvedPipeline(input.Prepared.resolved, true)
}
func validateRunInput(input RunInput) error {
if input.Pipeline.ID == "" {
func validateResolvedPipeline(pipeline ResolvedPipeline, rejectTyped bool) error {
if pipeline.ID == "" {
return fmt.Errorf("resolved pipeline id must not be empty")
}
if input.Pipeline.Digest == "" {
if pipeline.Digest == "" {
return fmt.Errorf("resolved pipeline digest must not be empty")
}
if input.Pipeline.Input.Module == "" {
if pipeline.Input.Module == "" {
return fmt.Errorf("resolved pipeline input module must not be empty")
}
if input.Pipeline.Chunk.Module == "" {
if pipeline.Chunk.Module == "" {
return fmt.Errorf("resolved pipeline chunk module must not be empty")
}
if input.Pipeline.Output.Module == "" {
if pipeline.Output.Module == "" {
return fmt.Errorf("resolved pipeline output module must not be empty")
}
if len(input.Pipeline.ArtifactLanes) == 0 {
if len(pipeline.ArtifactLanes) == 0 {
return fmt.Errorf("resolved pipeline artifact lanes must not be empty")
}
for _, lane := range input.Pipeline.ArtifactLanes {
for _, lane := range pipeline.ArtifactLanes {
if lane.ID == "" {
return fmt.Errorf("resolved pipeline artifact lane id must not be empty")
}
if lane.ArtifactKind != "" {
if rejectTyped && lane.ArtifactKind != "" {
return fmt.Errorf("resolved pipeline lane %q uses typed artifact kind %q, which the legacy raw runner cannot execute", lane.ID, lane.ArtifactKind)
}
if lane.Extract.Module == "" {
@@ -1169,7 +1136,7 @@ func manifestFromPipeline(input RunInput) artifacts.RunManifest {
runID = fmt.Sprintf("run-%d", startedAt.UnixNano())
}
pipeline := input.Pipeline
pipeline := input.pipeline
manifest := artifacts.RunManifest{
PipelineID: pipeline.ID,
PipelineDigest: pipeline.Digest,