eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update the debug workflow to provide raw LLM output

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-07 23:08:14 -05:00
parent ae65b95374
commit 3011dd91ca
10 changed files with 476 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
internal/framework/pipeline/runner.go
View File

@@ -205,7 +205,9 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}
chunksAccepted, chunkRejection, err = runWithRetry(ctx, input.Pipeline.Chunk.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
attemptStarted := time.Now().UTC()
chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
attemptPath := path.Join("chunk", fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
chunkResult, err := chunker.Chunk(attemptCtx, contracts.ChunkRequest{
Source: doc,
SourceInput: sourceInput.Clone(),
SessionID: sessionID,
@@ -216,25 +218,44 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
Metadata: input.Metadata,
})
if err != nil {
_ = writeDebugTimed(debugRecorder, path.Join("chunk", fmt.Sprintf("attempt-%02d.json", attempt)), debugTimedEnvelope{
_ = writeDebugTimed(debugRecorder, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Error: err.Error(),
})
}, llmScope))
return false, nil, fmt.Errorf("chunk source with chunker %q: %w", chunker.Key(), err)
}
if len(chunkResult.Chunks) == 0 {
return false, nil, fmt.Errorf("chunker %q returned no chunks", chunker.Key())
err := fmt.Errorf("chunker %q returned no chunks", chunker.Key())
_ = writeDebugTimed(debugRecorder, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Error: err.Error(),
}, llmScope))
return false, nil, err
}
chunks, err := validateAndCanonicalizeChunkResult(doc, chunkResult.Chunks)
if err != nil {
return false, nil, fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
err := fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
_ = writeDebugTimed(debugRecorder, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"warnings": cloneWarnings(chunkResult.Warnings),
},
Error: err.Error(),
}, llmScope))
return false, nil, err
}
validationWarnings, rejection, err := r.validateChunksRaw(ctx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.Pipeline.ChunkReferences.ReferenceSet, input.LLMClient, input.Metadata, input.Pipeline.ValidatorChains, attempt, input.Debug)
validationWarnings, rejection, err := r.validateChunksRaw(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.Pipeline.ChunkReferences.ReferenceSet, input.LLMClient, input.Metadata, input.Pipeline.ValidatorChains, attempt, input.Debug)
if err != nil || rejection != nil {
_ = writeDebugTimed(debugRecorder, path.Join("chunk", fmt.Sprintf("attempt-%02d.json", attempt)), debugTimedEnvelope{
_ = writeDebugTimed(debugRecorder, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
Attempt: attempt,
@@ -244,12 +265,12 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
"warnings": append(cloneWarnings(chunkResult.Warnings), validationWarnings...),
"rejection": debugRejectedOutputPtr(rejection),
},
})
}, llmScope))
return false, rejection, err
}
canonicalChunks = chunks
chunkWarnings = append(cloneWarnings(chunkResult.Warnings), validationWarnings...)
if err := writeDebugTimed(debugRecorder, path.Join("chunk", fmt.Sprintf("attempt-%02d.json", attempt)), debugTimedEnvelope{
if err := writeDebugTimed(debugRecorder, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
Attempt: attempt,
@@ -258,7 +279,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
"chunks": debugSourceChunkEnvelopes(chunks),
"warnings": chunkWarnings,
},
}); err != nil {
}, llmScope)); err != nil {
return false, nil, err
}
return true, nil, nil
@@ -419,7 +440,10 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
var acceptedOutput contracts.ExtractOutput
var acceptedWarnings []contracts.Warning
accepted, rejection, err := runWithRetry(ctx, lane.Extract.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
result, err := extractor.Extract(ctx, contracts.ExtractionRequest{
attemptStarted := time.Now().UTC()
attemptPath := path.Join("extract", debugPathComponent(lane.ID), fmt.Sprintf("chunk-%06d-attempt-%02d", chunk.Index+1, attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
result, err := extractor.Extract(attemptCtx, contracts.ExtractionRequest{
Source: doc,
Chunk: &chunk,
SourceInput: chunkInputMaterial(sourceInput, chunk),
@@ -431,6 +455,14 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
Metadata: input.Metadata,
})
if err != nil {
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageExtract),
LaneID: lane.ID,
ModuleKey: extractor.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Error: err.Error(),
}, llmScope))
return false, nil, fmt.Errorf("extract lane %q chunk %q with extractor %q: %w", lane.ID, chunk.ID, extractor.Key(), err)
}
extractOutput := result.Output
@@ -440,7 +472,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
extractOutput.ChunkID = chunk.ID
extractOutput.ChunkIndex = chunk.Index
extractOutput.Payload.Warnings = append(extractOutput.Payload.Warnings, result.Warnings...)
validationWarnings, rejection, err := r.validateRaw(ctx, rawValidationTarget{
validationWarnings, rejection, err := r.validateRaw(attemptCtx, rawValidationTarget{
stage: StageExtract,
laneID: lane.ID,
moduleKey: extractor.Key(),
@@ -461,10 +493,35 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
debug: input.Debug,
})
if err != nil || rejection != nil {
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageExtract),
LaneID: lane.ID,
ModuleKey: extractor.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"output": debugExtractOutputEnvelope(extractOutput),
"warnings": append(cloneWarnings(result.Warnings), validationWarnings...),
"rejection": debugRejectedOutputPtr(rejection),
},
}, llmScope))
return false, rejection, err
}
acceptedOutput = cloneExtractOutput(extractOutput)
acceptedWarnings = append(cloneWarnings(result.Warnings), validationWarnings...)
if err := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageExtract),
LaneID: lane.ID,
ModuleKey: extractor.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"output": debugExtractOutputEnvelope(extractOutput),
"warnings": acceptedWarnings,
},
}, llmScope)); err != nil {
return false, nil, err
}
return true, nil, nil
})
if err != nil {
@@ -534,7 +591,10 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
}
mergeAccepted, mergeRejection, err := runWithRetry(ctx, lane.Merge.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
mergeResult, err := merger.Merge(ctx, contracts.MergeRequest{
attemptStarted := time.Now().UTC()
attemptPath := path.Join("merge", debugPathComponent(lane.ID), fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
mergeResult, err := merger.Merge(attemptCtx, contracts.MergeRequest{
Source: doc,
LaneID: lane.ID,
ExtractOutputs: cloneExtractOutputs(extractOutputs),
@@ -547,6 +607,14 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
Metadata: input.Metadata,
})
if err != nil {
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageMerge),
LaneID: lane.ID,
ModuleKey: merger.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Error: err.Error(),
}, llmScope))
return false, nil, fmt.Errorf("merge lane %q with merger %q: %w", lane.ID, merger.Key(), err)
}
mergeOutput := mergeResult.Output
@@ -554,7 +622,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
mergeOutput.MergerKey = merger.Key()
mergeOutput.SourceID = doc.ID
mergeOutput.Payload.Warnings = append(mergeOutput.Payload.Warnings, mergeResult.Warnings...)
validationWarnings, rejection, err := r.validateRaw(ctx, rawValidationTarget{
validationWarnings, rejection, err := r.validateRaw(attemptCtx, rawValidationTarget{
stage: StageMerge,
laneID: lane.ID,
moduleKey: merger.Key(),
@@ -573,10 +641,35 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
debug: input.Debug,
})
if err != nil || rejection != nil {
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageMerge),
LaneID: lane.ID,
ModuleKey: merger.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"output": debugMergeOutputEnvelope(mergeOutput),
"warnings": append(cloneWarnings(mergeResult.Warnings), validationWarnings...),
"rejection": debugRejectedOutputPtr(rejection),
},
}, llmScope))
return false, rejection, err
}
acceptedMerge = cloneMergeOutput(mergeOutput)
mergeWarnings = append(cloneWarnings(mergeResult.Warnings), validationWarnings...)
if err := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageMerge),
LaneID: lane.ID,
ModuleKey: merger.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"output": debugMergeOutputEnvelope(mergeOutput),
"warnings": mergeWarnings,
},
}, llmScope)); err != nil {
return false, nil, err
}
return true, nil, nil
})
if err != nil {
@@ -654,7 +747,10 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
}
normalizeAccepted, normalizeRejection, err := runWithRetry(ctx, lane.Normalize.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
normalizeResult, err := normalizer.Normalize(ctx, contracts.NormalizeRequest{
attemptStarted := time.Now().UTC()
attemptPath := path.Join("normalize", debugPathComponent(lane.ID), fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
normalizeResult, err := normalizer.Normalize(attemptCtx, contracts.NormalizeRequest{
Source: doc,
LaneID: lane.ID,
MergeOutput: cloneMergeOutput(acceptedMerge),
@@ -667,6 +763,14 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
Metadata: input.Metadata,
})
if err != nil {
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageNormalize),
LaneID: lane.ID,
ModuleKey: normalizer.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Error: err.Error(),
}, llmScope))
return false, nil, fmt.Errorf("normalize lane %q with normalizer %q: %w", lane.ID, normalizer.Key(), err)
}
normalizeOutput := normalizeResult.Output
@@ -674,7 +778,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
normalizeOutput.NormalizerKey = normalizer.Key()
normalizeOutput.SourceID = doc.ID
normalizeOutput.Payload.Warnings = append(normalizeOutput.Payload.Warnings, normalizeResult.Warnings...)
validationWarnings, rejection, err := r.validateRaw(ctx, rawValidationTarget{
validationWarnings, rejection, err := r.validateRaw(attemptCtx, rawValidationTarget{
stage: StageNormalize,
laneID: lane.ID,
moduleKey: normalizer.Key(),
@@ -693,10 +797,35 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints Checkp
debug: input.Debug,
})
if err != nil || rejection != nil {
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageNormalize),
LaneID: lane.ID,
ModuleKey: normalizer.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"output": debugNormalizeOutputEnvelope(normalizeOutput),
"warnings": append(cloneWarnings(normalizeResult.Warnings), validationWarnings...),
"rejection": debugRejectedOutputPtr(rejection),
},
}, llmScope))
return false, rejection, err
}
acceptedNormalize = cloneNormalizeOutput(normalizeOutput)
normalizeWarnings = append(cloneWarnings(normalizeResult.Warnings), validationWarnings...)
if err := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageNormalize),
LaneID: lane.ID,
ModuleKey: normalizer.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"output": debugNormalizeOutputEnvelope(normalizeOutput),
"warnings": normalizeWarnings,
},
}, llmScope)); err != nil {
return false, nil, err
}
return true, nil, nil
})
if err != nil {
@@ -845,14 +974,16 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([
}
var warnings []contracts.Warning
for _, validatorBinding := range chain.Validators {
for index, validatorBinding := range chain.Validators {
validator, err := r.registries.Validators.Build(validatorBinding.Binding.Module)
if err != nil {
return nil, nil, fmt.Errorf("build validator %q: %w", validatorBinding.Binding.Module, err)
}
request := target.validationRequest(validatorBinding.Binding)
started := time.Now().UTC()
result, err := validator.Validate(ctx, request)
attemptPath := path.Join("validate", debugPathComponent(string(target.stage)), debugPathComponent(target.laneID), debugPathComponent(target.moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", index+1, debugPathComponent(validator.Name()), target.attempt))
validatorCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
result, err := validator.Validate(validatorCtx, request)
debugPayload := debugValidationCall{
ValidatorName: validator.Name(),
Request: debugValidationRequestEnvelope(request),
@@ -861,7 +992,7 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([
if err != nil {
debugPayload.Error = err.Error()
}
if debugErr := writeDebugTimed(target.debug, path.Join("validate", debugPathComponent(string(target.stage)), debugPathComponent(target.laneID), debugPathComponent(target.moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d.json", len(warnings)+1, debugPathComponent(validator.Name()), target.attempt)), debugTimedEnvelope{
if debugErr := writeDebugTimed(target.debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(target.stage),
LaneID: target.laneID,
ModuleKey: target.moduleKey,
@@ -869,7 +1000,7 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([
StartedAt: started,
Payload: debugPayload,
Error: debugPayload.Error,
}); debugErr != nil {
}, llmScope)); debugErr != nil {
return nil, nil, fmt.Errorf("write validation debug artifact: %w", debugErr)
}
if err != nil {