eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Record every retry attempt outcome

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 15:23:27 +00:00
parent 35bffdf336
commit 236ccc62ad
9 changed files with 387 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
internal/framework/pipeline/runner.go
View File

@@ -5,6 +5,7 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"mime"
"path"
@@ -207,6 +208,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
attemptStarted := time.Now().UTC()
attemptPath := path.Join("chunk", fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
terminal := newAttemptTerminalRecorder(debugRecorder, attemptPath, "chunk", llmScope, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: chunker.Key(), Attempt: attempt, StartedAt: attemptStarted})
chunkResult, err := chunker.Chunk(attemptCtx, contracts.ChunkRequest{
Source: doc,
SourceInput: sourceInput.Clone(),
@@ -216,68 +218,32 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
Metadata: input.Metadata,
})
if err != nil {
_ = writeDebugAttempt(debugRecorder, attemptPath, debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Error: err.Error(),
}, llmScope)
return false, nil, fmt.Errorf("chunk source with chunker %q: %w", chunker.Key(), err)
attemptErr := fmt.Errorf("chunk source with chunker %q: %w", chunker.Key(), err)
return false, nil, terminal.record(nil, attemptErr)
}
if len(chunkResult.Chunks) == 0 {
err := fmt.Errorf("chunker %q returned no chunks", chunker.Key())
_ = writeDebugAttempt(debugRecorder, attemptPath, debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Error: err.Error(),
}, llmScope)
return false, nil, err
attemptErr := fmt.Errorf("chunker %q returned no chunks", chunker.Key())
return false, nil, terminal.record(nil, attemptErr)
}
chunks, err := validateAndCanonicalizeChunkResult(doc, chunkResult.Chunks)
if err != nil {
err := fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
_ = writeDebugAttempt(debugRecorder, attemptPath, debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"warnings": cloneWarnings(chunkResult.Warnings),
},
Error: err.Error(),
}, llmScope)
return false, nil, err
attemptErr := fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
payload := map[string]any{"warnings": debugWarningEnvelopes(chunkResult.Warnings)}
return false, nil, terminal.record(payload, attemptErr)
}
validationWarnings, rejection, err := r.validateChunks(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.Metadata, input.Prepared.chunkValidators, attempt, input.Debug)
attemptWarnings := append(cloneWarnings(chunkResult.Warnings), validationWarnings...)
payload := map[string]any{
"chunks": debugSourceChunkEnvelopes(chunks),
"warnings": debugWarningEnvelopes(attemptWarnings),
"rejection": debugRejectedOutputPtr(rejection),
}
if err != nil || rejection != nil {
_ = writeDebugAttempt(debugRecorder, attemptPath, debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"chunks": debugSourceChunkEnvelopes(chunks),
"warnings": append(cloneWarnings(chunkResult.Warnings), validationWarnings...),
"rejection": debugRejectedOutputPtr(rejection),
},
}, llmScope)
return false, rejection, err
return false, rejection, terminal.record(payload, err)
}
canonicalChunks = chunks
chunkWarnings = append(cloneWarnings(chunkResult.Warnings), validationWarnings...)
if err := writeDebugAttempt(debugRecorder, attemptPath, debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"chunks": debugSourceChunkEnvelopes(chunks),
"warnings": chunkWarnings,
},
}, llmScope); err != nil {
chunkWarnings = attemptWarnings
if err := terminal.record(payload, nil); err != nil {
return false, nil, err
}
return true, nil, nil
@@ -446,11 +412,15 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
if err != nil {
debugCall.Error = err.Error()
}
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall, Error: debugCall.Error}, llmScope); debugErr != nil {
return nil, nil, debugErr
}
if err != nil {
return nil, nil, fmt.Errorf("validate chunks with validator %q: %w", binding.Module, err)
validationErr := fmt.Errorf("validate chunks with validator %q: %w", binding.Module, err)
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall, Error: debugCall.Error}, llmScope); debugErr != nil {
return warnings, nil, errors.Join(validationErr, fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr))
}
return warnings, nil, validationErr
}
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall}, llmScope); debugErr != nil {
return warnings, nil, fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr)
}
if !result.Approved {
reason := result.ReasonCode
@@ -461,7 +431,7 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
if message == "" {
message = "output rejected"
}
return nil, &contracts.RejectedOutput{Stage: string(StageChunk), ModuleKey: moduleKey, ValidatorName: binding.Module, ReasonCode: reason, Message: message, AttemptCount: attempt, DiagnosticArtifactPath: result.DiagnosticArtifactPath}, nil
return warnings, &contracts.RejectedOutput{Stage: string(StageChunk), ModuleKey: moduleKey, ValidatorName: binding.Module, ReasonCode: reason, Message: message, AttemptCount: attempt, DiagnosticArtifactPath: result.DiagnosticArtifactPath}, nil
}
warnings = append(warnings, result.Warnings...)
}