eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Generate and materialize canonical chunk plans

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 23:57:59 +00:00
parent 3bfac14397
commit 7844c0a93f
23 changed files with 396 additions and 540 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
internal/framework/pipeline/runner.go
View File

@@ -175,6 +175,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
chunker := input.Prepared.chunker
attachModuleManifestMetadata(&output, "chunker", chunker)
var canonicalChunks []source.Chunk
var generatedPlan *source.ChunkPlan
var chunkWarnings []contracts.Warning
chunkCheckpoint, chunkDecision := checkpointLoader.Chunk(chunker.Key(), doc.Digest)
recordCheckpointEvent(&output, checkpointLoader, string(StageChunk), "", chunker.Key(), chunkDecision)
@@ -209,7 +210,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
attemptPath := path.Join("chunk", fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
terminal := newAttemptTerminalRecorder(debugRecorder, attemptPath, "chunk", llmScope, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: chunker.Key(), Attempt: attempt, StartedAt: attemptStarted})
chunkResult, err := chunker.Chunk(attemptCtx, contracts.ChunkRequest{
chunkResult, err := chunker.Plan(attemptCtx, contracts.ChunkRequest{
Source: doc,
SourceInput: sourceInput.Clone(),
SessionID: sessionID,
@@ -221,27 +222,25 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
attemptErr := fmt.Errorf("chunk source with chunker %q: %w", chunker.Key(), err)
return false, nil, terminal.record(nil, attemptErr)
}
if len(chunkResult.Chunks) == 0 {
attemptErr := fmt.Errorf("chunker %q returned no chunks", chunker.Key())
return false, nil, terminal.record(nil, attemptErr)
}
chunks, err := validateAndCanonicalizeChunkResult(doc, chunkResult.Chunks)
plan, chunks, err := validateAndMaterializeChunkPlan(doc, chunkResult.Plan)
if err != nil {
attemptErr := fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
payload := map[string]any{"warnings": debugWarningEnvelopes(chunkResult.Warnings)}
attemptErr := fmt.Errorf("validate chunk plan from chunker %q: %w", chunker.Key(), err)
payload := map[string]any{"plan": debugChunkPlanEnvelope(chunkResult.Plan), "warnings": debugWarningEnvelopes(chunkResult.Warnings)}
return false, nil, terminal.record(payload, attemptErr)
}
validationWarnings, rejection, err := r.validateChunks(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.Metadata, input.Prepared.chunkValidators, attempt, input.Debug)
attemptWarnings := append(cloneWarnings(chunkResult.Warnings), validationWarnings...)
payload := map[string]any{
"chunks": debugSourceChunkEnvelopes(chunks),
"warnings": debugWarningEnvelopes(attemptWarnings),
"rejection": debugRejectedOutputPtr(rejection),
"plan": debugChunkPlanEnvelope(plan),
"materialized_chunks": debugSourceChunkEnvelopes(chunks),
"warnings": debugWarningEnvelopes(attemptWarnings),
"rejection": debugRejectedOutputPtr(rejection),
}
if err != nil || rejection != nil {
return false, rejection, terminal.record(payload, err)
}
canonicalChunks = chunks
generatedPlan = &plan
chunkWarnings = attemptWarnings
if err := terminal.record(payload, nil); err != nil {
return false, nil, err
@@ -265,10 +264,13 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}
}
chunkDebugPayload := map[string]any{
"reused": chunkDecision.Reused,
"accepted": chunksAccepted,
"chunks": debugSourceChunkEnvelopes(canonicalChunks),
"warnings": chunkWarnings,
"reused": chunkDecision.Reused,
"accepted": chunksAccepted,
"materialized_chunks": debugSourceChunkEnvelopes(canonicalChunks),
"warnings": chunkWarnings,
}
if generatedPlan != nil {
chunkDebugPayload["plan"] = debugChunkPlanEnvelope(*generatedPlan)
}
if chunkRejection != nil {
chunkDebugPayload["rejection"] = debugRejectedOutputEnvelope(*chunkRejection)