eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement final fixes and close out the implemetation roadmap

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 23:16:16 -05:00
parent 604c7a7945
commit 9fa9154dda
14 changed files with 463 additions and 879 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
internal/framework/pipeline/runner_chunk_plan.go
View File

@@ -58,7 +58,9 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
case ChunkPlanHit:
plan, chunks, validationErr := validateAndMaterializeChunkPlan(doc, record.Plan)
if validationErr == nil {
result.setCandidate(record, "reused")
if err := result.setCandidate(record, "reused"); err != nil {
return result, fmt.Errorf("clone reused chunk plan record: %w", err)
}
validationWarnings, rejection, err := r.validateChunks(ctx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.Metadata, input.Prepared.chunkValidators, 1, input.Debug)
result.plan = &plan
result.chunks = chunks
@@ -87,10 +89,14 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
attemptPath := path.Join("chunk", fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
terminal := newAttemptTerminalRecorder(input.Debug, attemptPath, "chunk", llmScope, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: chunker.Key(), Attempt: attempt, StartedAt: attemptStarted})
requestMetadata, metadataErr := cloneMetadata(input.Metadata)
if metadataErr != nil {
return false, nil, terminal.record(nil, fmt.Errorf("clone chunk request metadata: %w", metadataErr))
}
chunkResult, callErr := chunker.Plan(attemptCtx, contracts.ChunkRequest{
Source: doc, SourceInput: sourceInput.Clone(), SessionID: sessionID,
References: CloneReferenceSet(input.pipeline.ChunkReferences.ReferenceSet),
LLMProfile: input.pipeline.Chunk.LLMProfile, Metadata: input.Metadata,
LLMProfile: input.pipeline.Chunk.LLMProfile, Metadata: requestMetadata,
})
if callErr != nil {
return false, nil, terminal.record(nil, fmt.Errorf("chunk source with chunker %q: %w", chunker.Key(), callErr))
@@ -105,7 +111,10 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
if digestErr != nil {
return false, nil, terminal.record(nil, fmt.Errorf("digest generated chunk plan: %w", digestErr))
}
producerMetadata, _ := moduleManifestMetadata(chunker)
producerMetadata, _, metadataErr := moduleManifestMetadata(chunker)
if metadataErr != nil {
return false, nil, terminal.record(nil, fmt.Errorf("clone chunker manifest metadata: %w", metadataErr))
}
profile := ""
if provider, ok := chunker.(contracts.ChunkExecutionClassProvider); ok && provider.ExecutionClass() == contracts.ExecutionClassLLMBacked {
profile = input.pipeline.Chunk.LLMProfile
@@ -116,7 +125,7 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
Producer: ChunkPlanProducer{
InputModule: input.Prepared.input.Key(), ChunkModule: chunker.Key(), LLMProfile: profile,
References: append([]artifacts.ReferenceProvenance(nil), referenceTargetProvenance(input.pipeline.ChunkReferences)...),
Metadata: cloneMetadata(producerMetadata),
Metadata: producerMetadata,
},
Warnings: cloneWarnings(chunkResult.Warnings), CreatedAt: time.Now().UTC(),
}
@@ -127,7 +136,9 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
if mode == ChunkCacheBypass {
action = "bypassed"
}
result.setCandidate(candidate, action)
if candidateErr := result.setCandidate(candidate, action); candidateErr != nil {
return false, nil, terminal.record(nil, fmt.Errorf("clone generated chunk plan record: %w", candidateErr))
}
validationWarnings, rejected, validationErr := r.validateChunks(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.Metadata, input.Prepared.chunkValidators, attempt, input.Debug)
attemptWarnings := append(cloneWarnings(chunkResult.Warnings), validationWarnings...)
payload := map[string]any{
@@ -161,7 +172,10 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
}
if mode == ChunkCacheAuto || mode == ChunkCacheRefresh {
record := cloneChunkPlanRecord(*result.record)
record, cloneErr := cloneChunkPlanRecord(*result.record)
if cloneErr != nil {
return result, fmt.Errorf("clone chunk plan record for publication: %w", cloneErr)
}
record.Warnings = cloneWarnings(producerWarnings)
if err := input.ChunkPlans.Save(record); err != nil {
return result, fmt.Errorf("save chunk plan: %w", err)
@@ -197,13 +211,17 @@ func chunkPlanLookupReason(status ChunkPlanStatus) string {
}
}
func (result *chunkPlanExecution) setCandidate(record ChunkPlanRecord, action string) {
cloned := cloneChunkPlanRecord(record)
func (result *chunkPlanExecution) setCandidate(record ChunkPlanRecord, action string) error {
cloned, err := cloneChunkPlanRecord(record)
if err != nil {
return err
}
result.record = &cloned
result.action = action
result.summary.Action = action
result.summary.SourceDigest = record.SourceDigest
result.summary.CandidateDigest = record.PlanDigest
return nil
}
func (result *chunkPlanExecution) setValidation(warnings []contracts.Warning, rejection *contracts.RejectedOutput, err error) {
@@ -219,27 +237,31 @@ func (result *chunkPlanExecution) setValidation(warnings []contracts.Warning, re
}
}
func cloneChunkPlanRecord(record ChunkPlanRecord) ChunkPlanRecord {
func cloneChunkPlanRecord(record ChunkPlanRecord) (ChunkPlanRecord, error) {
record.Plan = source.CloneChunkPlan(record.Plan)
record.Producer.References = append([]artifacts.ReferenceProvenance(nil), record.Producer.References...)
record.Producer.Metadata = cloneMetadata(record.Producer.Metadata)
metadata, err := cloneMetadata(record.Producer.Metadata)
if err != nil {
return ChunkPlanRecord{}, fmt.Errorf("clone chunk plan producer metadata: %w", err)
}
record.Producer.Metadata = metadata
record.Warnings = cloneWarnings(record.Warnings)
return record
return record, nil
}
func applyChunkPlanExecution(output *RunOutput, result chunkPlanExecution) {
func applyChunkPlanExecution(output *RunOutput, result chunkPlanExecution) error {
if output == nil {
return
return nil
}
summary := result.summary
output.ChunkPlan = &summary
if output.Manifest.ChunkPlan == nil {
return
return nil
}
manifest := output.Manifest.ChunkPlan
manifest.Action = result.action
if result.record == nil {
return
return nil
}
record := result.record
manifest.SourceDigest = record.SourceDigest
@@ -249,7 +271,12 @@ func applyChunkPlanExecution(output *RunOutput, result chunkPlanExecution) {
manifest.ProducerModule = record.Producer.ChunkModule
manifest.ProducerLLMProfile = record.Producer.LLMProfile
manifest.ProducerReferences = append([]artifacts.ReferenceProvenance(nil), record.Producer.References...)
manifest.ProducerMetadata = cloneMetadata(record.Producer.Metadata)
metadata, err := cloneMetadata(record.Producer.Metadata)
if err != nil {
return fmt.Errorf("clone chunk plan manifest producer metadata: %w", err)
}
manifest.ProducerMetadata = metadata
createdAt := record.CreatedAt
manifest.CreatedAt = &createdAt
return nil
}