eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Run D&D spell lanes through typed artifacts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 07:19:58 +00:00
parent 52e6b31408
commit 66de1a5520
17 changed files with 699 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
internal/framework/pipeline/runner.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"mime"
"path"
@@ -244,7 +245,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}, llmScope))
return false, nil, err
}
validationWarnings, rejection, err := r.validateChunksRaw(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.llmClient, input.Metadata, input.Prepared.chunkValidators, attempt, input.Debug)
validationWarnings, rejection, err := r.validateChunks(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.llmClient, input.Metadata, input.Prepared.chunkValidators, attempt, input.Debug)
if err != nil || rejection != nil {
_ = writeDebugTimed(debugRecorder, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageChunk),
@@ -376,6 +377,13 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}
func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, checkpointLoader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, prepared preparedLaneExecutor, output *RunOutput) error {
if prepared.typed != nil {
return r.runTypedLane(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, chunks, prepared, output)
}
return r.runLegacyLane(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, chunks, prepared, output)
}
func (r *Runner) runLegacyLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, checkpointLoader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, prepared preparedLaneExecutor, output *RunOutput) error {
lane := prepared.resolved
if prepared.legacy == nil {
return fmt.Errorf("resolved pipeline lane %q uses typed artifact kind %q, which the legacy raw runner cannot execute", lane.ID, lane.ArtifactKind)
@@ -950,6 +958,64 @@ func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocume
})
}
func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, llmClient contracts.StructuredLLMClient, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
allLegacy := true
for _, item := range prepared.validators {
if item.resolved.Target != ValidatorTargetLegacyRaw && item.resolved.Target != "" {
allLegacy = false
break
}
}
if allLegacy {
return r.validateChunksRaw(ctx, doc, moduleKey, chunks, sourceInput, sessionID, references, llmClient, metadata, prepared, attempt, debug)
}
content, err := json.Marshal(chunks)
if err != nil {
return nil, nil, fmt.Errorf("encode canonical chunks for validation: %w", err)
}
schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)}
var warnings []contracts.Warning
for index, item := range prepared.validators {
binding := item.resolved.Binding
started := time.Now().UTC()
attemptPath := path.Join("validate", debugPathComponent(string(StageChunk)), "", debugPathComponent(moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", index+1, debugPathComponent(binding.Module), attempt))
validatorCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
var result contracts.ValidationResult
switch item.resolved.Target {
case ValidatorTargetChunk:
result, err = item.chunk.Validate(validatorCtx, contracts.ChunkValidationRequest{ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: cloneMetadata(metadata), Chunks: cloneSourceChunks(chunks)})
case ValidatorTargetSerialized:
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: cloneMetadata(metadata), Chunks: cloneSourceChunks(chunks), Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...)})
default:
return nil, nil, fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module)
}
debugRequest := contracts.ValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: cloneMetadata(metadata), Chunks: cloneSourceChunks(chunks), Schema: contracts.ResponseSchema{ID: schema.ID, Name: schema.Name, Version: schema.Version, JSONSchema: append([]byte(nil), schema.JSONSchema...)}, Payload: contracts.RawPayload{Content: content, MediaType: "application/json"}}
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: debugValidationRequestEnvelope(debugRequest), Result: debugValidationResultEnvelope(result)}
if err != nil {
debugCall.Error = err.Error()
}
if debugErr := writeDebugTimed(debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall, Error: debugCall.Error}, llmScope)); debugErr != nil {
return nil, nil, debugErr
}
if err != nil {
return nil, nil, fmt.Errorf("validate chunks with validator %q: %w", binding.Module, err)
}
if !result.Approved {
reason := result.ReasonCode
if reason == "" {
reason = "raw_output_rejected"
}
message := result.Message
if message == "" {
message = "raw output rejected"
}
return nil, &contracts.RejectedOutput{Stage: string(StageChunk), ModuleKey: moduleKey, ValidatorName: binding.Module, ReasonCode: reason, Message: message, AttemptCount: attempt, DiagnosticArtifactPath: result.DiagnosticArtifactPath}, nil
}
warnings = append(warnings, result.Warnings...)
}
return warnings, nil, nil
}
func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([]contracts.Warning, *contracts.RejectedOutput, error) {
if len(target.prepared.validators) == 0 {
return nil, nil, nil
@@ -1070,10 +1136,10 @@ func validateRunInput(input RunInput) error {
if input.Prepared == nil {
return fmt.Errorf("prepared pipeline must not be nil")
}
return validateResolvedPipeline(input.Prepared.resolved, true)
return validateResolvedPipeline(input.Prepared.resolved)
}
func validateResolvedPipeline(pipeline ResolvedPipeline, rejectTyped bool) error {
func validateResolvedPipeline(pipeline ResolvedPipeline) error {
if pipeline.ID == "" {
return fmt.Errorf("resolved pipeline id must not be empty")
}
@@ -1096,9 +1162,6 @@ func validateResolvedPipeline(pipeline ResolvedPipeline, rejectTyped bool) error
if lane.ID == "" {
return fmt.Errorf("resolved pipeline artifact lane id must not be empty")
}
if rejectTyped && lane.ArtifactKind != "" {
return fmt.Errorf("resolved pipeline lane %q uses typed artifact kind %q, which the legacy raw runner cannot execute", lane.ID, lane.ArtifactKind)
}
if lane.Extract.Module == "" {
return fmt.Errorf("resolved pipeline lane %q extract module must not be empty", lane.ID)
}