Implement final fixes and close out the implemetation roadmap

This commit is contained in:
2026-07-17 23:16:16 -05:00
parent 604c7a7945
commit 9fa9154dda
14 changed files with 463 additions and 879 deletions

View File

@@ -11,6 +11,7 @@ import (
"os"
"path/filepath"
"strings"
"syscall"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
@@ -50,7 +51,7 @@ func (s *filesystemStore) Load(sourceDigest string) (pipeline.ChunkPlanRecord, p
}
defer root.Close()
state, err := inspectDirectory(root, digestDir)
digestRoot, state, err := openDigestRoot(root, digestDir, false)
if err != nil {
return pipeline.ChunkPlanRecord{}, pipeline.ChunkPlanDecision{}, fmt.Errorf("inspect chunk plan directory: %w", err)
}
@@ -60,11 +61,11 @@ func (s *filesystemStore) Load(sourceDigest string) (pipeline.ChunkPlanRecord, p
if state == entryRejected {
return invalidDecision()
}
defer digestRoot.Close()
target := planPath(digestDir)
state, err = inspectPlan(root, target)
data, state, err := readPlan(digestRoot)
if err != nil {
return pipeline.ChunkPlanRecord{}, pipeline.ChunkPlanDecision{}, fmt.Errorf("inspect chunk plan file: %w", err)
return pipeline.ChunkPlanRecord{}, pipeline.ChunkPlanDecision{}, fmt.Errorf("read chunk plan: %w", err)
}
if state == entryMissing {
return pipeline.ChunkPlanRecord{}, pipeline.ChunkPlanDecision{Status: pipeline.ChunkPlanMissing, Reason: lookupReason(pipeline.ChunkPlanMissing)}, nil
@@ -73,14 +74,6 @@ func (s *filesystemStore) Load(sourceDigest string) (pipeline.ChunkPlanRecord, p
return invalidDecision()
}
data, err := root.ReadFile(target)
if err != nil {
if os.IsNotExist(err) {
return pipeline.ChunkPlanRecord{}, pipeline.ChunkPlanDecision{Status: pipeline.ChunkPlanMissing, Reason: lookupReason(pipeline.ChunkPlanMissing)}, nil
}
return pipeline.ChunkPlanRecord{}, pipeline.ChunkPlanDecision{}, fmt.Errorf("read chunk plan: %w", err)
}
var record pipeline.ChunkPlanRecord
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.DisallowUnknownFields()
@@ -116,11 +109,15 @@ func (s *filesystemStore) Save(record pipeline.ChunkPlanRecord) error {
return fmt.Errorf("open chunk plan root: %w", err)
}
defer root.Close()
if err := ensureDirectory(root, digestDir); err != nil {
digestRoot, state, err := openDigestRoot(root, digestDir, true)
if err != nil {
return fmt.Errorf("prepare chunk plan directory: %w", err)
}
target := planPath(digestDir)
state, err := inspectPlan(root, target)
if state == entryRejected {
return fmt.Errorf("chunk plan directory has an unsupported type")
}
defer digestRoot.Close()
state, err = inspectPlan(digestRoot, planFileName)
if err != nil {
return fmt.Errorf("inspect chunk plan file: %w", err)
}
@@ -131,7 +128,7 @@ func (s *filesystemStore) Save(record pipeline.ChunkPlanRecord) error {
if writer == nil {
writer = writeAtomic
}
if err := writer(root, target, data); err != nil {
if err := writer(digestRoot, planFileName, data); err != nil {
return fmt.Errorf("write chunk plan: %w", err)
}
return nil
@@ -174,52 +171,89 @@ const (
entryRejected
)
func inspectDirectory(root *os.Root, digestDir string) (entryState, error) {
func inspectDirectory(root *os.Root, digestDir string) (entryState, os.FileInfo, error) {
info, err := root.Lstat(digestDir)
if err != nil {
if os.IsNotExist(err) {
return entryMissing, nil
return entryMissing, nil, nil
}
return entryPresent, err
return entryPresent, nil, err
}
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
return entryRejected, nil
return entryRejected, info, nil
}
return entryPresent, nil
return entryPresent, info, nil
}
func ensureDirectory(root *os.Root, digestDir string) error {
type digestOpenHooks struct {
BeforeOpen func() error
}
func openDigestRoot(root *os.Root, digestDir string, create bool) (*os.Root, entryState, error) {
return openDigestRootWithHooks(root, digestDir, create, digestOpenHooks{})
}
func openDigestRootWithHooks(root *os.Root, digestDir string, create bool, hooks digestOpenHooks) (*os.Root, entryState, error) {
for {
state, err := inspectDirectory(root, digestDir)
state, before, err := inspectDirectory(root, digestDir)
if err != nil {
return err
return nil, entryPresent, err
}
switch state {
case entryRejected:
return fmt.Errorf("chunk plan directory has an unsupported type")
return nil, entryRejected, nil
case entryMissing:
if !create {
return nil, entryMissing, nil
}
if err := root.Mkdir(digestDir, 0o700); err != nil && !errors.Is(err, os.ErrExist) {
return err
return nil, entryPresent, err
}
continue
}
directory, err := root.Open(digestDir)
if hooks.BeforeOpen != nil {
if err := hooks.BeforeOpen(); err != nil {
return nil, entryPresent, err
}
}
digestRoot, err := root.OpenRoot(digestDir)
if err != nil {
return err
return nil, entryPresent, err
}
info, statErr := directory.Stat()
if statErr == nil && !info.IsDir() {
statErr = fmt.Errorf("chunk plan directory has an unsupported type")
opened, statErr := digestRoot.Stat(".")
afterState, after, afterErr := inspectDirectory(root, digestDir)
if statErr != nil || afterErr != nil {
_ = digestRoot.Close()
if statErr != nil {
return nil, entryPresent, statErr
}
return nil, entryPresent, afterErr
}
if statErr == nil {
statErr = directory.Chmod(0o700)
if afterState != entryPresent || !os.SameFile(before, after) || !os.SameFile(opened, after) {
_ = digestRoot.Close()
if afterState == entryRejected {
return nil, entryRejected, nil
}
continue
}
closeErr := directory.Close()
if statErr != nil {
return statErr
if create {
directory, openErr := digestRoot.Open(".")
if openErr != nil {
_ = digestRoot.Close()
return nil, entryPresent, openErr
}
chmodErr := directory.Chmod(0o700)
closeErr := directory.Close()
if chmodErr != nil || closeErr != nil {
_ = digestRoot.Close()
if chmodErr != nil {
return nil, entryPresent, chmodErr
}
return nil, entryPresent, closeErr
}
}
return closeErr
return digestRoot, entryPresent, nil
}
}
@@ -237,8 +271,54 @@ func inspectPlan(root *os.Root, target string) (entryState, error) {
return entryPresent, nil
}
func planPath(digestDir string) string {
return digestDir + "/" + planFileName
type planReadHooks struct {
BeforeOpen func() error
}
func readPlan(root *os.Root) ([]byte, entryState, error) {
return readPlanWithHooks(root, planReadHooks{})
}
func readPlanWithHooks(root *os.Root, hooks planReadHooks) ([]byte, entryState, error) {
state, err := inspectPlan(root, planFileName)
if err != nil || state != entryPresent {
return nil, state, err
}
if hooks.BeforeOpen != nil {
if err := hooks.BeforeOpen(); err != nil {
return nil, entryPresent, err
}
}
file, err := root.OpenFile(planFileName, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
if err != nil {
if os.IsNotExist(err) {
return nil, entryMissing, nil
}
if errors.Is(err, syscall.ELOOP) {
return nil, entryRejected, nil
}
return nil, entryPresent, err
}
defer file.Close()
opened, err := file.Stat()
if err != nil {
return nil, entryPresent, err
}
if !opened.Mode().IsRegular() {
return nil, entryRejected, nil
}
currentState, currentErr := inspectPlan(root, planFileName)
if currentErr != nil {
return nil, entryPresent, currentErr
}
if currentState == entryRejected {
return nil, entryRejected, nil
}
data, err := io.ReadAll(file)
if err != nil {
return nil, entryPresent, err
}
return data, entryPresent, nil
}
func digestPathSegment(digest string) (string, error) {

View File

@@ -163,6 +163,61 @@ func TestFilesystemStoreRejectsSymlinkedEntries(t *testing.T) {
}
}
func TestOpenDigestRootRejectsEntryReplacedDuringOpen(t *testing.T) {
rootPath := t.TempDir()
digestDir := strings.TrimPrefix(testSourceDigest, "sha256:")
if err := os.Mkdir(filepath.Join(rootPath, digestDir), 0o700); err != nil {
t.Fatal(err)
}
if err := os.Mkdir(filepath.Join(rootPath, "redirect"), 0o700); err != nil {
t.Fatal(err)
}
root, err := os.OpenRoot(rootPath)
if err != nil {
t.Fatal(err)
}
defer root.Close()
traced := false
opened, state, err := openDigestRootWithHooks(root, digestDir, false, digestOpenHooks{BeforeOpen: func() error {
if traced {
return nil
}
traced = true
if err := os.Rename(filepath.Join(rootPath, digestDir), filepath.Join(rootPath, "original")); err != nil {
return err
}
return os.Symlink("redirect", filepath.Join(rootPath, digestDir))
}})
if opened != nil {
_ = opened.Close()
}
if err != nil || state != entryRejected {
t.Fatalf("openDigestRootWithHooks() root=%v state=%v error=%v, want nil/rejected/nil", opened, state, err)
}
}
func TestReadPlanRejectsEntryReplacedDuringOpen(t *testing.T) {
rootPath := t.TempDir()
writeFile(t, filepath.Join(rootPath, planFileName), []byte("original"), 0o600)
writeFile(t, filepath.Join(rootPath, "redirect.json"), []byte("redirect"), 0o600)
root, err := os.OpenRoot(rootPath)
if err != nil {
t.Fatal(err)
}
defer root.Close()
data, state, err := readPlanWithHooks(root, planReadHooks{BeforeOpen: func() error {
if err := os.Remove(filepath.Join(rootPath, planFileName)); err != nil {
return err
}
return os.Symlink("redirect.json", filepath.Join(rootPath, planFileName))
}})
if err != nil || state != entryRejected || data != nil {
t.Fatalf("readPlanWithHooks() data=%q state=%v error=%v, want nil/rejected/nil", data, state, err)
}
}
func TestFilesystemStoreRejectsUnexpectedEntryTypes(t *testing.T) {
tests := []struct {
name string

View File

@@ -67,7 +67,7 @@ type artifactCodecEntry struct {
valueType reflect.Type
encode func(any) ([]byte, error)
encodeCandidate func(any) ([]byte, error)
metadata func(any) map[string]any
metadata func(any) (map[string]any, error)
decode func([]byte) (any, error)
}
@@ -132,10 +132,10 @@ func RegisterArtifactCodec[T any](registry *ArtifactCodecRegistry, codec contrac
return append([]byte(nil), content...), nil
}
if provider, ok := any(codec).(interface{ Metadata(T) map[string]any }); ok {
entry.metadata = func(value any) map[string]any {
entry.metadata = func(value any) (map[string]any, error) {
typed, err := exactTypedValue[T]("artifact metadata", value)
if err != nil {
return nil
return nil, err
}
return cloneMetadata(provider.Metadata(typed))
}

View File

@@ -29,7 +29,10 @@ func TestChunkCanonicalizationAndClonePreserveAnnotationScopes(t *testing.T) {
t.Fatalf("plan annotation = %q", got)
}
cloned := cloneSourceChunk(chunks[0])
cloned, err := cloneSourceChunk(chunks[0])
if err != nil {
t.Fatal(err)
}
cloned.Annotations["shared"][0] = '['
cloned.PlanAnnotations["shared"][0] = '['
if string(chunks[0].Annotations["shared"]) != `{"range":true}` || string(chunks[0].PlanAnnotations["shared"]) != `{"plan":true}` {

View File

@@ -21,12 +21,16 @@ func validateAndMaterializeChunkPlan(doc *source.SourceDocument, plan source.Chu
return canonical, chunks, nil
}
func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit {
func cloneSourceUnit(unit source.SourceUnit) (source.SourceUnit, error) {
metadata, err := cloneMetadata(unit.Metadata)
if err != nil {
return source.SourceUnit{}, fmt.Errorf("clone source unit %d metadata: %w", unit.ID, err)
}
return source.SourceUnit{
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Ref: unit.Ref,
Metadata: cloneMetadata(unit.Metadata),
}
Metadata: metadata,
}, nil
}

View File

@@ -471,7 +471,7 @@ func debugSourceDocumentEnvelope(doc *source.SourceDocument) *debugSourceDocumen
Kind: doc.Kind,
Format: doc.Format,
Digest: doc.Digest,
Units: cloneSourceUnits(doc.Units),
Units: cloneSourceUnitsForDebug(doc.Units),
Metadata: redactSensitiveMap(doc.Metadata),
}
}
@@ -483,13 +483,21 @@ func debugSourceChunkEnvelope(chunk source.Chunk) debugSourceChunk {
Index: chunk.Index,
Ref: chunk.Ref,
Content: debugContentEnvelope(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
Units: cloneSourceUnits(chunk.Units),
Units: cloneSourceUnitsForDebug(chunk.Units),
Metadata: redactSensitiveMap(chunk.Metadata),
Annotations: source.CloneChunkAnnotations(chunk.Annotations),
PlanAnnotations: source.CloneChunkAnnotations(chunk.PlanAnnotations),
}
}
func cloneSourceUnitsForDebug(units []source.SourceUnit) []source.SourceUnit {
cloned, err := cloneSourceUnits(units)
if err != nil {
return nil
}
return cloned
}
func debugSourceChunkEnvelopes(chunks []source.Chunk) []debugSourceChunk {
if len(chunks) == 0 {
return nil

View File

@@ -78,6 +78,10 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
if err := validateRunInput(input); err != nil {
return output, err
}
input.Metadata, err = cloneMetadata(input.Metadata)
if err != nil {
return output, fmt.Errorf("clone run metadata: %w", err)
}
input.pipeline = input.Prepared.resolved
input.llmClient = input.Prepared.dependencies.LLM
@@ -121,7 +125,9 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}
adapter := input.Prepared.input
attachModuleManifestMetadata(&output, "input", adapter)
if err := attachModuleManifestMetadata(&output, "input", adapter); err != nil {
return failOutput(output), err
}
sourceCheckpoint, sourceDecision := checkpointLoader.Source(adapter.Key())
recordCheckpointEvent(&output, checkpointLoader, "source", "", adapter.Key(), sourceDecision)
doc := sourceCheckpoint.Document
@@ -144,12 +150,16 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
if err := checkpoints.SourceRunning(adapter.Key()); err != nil {
return failOutput(output), fmt.Errorf("write source checkpoint: %w", err)
}
requestMetadata, metadataErr := cloneMetadata(input.Metadata)
if metadataErr != nil {
return failOutput(output), fmt.Errorf("clone input adapter metadata: %w", metadataErr)
}
doc, err = adapter.Parse(ctx, contracts.ParseRequest{
SourceID: input.SourceID,
Path: input.Path,
Raw: input.RawInput,
LLMProfile: input.pipeline.Input.LLMProfile,
Metadata: input.Metadata,
Metadata: requestMetadata,
})
if err != nil {
_ = checkpoints.SourceFailed(adapter.Key(), err)
@@ -177,11 +187,16 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}
sourceInput := sourceInputMaterial(input.Path, input.RawInput)
sessionID := resolvedSessionID(input.SessionID, doc.ID)
output.Manifest.Metadata = manifestMetadataWithSessionID(output.Manifest.Metadata, sessionID)
output.Manifest.Metadata, err = manifestMetadataWithSessionID(output.Manifest.Metadata, sessionID)
if err != nil {
return failOutput(output), err
}
output.Manifest.SourceDigests = []string{doc.Digest}
chunker := input.Prepared.chunker
attachModuleManifestMetadata(&output, "chunker", chunker)
if err := attachModuleManifestMetadata(&output, "chunker", chunker); err != nil {
return failOutput(output), err
}
chunkStarted := time.Now().UTC()
chunkMode := effectiveChunkCacheMode(input.ChunkCacheMode)
if err := writeDebugTimed(debugRecorder, "chunk/input.json", debugTimedEnvelope{
@@ -199,7 +214,9 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err)
}
chunkResult, err := r.runChunkPlan(ctx, input, doc, sourceInput, sessionID)
applyChunkPlanExecution(&output, chunkResult)
if applyErr := applyChunkPlanExecution(&output, chunkResult); applyErr != nil {
return failOutput(output), applyErr
}
if err != nil {
return failOutput(output), err
}
@@ -233,7 +250,9 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
if chunkResult.accepted {
laneOutput, laneErr := r.runLanes(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, chunkResult.chunks)
mergeLaneOutput(&output, laneOutput)
if err := mergeLaneOutput(&output, laneOutput); err != nil {
return failOutput(output), err
}
if laneErr != nil {
return failOutput(output), laneErr
}
@@ -248,7 +267,9 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
output.Manifest.CompletedAt = timePtr(time.Now().UTC())
encoder := input.Prepared.output
attachModuleManifestMetadata(&output, "output", encoder)
if err := attachModuleManifestMetadata(&output, "output", encoder); err != nil {
return failOutput(output), err
}
outputStarted := time.Now().UTC()
if err := writeDebugTimed(debugRecorder, "output/input.json", debugTimedEnvelope{
Stage: string(StageOutput),
@@ -265,13 +286,17 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}); err != nil {
return failOutput(output), fmt.Errorf("write output debug artifact: %w", err)
}
outputMetadata, err := cloneMetadata(input.Metadata)
if err != nil {
return failOutput(output), fmt.Errorf("clone output encoder metadata: %w", err)
}
encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
Manifest: output.Manifest,
NormalizeOutputs: cloneSerializedOutputs(output.NormalizeOutputs),
Rejected: cloneRejectedOutputs(output.Rejected),
Warnings: output.Warnings,
LLMProfile: input.pipeline.Output.LLMProfile,
Metadata: input.Metadata,
Metadata: outputMetadata,
})
output.Warnings = append(output.Warnings, encoded.Warnings...)
if err != nil {
@@ -351,11 +376,19 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
attemptPath := path.Join("validate", debugPathComponent(string(StageChunk)), "", debugPathComponent(moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", index+1, debugPathComponent(binding.Module), attempt))
validatorCtx, llmScope := withIsolatedDebugLLMScope(ctx, attemptPath)
var result contracts.ValidationResult
requestMetadata, cloneErr := cloneMetadata(metadata)
if cloneErr != nil {
return nil, nil, fmt.Errorf("clone chunk validation metadata: %w", cloneErr)
}
requestChunks, cloneErr := cloneSourceChunks(chunks)
if cloneErr != nil {
return nil, nil, fmt.Errorf("clone chunks for validation: %w", cloneErr)
}
switch item.resolved.Target {
case ValidatorTargetChunk:
result, err = item.chunk.Validate(validatorCtx, contracts.ChunkValidationRequest{ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: cloneMetadata(metadata), Chunks: cloneSourceChunks(chunks)})
result, err = item.chunk.Validate(validatorCtx, contracts.ChunkValidationRequest{ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: requestMetadata, Chunks: requestChunks})
case ValidatorTargetSerialized:
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: cloneMetadata(metadata), Chunks: cloneSourceChunks(chunks), Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...)})
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: requestMetadata, Chunks: requestChunks, Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...)})
default:
return nil, nil, fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module)
}
@@ -616,31 +649,38 @@ func rejectedOutputManifests(rejected []contracts.RejectedOutput) []artifacts.Re
return manifests
}
func attachModuleManifestMetadata(output *RunOutput, moduleKey string, module any) {
func attachModuleManifestMetadata(output *RunOutput, moduleKey string, module any) error {
if output == nil {
return
return nil
}
metadata, ok, err := moduleManifestMetadata(module)
if err != nil {
return fmt.Errorf("clone manifest metadata for module %q: %w", moduleKey, err)
}
metadata, ok := moduleManifestMetadata(module)
if !ok {
return
return nil
}
if output.Manifest.ModuleMetadata == nil {
output.Manifest.ModuleMetadata = make(map[string]map[string]any)
}
output.Manifest.ModuleMetadata[moduleKey] = metadata
return nil
}
func moduleManifestMetadata(module any) (map[string]any, bool) {
func moduleManifestMetadata(module any) (map[string]any, bool, error) {
provider, ok := module.(contracts.ManifestMetadataProvider)
if !ok {
return nil, false
return nil, false, nil
}
moduleMetadata := cloneMetadata(provider.ManifestMetadata())
if len(moduleMetadata) == 0 {
return nil, false
moduleMetadata, err := cloneMetadata(provider.ManifestMetadata())
if err != nil {
return nil, false, err
}
return moduleMetadata, true
if len(moduleMetadata) == 0 {
return nil, false, nil
}
return moduleMetadata, true, nil
}
func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) {
@@ -678,12 +718,8 @@ func validateOutputFileName(name string) error {
return nil
}
func cloneMetadata(metadata map[string]any) map[string]any {
cloned, err := source.CloneMetadata(metadata)
if err != nil {
return metadata
}
return cloned
func cloneMetadata(metadata map[string]any) (map[string]any, error) {
return source.CloneMetadata(metadata)
}
func cloneLLMProfiles(profiles []artifacts.LLMProfileManifest) []artifacts.LLMProfileManifest {
@@ -785,16 +821,19 @@ func resolvedSessionID(explicit string, sourceDocumentID string) string {
return strings.TrimSpace(sourceDocumentID)
}
func manifestMetadataWithSessionID(metadata map[string]any, sessionID string) map[string]any {
out := cloneMetadata(metadata)
func manifestMetadataWithSessionID(metadata map[string]any, sessionID string) (map[string]any, error) {
out, err := cloneMetadata(metadata)
if err != nil {
return nil, fmt.Errorf("clone run manifest metadata: %w", err)
}
if strings.TrimSpace(sessionID) == "" {
return out
return out, nil
}
if out == nil {
out = make(map[string]any)
}
out["session_id"] = sessionID
return out
return out, nil
}
func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
@@ -804,43 +843,61 @@ func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
return append([]contracts.Warning(nil), warnings...)
}
func cloneSourceChunkPtr(chunk *source.Chunk) *source.Chunk {
func cloneSourceChunkPtr(chunk *source.Chunk) (*source.Chunk, error) {
if chunk == nil {
return nil
return nil, nil
}
cloned := cloneSourceChunk(*chunk)
return &cloned
cloned, err := cloneSourceChunk(*chunk)
if err != nil {
return nil, err
}
return &cloned, nil
}
func cloneSourceChunk(chunk source.Chunk) source.Chunk {
func cloneSourceChunk(chunk source.Chunk) (source.Chunk, error) {
chunk.Content = append([]byte(nil), chunk.Content...)
chunk.Units = cloneSourceUnits(chunk.Units)
chunk.Metadata = cloneMetadata(chunk.Metadata)
var err error
chunk.Units, err = cloneSourceUnits(chunk.Units)
if err != nil {
return source.Chunk{}, err
}
chunk.Metadata, err = cloneMetadata(chunk.Metadata)
if err != nil {
return source.Chunk{}, fmt.Errorf("clone chunk metadata: %w", err)
}
chunk.Annotations = source.CloneChunkAnnotations(chunk.Annotations)
chunk.PlanAnnotations = source.CloneChunkAnnotations(chunk.PlanAnnotations)
return chunk
return chunk, nil
}
func cloneSourceChunks(chunks []source.Chunk) []source.Chunk {
func cloneSourceChunks(chunks []source.Chunk) ([]source.Chunk, error) {
if len(chunks) == 0 {
return nil
return nil, nil
}
out := make([]source.Chunk, 0, len(chunks))
for _, chunk := range chunks {
out = append(out, cloneSourceChunk(chunk))
cloned, err := cloneSourceChunk(chunk)
if err != nil {
return nil, err
}
out = append(out, cloned)
}
return out
return out, nil
}
func cloneSourceUnits(units []source.SourceUnit) []source.SourceUnit {
func cloneSourceUnits(units []source.SourceUnit) ([]source.SourceUnit, error) {
if len(units) == 0 {
return nil
return nil, nil
}
out := make([]source.SourceUnit, 0, len(units))
for _, unit := range units {
out = append(out, cloneSourceUnit(unit))
cloned, err := cloneSourceUnit(unit)
if err != nil {
return nil, err
}
out = append(out, cloned)
}
return out
return out, nil
}
func cloneSerializedOutputs(outputs []contracts.SerializedOutput) []contracts.SerializedOutput {

View File

@@ -58,7 +58,9 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
case ChunkPlanHit:
plan, chunks, validationErr := validateAndMaterializeChunkPlan(doc, record.Plan)
if validationErr == nil {
result.setCandidate(record, "reused")
if err := result.setCandidate(record, "reused"); err != nil {
return result, fmt.Errorf("clone reused chunk plan record: %w", err)
}
validationWarnings, rejection, err := r.validateChunks(ctx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.Metadata, input.Prepared.chunkValidators, 1, input.Debug)
result.plan = &plan
result.chunks = chunks
@@ -87,10 +89,14 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
attemptPath := path.Join("chunk", fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
terminal := newAttemptTerminalRecorder(input.Debug, attemptPath, "chunk", llmScope, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: chunker.Key(), Attempt: attempt, StartedAt: attemptStarted})
requestMetadata, metadataErr := cloneMetadata(input.Metadata)
if metadataErr != nil {
return false, nil, terminal.record(nil, fmt.Errorf("clone chunk request metadata: %w", metadataErr))
}
chunkResult, callErr := chunker.Plan(attemptCtx, contracts.ChunkRequest{
Source: doc, SourceInput: sourceInput.Clone(), SessionID: sessionID,
References: CloneReferenceSet(input.pipeline.ChunkReferences.ReferenceSet),
LLMProfile: input.pipeline.Chunk.LLMProfile, Metadata: input.Metadata,
LLMProfile: input.pipeline.Chunk.LLMProfile, Metadata: requestMetadata,
})
if callErr != nil {
return false, nil, terminal.record(nil, fmt.Errorf("chunk source with chunker %q: %w", chunker.Key(), callErr))
@@ -105,7 +111,10 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
if digestErr != nil {
return false, nil, terminal.record(nil, fmt.Errorf("digest generated chunk plan: %w", digestErr))
}
producerMetadata, _ := moduleManifestMetadata(chunker)
producerMetadata, _, metadataErr := moduleManifestMetadata(chunker)
if metadataErr != nil {
return false, nil, terminal.record(nil, fmt.Errorf("clone chunker manifest metadata: %w", metadataErr))
}
profile := ""
if provider, ok := chunker.(contracts.ChunkExecutionClassProvider); ok && provider.ExecutionClass() == contracts.ExecutionClassLLMBacked {
profile = input.pipeline.Chunk.LLMProfile
@@ -116,7 +125,7 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
Producer: ChunkPlanProducer{
InputModule: input.Prepared.input.Key(), ChunkModule: chunker.Key(), LLMProfile: profile,
References: append([]artifacts.ReferenceProvenance(nil), referenceTargetProvenance(input.pipeline.ChunkReferences)...),
Metadata: cloneMetadata(producerMetadata),
Metadata: producerMetadata,
},
Warnings: cloneWarnings(chunkResult.Warnings), CreatedAt: time.Now().UTC(),
}
@@ -127,7 +136,9 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
if mode == ChunkCacheBypass {
action = "bypassed"
}
result.setCandidate(candidate, action)
if candidateErr := result.setCandidate(candidate, action); candidateErr != nil {
return false, nil, terminal.record(nil, fmt.Errorf("clone generated chunk plan record: %w", candidateErr))
}
validationWarnings, rejected, validationErr := r.validateChunks(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.Metadata, input.Prepared.chunkValidators, attempt, input.Debug)
attemptWarnings := append(cloneWarnings(chunkResult.Warnings), validationWarnings...)
payload := map[string]any{
@@ -161,7 +172,10 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
}
if mode == ChunkCacheAuto || mode == ChunkCacheRefresh {
record := cloneChunkPlanRecord(*result.record)
record, cloneErr := cloneChunkPlanRecord(*result.record)
if cloneErr != nil {
return result, fmt.Errorf("clone chunk plan record for publication: %w", cloneErr)
}
record.Warnings = cloneWarnings(producerWarnings)
if err := input.ChunkPlans.Save(record); err != nil {
return result, fmt.Errorf("save chunk plan: %w", err)
@@ -197,13 +211,17 @@ func chunkPlanLookupReason(status ChunkPlanStatus) string {
}
}
func (result *chunkPlanExecution) setCandidate(record ChunkPlanRecord, action string) {
cloned := cloneChunkPlanRecord(record)
func (result *chunkPlanExecution) setCandidate(record ChunkPlanRecord, action string) error {
cloned, err := cloneChunkPlanRecord(record)
if err != nil {
return err
}
result.record = &cloned
result.action = action
result.summary.Action = action
result.summary.SourceDigest = record.SourceDigest
result.summary.CandidateDigest = record.PlanDigest
return nil
}
func (result *chunkPlanExecution) setValidation(warnings []contracts.Warning, rejection *contracts.RejectedOutput, err error) {
@@ -219,27 +237,31 @@ func (result *chunkPlanExecution) setValidation(warnings []contracts.Warning, re
}
}
func cloneChunkPlanRecord(record ChunkPlanRecord) ChunkPlanRecord {
func cloneChunkPlanRecord(record ChunkPlanRecord) (ChunkPlanRecord, error) {
record.Plan = source.CloneChunkPlan(record.Plan)
record.Producer.References = append([]artifacts.ReferenceProvenance(nil), record.Producer.References...)
record.Producer.Metadata = cloneMetadata(record.Producer.Metadata)
metadata, err := cloneMetadata(record.Producer.Metadata)
if err != nil {
return ChunkPlanRecord{}, fmt.Errorf("clone chunk plan producer metadata: %w", err)
}
record.Producer.Metadata = metadata
record.Warnings = cloneWarnings(record.Warnings)
return record
return record, nil
}
func applyChunkPlanExecution(output *RunOutput, result chunkPlanExecution) {
func applyChunkPlanExecution(output *RunOutput, result chunkPlanExecution) error {
if output == nil {
return
return nil
}
summary := result.summary
output.ChunkPlan = &summary
if output.Manifest.ChunkPlan == nil {
return
return nil
}
manifest := output.Manifest.ChunkPlan
manifest.Action = result.action
if result.record == nil {
return
return nil
}
record := result.record
manifest.SourceDigest = record.SourceDigest
@@ -249,7 +271,12 @@ func applyChunkPlanExecution(output *RunOutput, result chunkPlanExecution) {
manifest.ProducerModule = record.Producer.ChunkModule
manifest.ProducerLLMProfile = record.Producer.LLMProfile
manifest.ProducerReferences = append([]artifacts.ReferenceProvenance(nil), record.Producer.References...)
manifest.ProducerMetadata = cloneMetadata(record.Producer.Metadata)
metadata, err := cloneMetadata(record.Producer.Metadata)
if err != nil {
return fmt.Errorf("clone chunk plan manifest producer metadata: %w", err)
}
manifest.ProducerMetadata = metadata
createdAt := record.CreatedAt
manifest.CreatedAt = &createdAt
return nil
}

View File

@@ -438,6 +438,20 @@ func TestRunnerStoresProducerProvenanceAndProducerWarnings(t *testing.T) {
}
}
func TestRunnerRejectsUncloneableModuleManifestMetadata(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
cyclic := make(map[string]any)
cyclic["self"] = cyclic
prepared.chunker = manifestChunker{
terminalChunker: terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan},
metadata: map[string]any{"cyclic": cyclic},
}
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
if err == nil || !strings.Contains(err.Error(), `clone manifest metadata for module "chunker": metadata.cyclic.self contains a cycle`) {
t.Fatalf("Run() error = %v, want contextual module metadata clone failure", err)
}
}
func TestRunnerOmitsProducerProfileForDeterministicChunker(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.Chunk.LLMProfile = "configured-but-unused"

View File

@@ -79,7 +79,9 @@ func (r *Runner) runLanes(parent context.Context, input RunInput, checkpoints Ch
if prepared.typed == nil {
return output, fmt.Errorf("typed lane %q executor is not prepared", prepared.resolved.ID)
}
setTypedLaneManifestMetadata(&output, prepared.resolved.ID, prepared.typed.extractor, prepared.typed.merger, prepared.typed.normalizer)
if err := setTypedLaneManifestMetadata(&output, prepared.resolved.ID, prepared.typed.extractor, prepared.typed.merger, prepared.typed.normalizer); err != nil {
return output, err
}
state, err := prepareLaneExtract(input, loader, doc, chunks, i, prepared)
if err != nil {
return output, err
@@ -212,7 +214,9 @@ func (r *Runner) runLanes(parent context.Context, input RunInput, checkpoints Ch
close(continuations)
continuationWorkers.Wait()
for i := range completedOutputs {
mergeLaneOutput(&output, completedOutputs[i])
if err := mergeLaneOutput(&output, completedOutputs[i]); err != nil {
return output, err
}
}
if err := selectRunError(parent, runErrors); err != nil {
return output, err
@@ -244,7 +248,10 @@ func prepareLaneExtract(input RunInput, loader CheckpointLoader, doc *source.Sou
if decodeErr != nil {
return nil, fmt.Errorf("decode extract checkpoint for lane %q: %w", lane.ID, decodeErr)
}
stored = hydrateCheckpointArtifact(typed.codec, stored, value)
stored, decodeErr = hydrateCheckpointArtifact(typed.codec, stored, value)
if decodeErr != nil {
return nil, fmt.Errorf("hydrate extract checkpoint for lane %q: %w", lane.ID, decodeErr)
}
artifact := erasedExtractArtifact{LaneID: lane.ID, ExtractorKey: lane.Extract.Module, SourceID: doc.ID, ChunkID: stored.ChunkID, ChunkIndex: stored.ChunkIndex, ChunkRef: stored.ChunkRef, Value: value}
if stored.ChunkIndex >= 0 && stored.ChunkIndex < len(chunks) && artifact.ChunkRef == (source.SourceRef{}) {
artifact.ChunkRef = chunks[stored.ChunkIndex].Ref
@@ -258,9 +265,14 @@ func prepareLaneExtract(input RunInput, loader CheckpointLoader, doc *source.Sou
}
func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, job extractJob) extractJobResult {
state, chunk := job.lane, cloneSourceChunk(job.chunk)
state := job.lane
chunk, cloneErr := cloneSourceChunk(job.chunk)
lane, typed := state.prepared.resolved, state.prepared.typed
result := extractJobResult{laneIndex: state.index, chunkIndex: chunk.Index}
result := extractJobResult{laneIndex: state.index, chunkIndex: job.chunk.Index}
if cloneErr != nil {
result.err = fmt.Errorf("clone chunk %q for extraction: %w", job.chunk.ID, cloneErr)
return result
}
var accepted erasedExtractArtifact
var serialized CheckpointArtifact
var acceptedWarnings []contracts.Warning
@@ -269,7 +281,11 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.
attemptPath := path.Join("extract", debugPathComponent(lane.ID), fmt.Sprintf("chunk-%06d", chunk.Index+1), fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
terminal := newAttemptTerminalRecorder(input.Debug, attemptPath, "extract", llmScope, debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, Attempt: attempt, StartedAt: started})
extracted, callErr := typed.extract(attemptCtx, typed.extractor, contracts.TypedExtractionRequest{Source: doc, Chunk: &chunk, SourceInput: chunkInputMaterial(sourceInput, chunk), SessionID: sessionID, References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet), LLMProfile: lane.Extract.LLMProfile, Metadata: cloneMetadata(input.Metadata)})
requestMetadata, metadataErr := cloneMetadata(input.Metadata)
if metadataErr != nil {
return false, nil, terminal.record(nil, fmt.Errorf("clone extract request metadata: %w", metadataErr))
}
extracted, callErr := typed.extract(attemptCtx, typed.extractor, contracts.TypedExtractionRequest{Source: doc, Chunk: &chunk, SourceInput: chunkInputMaterial(sourceInput, chunk), SessionID: sessionID, References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet), LLMProfile: lane.Extract.LLMProfile, Metadata: requestMetadata})
if callErr != nil {
attemptErr := fmt.Errorf("extract lane %q chunk %q with extractor %q: %w", lane.ID, chunk.ID, lane.Extract.Module, callErr)
return false, nil, terminal.record(nil, attemptErr)
@@ -414,9 +430,9 @@ func selectRunError(parent context.Context, values []orderedRunError) error {
return filtered[0].err
}
func mergeLaneOutput(dst *RunOutput, src RunOutput) {
func mergeLaneOutput(dst *RunOutput, src RunOutput) error {
if dst == nil {
return
return nil
}
dst.NormalizeOutputs = append(dst.NormalizeOutputs, cloneSerializedOutputs(src.NormalizeOutputs)...)
dst.Rejected = append(dst.Rejected, cloneRejectedOutputs(src.Rejected)...)
@@ -425,8 +441,13 @@ func mergeLaneOutput(dst *RunOutput, src RunOutput) {
for i := range dst.Manifest.ArtifactLanes {
for j := range src.Manifest.ArtifactLanes {
if dst.Manifest.ArtifactLanes[i].ID == src.Manifest.ArtifactLanes[j].ID && src.Manifest.ArtifactLanes[j].Metadata != nil {
dst.Manifest.ArtifactLanes[i].Metadata = cloneMetadata(src.Manifest.ArtifactLanes[j].Metadata)
metadata, err := cloneMetadata(src.Manifest.ArtifactLanes[j].Metadata)
if err != nil {
return fmt.Errorf("clone lane %q manifest metadata: %w", dst.Manifest.ArtifactLanes[i].ID, err)
}
dst.Manifest.ArtifactLanes[i].Metadata = metadata
}
}
}
return nil
}

View File

@@ -135,6 +135,20 @@ func TestRunnerPassesIndependentConcreteMetadataToValidatorsAndExtractors(t *tes
}
}
func TestRunnerRejectsMetadataThatCannotBeCloned(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
cyclic := make(map[string]any)
cyclic["self"] = cyclic
_, err := New().Run(context.Background(), RunInput{
Prepared: prepared,
RawInput: []byte("input"),
Metadata: map[string]any{"cyclic": cyclic},
})
if err == nil || !strings.Contains(err.Error(), "clone run metadata: metadata.cyclic.self contains a cycle") {
t.Fatalf("Run() error = %v, want contextual metadata clone failure", err)
}
}
func TestRunnerContinuesFromFreshAndReusedExtractResults(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
extractCalls := 0

View File

@@ -30,14 +30,18 @@ func cloneCheckpointArtifact(output CheckpointArtifact) CheckpointArtifact {
return output
}
func hydrateCheckpointArtifact(codec artifactCodecEntry, output CheckpointArtifact, value any) CheckpointArtifact {
func hydrateCheckpointArtifact(codec artifactCodecEntry, output CheckpointArtifact, value any) (CheckpointArtifact, error) {
output.Artifact.Schema = contracts.CloneArtifactSchema(codec.spec.Schema)
if codec.metadata != nil {
output.Artifact.Metadata = cloneMetadata(codec.metadata(value))
metadata, err := codec.metadata(value)
if err != nil {
return CheckpointArtifact{}, fmt.Errorf("clone artifact metadata: %w", err)
}
output.Artifact.Metadata = metadata
} else {
output.Artifact.Metadata = nil
}
return output
return output, nil
}
func artifactCheckpointDigests(outputs []CheckpointArtifact) []CheckpointFingerprint {
values := make([]CheckpointFingerprint, 0, len(outputs))
@@ -82,9 +86,12 @@ func serializeArtifact(codec artifactCodecEntry, value any, candidate bool) (con
schema := codec.spec.Schema
metadata := map[string]any(nil)
if codec.metadata != nil {
metadata = codec.metadata(value)
metadata, err = codec.metadata(value)
if err != nil {
return contracts.SerializedArtifact{}, fmt.Errorf("clone artifact metadata: %w", err)
}
}
return contracts.SerializedArtifact{Kind: codec.spec.Kind, Schema: contracts.CloneArtifactSchema(schema), MediaType: codec.spec.MediaType, Content: append([]byte(nil), content...), Metadata: cloneMetadata(metadata)}, nil
return contracts.SerializedArtifact{Kind: codec.spec.Kind, Schema: contracts.CloneArtifactSchema(schema), MediaType: codec.spec.MediaType, Content: append([]byte(nil), content...), Metadata: metadata}, nil
}
func decodeCheckpointArtifact(codec artifactCodecEntry, artifact CheckpointArtifact) (any, error) {
@@ -139,7 +146,9 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoi
if typed == nil {
return fmt.Errorf("typed lane %q executor is not prepared", lane.ID)
}
setTypedLaneManifestMetadata(output, lane.ID, typed.extractor, typed.merger, typed.normalizer)
if err := setTypedLaneManifestMetadata(output, lane.ID, typed.extractor, typed.merger, typed.normalizer); err != nil {
return err
}
mergeInputs := make([]contracts.ExtractArtifact[any], len(extracts.accepted))
for i, value := range extracts.accepted {
@@ -165,7 +174,10 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoi
return fmt.Errorf("decode merge checkpoint for lane %q: %w", lane.ID, decodeErr)
}
merged = erasedMergeArtifact{LaneID: lane.ID, MergerKey: lane.Merge.Module, SourceID: doc.ID, Value: value}
serializedMerge = hydrateCheckpointArtifact(typed.codec, cloneCheckpointArtifact(mergeCP.Output), value)
serializedMerge, decodeErr = hydrateCheckpointArtifact(typed.codec, cloneCheckpointArtifact(mergeCP.Output), value)
if decodeErr != nil {
return fmt.Errorf("hydrate merge checkpoint for lane %q: %w", lane.ID, decodeErr)
}
mergeWarnings = cloneWarnings(mergeCP.Warnings)
output.Warnings = append(output.Warnings, mergeWarnings...)
} else {
@@ -177,7 +189,11 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoi
attemptPath := path.Join("merge", debugPathComponent(lane.ID), fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
terminal := newAttemptTerminalRecorder(input.Debug, attemptPath, "merge", llmScope, debugTimedEnvelope{Stage: string(StageMerge), LaneID: lane.ID, ModuleKey: lane.Merge.Module, Attempt: attempt, StartedAt: started})
result, callErr := typed.merge(attemptCtx, typed.merger, contracts.TypedMergeRequest[any]{Source: doc, LaneID: lane.ID, ExtractOutputs: mergeInputs, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(lane.MergeReferences.ReferenceSet), LLMProfile: lane.Merge.LLMProfile, Metadata: cloneMetadata(input.Metadata)})
requestMetadata, metadataErr := cloneMetadata(input.Metadata)
if metadataErr != nil {
return false, nil, terminal.record(nil, fmt.Errorf("clone merge request metadata: %w", metadataErr))
}
result, callErr := typed.merge(attemptCtx, typed.merger, contracts.TypedMergeRequest[any]{Source: doc, LaneID: lane.ID, ExtractOutputs: mergeInputs, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(lane.MergeReferences.ReferenceSet), LLMProfile: lane.Merge.LLMProfile, Metadata: requestMetadata})
if callErr != nil {
attemptErr := fmt.Errorf("merge lane %q with merger %q: %w", lane.ID, lane.Merge.Module, callErr)
return false, nil, terminal.record(nil, attemptErr)
@@ -246,7 +262,11 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoi
if decodeErr != nil {
return fmt.Errorf("decode normalize checkpoint for lane %q: %w", lane.ID, decodeErr)
}
serializedNormalize, normalizeWarnings = hydrateCheckpointArtifact(typed.codec, cloneCheckpointArtifact(normalizeCP.Output), value), cloneWarnings(normalizeCP.Warnings)
serializedNormalize, decodeErr = hydrateCheckpointArtifact(typed.codec, cloneCheckpointArtifact(normalizeCP.Output), value)
if decodeErr != nil {
return fmt.Errorf("hydrate normalize checkpoint for lane %q: %w", lane.ID, decodeErr)
}
normalizeWarnings = cloneWarnings(normalizeCP.Warnings)
output.Warnings = append(output.Warnings, normalizeWarnings...)
} else {
if err := checkpoints.NormalizeRunning(lane.ID, lane.Normalize.Module, normalizeDeps); err != nil {
@@ -257,7 +277,11 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoi
attemptPath := path.Join("normalize", debugPathComponent(lane.ID), fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
terminal := newAttemptTerminalRecorder(input.Debug, attemptPath, "normalize", llmScope, debugTimedEnvelope{Stage: string(StageNormalize), LaneID: lane.ID, ModuleKey: lane.Normalize.Module, Attempt: attempt, StartedAt: started})
result, callErr := typed.normalize(attemptCtx, typed.normalizer, contracts.TypedNormalizeRequest[any]{Source: doc, LaneID: lane.ID, MergeOutput: contracts.MergeArtifact[any]{LaneID: lane.ID, MergerKey: lane.Merge.Module, SourceID: doc.ID, Value: merged.Value}, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(lane.NormalizeReferences.ReferenceSet), LLMProfile: lane.Normalize.LLMProfile, Metadata: cloneMetadata(input.Metadata)})
requestMetadata, metadataErr := cloneMetadata(input.Metadata)
if metadataErr != nil {
return false, nil, terminal.record(nil, fmt.Errorf("clone normalize request metadata: %w", metadataErr))
}
result, callErr := typed.normalize(attemptCtx, typed.normalizer, contracts.TypedNormalizeRequest[any]{Source: doc, LaneID: lane.ID, MergeOutput: contracts.MergeArtifact[any]{LaneID: lane.ID, MergerKey: lane.Merge.Module, SourceID: doc.ID, Value: merged.Value}, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(lane.NormalizeReferences.ReferenceSet), LLMProfile: lane.Normalize.LLMProfile, Metadata: requestMetadata})
if callErr != nil {
attemptErr := fmt.Errorf("normalize lane %q with normalizer %q: %w", lane.ID, lane.Normalize.Module, callErr)
return false, nil, terminal.record(nil, attemptErr)
@@ -309,9 +333,9 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoi
return nil
}
func setTypedLaneManifestMetadata(output *RunOutput, laneID string, extractor, merger, normalizer any) {
func setTypedLaneManifestMetadata(output *RunOutput, laneID string, extractor, merger, normalizer any) error {
if output == nil {
return
return nil
}
for i := range output.Manifest.ArtifactLanes {
if output.Manifest.ArtifactLanes[i].ID != laneID {
@@ -322,15 +346,20 @@ func setTypedLaneManifestMetadata(output *RunOutput, laneID string, extractor, m
name string
module any
}{{"extractor", extractor}, {"merger", merger}, {"normalizer", normalizer}} {
if value, ok := moduleManifestMetadata(item.module); ok {
value, ok, err := moduleManifestMetadata(item.module)
if err != nil {
return fmt.Errorf("clone %s manifest metadata for lane %q: %w", item.name, laneID, err)
}
if ok {
metadata[item.name] = value
}
}
if len(metadata) > 0 {
output.Manifest.ArtifactLanes[i].Metadata = metadata
}
return
return nil
}
return nil
}
func (r *Runner) validateTypedArtifact(ctx context.Context, codec artifactCodecEntry, target typedValidationTarget, chain preparedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
@@ -349,17 +378,32 @@ func (r *Runner) validateTypedArtifact(ctx context.Context, codec artifactCodecE
started := time.Now().UTC()
attemptPath := path.Join("validate", debugPathComponent(string(target.stage)), debugPathComponent(target.laneID), debugPathComponent(target.moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", index+1, debugPathComponent(binding.Module), attempt))
validatorCtx, llmScope := withIsolatedDebugLLMScope(ctx, attemptPath)
requestTarget := target
requestTarget.sourceInput = target.sourceInput.Clone()
requestTarget.references = CloneReferenceSet(target.references)
requestTarget.metadata, err = cloneMetadata(target.metadata)
if err != nil {
return nil, nil, fmt.Errorf("clone typed validation metadata: %w", err)
}
requestTarget.chunk, err = cloneSourceChunkPtr(target.chunk)
if err != nil {
return nil, nil, fmt.Errorf("clone typed validation chunk: %w", err)
}
requestTarget.chunks, err = cloneSourceChunks(target.chunks)
if err != nil {
return nil, nil, fmt.Errorf("clone typed validation chunks: %w", err)
}
switch item.resolved.Target {
case ValidatorTargetTyped:
target.llmProfile = binding.LLMProfile
result, err = item.typedValidate(validatorCtx, item.typed, target)
requestTarget.llmProfile = binding.LLMProfile
result, err = item.typedValidate(validatorCtx, item.typed, requestTarget)
case ValidatorTargetSerialized:
artifact, encodeErr := validationCandidateArtifact(codec, target)
if encodeErr != nil {
err = encodeErr
break
}
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, Source: target.source, SourceID: target.sourceID, SourceInput: target.sourceInput.Clone(), SessionID: target.sessionID, References: CloneReferenceSet(target.references), LLMProfile: binding.LLMProfile, Metadata: cloneMetadata(target.metadata), Chunk: cloneSourceChunkPtr(target.chunk), Chunks: cloneSourceChunks(target.chunks), Schema: contracts.CloneArtifactSchema(artifact.Artifact.Schema), MediaType: artifact.Artifact.MediaType, Content: append([]byte(nil), artifact.Artifact.Content...)})
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, Source: target.source, SourceID: target.sourceID, SourceInput: requestTarget.sourceInput, SessionID: target.sessionID, References: requestTarget.references, LLMProfile: binding.LLMProfile, Metadata: requestTarget.metadata, Chunk: requestTarget.chunk, Chunks: requestTarget.chunks, Schema: contracts.CloneArtifactSchema(artifact.Artifact.Schema), MediaType: artifact.Artifact.MediaType, Content: append([]byte(nil), artifact.Artifact.Content...)})
default:
return nil, nil, fmt.Errorf("validator %q is incompatible with typed artifact validation", binding.Module)
}