|
|
|
|
@@ -2,6 +2,8 @@ package pipeline
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"context"
|
|
|
|
|
"crypto/sha256"
|
|
|
|
|
"encoding/hex"
|
|
|
|
|
"fmt"
|
|
|
|
|
"path"
|
|
|
|
|
"sort"
|
|
|
|
|
@@ -11,50 +13,133 @@ import (
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
// migrationRawArtifact serializes a typed value into the existing raw
|
|
|
|
|
// checkpoint, debug, and output envelopes. It is removed when those boundaries
|
|
|
|
|
// consume SerializedArtifact directly.
|
|
|
|
|
func migrationRawArtifact(codec artifactCodecEntry, value any) (contracts.ResponseSchema, contracts.RawPayload, error) {
|
|
|
|
|
content, err := codec.encodeCandidate(value)
|
|
|
|
|
func loadArtifactExtract(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (ArtifactExtractCheckpoint, CheckpointDecision) {
|
|
|
|
|
typed, ok := loader.(ArtifactCheckpointLoader)
|
|
|
|
|
if !ok {
|
|
|
|
|
return ArtifactExtractCheckpoint{}, CheckpointDecision{Reason: "artifact checkpoint loading is unavailable"}
|
|
|
|
|
}
|
|
|
|
|
return typed.ArtifactExtract(laneID, moduleKey, deps)
|
|
|
|
|
}
|
|
|
|
|
func loadArtifactMerge(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (ArtifactMergeCheckpoint, CheckpointDecision) {
|
|
|
|
|
typed, ok := loader.(ArtifactCheckpointLoader)
|
|
|
|
|
if !ok {
|
|
|
|
|
return ArtifactMergeCheckpoint{}, CheckpointDecision{Reason: "artifact checkpoint loading is unavailable"}
|
|
|
|
|
}
|
|
|
|
|
return typed.ArtifactMerge(laneID, moduleKey, deps)
|
|
|
|
|
}
|
|
|
|
|
func loadArtifactNormalize(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (ArtifactNormalizeCheckpoint, CheckpointDecision) {
|
|
|
|
|
typed, ok := loader.(ArtifactCheckpointLoader)
|
|
|
|
|
if !ok {
|
|
|
|
|
return ArtifactNormalizeCheckpoint{}, CheckpointDecision{Reason: "artifact checkpoint loading is unavailable"}
|
|
|
|
|
}
|
|
|
|
|
return typed.ArtifactNormalize(laneID, moduleKey, deps)
|
|
|
|
|
}
|
|
|
|
|
func recordArtifactExtract(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []ArtifactCheckpointOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
|
|
|
|
|
typed, ok := recorder.(ArtifactCheckpointRecorder)
|
|
|
|
|
if !ok {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
return typed.ArtifactExtractSucceeded(laneID, moduleKey, deps, outputs, rejected, warnings)
|
|
|
|
|
}
|
|
|
|
|
func recordArtifactMerge(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, output ArtifactCheckpointOutput, warnings []contracts.Warning) error {
|
|
|
|
|
typed, ok := recorder.(ArtifactCheckpointRecorder)
|
|
|
|
|
if !ok {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
return typed.ArtifactMergeSucceeded(laneID, moduleKey, deps, output, warnings)
|
|
|
|
|
}
|
|
|
|
|
func recordArtifactNormalize(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, output ArtifactCheckpointOutput, warnings []contracts.Warning) error {
|
|
|
|
|
typed, ok := recorder.(ArtifactCheckpointRecorder)
|
|
|
|
|
if !ok {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
return typed.ArtifactNormalizeSucceeded(laneID, moduleKey, deps, output, warnings)
|
|
|
|
|
}
|
|
|
|
|
func cloneArtifactCheckpointOutput(output ArtifactCheckpointOutput) ArtifactCheckpointOutput {
|
|
|
|
|
output.Artifact = contracts.CloneSerializedArtifact(output.Artifact)
|
|
|
|
|
return output
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func hydrateCheckpointArtifact(codec artifactCodecEntry, output ArtifactCheckpointOutput, value any) ArtifactCheckpointOutput {
|
|
|
|
|
output.Artifact.Schema = contracts.CloneArtifactSchema(codec.spec.Schema)
|
|
|
|
|
if codec.metadata != nil {
|
|
|
|
|
output.Artifact.Metadata = cloneMetadata(codec.metadata(value))
|
|
|
|
|
} else {
|
|
|
|
|
output.Artifact.Metadata = nil
|
|
|
|
|
}
|
|
|
|
|
return output
|
|
|
|
|
}
|
|
|
|
|
func artifactCheckpointDigests(outputs []ArtifactCheckpointOutput) []CheckpointFingerprint {
|
|
|
|
|
values := make([]CheckpointFingerprint, 0, len(outputs))
|
|
|
|
|
for i, output := range outputs {
|
|
|
|
|
sum := sha256.Sum256(output.Artifact.Content)
|
|
|
|
|
values = append(values, CheckpointFingerprint{Name: fmt.Sprintf("artifact[%d]", i), Value: "sha256:" + hex.EncodeToString(sum[:])})
|
|
|
|
|
}
|
|
|
|
|
return normalizeCheckpointFingerprints(values)
|
|
|
|
|
}
|
|
|
|
|
func debugArtifactCheckpointOutput(output ArtifactCheckpointOutput) map[string]any {
|
|
|
|
|
artifact := output.Artifact
|
|
|
|
|
schema := contracts.CloneArtifactSchema(artifact.Schema)
|
|
|
|
|
digest := output.SchemaDigest
|
|
|
|
|
if digest == "" {
|
|
|
|
|
digest = contracts.DigestArtifactSchema(schema)
|
|
|
|
|
}
|
|
|
|
|
schema.JSONSchema = nil
|
|
|
|
|
content := debugContentEnvelope(artifact.Content, artifact.MediaType, artifact.Metadata, nil)
|
|
|
|
|
content.ContentDigest = debugContentDigest(artifact.Content)
|
|
|
|
|
return map[string]any{"lane_id": output.LaneID, "module_key": output.ModuleKey, "source_id": output.SourceID, "chunk_id": output.ChunkID, "chunk_index": output.ChunkIndex, "chunk_ref": output.ChunkRef, "artifact_kind": artifact.Kind, "schema": schema, "schema_digest": digest, "content": content}
|
|
|
|
|
}
|
|
|
|
|
func debugArtifactCheckpointOutputs(outputs []ArtifactCheckpointOutput) []map[string]any {
|
|
|
|
|
if len(outputs) == 0 {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
out := make([]map[string]any, 0, len(outputs))
|
|
|
|
|
for _, output := range outputs {
|
|
|
|
|
out = append(out, debugArtifactCheckpointOutput(output))
|
|
|
|
|
}
|
|
|
|
|
return out
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func serializeArtifact(codec artifactCodecEntry, value any, candidate bool) (contracts.SerializedArtifact, error) {
|
|
|
|
|
encode := codec.encode
|
|
|
|
|
if candidate {
|
|
|
|
|
encode = codec.encodeCandidate
|
|
|
|
|
}
|
|
|
|
|
content, err := encode(value)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return contracts.ResponseSchema{}, contracts.RawPayload{}, err
|
|
|
|
|
return contracts.SerializedArtifact{}, err
|
|
|
|
|
}
|
|
|
|
|
schema := codec.spec.Schema
|
|
|
|
|
metadata := map[string]any(nil)
|
|
|
|
|
if codec.metadata != nil {
|
|
|
|
|
metadata = codec.metadata(value)
|
|
|
|
|
}
|
|
|
|
|
return contracts.ResponseSchema{ID: schema.ID, Name: schema.Name, Version: schema.Version, JSONSchema: append([]byte(nil), schema.JSONSchema...)}, contracts.RawPayload{Content: content, MediaType: codec.spec.MediaType, Metadata: metadata}, nil
|
|
|
|
|
return contracts.SerializedArtifact{Kind: codec.spec.Kind, Schema: contracts.CloneArtifactSchema(schema), MediaType: codec.spec.MediaType, Content: append([]byte(nil), content...), Metadata: cloneMetadata(metadata)}, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// migrationDecodeArtifact restores a typed value from an existing raw
|
|
|
|
|
// checkpoint envelope through the registered codec.
|
|
|
|
|
func migrationDecodeArtifact(codec artifactCodecEntry, payload contracts.RawPayload) (any, error) {
|
|
|
|
|
return codec.decode(append([]byte(nil), payload.Content...))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func migrationExtractOutput(codec artifactCodecEntry, artifact erasedExtractArtifact) (contracts.ExtractOutput, error) {
|
|
|
|
|
schema, payload, err := migrationRawArtifact(codec, artifact.Value)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return contracts.ExtractOutput{}, err
|
|
|
|
|
func decodeCheckpointArtifact(codec artifactCodecEntry, artifact ArtifactCheckpointOutput) (any, error) {
|
|
|
|
|
expectedDigest := contracts.DigestArtifactSchema(codec.spec.Schema)
|
|
|
|
|
if artifact.Artifact.Kind != codec.spec.Kind {
|
|
|
|
|
return nil, fmt.Errorf("artifact kind %q does not match codec %q", artifact.Artifact.Kind, codec.spec.Kind)
|
|
|
|
|
}
|
|
|
|
|
return contracts.ExtractOutput{LaneID: artifact.LaneID, ExtractorKey: artifact.ExtractorKey, SourceID: artifact.SourceID, ChunkID: artifact.ChunkID, ChunkIndex: artifact.ChunkIndex, Schema: schema, Payload: payload}, nil
|
|
|
|
|
if artifact.Artifact.Schema.ID != codec.spec.Schema.ID || artifact.Artifact.Schema.Version != codec.spec.Schema.Version {
|
|
|
|
|
return nil, fmt.Errorf("artifact schema %q version %q does not match codec schema %q version %q", artifact.Artifact.Schema.ID, artifact.Artifact.Schema.Version, codec.spec.Schema.ID, codec.spec.Schema.Version)
|
|
|
|
|
}
|
|
|
|
|
if artifact.SchemaDigest != expectedDigest {
|
|
|
|
|
return nil, fmt.Errorf("artifact schema digest %q does not match codec schema digest %q", artifact.SchemaDigest, expectedDigest)
|
|
|
|
|
}
|
|
|
|
|
if artifact.Artifact.MediaType != codec.spec.MediaType {
|
|
|
|
|
return nil, fmt.Errorf("artifact media type %q does not match codec media type %q", artifact.Artifact.MediaType, codec.spec.MediaType)
|
|
|
|
|
}
|
|
|
|
|
return codec.decode(append([]byte(nil), artifact.Artifact.Content...))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func migrationMergeOutput(codec artifactCodecEntry, artifact erasedMergeArtifact) (contracts.MergeOutput, error) {
|
|
|
|
|
schema, payload, err := migrationRawArtifact(codec, artifact.Value)
|
|
|
|
|
func checkpointArtifact(codec artifactCodecEntry, laneID, moduleKey, sourceID string, value any) (ArtifactCheckpointOutput, error) {
|
|
|
|
|
serialized, err := serializeArtifact(codec, value, false)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return contracts.MergeOutput{}, err
|
|
|
|
|
return ArtifactCheckpointOutput{}, err
|
|
|
|
|
}
|
|
|
|
|
return contracts.MergeOutput{LaneID: artifact.LaneID, MergerKey: artifact.MergerKey, SourceID: artifact.SourceID, Schema: schema, Payload: payload}, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func migrationNormalizeOutput(codec artifactCodecEntry, laneID, key, sourceID string, value any) (contracts.NormalizeOutput, error) {
|
|
|
|
|
schema, payload, err := migrationRawArtifact(codec, value)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return contracts.NormalizeOutput{}, err
|
|
|
|
|
}
|
|
|
|
|
return contracts.NormalizeOutput{LaneID: laneID, NormalizerKey: key, SourceID: sourceID, Schema: schema, Payload: payload}, nil
|
|
|
|
|
return ArtifactCheckpointOutput{LaneID: laneID, ModuleKey: moduleKey, SourceID: sourceID, Artifact: serialized, SchemaDigest: contracts.DigestArtifactSchema(serialized.Schema)}, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, prepared preparedLaneExecutor, output *RunOutput) error {
|
|
|
|
|
@@ -65,7 +150,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
|
|
|
|
setTypedLaneManifestMetadata(output, lane.ID, typed.extractor, typed.merger, typed.normalizer)
|
|
|
|
|
|
|
|
|
|
values := make([]erasedExtractArtifact, 0, len(chunks))
|
|
|
|
|
rawExtracts := make([]contracts.ExtractOutput, 0, len(chunks))
|
|
|
|
|
serializedExtracts := make([]ArtifactCheckpointOutput, 0, len(chunks))
|
|
|
|
|
extractWarnings := []contracts.Warning{}
|
|
|
|
|
rejectedStart := len(output.Rejected)
|
|
|
|
|
chunksDigest, err := joinedChunkDigest(chunks)
|
|
|
|
|
@@ -73,23 +158,32 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
|
|
|
|
return fmt.Errorf("digest chunks for lane %q: %w", lane.ID, err)
|
|
|
|
|
}
|
|
|
|
|
extractDeps := digestFingerprints("chunks", chunksDigest)
|
|
|
|
|
cp, decision := loader.Extract(lane.ID, lane.Extract.Module, extractDeps)
|
|
|
|
|
cp, decision := loadArtifactExtract(loader, lane.ID, lane.Extract.Module, extractDeps)
|
|
|
|
|
if decision.Reused {
|
|
|
|
|
for _, stored := range cp.Outputs {
|
|
|
|
|
if _, decodeErr := decodeCheckpointArtifact(typed.codec, stored); decodeErr != nil {
|
|
|
|
|
decision = CheckpointDecision{Reason: "extract artifact checkpoint codec is incompatible: " + decodeErr.Error()}
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
recordCheckpointEvent(output, loader, string(StageExtract), lane.ID, lane.Extract.Module, decision)
|
|
|
|
|
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": decision.Reused, "decision": decision, "source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "options": redactSensitiveMap(lane.Extract.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
if decision.Reused {
|
|
|
|
|
for _, raw := range cp.Outputs {
|
|
|
|
|
value, decodeErr := migrationDecodeArtifact(typed.codec, raw.Payload)
|
|
|
|
|
for _, stored := range cp.Outputs {
|
|
|
|
|
value, decodeErr := decodeCheckpointArtifact(typed.codec, stored)
|
|
|
|
|
if decodeErr != nil {
|
|
|
|
|
return fmt.Errorf("decode extract checkpoint for lane %q: %w", lane.ID, decodeErr)
|
|
|
|
|
}
|
|
|
|
|
artifact := erasedExtractArtifact{LaneID: lane.ID, ExtractorKey: lane.Extract.Module, SourceID: doc.ID, ChunkID: raw.ChunkID, ChunkIndex: raw.ChunkIndex, Value: value}
|
|
|
|
|
if raw.ChunkIndex >= 0 && raw.ChunkIndex < len(chunks) {
|
|
|
|
|
artifact.ChunkRef = chunks[raw.ChunkIndex].Ref
|
|
|
|
|
stored = hydrateCheckpointArtifact(typed.codec, stored, value)
|
|
|
|
|
artifact := erasedExtractArtifact{LaneID: lane.ID, ExtractorKey: lane.Extract.Module, SourceID: doc.ID, ChunkID: stored.ChunkID, ChunkIndex: stored.ChunkIndex, ChunkRef: stored.ChunkRef, Value: value}
|
|
|
|
|
if stored.ChunkIndex >= 0 && stored.ChunkIndex < len(chunks) && artifact.ChunkRef == (source.SourceRef{}) {
|
|
|
|
|
artifact.ChunkRef = chunks[stored.ChunkIndex].Ref
|
|
|
|
|
}
|
|
|
|
|
values = append(values, artifact)
|
|
|
|
|
rawExtracts = append(rawExtracts, cloneExtractOutput(raw))
|
|
|
|
|
serializedExtracts = append(serializedExtracts, cloneArtifactCheckpointOutput(stored))
|
|
|
|
|
}
|
|
|
|
|
extractWarnings = cloneWarnings(cp.Warnings)
|
|
|
|
|
output.Warnings = append(output.Warnings, extractWarnings...)
|
|
|
|
|
@@ -101,7 +195,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
|
|
|
|
for i := range chunks {
|
|
|
|
|
chunk := chunks[i]
|
|
|
|
|
var accepted erasedExtractArtifact
|
|
|
|
|
var rawAccepted contracts.ExtractOutput
|
|
|
|
|
var serializedAccepted ArtifactCheckpointOutput
|
|
|
|
|
var acceptedWarnings []contracts.Warning
|
|
|
|
|
ok, rejection, runErr := runWithRetry(ctx, lane.Extract.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
|
|
|
|
started := time.Now().UTC()
|
|
|
|
|
@@ -117,14 +211,14 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
|
|
|
|
if validateErr != nil || rejected != nil {
|
|
|
|
|
return false, rejected, validateErr
|
|
|
|
|
}
|
|
|
|
|
raw, encodeErr := migrationExtractOutput(typed.codec, artifact)
|
|
|
|
|
stored, encodeErr := checkpointArtifact(typed.codec, artifact.LaneID, artifact.ExtractorKey, artifact.SourceID, artifact.Value)
|
|
|
|
|
if encodeErr != nil {
|
|
|
|
|
return false, nil, encodeErr
|
|
|
|
|
}
|
|
|
|
|
raw.Payload.Warnings = append(raw.Payload.Warnings, cloneWarnings(result.Warnings)...)
|
|
|
|
|
accepted, rawAccepted = artifact, raw
|
|
|
|
|
stored.ChunkID, stored.ChunkIndex, stored.ChunkRef = artifact.ChunkID, artifact.ChunkIndex, artifact.ChunkRef
|
|
|
|
|
accepted, serializedAccepted = artifact, stored
|
|
|
|
|
acceptedWarnings = append(cloneWarnings(result.Warnings), warnings...)
|
|
|
|
|
if debugErr := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, Attempt: attempt, StartedAt: started, Payload: map[string]any{"output": debugExtractOutputEnvelope(raw), "warnings": debugWarningEnvelopes(acceptedWarnings)}}, llmScope)); debugErr != nil {
|
|
|
|
|
if debugErr := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, Attempt: attempt, StartedAt: started, Payload: map[string]any{"output": debugArtifactCheckpointOutput(stored), "warnings": debugWarningEnvelopes(acceptedWarnings)}}, llmScope)); debugErr != nil {
|
|
|
|
|
return false, nil, debugErr
|
|
|
|
|
}
|
|
|
|
|
return true, nil, nil
|
|
|
|
|
@@ -138,17 +232,17 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
values = append(values, accepted)
|
|
|
|
|
rawExtracts = append(rawExtracts, rawAccepted)
|
|
|
|
|
serializedExtracts = append(serializedExtracts, serializedAccepted)
|
|
|
|
|
extractWarnings = append(extractWarnings, acceptedWarnings...)
|
|
|
|
|
output.Warnings = append(output.Warnings, acceptedWarnings...)
|
|
|
|
|
}
|
|
|
|
|
if err := checkpoints.ExtractSucceeded(lane.ID, lane.Extract.Module, extractDeps, rawExtracts, cloneRejectedOutputs(output.Rejected[rejectedStart:]), extractWarnings); err != nil {
|
|
|
|
|
if err := recordArtifactExtract(checkpoints, lane.ID, lane.Extract.Module, extractDeps, serializedExtracts, cloneRejectedOutputs(output.Rejected[rejectedStart:]), extractWarnings); err != nil {
|
|
|
|
|
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
sort.SliceStable(values, func(i, j int) bool { return values[i].ChunkIndex < values[j].ChunkIndex })
|
|
|
|
|
sort.SliceStable(rawExtracts, func(i, j int) bool { return rawExtracts[i].ChunkIndex < rawExtracts[j].ChunkIndex })
|
|
|
|
|
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": decision.Reused, "outputs": debugExtractOutputEnvelopes(rawExtracts), "rejected": debugRejectedOutputEnvelopes(output.Rejected[rejectedStart:]), "warnings": debugWarningEnvelopes(extractWarnings)}}); err != nil {
|
|
|
|
|
sort.SliceStable(serializedExtracts, func(i, j int) bool { return serializedExtracts[i].ChunkIndex < serializedExtracts[j].ChunkIndex })
|
|
|
|
|
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": decision.Reused, "outputs": debugArtifactCheckpointOutputs(serializedExtracts), "rejected": debugRejectedOutputEnvelopes(output.Rejected[rejectedStart:]), "warnings": debugWarningEnvelopes(extractWarnings)}}); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
if len(values) == 0 {
|
|
|
|
|
@@ -159,22 +253,27 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
|
|
|
|
for i, value := range values {
|
|
|
|
|
mergeInputs[i] = contracts.ExtractArtifact[any]{LaneID: value.LaneID, ExtractorKey: value.ExtractorKey, SourceID: value.SourceID, ChunkID: value.ChunkID, ChunkIndex: value.ChunkIndex, ChunkRef: value.ChunkRef, Value: value.Value}
|
|
|
|
|
}
|
|
|
|
|
mergeDeps := rawOutputDigests(extractPayloads(rawExtracts))
|
|
|
|
|
mergeCP, mergeDecision := loader.Merge(lane.ID, lane.Merge.Module, mergeDeps)
|
|
|
|
|
mergeDeps := artifactCheckpointDigests(serializedExtracts)
|
|
|
|
|
mergeCP, mergeDecision := loadArtifactMerge(loader, lane.ID, lane.Merge.Module, mergeDeps)
|
|
|
|
|
if mergeDecision.Reused {
|
|
|
|
|
if _, decodeErr := decodeCheckpointArtifact(typed.codec, mergeCP.Output); decodeErr != nil {
|
|
|
|
|
mergeDecision = CheckpointDecision{Reason: "merge artifact checkpoint codec is incompatible: " + decodeErr.Error()}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
recordCheckpointEvent(output, loader, string(StageMerge), lane.ID, lane.Merge.Module, mergeDecision)
|
|
|
|
|
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageMerge), LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": mergeDecision.Reused, "decision": mergeDecision, "source": debugSourceDocumentEnvelope(doc), "extract_outputs": debugExtractOutputEnvelopes(rawExtracts), "options": redactSensitiveMap(lane.Merge.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
|
|
|
|
|
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageMerge), LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": mergeDecision.Reused, "decision": mergeDecision, "source": debugSourceDocumentEnvelope(doc), "extract_outputs": debugArtifactCheckpointOutputs(serializedExtracts), "options": redactSensitiveMap(lane.Merge.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
var merged erasedMergeArtifact
|
|
|
|
|
var rawMerge contracts.MergeOutput
|
|
|
|
|
var serializedMerge ArtifactCheckpointOutput
|
|
|
|
|
var mergeWarnings []contracts.Warning
|
|
|
|
|
if mergeDecision.Reused {
|
|
|
|
|
value, decodeErr := migrationDecodeArtifact(typed.codec, mergeCP.Output.Payload)
|
|
|
|
|
value, decodeErr := decodeCheckpointArtifact(typed.codec, mergeCP.Output)
|
|
|
|
|
if decodeErr != nil {
|
|
|
|
|
return fmt.Errorf("decode merge checkpoint for lane %q: %w", lane.ID, decodeErr)
|
|
|
|
|
}
|
|
|
|
|
merged = erasedMergeArtifact{LaneID: lane.ID, MergerKey: lane.Merge.Module, SourceID: doc.ID, Value: value}
|
|
|
|
|
rawMerge = cloneMergeOutput(mergeCP.Output)
|
|
|
|
|
serializedMerge = hydrateCheckpointArtifact(typed.codec, cloneArtifactCheckpointOutput(mergeCP.Output), value)
|
|
|
|
|
mergeWarnings = cloneWarnings(mergeCP.Warnings)
|
|
|
|
|
output.Warnings = append(output.Warnings, mergeWarnings...)
|
|
|
|
|
} else {
|
|
|
|
|
@@ -191,12 +290,11 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
|
|
|
|
if validateErr != nil || rejected != nil {
|
|
|
|
|
return false, rejected, validateErr
|
|
|
|
|
}
|
|
|
|
|
raw, encodeErr := migrationMergeOutput(typed.codec, candidate)
|
|
|
|
|
stored, encodeErr := checkpointArtifact(typed.codec, candidate.LaneID, candidate.MergerKey, candidate.SourceID, candidate.Value)
|
|
|
|
|
if encodeErr != nil {
|
|
|
|
|
return false, nil, encodeErr
|
|
|
|
|
}
|
|
|
|
|
raw.Payload.Warnings = append(raw.Payload.Warnings, cloneWarnings(result.Warnings)...)
|
|
|
|
|
merged, rawMerge = candidate, raw
|
|
|
|
|
merged, serializedMerge = candidate, stored
|
|
|
|
|
mergeWarnings = append(cloneWarnings(result.Warnings), warnings...)
|
|
|
|
|
return true, nil, nil
|
|
|
|
|
})
|
|
|
|
|
@@ -212,28 +310,33 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
output.Warnings = append(output.Warnings, mergeWarnings...)
|
|
|
|
|
if err := checkpoints.MergeSucceeded(lane.ID, lane.Merge.Module, mergeDeps, rawMerge, mergeWarnings); err != nil {
|
|
|
|
|
if err := recordArtifactMerge(checkpoints, lane.ID, lane.Merge.Module, mergeDeps, serializedMerge, mergeWarnings); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageMerge), LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": mergeDecision.Reused, "accepted": true, "output": debugMergeOutputEnvelope(rawMerge), "warnings": debugWarningEnvelopes(mergeWarnings)}}); err != nil {
|
|
|
|
|
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageMerge), LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": mergeDecision.Reused, "accepted": true, "output": debugArtifactCheckpointOutput(serializedMerge), "warnings": debugWarningEnvelopes(mergeWarnings)}}); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
normalizeDeps := rawOutputDigests([]contracts.RawPayload{rawMerge.Payload})
|
|
|
|
|
normalizeCP, normalizeDecision := loader.Normalize(lane.ID, lane.Normalize.Module, normalizeDeps)
|
|
|
|
|
normalizeDeps := artifactCheckpointDigests([]ArtifactCheckpointOutput{serializedMerge})
|
|
|
|
|
normalizeCP, normalizeDecision := loadArtifactNormalize(loader, lane.ID, lane.Normalize.Module, normalizeDeps)
|
|
|
|
|
if normalizeDecision.Reused {
|
|
|
|
|
if _, decodeErr := decodeCheckpointArtifact(typed.codec, normalizeCP.Output); decodeErr != nil {
|
|
|
|
|
normalizeDecision = CheckpointDecision{Reason: "normalize artifact checkpoint codec is incompatible: " + decodeErr.Error()}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
recordCheckpointEvent(output, loader, string(StageNormalize), lane.ID, lane.Normalize.Module, normalizeDecision)
|
|
|
|
|
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageNormalize), LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": normalizeDecision.Reused, "decision": normalizeDecision, "source": debugSourceDocumentEnvelope(doc), "merge_output": debugMergeOutputEnvelope(rawMerge), "options": redactSensitiveMap(lane.Normalize.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
|
|
|
|
|
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageNormalize), LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": normalizeDecision.Reused, "decision": normalizeDecision, "source": debugSourceDocumentEnvelope(doc), "merge_output": debugArtifactCheckpointOutput(serializedMerge), "options": redactSensitiveMap(lane.Normalize.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
var rawNormalize contracts.NormalizeOutput
|
|
|
|
|
var serializedNormalize ArtifactCheckpointOutput
|
|
|
|
|
var normalizeWarnings []contracts.Warning
|
|
|
|
|
if normalizeDecision.Reused {
|
|
|
|
|
_, decodeErr := migrationDecodeArtifact(typed.codec, normalizeCP.Output.Payload)
|
|
|
|
|
value, decodeErr := decodeCheckpointArtifact(typed.codec, normalizeCP.Output)
|
|
|
|
|
if decodeErr != nil {
|
|
|
|
|
return fmt.Errorf("decode normalize checkpoint for lane %q: %w", lane.ID, decodeErr)
|
|
|
|
|
}
|
|
|
|
|
rawNormalize, normalizeWarnings = cloneNormalizeOutput(normalizeCP.Output), cloneWarnings(normalizeCP.Warnings)
|
|
|
|
|
serializedNormalize, normalizeWarnings = hydrateCheckpointArtifact(typed.codec, cloneArtifactCheckpointOutput(normalizeCP.Output), value), cloneWarnings(normalizeCP.Warnings)
|
|
|
|
|
output.Warnings = append(output.Warnings, normalizeWarnings...)
|
|
|
|
|
} else {
|
|
|
|
|
if err := checkpoints.NormalizeRunning(lane.ID, lane.Normalize.Module, normalizeDeps); err != nil {
|
|
|
|
|
@@ -248,12 +351,11 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
|
|
|
|
if validateErr != nil || rejected != nil {
|
|
|
|
|
return false, rejected, validateErr
|
|
|
|
|
}
|
|
|
|
|
raw, encodeErr := migrationNormalizeOutput(typed.codec, lane.ID, lane.Normalize.Module, doc.ID, result.Value)
|
|
|
|
|
stored, encodeErr := checkpointArtifact(typed.codec, lane.ID, lane.Normalize.Module, doc.ID, result.Value)
|
|
|
|
|
if encodeErr != nil {
|
|
|
|
|
return false, nil, encodeErr
|
|
|
|
|
}
|
|
|
|
|
raw.Payload.Warnings = append(raw.Payload.Warnings, cloneWarnings(result.Warnings)...)
|
|
|
|
|
rawNormalize = raw
|
|
|
|
|
serializedNormalize = stored
|
|
|
|
|
normalizeWarnings = append(cloneWarnings(result.Warnings), warnings...)
|
|
|
|
|
return true, nil, nil
|
|
|
|
|
})
|
|
|
|
|
@@ -269,14 +371,14 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
output.Warnings = append(output.Warnings, normalizeWarnings...)
|
|
|
|
|
if err := checkpoints.NormalizeSucceeded(lane.ID, lane.Normalize.Module, normalizeDeps, rawNormalize, normalizeWarnings); err != nil {
|
|
|
|
|
if err := recordArtifactNormalize(checkpoints, lane.ID, lane.Normalize.Module, normalizeDeps, serializedNormalize, normalizeWarnings); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageNormalize), LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": normalizeDecision.Reused, "accepted": true, "output": debugNormalizeOutputEnvelope(rawNormalize), "warnings": debugWarningEnvelopes(normalizeWarnings)}}); err != nil {
|
|
|
|
|
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageNormalize), LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": normalizeDecision.Reused, "accepted": true, "output": debugArtifactCheckpointOutput(serializedNormalize), "warnings": debugWarningEnvelopes(normalizeWarnings)}}); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
output.NormalizeOutputs = append(output.NormalizeOutputs, rawNormalize)
|
|
|
|
|
output.NormalizeOutputs = append(output.NormalizeOutputs, contracts.SerializedOutput{LaneID: lane.ID, NormalizerKey: lane.Normalize.Module, SourceID: doc.ID, Artifact: contracts.CloneSerializedArtifact(serializedNormalize.Artifact)})
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@@ -318,18 +420,17 @@ func (r *Runner) validateTypedArtifact(ctx context.Context, codec artifactCodecE
|
|
|
|
|
target.llmProfile = binding.LLMProfile
|
|
|
|
|
result, err = item.typedValidate(validatorCtx, item.typed, target)
|
|
|
|
|
case ValidatorTargetSerialized:
|
|
|
|
|
schema, payload, encodeErr := migrationRawArtifact(codec, target.value)
|
|
|
|
|
artifact, encodeErr := serializeArtifact(codec, target.value, true)
|
|
|
|
|
if encodeErr != nil {
|
|
|
|
|
err = encodeErr
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, Source: target.source, SourceID: target.sourceID, SourceInput: target.sourceInput.Clone(), SessionID: target.sessionID, References: CloneReferenceSet(target.references), LLMProfile: binding.LLMProfile, Metadata: cloneMetadata(target.metadata), Chunk: cloneSourceChunkPtr(target.chunk), Chunks: cloneSourceChunks(target.chunks), Schema: contracts.ArtifactSchema{ID: schema.ID, Name: schema.Name, Version: schema.Version, JSONSchema: append([]byte(nil), schema.JSONSchema...)}, MediaType: payload.MediaType, Content: append([]byte(nil), payload.Content...)})
|
|
|
|
|
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, Source: target.source, SourceID: target.sourceID, SourceInput: target.sourceInput.Clone(), SessionID: target.sessionID, References: CloneReferenceSet(target.references), LLMProfile: binding.LLMProfile, Metadata: cloneMetadata(target.metadata), Chunk: cloneSourceChunkPtr(target.chunk), Chunks: cloneSourceChunks(target.chunks), Schema: contracts.CloneArtifactSchema(artifact.Schema), MediaType: artifact.MediaType, Content: append([]byte(nil), artifact.Content...)})
|
|
|
|
|
default:
|
|
|
|
|
return nil, nil, fmt.Errorf("validator %q is incompatible with typed artifact validation", binding.Module)
|
|
|
|
|
}
|
|
|
|
|
schema, payload, _ := migrationRawArtifact(codec, target.value)
|
|
|
|
|
debugRequest := contracts.ValidationRequest{Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, Source: target.source, SourceID: target.sourceID, SourceInput: target.sourceInput.Clone(), SessionID: target.sessionID, References: CloneReferenceSet(target.references), LLMProfile: binding.LLMProfile, Metadata: cloneMetadata(target.metadata), Chunk: cloneSourceChunkPtr(target.chunk), Chunks: cloneSourceChunks(target.chunks), Schema: schema, Payload: payload}
|
|
|
|
|
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: debugValidationRequestEnvelope(debugRequest), Result: debugValidationResultEnvelope(result)}
|
|
|
|
|
artifact, _ := serializeArtifact(codec, target.value, true)
|
|
|
|
|
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(target.stage), "lane_id": target.laneID, "module_key": target.moduleKey, "source_id": target.sourceID, "artifact": debugArtifactCheckpointOutput(ArtifactCheckpointOutput{Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(artifact.Schema)}), "metadata": redactSensitiveMap(target.metadata)}, Result: debugValidationResultEnvelope(result)}
|
|
|
|
|
if err != nil {
|
|
|
|
|
debugCall.Error = err.Error()
|
|
|
|
|
}
|
|
|
|
|
@@ -342,11 +443,11 @@ func (r *Runner) validateTypedArtifact(ctx context.Context, codec artifactCodecE
|
|
|
|
|
if !result.Approved {
|
|
|
|
|
reason := result.ReasonCode
|
|
|
|
|
if reason == "" {
|
|
|
|
|
reason = "raw_output_rejected"
|
|
|
|
|
reason = "artifact_rejected"
|
|
|
|
|
}
|
|
|
|
|
message := result.Message
|
|
|
|
|
if message == "" {
|
|
|
|
|
message = "raw output rejected"
|
|
|
|
|
message = "artifact rejected"
|
|
|
|
|
}
|
|
|
|
|
return nil, &contracts.RejectedOutput{Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, ChunkID: func() string {
|
|
|
|
|
if target.chunk != nil {
|
|
|
|
|
|