diff --git a/docs/integrations/json-output.md b/docs/integrations/json-output.md index f7395dc..e212f5e 100644 --- a/docs/integrations/json-output.md +++ b/docs/integrations/json-output.md @@ -100,8 +100,9 @@ Fields with empty values may be omitted by JSON encoding. `source_digests` contains source document digests only. Bound references are recorded separately under `references`, which contains provenance only: -lane ID, slot name, origin type and URI, digest, media type, byte size, and -binding source. Reference content is not written to durable output. +target stage, lane ID when present, slot name, origin type and URI, digest, +media type, byte size, and binding source. Reference content is not written to +durable output. When references are bound, the manifest section has this shape: @@ -109,6 +110,7 @@ When references are bound, the manifest section has this shape: { "references": [ { + "stage": "extract", "lane_id": "spells", "slot_name": "roster", "origin_type": "file", diff --git a/docs/internal/pipeline.md b/docs/internal/pipeline.md index 1b17ee8..a63e31a 100644 --- a/docs/internal/pipeline.md +++ b/docs/internal/pipeline.md @@ -37,8 +37,9 @@ bindings override or add lane bindings, runtime `--reference` requests override config bindings, and runtime unbinds remove optional bindings. Flat runtime slot names are resolved only when exactly one selected lane declares the slot; otherwise the CLI requires `lane.slot`. Resolution validates bindings against -extractor specs and records lane-scoped binding metadata. It does not read -reference files or include reference bytes in source digests. +extractor specs and stores the bindings in target-aware resolved reference +holders. It does not read reference files or include reference bytes in source +digests. During run preparation, resolved file references are materialized before any LLM-backed pipeline work. Config bindings resolve relative to the config file, @@ -51,8 +52,8 @@ bound files. Media-type acceptance is checked only when a slot declares `AcceptedMediaTypes`; unknown extensions are recorded as `application/octet-stream`. Reference content is omitted from diagnostics and manifests. The CLI writes provenance-only resolved reference diagnostics, and -the run manifest records lane-scoped reference provenance separately from source -digests. +the run manifest records target-stage reference provenance separately from +source digests. Prompt bundles can declare reference slots and use `reference` and `hasreference` template functions. Bundle loading validates string-literal slot diff --git a/docs/operations.md b/docs/operations.md index ef6239f..0ae64cc 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -63,9 +63,9 @@ Implemented diagnostics artifacts: path, selected lanes, run ID, and pipeline digest when available. - `effective-config.json`: resolved config with API keys redacted. - `resolved-pipeline.json`: resolved module bindings and pipeline digest. -- `resolved-references.json`: lane-scoped resolved reference provenance, - including origin, digest, media type, byte size, and binding source, without - reference content. +- `resolved-references.json`: resolved reference provenance, including target + stage, lane ID when present, origin, digest, media type, byte size, and + binding source, without reference content. - `run-manifest.json`: the same run manifest written to durable output when it is available, including top-level module metadata when present. - `warnings.json`: warning list. diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 50e7198..e77ba2f 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -769,7 +769,7 @@ func TestRunPipelineReferenceFlagBindsUnambiguousSlot(t *testing.T) { t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String()) } resolved := readResolvedPipeline(t, diagnosticsDir) - refs := resolved.ArtifactLanes[0].References + refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings want := []pipeline.ReferenceBinding{ {LaneID: "events", SlotName: "roster", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI}, } @@ -810,12 +810,12 @@ func TestRunPipelineReferenceFlagBindsLaneQualifiedSlot(t *testing.T) { } resolved := readResolvedPipeline(t, diagnosticsDir) events := resolvedArtifactLane(t, resolved, "events") - if len(events.References) != 0 { - t.Fatalf("events references = %#v, want none", events.References) + if len(events.ExtractReferences.Bindings) != 0 { + t.Fatalf("events references = %#v, want none", events.ExtractReferences.Bindings) } notes := resolvedArtifactLane(t, resolved, "notes") - if len(notes.References) != 1 || notes.References[0].Source != referencePath { - t.Fatalf("notes references = %#v, want lane-qualified binding", notes.References) + if len(notes.ExtractReferences.Bindings) != 1 || notes.ExtractReferences.Bindings[0].Source != referencePath { + t.Fatalf("notes references = %#v, want lane-qualified binding", notes.ExtractReferences.Bindings) } } @@ -914,7 +914,7 @@ func TestRunPipelineWithoutReferenceRemovesOptionalConfigBinding(t *testing.T) { t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String()) } resolved := readResolvedPipeline(t, diagnosticsDir) - if refs := resolved.ArtifactLanes[0].References; len(refs) != 0 { + if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 { t.Fatalf("references = %#v, want unbound optional slot", refs) } } diff --git a/internal/core/artifacts/artifacts.go b/internal/core/artifacts/artifacts.go index 5e0737d..e03981a 100644 --- a/internal/core/artifacts/artifacts.go +++ b/internal/core/artifacts/artifacts.go @@ -49,7 +49,8 @@ type LLMProfileManifest struct { } type ReferenceProvenance struct { - LaneID string `json:"lane_id"` + Stage string `json:"stage,omitempty"` + LaneID string `json:"lane_id,omitempty"` SlotName string `json:"slot_name"` OriginType string `json:"origin_type"` OriginURI string `json:"origin_uri,omitempty"` diff --git a/internal/core/artifacts/artifacts_test.go b/internal/core/artifacts/artifacts_test.go index 67530f6..cf7faf9 100644 --- a/internal/core/artifacts/artifacts_test.go +++ b/internal/core/artifacts/artifacts_test.go @@ -187,6 +187,7 @@ func TestRunManifestIncludesReferenceProvenance(t *testing.T) { manifest := RunManifest{ References: []ReferenceProvenance{ { + Stage: "extract", LaneID: "events", SlotName: "roster", OriginType: "file", @@ -212,7 +213,7 @@ func TestRunManifestIncludesReferenceProvenance(t *testing.T) { t.Fatalf("len(References) = %d, want 1", len(got.References)) } reference := got.References[0] - if reference.LaneID != "events" || reference.SlotName != "roster" || reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" { + if reference.Stage != "extract" || reference.LaneID != "events" || reference.SlotName != "roster" || reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" { t.Fatalf("reference provenance = %#v, want lane-scoped origin details", reference) } if reference.Digest != "sha256:reference" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != 12 || reference.BindingSource != "config" { diff --git a/internal/core/config/redaction.go b/internal/core/config/redaction.go index ded3ae8..3ab6f50 100644 --- a/internal/core/config/redaction.go +++ b/internal/core/config/redaction.go @@ -34,6 +34,7 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli out := in out.Input = cloneModuleBinding(in.Input) out.Chunk = cloneModuleBinding(in.Chunk) + out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences) out.Output = cloneModuleBinding(in.Output) if len(in.ArtifactLanes) > 0 { out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes)) @@ -49,8 +50,8 @@ func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.Resolv out.Extract = cloneModuleBinding(in.Extract) out.Merge = cloneModuleBinding(in.Merge) out.Normalize = cloneModuleBinding(in.Normalize) - out.References = append([]pipeline.ReferenceBinding(nil), in.References...) - out.ReferenceSet = pipeline.CloneReferenceSet(in.ReferenceSet) + out.ExtractReferences = pipeline.CloneReferenceTarget(in.ExtractReferences) + out.NormalizeReferences = pipeline.CloneReferenceTarget(in.NormalizeReferences) if len(in.Validators) > 0 { out.Validators = make([]pipeline.ModuleBinding, len(in.Validators)) for i, binding := range in.Validators { diff --git a/internal/core/config/redaction_test.go b/internal/core/config/redaction_test.go index d218ac0..687583f 100644 --- a/internal/core/config/redaction_test.go +++ b/internal/core/config/redaction_test.go @@ -108,8 +108,31 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T) if effective.ResolvedPipeline.ArtifactLanes[0].Extract.Options["temperature"] != 0.2 { t.Fatalf("expected resolved pipeline options to be copied") } - payload.ResolvedPipeline.ArtifactLanes[0].References[0].Source = "./changed.yml" - if effective.ResolvedPipeline.ArtifactLanes[0].References[0].Source != "./roster.yml" { + payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source = "./changed.yml" + if effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source != "./roster.yml" { t.Fatalf("expected resolved pipeline references to be copied") } + + effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{ + Slots: map[string]contracts.ResolvedReferenceSlot{ + "roster": { + Slot: contracts.ReferenceSlot{Name: "roster"}, + Items: []contracts.ReferenceItem{ + { + SlotName: "roster", + Content: []byte("reference content"), + }, + }, + }, + }, + } + payload, ok = effective.RedactedDiagnosticsPayload().(EffectiveConfig) + if !ok { + t.Fatalf("expected EffectiveConfig payload, got %T", effective.RedactedDiagnosticsPayload()) + } + payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content[0] = 'X' + got := effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content + if string(got) != "reference content" { + t.Fatalf("expected materialized reference content to be copied, got %q", got) + } } diff --git a/internal/framework/pipeline/profile.go b/internal/framework/pipeline/profile.go index 4654296..8e3fa5a 100644 --- a/internal/framework/pipeline/profile.go +++ b/internal/framework/pipeline/profile.go @@ -60,23 +60,32 @@ type ReferenceUnbind struct { SlotName string `json:"slot_name"` } -type ResolvedArtifactLane struct { - ID string - Extract ModuleBinding - Merge ModuleBinding - Normalize ModuleBinding - Validators []ModuleBinding - References []ReferenceBinding `json:"references,omitempty"` +type ResolvedReferenceTarget struct { + Stage ModuleStage `json:"stage"` + LaneID string `json:"lane_id,omitempty"` + Module string `json:"module"` + Bindings []ReferenceBinding `json:"bindings,omitempty"` ReferenceSet contracts.ReferenceSet `json:"-"` } +type ResolvedArtifactLane struct { + ID string + Extract ModuleBinding + Merge ModuleBinding + Normalize ModuleBinding + Validators []ModuleBinding + ExtractReferences ResolvedReferenceTarget `json:"extract_references"` + NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"` +} + type ResolvedPipeline struct { - ID string - Digest string - Input ModuleBinding - Chunk ModuleBinding - ArtifactLanes []ResolvedArtifactLane - Output ModuleBinding + ID string + Digest string + Input ModuleBinding + Chunk ModuleBinding + ChunkReferences ResolvedReferenceTarget `json:"chunk_references"` + ArtifactLanes []ResolvedArtifactLane + Output ModuleBinding } type ModuleCatalog struct { @@ -140,10 +149,11 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo } resolved := ResolvedPipeline{ - ID: pipelineID, - Input: input, - Chunk: chunk, - Output: resolveBinding(profile.Output, DefaultOutputModule), + ID: pipelineID, + Input: input, + Chunk: chunk, + ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, nil), + Output: resolveBinding(profile.Output, DefaultOutputModule), } outputCapabilities := capabilities.clone() @@ -206,7 +216,7 @@ func resolveArtifactLane( if err != nil { return ResolvedArtifactLane{}, nil, err } - lane.References = references + lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references) capabilities.add(extractSpec.Provides...) mergeSpec, err := mergerSpec(catalog, lane.Merge.Module) @@ -225,6 +235,7 @@ func resolveArtifactLane( if missing, ok := capabilities.missing(normalizeSpec.Requires); ok { return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing) } + lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, nil) capabilities.add(normalizeSpec.Provides...) for _, validator := range lane.Validators { @@ -241,6 +252,15 @@ func resolveArtifactLane( return lane, capabilities, nil } +func referenceTarget(stage ModuleStage, laneID string, module string, bindings []ReferenceBinding) ResolvedReferenceTarget { + return ResolvedReferenceTarget{ + Stage: stage, + LaneID: strings.TrimSpace(laneID), + Module: strings.TrimSpace(module), + Bindings: append([]ReferenceBinding(nil), bindings...), + } +} + func validatePipelineReferenceDefaults( pipelineID string, pipelineReferences map[string]string, @@ -532,17 +552,19 @@ func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneP func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) { withoutDigest := struct { - ID string - Input ModuleBinding - Chunk ModuleBinding - ArtifactLanes []ResolvedArtifactLane - Output ModuleBinding + ID string + Input ModuleBinding + Chunk ModuleBinding + ChunkReferences ResolvedReferenceTarget + ArtifactLanes []ResolvedArtifactLane + Output ModuleBinding }{ - ID: resolved.ID, - Input: resolved.Input, - Chunk: resolved.Chunk, - ArtifactLanes: resolved.ArtifactLanes, - Output: resolved.Output, + ID: resolved.ID, + Input: resolved.Input, + Chunk: resolved.Chunk, + ChunkReferences: resolved.ChunkReferences, + ArtifactLanes: resolved.ArtifactLanes, + Output: resolved.Output, } encoded, err := json.Marshal(withoutDigest) if err != nil { diff --git a/internal/framework/pipeline/profile_test.go b/internal/framework/pipeline/profile_test.go index aa92594..96e08d2 100644 --- a/internal/framework/pipeline/profile_test.go +++ b/internal/framework/pipeline/profile_test.go @@ -154,16 +154,25 @@ func TestResolvePipelineAppliesReferenceBindings(t *testing.T) { } events := resolvedLane(t, resolved.ArtifactLanes, "events") + if events.ExtractReferences.Stage != StageExtract || events.ExtractReferences.LaneID != "events" || events.ExtractReferences.Module != "event-extractor" { + t.Fatalf("extract reference target = %#v, want event extractor target", events.ExtractReferences) + } + if events.NormalizeReferences.Stage != StageNormalize || events.NormalizeReferences.LaneID != "events" || events.NormalizeReferences.Module != DefaultNormalizeModule { + t.Fatalf("normalize reference target = %#v, want event normalizer target", events.NormalizeReferences) + } + if resolved.ChunkReferences.Stage != StageChunk || resolved.ChunkReferences.Module != DefaultChunkModule { + t.Fatalf("chunk reference target = %#v, want chunk target", resolved.ChunkReferences) + } want := []ReferenceBinding{ {LaneID: "events", SlotName: "lore", Source: "./lore.md", BindingSource: contracts.ReferenceBindingSourceConfig}, {LaneID: "events", SlotName: "roster", Source: "./lane-roster.yml", BindingSource: contracts.ReferenceBindingSourceConfig}, } - if !reflect.DeepEqual(events.References, want) { - t.Fatalf("events references = %#v, want %#v", events.References, want) + if !reflect.DeepEqual(events.ExtractReferences.Bindings, want) { + t.Fatalf("events references = %#v, want %#v", events.ExtractReferences.Bindings, want) } summaries := resolvedLane(t, resolved.ArtifactLanes, "summaries") - if len(summaries.References) != 0 { - t.Fatalf("summaries references = %#v, want none", summaries.References) + if len(summaries.ExtractReferences.Bindings) != 0 { + t.Fatalf("summaries references = %#v, want none", summaries.ExtractReferences.Bindings) } } @@ -184,7 +193,7 @@ func TestResolvePipelineAllowsPipelineReferenceDeclaredOnlyByUnselectedLane(t *t if err != nil { t.Fatalf("ResolvePipeline() error = %v, want nil", err) } - if refs := resolved.ArtifactLanes[0].References; len(refs) != 0 { + if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 { t.Fatalf("selected lane references = %#v, want none", refs) } } @@ -284,7 +293,7 @@ func TestResolvePipelineUsesReferenceSlotsFromSpecWithoutConstructingExtractor(t if err != nil { t.Fatalf("ResolvePipeline() error = %v, want nil", err) } - if got := resolved.ArtifactLanes[0].References[0].Source; got != "./roster.yml" { + if got := resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].Source; got != "./roster.yml" { t.Fatalf("reference source = %q, want ./roster.yml", got) } } diff --git a/internal/framework/pipeline/references.go b/internal/framework/pipeline/references.go index 9814428..de2199b 100644 --- a/internal/framework/pipeline/references.go +++ b/internal/framework/pipeline/references.go @@ -29,6 +29,7 @@ type ReferenceMaterializationOptions struct { func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, options ReferenceMaterializationOptions) (ResolvedPipeline, []contracts.Warning, error) { out := resolved + out.ChunkReferences = CloneReferenceTarget(resolved.ChunkReferences) if len(resolved.ArtifactLanes) == 0 { return out, nil, nil } @@ -37,32 +38,31 @@ func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, opt out.ArtifactLanes = make([]ResolvedArtifactLane, len(resolved.ArtifactLanes)) for i, lane := range resolved.ArtifactLanes { materializedLane := lane - referenceSet, laneWarnings, err := materializeLaneReferences(resolved.ID, lane, catalog, options) + materializedLane.ExtractReferences = CloneReferenceTarget(lane.ExtractReferences) + materializedLane.NormalizeReferences = CloneReferenceTarget(lane.NormalizeReferences) + referenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.ExtractReferences, catalog, options) if err != nil { return ResolvedPipeline{}, nil, err } - materializedLane.ReferenceSet = referenceSet + materializedLane.ExtractReferences.ReferenceSet = referenceSet out.ArtifactLanes[i] = materializedLane warnings = append(warnings, laneWarnings...) } return out, warnings, nil } -func materializeLaneReferences( +func materializeReferenceTarget( pipelineID string, - lane ResolvedArtifactLane, + target ResolvedReferenceTarget, catalog ModuleCatalog, options ReferenceMaterializationOptions, ) (contracts.ReferenceSet, []contracts.Warning, error) { - if len(lane.References) == 0 { + if len(target.Bindings) == 0 { return contracts.ReferenceSet{}, nil, nil } - if catalog.Extractors == nil { - return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", pipelineID, lane.ID, lane.Extract.Module, lane.Extract.Module) - } - spec, ok := catalog.Extractors.Spec(lane.Extract.Module) - if !ok { - return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", pipelineID, lane.ID, lane.Extract.Module, lane.Extract.Module) + spec, err := referenceTargetSpec(target, catalog) + if err != nil { + return contracts.ReferenceSet{}, nil, fmt.Errorf("%s: %w", referenceTargetContext(pipelineID, target), err) } slotByName := make(map[string]contracts.ReferenceSlot, len(spec.ReferenceSlots)) @@ -70,38 +70,38 @@ func materializeLaneReferences( slotByName[slot.Name] = slot } - set := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(lane.References))} + set := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(target.Bindings))} var warnings []contracts.Warning - for _, binding := range lane.References { + for _, binding := range target.Bindings { slotName := strings.TrimSpace(binding.SlotName) slot, ok := slotByName[slotName] if !ok { - return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q is not declared by extractor %q", pipelineID, lane.ID, slotName, lane.Extract.Module) + return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q is not declared by %s module %q", referenceTargetContext(pipelineID, target), slotName, target.Stage, target.Module) } path, err := referencePath(binding, options) if err != nil { - return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q: %w", pipelineID, lane.ID, slotName, binding.Source, err) + return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q: %w", referenceTargetContext(pipelineID, target), slotName, binding.Source, err) } content, err := os.ReadFile(path) if err != nil { - return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q read %q: %w", pipelineID, lane.ID, slotName, path, err) + return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q read %q: %w", referenceTargetContext(pipelineID, target), slotName, path, err) } if !utf8.Valid(content) { - return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q must be UTF-8 text", pipelineID, lane.ID, slotName, path) + return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q must be UTF-8 text", referenceTargetContext(pipelineID, target), slotName, path) } mediaType := referenceMediaTypeForPath(path) if !referenceMediaTypeAccepted(mediaType, slot.AcceptedMediaTypes) { - return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q media type %q is not accepted", pipelineID, lane.ID, slotName, path, mediaType) + return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q media type %q is not accepted", referenceTargetContext(pipelineID, target), slotName, path, mediaType) } if slot.MaxBytes > 0 && int64(len(content)) > slot.MaxBytes { - return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q is %d bytes, limit %d", pipelineID, lane.ID, slotName, path, len(content), slot.MaxBytes) + return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q is %d bytes, limit %d", referenceTargetContext(pipelineID, target), slotName, path, len(content), slot.MaxBytes) } if len(content) == 0 { warnings = append(warnings, contracts.Warning{ - Scope: fmt.Sprintf("pipeline.%s.lane.%s.reference.%s", pipelineID, lane.ID, slotName), + Scope: referenceWarningScope(pipelineID, target, slotName), ReasonCode: "empty_reference", - Message: fmt.Sprintf("reference slot %q for lane %q is bound to an empty file", slotName, lane.ID), + Message: fmt.Sprintf("reference slot %q for %s is bound to an empty file", slotName, referenceTargetLabel(target)), }) } @@ -122,6 +122,40 @@ func materializeLaneReferences( return set, warnings, nil } +func referenceTargetSpec(target ResolvedReferenceTarget, catalog ModuleCatalog) (ModuleSpec, error) { + switch target.Stage { + case StageChunk: + return registrySpec(catalog.Chunkers, target.Module) + case StageExtract: + return registrySpec(catalog.Extractors, target.Module) + case StageNormalize: + return registrySpec(catalog.Normalizers, target.Module) + default: + return ModuleSpec{}, fmt.Errorf("reference target stage %q is not supported", target.Stage) + } +} + +func referenceTargetContext(pipelineID string, target ResolvedReferenceTarget) string { + if target.LaneID != "" { + return fmt.Sprintf("pipeline %q lane %q %s module %q", pipelineID, target.LaneID, target.Stage, target.Module) + } + return fmt.Sprintf("pipeline %q %s module %q", pipelineID, target.Stage, target.Module) +} + +func referenceTargetLabel(target ResolvedReferenceTarget) string { + if target.LaneID != "" { + return fmt.Sprintf("lane %q %s target", target.LaneID, target.Stage) + } + return fmt.Sprintf("%s target", target.Stage) +} + +func referenceWarningScope(pipelineID string, target ResolvedReferenceTarget, slotName string) string { + if target.LaneID != "" { + return fmt.Sprintf("pipeline.%s.lane.%s.%s.reference.%s", pipelineID, target.LaneID, target.Stage, slotName) + } + return fmt.Sprintf("pipeline.%s.%s.reference.%s", pipelineID, target.Stage, slotName) +} + func referenceMediaTypeForPath(path string) string { extension := strings.ToLower(filepath.Ext(path)) mediaType := mime.TypeByExtension(extension) @@ -230,31 +264,47 @@ func CloneReferenceSet(in contracts.ReferenceSet) contracts.ReferenceSet { return out } +func CloneReferenceTarget(in ResolvedReferenceTarget) ResolvedReferenceTarget { + out := in + out.Bindings = append([]ReferenceBinding(nil), in.Bindings...) + out.ReferenceSet = CloneReferenceSet(in.ReferenceSet) + return out +} + func ReferenceProvenance(resolved ResolvedPipeline) []artifacts.ReferenceProvenance { provenance := []artifacts.ReferenceProvenance{} + provenance = append(provenance, referenceTargetProvenance(resolved.ChunkReferences)...) for _, lane := range resolved.ArtifactLanes { - if len(lane.ReferenceSet.Slots) == 0 { - continue - } - slotNames := make([]string, 0, len(lane.ReferenceSet.Slots)) - for slotName := range lane.ReferenceSet.Slots { - slotNames = append(slotNames, slotName) - } - sort.Strings(slotNames) - for _, slotName := range slotNames { - slot := lane.ReferenceSet.Slots[slotName] - for _, item := range slot.Items { - provenance = append(provenance, artifacts.ReferenceProvenance{ - LaneID: lane.ID, - SlotName: item.SlotName, - OriginType: item.Origin.Type, - OriginURI: item.Origin.URI, - Digest: item.Digest, - MediaType: item.MediaType, - SizeBytes: item.SizeBytes, - BindingSource: item.BindingSource, - }) - } + provenance = append(provenance, referenceTargetProvenance(lane.ExtractReferences)...) + provenance = append(provenance, referenceTargetProvenance(lane.NormalizeReferences)...) + } + return provenance +} + +func referenceTargetProvenance(target ResolvedReferenceTarget) []artifacts.ReferenceProvenance { + if len(target.ReferenceSet.Slots) == 0 { + return nil + } + provenance := []artifacts.ReferenceProvenance{} + slotNames := make([]string, 0, len(target.ReferenceSet.Slots)) + for slotName := range target.ReferenceSet.Slots { + slotNames = append(slotNames, slotName) + } + sort.Strings(slotNames) + for _, slotName := range slotNames { + slot := target.ReferenceSet.Slots[slotName] + for _, item := range slot.Items { + provenance = append(provenance, artifacts.ReferenceProvenance{ + Stage: string(target.Stage), + LaneID: target.LaneID, + SlotName: item.SlotName, + OriginType: item.Origin.Type, + OriginURI: item.Origin.URI, + Digest: item.Digest, + MediaType: item.MediaType, + SizeBytes: item.SizeBytes, + BindingSource: item.BindingSource, + }) } } return provenance diff --git a/internal/framework/pipeline/references_test.go b/internal/framework/pipeline/references_test.go index a4e4d0d..d529bfb 100644 --- a/internal/framework/pipeline/references_test.go +++ b/internal/framework/pipeline/references_test.go @@ -54,12 +54,12 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) { t.Fatalf("MaterializeReferences(second) error = %v, want nil", err) } - referenceSet := first.ArtifactLanes[0].ReferenceSet + referenceSet := first.ArtifactLanes[0].ExtractReferences.ReferenceSet roster := referenceSet.Slots["roster"].Items[0] if string(roster.Content) != "config text" { t.Fatalf("roster content = %q, want config text", roster.Content) } - if roster.Digest != referenceDigest([]byte("config text")) || roster.Digest != second.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0].Digest { + if roster.Digest != referenceDigest([]byte("config text")) || roster.Digest != second.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Digest { t.Fatalf("roster digest = %q, want stable digest", roster.Digest) } if roster.BindingSource != contracts.ReferenceBindingSourceConfig { @@ -87,10 +87,10 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) { if len(provenance) != 2 { t.Fatalf("ReferenceProvenance() = %#v, want two entries", provenance) } - if provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest { + if provenance[0].Stage != string(StageExtract) || provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest { t.Fatalf("ReferenceProvenance()[0] = %#v, want sorted glossary provenance", provenance[0]) } - if provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest { + if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest { t.Fatalf("ReferenceProvenance()[1] = %#v, want roster provenance", provenance[1]) } @@ -129,7 +129,7 @@ func TestMaterializeReferencesAllowsAnyMediaTypeWhenSlotDoesNotRestrictIt(t *tes if err != nil { t.Fatalf("MaterializeReferences() error = %v, want nil", err) } - item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0] + item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0] if item.MediaType != unknownMediaType { t.Fatalf("MediaType = %q, want %q", item.MediaType, unknownMediaType) } @@ -148,7 +148,7 @@ func TestMaterializeReferencesAcceptsDeclaredMarkdownMediaType(t *testing.T) { if err != nil { t.Fatalf("MaterializeReferences() error = %v, want nil", err) } - item := materialized.ArtifactLanes[0].ReferenceSet.Slots["glossary"].Items[0] + item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["glossary"].Items[0] if item.MediaType != "text/markdown" { t.Fatalf("MediaType = %q, want text/markdown", item.MediaType) } @@ -167,7 +167,7 @@ func TestMaterializeReferencesAcceptsDeclaredJSONMediaType(t *testing.T) { if err != nil { t.Fatalf("MaterializeReferences() error = %v, want nil", err) } - item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0] + item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0] if item.MediaType != "application/json" { t.Fatalf("MediaType = %q, want application/json", item.MediaType) } @@ -201,7 +201,7 @@ func TestMaterializeReferencesMatchesAcceptedMediaTypesIgnoringParameters(t *tes if err != nil { t.Fatalf("MaterializeReferences() error = %v, want nil", err) } - item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0] + item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0] if item.MediaType != referenceMediaType { t.Fatalf("MediaType = %q, want %q", item.MediaType, referenceMediaType) } @@ -222,7 +222,7 @@ func TestMaterializeReferencesWarnsForEmptyFiles(t *testing.T) { if len(warnings) != 1 || warnings[0].ReasonCode != "empty_reference" { t.Fatalf("warnings = %#v, want empty reference warning", warnings) } - item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0] + item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0] if item.SizeBytes != 0 || item.Digest != referenceDigest(nil) { t.Fatalf("empty item = %#v, want zero size and empty digest", item) } @@ -255,7 +255,7 @@ func resolvedPipelineWithReference(t *testing.T, slotName, source, bindingSource t.Fatalf("ResolvePipeline() error = %v, want nil", err) } if bindingSource != contracts.ReferenceBindingSourceConfig { - resolved.ArtifactLanes[0].References[0].BindingSource = bindingSource + resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].BindingSource = bindingSource } return resolved } diff --git a/internal/framework/pipeline/runner.go b/internal/framework/pipeline/runner.go index 1136a86..6a6d5ff 100644 --- a/internal/framework/pipeline/runner.go +++ b/internal/framework/pipeline/runner.go @@ -186,7 +186,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source result, err := extractor.Extract(ctx, contracts.ExtractionRequest{ Source: doc, Chunk: &chunk, - References: CloneReferenceSet(lane.ReferenceSet), + References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet), LLMClient: input.LLMClient, LLMProfile: lane.Extract.LLMProfile, Options: cloneOptions(lane.Extract.Options), diff --git a/internal/framework/pipeline/runner_test.go b/internal/framework/pipeline/runner_test.go index 0d2bc75..63d77c4 100644 --- a/internal/framework/pipeline/runner_test.go +++ b/internal/framework/pipeline/runner_test.go @@ -566,7 +566,7 @@ func TestRunPassesModuleBindingConfigToStageRequests(t *testing.T) { func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) { modules := defaultRunnerModules() pipeline := resolvedPipeline() - pipeline.ArtifactLanes[0].ReferenceSet = contracts.ReferenceSet{ + pipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{ Slots: map[string]contracts.ResolvedReferenceSlot{ "roster": { Slot: contracts.ReferenceSlot{Name: "roster"}, @@ -596,7 +596,7 @@ func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) { t.Fatalf("reference content = %q, want reference text", item.Content) } item.Content[0] = 'R' - if got := string(pipeline.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0].Content); got != "reference text" { + if got := string(pipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content); got != "reference text" { t.Fatalf("runner mutated reference set content = %q", got) } } @@ -965,7 +965,7 @@ func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) { func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) { resolved := resolvedPipelineWithValidators("configured") - resolved.ArtifactLanes[0].ReferenceSet = contracts.ReferenceSet{ + resolved.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{ Slots: map[string]contracts.ResolvedReferenceSlot{ "roster": { Slot: contracts.ReferenceSlot{Name: "roster"}, @@ -1003,7 +1003,7 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) { t.Fatalf("References = %#v, want one reference provenance entry", manifest.References) } reference := manifest.References[0] - if reference.LaneID != "alpha" || reference.SlotName != "roster" || reference.Digest != "sha256:reference" { + if reference.Stage != string(StageExtract) || reference.LaneID != "alpha" || reference.SlotName != "roster" || reference.Digest != "sha256:reference" { t.Fatalf("reference provenance = %#v, want lane slot digest", reference) } if reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != int64(len("reference content")) || reference.BindingSource != contracts.ReferenceBindingSourceConfig { @@ -1161,16 +1161,19 @@ func TestRunRejectsNilDefaultValidator(t *testing.T) { func resolvedPipeline() ResolvedPipeline { return ResolvedPipeline{ - ID: "pipeline-1", - Digest: "sha256:pipeline", - Input: Binding("input"), - Chunk: Binding("chunk"), + ID: "pipeline-1", + Digest: "sha256:pipeline", + Input: Binding("input"), + Chunk: Binding("chunk"), + ChunkReferences: referenceTarget(StageChunk, "", "chunk", nil), ArtifactLanes: []ResolvedArtifactLane{ { - ID: "alpha", - Extract: Binding("extract-alpha"), - Merge: Binding("merge"), - Normalize: Binding("normalize"), + ID: "alpha", + Extract: Binding("extract-alpha"), + Merge: Binding("merge"), + Normalize: Binding("normalize"), + ExtractReferences: referenceTarget(StageExtract, "alpha", "extract-alpha", nil), + NormalizeReferences: referenceTarget(StageNormalize, "alpha", "normalize", nil), }, }, Output: Binding("output"), diff --git a/internal/framework/pipeline/testdata/walking_skeleton_output.json b/internal/framework/pipeline/testdata/walking_skeleton_output.json index 1258f1c..5b61cb6 100644 --- a/internal/framework/pipeline/testdata/walking_skeleton_output.json +++ b/internal/framework/pipeline/testdata/walking_skeleton_output.json @@ -1,7 +1,7 @@ { "manifest": { "pipeline_id": "walking-skeleton", - "pipeline_digest": "sha256:5df1e501a2307ef75bbfeb59d315b3710571d52e5466a9c7f8320248740e6fca", + "pipeline_digest": "sha256:25084e39a0cadace375c896551d1752413755c1a6772f6b24cfe91c029ea2631", "validation_status": "approved", "artifact_lanes": [ { diff --git a/internal/modules/extract/dnd/spells/runner_test.go b/internal/modules/extract/dnd/spells/runner_test.go index dc36b44..90c010f 100644 --- a/internal/modules/extract/dnd/spells/runner_test.go +++ b/internal/modules/extract/dnd/spells/runner_test.go @@ -114,7 +114,7 @@ func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) raw := readDNDSpellsFixture(t) expectedDoc := parseDNDSpellsFixture(t, raw) resolved := resolveDNDSpellsPipeline(t) - resolved.ResolvedPipeline.ArtifactLanes[0].ReferenceSet = dndSpellsReferenceSet( + resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet( "Aria: party cleric\nBorin: fighter", "Fire Bolt: evocation cantrip", ) @@ -168,7 +168,7 @@ func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) func TestRunnerDoesNotExtractSpellMentionedOnlyInRoster(t *testing.T) { raw := readDNDSpellsFixture(t) resolved := resolveDNDSpellsPipeline(t) - resolved.ResolvedPipeline.ArtifactLanes[0].ReferenceSet = dndSpellsReferenceSet( + resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet( "Mira: wizard who can cast Lightning Bolt", "", ) diff --git a/internal/modules/output/json/encoder_test.go b/internal/modules/output/json/encoder_test.go index f583b13..9d00cfd 100644 --- a/internal/modules/output/json/encoder_test.go +++ b/internal/modules/output/json/encoder_test.go @@ -148,6 +148,7 @@ func TestEncodeIncludesManifestReferences(t *testing.T) { RunID: "run-1", References: []artifacts.ReferenceProvenance{ { + Stage: "extract", LaneID: "events", SlotName: "roster", OriginType: "file",