Introduce target-aware resolved reference storage

This commit is contained in:
2026-07-05 16:12:38 +00:00
parent 39e49d7f77
commit 9278797aa9
17 changed files with 238 additions and 124 deletions

View File

@@ -100,8 +100,9 @@ Fields with empty values may be omitted by JSON encoding.
`source_digests` contains source document digests only. Bound references are `source_digests` contains source document digests only. Bound references are
recorded separately under `references`, which contains provenance only: recorded separately under `references`, which contains provenance only:
lane ID, slot name, origin type and URI, digest, media type, byte size, and target stage, lane ID when present, slot name, origin type and URI, digest,
binding source. Reference content is not written to durable output. media type, byte size, and binding source. Reference content is not written to
durable output.
When references are bound, the manifest section has this shape: When references are bound, the manifest section has this shape:
@@ -109,6 +110,7 @@ When references are bound, the manifest section has this shape:
{ {
"references": [ "references": [
{ {
"stage": "extract",
"lane_id": "spells", "lane_id": "spells",
"slot_name": "roster", "slot_name": "roster",
"origin_type": "file", "origin_type": "file",

View File

@@ -37,8 +37,9 @@ bindings override or add lane bindings, runtime `--reference` requests override
config bindings, and runtime unbinds remove optional bindings. Flat runtime slot config bindings, and runtime unbinds remove optional bindings. Flat runtime slot
names are resolved only when exactly one selected lane declares the slot; names are resolved only when exactly one selected lane declares the slot;
otherwise the CLI requires `lane.slot`. Resolution validates bindings against otherwise the CLI requires `lane.slot`. Resolution validates bindings against
extractor specs and records lane-scoped binding metadata. It does not read extractor specs and stores the bindings in target-aware resolved reference
reference files or include reference bytes in source digests. holders. It does not read reference files or include reference bytes in source
digests.
During run preparation, resolved file references are materialized before any During run preparation, resolved file references are materialized before any
LLM-backed pipeline work. Config bindings resolve relative to the config file, LLM-backed pipeline work. Config bindings resolve relative to the config file,
@@ -51,8 +52,8 @@ bound files. Media-type acceptance is checked only when a slot declares
`AcceptedMediaTypes`; unknown extensions are recorded as `AcceptedMediaTypes`; unknown extensions are recorded as
`application/octet-stream`. Reference content is omitted from diagnostics and `application/octet-stream`. Reference content is omitted from diagnostics and
manifests. The CLI writes provenance-only resolved reference diagnostics, and manifests. The CLI writes provenance-only resolved reference diagnostics, and
the run manifest records lane-scoped reference provenance separately from source the run manifest records target-stage reference provenance separately from
digests. source digests.
Prompt bundles can declare reference slots and use `reference` and Prompt bundles can declare reference slots and use `reference` and
`hasreference` template functions. Bundle loading validates string-literal slot `hasreference` template functions. Bundle loading validates string-literal slot

View File

@@ -63,9 +63,9 @@ Implemented diagnostics artifacts:
path, selected lanes, run ID, and pipeline digest when available. path, selected lanes, run ID, and pipeline digest when available.
- `effective-config.json`: resolved config with API keys redacted. - `effective-config.json`: resolved config with API keys redacted.
- `resolved-pipeline.json`: resolved module bindings and pipeline digest. - `resolved-pipeline.json`: resolved module bindings and pipeline digest.
- `resolved-references.json`: lane-scoped resolved reference provenance, - `resolved-references.json`: resolved reference provenance, including target
including origin, digest, media type, byte size, and binding source, without stage, lane ID when present, origin, digest, media type, byte size, and
reference content. binding source, without reference content.
- `run-manifest.json`: the same run manifest written to durable output when it - `run-manifest.json`: the same run manifest written to durable output when it
is available, including top-level module metadata when present. is available, including top-level module metadata when present.
- `warnings.json`: warning list. - `warnings.json`: warning list.

View File

@@ -769,7 +769,7 @@ func TestRunPipelineReferenceFlagBindsUnambiguousSlot(t *testing.T) {
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String()) t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
} }
resolved := readResolvedPipeline(t, diagnosticsDir) resolved := readResolvedPipeline(t, diagnosticsDir)
refs := resolved.ArtifactLanes[0].References refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings
want := []pipeline.ReferenceBinding{ want := []pipeline.ReferenceBinding{
{LaneID: "events", SlotName: "roster", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI}, {LaneID: "events", SlotName: "roster", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI},
} }
@@ -810,12 +810,12 @@ func TestRunPipelineReferenceFlagBindsLaneQualifiedSlot(t *testing.T) {
} }
resolved := readResolvedPipeline(t, diagnosticsDir) resolved := readResolvedPipeline(t, diagnosticsDir)
events := resolvedArtifactLane(t, resolved, "events") events := resolvedArtifactLane(t, resolved, "events")
if len(events.References) != 0 { if len(events.ExtractReferences.Bindings) != 0 {
t.Fatalf("events references = %#v, want none", events.References) t.Fatalf("events references = %#v, want none", events.ExtractReferences.Bindings)
} }
notes := resolvedArtifactLane(t, resolved, "notes") notes := resolvedArtifactLane(t, resolved, "notes")
if len(notes.References) != 1 || notes.References[0].Source != referencePath { if len(notes.ExtractReferences.Bindings) != 1 || notes.ExtractReferences.Bindings[0].Source != referencePath {
t.Fatalf("notes references = %#v, want lane-qualified binding", notes.References) t.Fatalf("notes references = %#v, want lane-qualified binding", notes.ExtractReferences.Bindings)
} }
} }
@@ -914,7 +914,7 @@ func TestRunPipelineWithoutReferenceRemovesOptionalConfigBinding(t *testing.T) {
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String()) t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
} }
resolved := readResolvedPipeline(t, diagnosticsDir) resolved := readResolvedPipeline(t, diagnosticsDir)
if refs := resolved.ArtifactLanes[0].References; len(refs) != 0 { if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("references = %#v, want unbound optional slot", refs) t.Fatalf("references = %#v, want unbound optional slot", refs)
} }
} }

View File

@@ -49,7 +49,8 @@ type LLMProfileManifest struct {
} }
type ReferenceProvenance struct { type ReferenceProvenance struct {
LaneID string `json:"lane_id"` Stage string `json:"stage,omitempty"`
LaneID string `json:"lane_id,omitempty"`
SlotName string `json:"slot_name"` SlotName string `json:"slot_name"`
OriginType string `json:"origin_type"` OriginType string `json:"origin_type"`
OriginURI string `json:"origin_uri,omitempty"` OriginURI string `json:"origin_uri,omitempty"`

View File

@@ -187,6 +187,7 @@ func TestRunManifestIncludesReferenceProvenance(t *testing.T) {
manifest := RunManifest{ manifest := RunManifest{
References: []ReferenceProvenance{ References: []ReferenceProvenance{
{ {
Stage: "extract",
LaneID: "events", LaneID: "events",
SlotName: "roster", SlotName: "roster",
OriginType: "file", OriginType: "file",
@@ -212,7 +213,7 @@ func TestRunManifestIncludesReferenceProvenance(t *testing.T) {
t.Fatalf("len(References) = %d, want 1", len(got.References)) t.Fatalf("len(References) = %d, want 1", len(got.References))
} }
reference := got.References[0] reference := got.References[0]
if reference.LaneID != "events" || reference.SlotName != "roster" || reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" { if reference.Stage != "extract" || reference.LaneID != "events" || reference.SlotName != "roster" || reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" {
t.Fatalf("reference provenance = %#v, want lane-scoped origin details", reference) t.Fatalf("reference provenance = %#v, want lane-scoped origin details", reference)
} }
if reference.Digest != "sha256:reference" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != 12 || reference.BindingSource != "config" { if reference.Digest != "sha256:reference" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != 12 || reference.BindingSource != "config" {

View File

@@ -34,6 +34,7 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli
out := in out := in
out.Input = cloneModuleBinding(in.Input) out.Input = cloneModuleBinding(in.Input)
out.Chunk = cloneModuleBinding(in.Chunk) out.Chunk = cloneModuleBinding(in.Chunk)
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)
out.Output = cloneModuleBinding(in.Output) out.Output = cloneModuleBinding(in.Output)
if len(in.ArtifactLanes) > 0 { if len(in.ArtifactLanes) > 0 {
out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes)) out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes))
@@ -49,8 +50,8 @@ func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.Resolv
out.Extract = cloneModuleBinding(in.Extract) out.Extract = cloneModuleBinding(in.Extract)
out.Merge = cloneModuleBinding(in.Merge) out.Merge = cloneModuleBinding(in.Merge)
out.Normalize = cloneModuleBinding(in.Normalize) out.Normalize = cloneModuleBinding(in.Normalize)
out.References = append([]pipeline.ReferenceBinding(nil), in.References...) out.ExtractReferences = pipeline.CloneReferenceTarget(in.ExtractReferences)
out.ReferenceSet = pipeline.CloneReferenceSet(in.ReferenceSet) out.NormalizeReferences = pipeline.CloneReferenceTarget(in.NormalizeReferences)
if len(in.Validators) > 0 { if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators)) out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
for i, binding := range in.Validators { for i, binding := range in.Validators {

View File

@@ -108,8 +108,31 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.Options["temperature"] != 0.2 { if effective.ResolvedPipeline.ArtifactLanes[0].Extract.Options["temperature"] != 0.2 {
t.Fatalf("expected resolved pipeline options to be copied") t.Fatalf("expected resolved pipeline options to be copied")
} }
payload.ResolvedPipeline.ArtifactLanes[0].References[0].Source = "./changed.yml" payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source = "./changed.yml"
if effective.ResolvedPipeline.ArtifactLanes[0].References[0].Source != "./roster.yml" { if effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source != "./roster.yml" {
t.Fatalf("expected resolved pipeline references to be copied") t.Fatalf("expected resolved pipeline references to be copied")
} }
effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{
SlotName: "roster",
Content: []byte("reference content"),
},
},
},
},
}
payload, ok = effective.RedactedDiagnosticsPayload().(EffectiveConfig)
if !ok {
t.Fatalf("expected EffectiveConfig payload, got %T", effective.RedactedDiagnosticsPayload())
}
payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content[0] = 'X'
got := effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content
if string(got) != "reference content" {
t.Fatalf("expected materialized reference content to be copied, got %q", got)
}
} }

View File

@@ -60,14 +60,22 @@ type ReferenceUnbind struct {
SlotName string `json:"slot_name"` SlotName string `json:"slot_name"`
} }
type ResolvedReferenceTarget struct {
Stage ModuleStage `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
Module string `json:"module"`
Bindings []ReferenceBinding `json:"bindings,omitempty"`
ReferenceSet contracts.ReferenceSet `json:"-"`
}
type ResolvedArtifactLane struct { type ResolvedArtifactLane struct {
ID string ID string
Extract ModuleBinding Extract ModuleBinding
Merge ModuleBinding Merge ModuleBinding
Normalize ModuleBinding Normalize ModuleBinding
Validators []ModuleBinding Validators []ModuleBinding
References []ReferenceBinding `json:"references,omitempty"` ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
ReferenceSet contracts.ReferenceSet `json:"-"` NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
} }
type ResolvedPipeline struct { type ResolvedPipeline struct {
@@ -75,6 +83,7 @@ type ResolvedPipeline struct {
Digest string Digest string
Input ModuleBinding Input ModuleBinding
Chunk ModuleBinding Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
ArtifactLanes []ResolvedArtifactLane ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding Output ModuleBinding
} }
@@ -143,6 +152,7 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
ID: pipelineID, ID: pipelineID,
Input: input, Input: input,
Chunk: chunk, Chunk: chunk,
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, nil),
Output: resolveBinding(profile.Output, DefaultOutputModule), Output: resolveBinding(profile.Output, DefaultOutputModule),
} }
outputCapabilities := capabilities.clone() outputCapabilities := capabilities.clone()
@@ -206,7 +216,7 @@ func resolveArtifactLane(
if err != nil { if err != nil {
return ResolvedArtifactLane{}, nil, err return ResolvedArtifactLane{}, nil, err
} }
lane.References = references lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
capabilities.add(extractSpec.Provides...) capabilities.add(extractSpec.Provides...)
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module) mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
@@ -225,6 +235,7 @@ func resolveArtifactLane(
if missing, ok := capabilities.missing(normalizeSpec.Requires); ok { if missing, ok := capabilities.missing(normalizeSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing) return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing)
} }
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, nil)
capabilities.add(normalizeSpec.Provides...) capabilities.add(normalizeSpec.Provides...)
for _, validator := range lane.Validators { for _, validator := range lane.Validators {
@@ -241,6 +252,15 @@ func resolveArtifactLane(
return lane, capabilities, nil return lane, capabilities, nil
} }
func referenceTarget(stage ModuleStage, laneID string, module string, bindings []ReferenceBinding) ResolvedReferenceTarget {
return ResolvedReferenceTarget{
Stage: stage,
LaneID: strings.TrimSpace(laneID),
Module: strings.TrimSpace(module),
Bindings: append([]ReferenceBinding(nil), bindings...),
}
}
func validatePipelineReferenceDefaults( func validatePipelineReferenceDefaults(
pipelineID string, pipelineID string,
pipelineReferences map[string]string, pipelineReferences map[string]string,
@@ -535,12 +555,14 @@ func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
ID string ID string
Input ModuleBinding Input ModuleBinding
Chunk ModuleBinding Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget
ArtifactLanes []ResolvedArtifactLane ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding Output ModuleBinding
}{ }{
ID: resolved.ID, ID: resolved.ID,
Input: resolved.Input, Input: resolved.Input,
Chunk: resolved.Chunk, Chunk: resolved.Chunk,
ChunkReferences: resolved.ChunkReferences,
ArtifactLanes: resolved.ArtifactLanes, ArtifactLanes: resolved.ArtifactLanes,
Output: resolved.Output, Output: resolved.Output,
} }

View File

@@ -154,16 +154,25 @@ func TestResolvePipelineAppliesReferenceBindings(t *testing.T) {
} }
events := resolvedLane(t, resolved.ArtifactLanes, "events") events := resolvedLane(t, resolved.ArtifactLanes, "events")
if events.ExtractReferences.Stage != StageExtract || events.ExtractReferences.LaneID != "events" || events.ExtractReferences.Module != "event-extractor" {
t.Fatalf("extract reference target = %#v, want event extractor target", events.ExtractReferences)
}
if events.NormalizeReferences.Stage != StageNormalize || events.NormalizeReferences.LaneID != "events" || events.NormalizeReferences.Module != DefaultNormalizeModule {
t.Fatalf("normalize reference target = %#v, want event normalizer target", events.NormalizeReferences)
}
if resolved.ChunkReferences.Stage != StageChunk || resolved.ChunkReferences.Module != DefaultChunkModule {
t.Fatalf("chunk reference target = %#v, want chunk target", resolved.ChunkReferences)
}
want := []ReferenceBinding{ want := []ReferenceBinding{
{LaneID: "events", SlotName: "lore", Source: "./lore.md", BindingSource: contracts.ReferenceBindingSourceConfig}, {LaneID: "events", SlotName: "lore", Source: "./lore.md", BindingSource: contracts.ReferenceBindingSourceConfig},
{LaneID: "events", SlotName: "roster", Source: "./lane-roster.yml", BindingSource: contracts.ReferenceBindingSourceConfig}, {LaneID: "events", SlotName: "roster", Source: "./lane-roster.yml", BindingSource: contracts.ReferenceBindingSourceConfig},
} }
if !reflect.DeepEqual(events.References, want) { if !reflect.DeepEqual(events.ExtractReferences.Bindings, want) {
t.Fatalf("events references = %#v, want %#v", events.References, want) t.Fatalf("events references = %#v, want %#v", events.ExtractReferences.Bindings, want)
} }
summaries := resolvedLane(t, resolved.ArtifactLanes, "summaries") summaries := resolvedLane(t, resolved.ArtifactLanes, "summaries")
if len(summaries.References) != 0 { if len(summaries.ExtractReferences.Bindings) != 0 {
t.Fatalf("summaries references = %#v, want none", summaries.References) t.Fatalf("summaries references = %#v, want none", summaries.ExtractReferences.Bindings)
} }
} }
@@ -184,7 +193,7 @@ func TestResolvePipelineAllowsPipelineReferenceDeclaredOnlyByUnselectedLane(t *t
if err != nil { if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err) t.Fatalf("ResolvePipeline() error = %v, want nil", err)
} }
if refs := resolved.ArtifactLanes[0].References; len(refs) != 0 { if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("selected lane references = %#v, want none", refs) t.Fatalf("selected lane references = %#v, want none", refs)
} }
} }
@@ -284,7 +293,7 @@ func TestResolvePipelineUsesReferenceSlotsFromSpecWithoutConstructingExtractor(t
if err != nil { if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err) t.Fatalf("ResolvePipeline() error = %v, want nil", err)
} }
if got := resolved.ArtifactLanes[0].References[0].Source; got != "./roster.yml" { if got := resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].Source; got != "./roster.yml" {
t.Fatalf("reference source = %q, want ./roster.yml", got) t.Fatalf("reference source = %q, want ./roster.yml", got)
} }
} }

View File

@@ -29,6 +29,7 @@ type ReferenceMaterializationOptions struct {
func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, options ReferenceMaterializationOptions) (ResolvedPipeline, []contracts.Warning, error) { func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, options ReferenceMaterializationOptions) (ResolvedPipeline, []contracts.Warning, error) {
out := resolved out := resolved
out.ChunkReferences = CloneReferenceTarget(resolved.ChunkReferences)
if len(resolved.ArtifactLanes) == 0 { if len(resolved.ArtifactLanes) == 0 {
return out, nil, nil return out, nil, nil
} }
@@ -37,32 +38,31 @@ func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, opt
out.ArtifactLanes = make([]ResolvedArtifactLane, len(resolved.ArtifactLanes)) out.ArtifactLanes = make([]ResolvedArtifactLane, len(resolved.ArtifactLanes))
for i, lane := range resolved.ArtifactLanes { for i, lane := range resolved.ArtifactLanes {
materializedLane := lane materializedLane := lane
referenceSet, laneWarnings, err := materializeLaneReferences(resolved.ID, lane, catalog, options) materializedLane.ExtractReferences = CloneReferenceTarget(lane.ExtractReferences)
materializedLane.NormalizeReferences = CloneReferenceTarget(lane.NormalizeReferences)
referenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.ExtractReferences, catalog, options)
if err != nil { if err != nil {
return ResolvedPipeline{}, nil, err return ResolvedPipeline{}, nil, err
} }
materializedLane.ReferenceSet = referenceSet materializedLane.ExtractReferences.ReferenceSet = referenceSet
out.ArtifactLanes[i] = materializedLane out.ArtifactLanes[i] = materializedLane
warnings = append(warnings, laneWarnings...) warnings = append(warnings, laneWarnings...)
} }
return out, warnings, nil return out, warnings, nil
} }
func materializeLaneReferences( func materializeReferenceTarget(
pipelineID string, pipelineID string,
lane ResolvedArtifactLane, target ResolvedReferenceTarget,
catalog ModuleCatalog, catalog ModuleCatalog,
options ReferenceMaterializationOptions, options ReferenceMaterializationOptions,
) (contracts.ReferenceSet, []contracts.Warning, error) { ) (contracts.ReferenceSet, []contracts.Warning, error) {
if len(lane.References) == 0 { if len(target.Bindings) == 0 {
return contracts.ReferenceSet{}, nil, nil return contracts.ReferenceSet{}, nil, nil
} }
if catalog.Extractors == nil { spec, err := referenceTargetSpec(target, catalog)
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", pipelineID, lane.ID, lane.Extract.Module, lane.Extract.Module) if err != nil {
} return contracts.ReferenceSet{}, nil, fmt.Errorf("%s: %w", referenceTargetContext(pipelineID, target), err)
spec, ok := catalog.Extractors.Spec(lane.Extract.Module)
if !ok {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", pipelineID, lane.ID, lane.Extract.Module, lane.Extract.Module)
} }
slotByName := make(map[string]contracts.ReferenceSlot, len(spec.ReferenceSlots)) slotByName := make(map[string]contracts.ReferenceSlot, len(spec.ReferenceSlots))
@@ -70,38 +70,38 @@ func materializeLaneReferences(
slotByName[slot.Name] = slot slotByName[slot.Name] = slot
} }
set := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(lane.References))} set := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(target.Bindings))}
var warnings []contracts.Warning var warnings []contracts.Warning
for _, binding := range lane.References { for _, binding := range target.Bindings {
slotName := strings.TrimSpace(binding.SlotName) slotName := strings.TrimSpace(binding.SlotName)
slot, ok := slotByName[slotName] slot, ok := slotByName[slotName]
if !ok { if !ok {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q is not declared by extractor %q", pipelineID, lane.ID, slotName, lane.Extract.Module) return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q is not declared by %s module %q", referenceTargetContext(pipelineID, target), slotName, target.Stage, target.Module)
} }
path, err := referencePath(binding, options) path, err := referencePath(binding, options)
if err != nil { if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q: %w", pipelineID, lane.ID, slotName, binding.Source, err) return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q: %w", referenceTargetContext(pipelineID, target), slotName, binding.Source, err)
} }
content, err := os.ReadFile(path) content, err := os.ReadFile(path)
if err != nil { if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q read %q: %w", pipelineID, lane.ID, slotName, path, err) return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q read %q: %w", referenceTargetContext(pipelineID, target), slotName, path, err)
} }
if !utf8.Valid(content) { if !utf8.Valid(content) {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q must be UTF-8 text", pipelineID, lane.ID, slotName, path) return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q must be UTF-8 text", referenceTargetContext(pipelineID, target), slotName, path)
} }
mediaType := referenceMediaTypeForPath(path) mediaType := referenceMediaTypeForPath(path)
if !referenceMediaTypeAccepted(mediaType, slot.AcceptedMediaTypes) { if !referenceMediaTypeAccepted(mediaType, slot.AcceptedMediaTypes) {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q media type %q is not accepted", pipelineID, lane.ID, slotName, path, mediaType) return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q media type %q is not accepted", referenceTargetContext(pipelineID, target), slotName, path, mediaType)
} }
if slot.MaxBytes > 0 && int64(len(content)) > slot.MaxBytes { if slot.MaxBytes > 0 && int64(len(content)) > slot.MaxBytes {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q is %d bytes, limit %d", pipelineID, lane.ID, slotName, path, len(content), slot.MaxBytes) return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q is %d bytes, limit %d", referenceTargetContext(pipelineID, target), slotName, path, len(content), slot.MaxBytes)
} }
if len(content) == 0 { if len(content) == 0 {
warnings = append(warnings, contracts.Warning{ warnings = append(warnings, contracts.Warning{
Scope: fmt.Sprintf("pipeline.%s.lane.%s.reference.%s", pipelineID, lane.ID, slotName), Scope: referenceWarningScope(pipelineID, target, slotName),
ReasonCode: "empty_reference", ReasonCode: "empty_reference",
Message: fmt.Sprintf("reference slot %q for lane %q is bound to an empty file", slotName, lane.ID), Message: fmt.Sprintf("reference slot %q for %s is bound to an empty file", slotName, referenceTargetLabel(target)),
}) })
} }
@@ -122,6 +122,40 @@ func materializeLaneReferences(
return set, warnings, nil return set, warnings, nil
} }
func referenceTargetSpec(target ResolvedReferenceTarget, catalog ModuleCatalog) (ModuleSpec, error) {
switch target.Stage {
case StageChunk:
return registrySpec(catalog.Chunkers, target.Module)
case StageExtract:
return registrySpec(catalog.Extractors, target.Module)
case StageNormalize:
return registrySpec(catalog.Normalizers, target.Module)
default:
return ModuleSpec{}, fmt.Errorf("reference target stage %q is not supported", target.Stage)
}
}
func referenceTargetContext(pipelineID string, target ResolvedReferenceTarget) string {
if target.LaneID != "" {
return fmt.Sprintf("pipeline %q lane %q %s module %q", pipelineID, target.LaneID, target.Stage, target.Module)
}
return fmt.Sprintf("pipeline %q %s module %q", pipelineID, target.Stage, target.Module)
}
func referenceTargetLabel(target ResolvedReferenceTarget) string {
if target.LaneID != "" {
return fmt.Sprintf("lane %q %s target", target.LaneID, target.Stage)
}
return fmt.Sprintf("%s target", target.Stage)
}
func referenceWarningScope(pipelineID string, target ResolvedReferenceTarget, slotName string) string {
if target.LaneID != "" {
return fmt.Sprintf("pipeline.%s.lane.%s.%s.reference.%s", pipelineID, target.LaneID, target.Stage, slotName)
}
return fmt.Sprintf("pipeline.%s.%s.reference.%s", pipelineID, target.Stage, slotName)
}
func referenceMediaTypeForPath(path string) string { func referenceMediaTypeForPath(path string) string {
extension := strings.ToLower(filepath.Ext(path)) extension := strings.ToLower(filepath.Ext(path))
mediaType := mime.TypeByExtension(extension) mediaType := mime.TypeByExtension(extension)
@@ -230,22 +264,39 @@ func CloneReferenceSet(in contracts.ReferenceSet) contracts.ReferenceSet {
return out return out
} }
func CloneReferenceTarget(in ResolvedReferenceTarget) ResolvedReferenceTarget {
out := in
out.Bindings = append([]ReferenceBinding(nil), in.Bindings...)
out.ReferenceSet = CloneReferenceSet(in.ReferenceSet)
return out
}
func ReferenceProvenance(resolved ResolvedPipeline) []artifacts.ReferenceProvenance { func ReferenceProvenance(resolved ResolvedPipeline) []artifacts.ReferenceProvenance {
provenance := []artifacts.ReferenceProvenance{} provenance := []artifacts.ReferenceProvenance{}
provenance = append(provenance, referenceTargetProvenance(resolved.ChunkReferences)...)
for _, lane := range resolved.ArtifactLanes { for _, lane := range resolved.ArtifactLanes {
if len(lane.ReferenceSet.Slots) == 0 { provenance = append(provenance, referenceTargetProvenance(lane.ExtractReferences)...)
continue provenance = append(provenance, referenceTargetProvenance(lane.NormalizeReferences)...)
} }
slotNames := make([]string, 0, len(lane.ReferenceSet.Slots)) return provenance
for slotName := range lane.ReferenceSet.Slots { }
func referenceTargetProvenance(target ResolvedReferenceTarget) []artifacts.ReferenceProvenance {
if len(target.ReferenceSet.Slots) == 0 {
return nil
}
provenance := []artifacts.ReferenceProvenance{}
slotNames := make([]string, 0, len(target.ReferenceSet.Slots))
for slotName := range target.ReferenceSet.Slots {
slotNames = append(slotNames, slotName) slotNames = append(slotNames, slotName)
} }
sort.Strings(slotNames) sort.Strings(slotNames)
for _, slotName := range slotNames { for _, slotName := range slotNames {
slot := lane.ReferenceSet.Slots[slotName] slot := target.ReferenceSet.Slots[slotName]
for _, item := range slot.Items { for _, item := range slot.Items {
provenance = append(provenance, artifacts.ReferenceProvenance{ provenance = append(provenance, artifacts.ReferenceProvenance{
LaneID: lane.ID, Stage: string(target.Stage),
LaneID: target.LaneID,
SlotName: item.SlotName, SlotName: item.SlotName,
OriginType: item.Origin.Type, OriginType: item.Origin.Type,
OriginURI: item.Origin.URI, OriginURI: item.Origin.URI,
@@ -256,6 +307,5 @@ func ReferenceProvenance(resolved ResolvedPipeline) []artifacts.ReferenceProvena
}) })
} }
} }
}
return provenance return provenance
} }

View File

@@ -54,12 +54,12 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
t.Fatalf("MaterializeReferences(second) error = %v, want nil", err) t.Fatalf("MaterializeReferences(second) error = %v, want nil", err)
} }
referenceSet := first.ArtifactLanes[0].ReferenceSet referenceSet := first.ArtifactLanes[0].ExtractReferences.ReferenceSet
roster := referenceSet.Slots["roster"].Items[0] roster := referenceSet.Slots["roster"].Items[0]
if string(roster.Content) != "config text" { if string(roster.Content) != "config text" {
t.Fatalf("roster content = %q, want config text", roster.Content) t.Fatalf("roster content = %q, want config text", roster.Content)
} }
if roster.Digest != referenceDigest([]byte("config text")) || roster.Digest != second.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0].Digest { if roster.Digest != referenceDigest([]byte("config text")) || roster.Digest != second.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Digest {
t.Fatalf("roster digest = %q, want stable digest", roster.Digest) t.Fatalf("roster digest = %q, want stable digest", roster.Digest)
} }
if roster.BindingSource != contracts.ReferenceBindingSourceConfig { if roster.BindingSource != contracts.ReferenceBindingSourceConfig {
@@ -87,10 +87,10 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
if len(provenance) != 2 { if len(provenance) != 2 {
t.Fatalf("ReferenceProvenance() = %#v, want two entries", provenance) t.Fatalf("ReferenceProvenance() = %#v, want two entries", provenance)
} }
if provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest { if provenance[0].Stage != string(StageExtract) || provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest {
t.Fatalf("ReferenceProvenance()[0] = %#v, want sorted glossary provenance", provenance[0]) t.Fatalf("ReferenceProvenance()[0] = %#v, want sorted glossary provenance", provenance[0])
} }
if provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest { if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest {
t.Fatalf("ReferenceProvenance()[1] = %#v, want roster provenance", provenance[1]) t.Fatalf("ReferenceProvenance()[1] = %#v, want roster provenance", provenance[1])
} }
@@ -129,7 +129,7 @@ func TestMaterializeReferencesAllowsAnyMediaTypeWhenSlotDoesNotRestrictIt(t *tes
if err != nil { if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err) t.Fatalf("MaterializeReferences() error = %v, want nil", err)
} }
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0] item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != unknownMediaType { if item.MediaType != unknownMediaType {
t.Fatalf("MediaType = %q, want %q", item.MediaType, unknownMediaType) t.Fatalf("MediaType = %q, want %q", item.MediaType, unknownMediaType)
} }
@@ -148,7 +148,7 @@ func TestMaterializeReferencesAcceptsDeclaredMarkdownMediaType(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err) t.Fatalf("MaterializeReferences() error = %v, want nil", err)
} }
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["glossary"].Items[0] item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["glossary"].Items[0]
if item.MediaType != "text/markdown" { if item.MediaType != "text/markdown" {
t.Fatalf("MediaType = %q, want text/markdown", item.MediaType) t.Fatalf("MediaType = %q, want text/markdown", item.MediaType)
} }
@@ -167,7 +167,7 @@ func TestMaterializeReferencesAcceptsDeclaredJSONMediaType(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err) t.Fatalf("MaterializeReferences() error = %v, want nil", err)
} }
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0] item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != "application/json" { if item.MediaType != "application/json" {
t.Fatalf("MediaType = %q, want application/json", item.MediaType) t.Fatalf("MediaType = %q, want application/json", item.MediaType)
} }
@@ -201,7 +201,7 @@ func TestMaterializeReferencesMatchesAcceptedMediaTypesIgnoringParameters(t *tes
if err != nil { if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err) t.Fatalf("MaterializeReferences() error = %v, want nil", err)
} }
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0] item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != referenceMediaType { if item.MediaType != referenceMediaType {
t.Fatalf("MediaType = %q, want %q", item.MediaType, referenceMediaType) t.Fatalf("MediaType = %q, want %q", item.MediaType, referenceMediaType)
} }
@@ -222,7 +222,7 @@ func TestMaterializeReferencesWarnsForEmptyFiles(t *testing.T) {
if len(warnings) != 1 || warnings[0].ReasonCode != "empty_reference" { if len(warnings) != 1 || warnings[0].ReasonCode != "empty_reference" {
t.Fatalf("warnings = %#v, want empty reference warning", warnings) t.Fatalf("warnings = %#v, want empty reference warning", warnings)
} }
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0] item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.SizeBytes != 0 || item.Digest != referenceDigest(nil) { if item.SizeBytes != 0 || item.Digest != referenceDigest(nil) {
t.Fatalf("empty item = %#v, want zero size and empty digest", item) t.Fatalf("empty item = %#v, want zero size and empty digest", item)
} }
@@ -255,7 +255,7 @@ func resolvedPipelineWithReference(t *testing.T, slotName, source, bindingSource
t.Fatalf("ResolvePipeline() error = %v, want nil", err) t.Fatalf("ResolvePipeline() error = %v, want nil", err)
} }
if bindingSource != contracts.ReferenceBindingSourceConfig { if bindingSource != contracts.ReferenceBindingSourceConfig {
resolved.ArtifactLanes[0].References[0].BindingSource = bindingSource resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].BindingSource = bindingSource
} }
return resolved return resolved
} }

View File

@@ -186,7 +186,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
result, err := extractor.Extract(ctx, contracts.ExtractionRequest{ result, err := extractor.Extract(ctx, contracts.ExtractionRequest{
Source: doc, Source: doc,
Chunk: &chunk, Chunk: &chunk,
References: CloneReferenceSet(lane.ReferenceSet), References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet),
LLMClient: input.LLMClient, LLMClient: input.LLMClient,
LLMProfile: lane.Extract.LLMProfile, LLMProfile: lane.Extract.LLMProfile,
Options: cloneOptions(lane.Extract.Options), Options: cloneOptions(lane.Extract.Options),

View File

@@ -566,7 +566,7 @@ func TestRunPassesModuleBindingConfigToStageRequests(t *testing.T) {
func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) { func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) {
modules := defaultRunnerModules() modules := defaultRunnerModules()
pipeline := resolvedPipeline() pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].ReferenceSet = contracts.ReferenceSet{ pipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{ Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": { "roster": {
Slot: contracts.ReferenceSlot{Name: "roster"}, Slot: contracts.ReferenceSlot{Name: "roster"},
@@ -596,7 +596,7 @@ func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) {
t.Fatalf("reference content = %q, want reference text", item.Content) t.Fatalf("reference content = %q, want reference text", item.Content)
} }
item.Content[0] = 'R' item.Content[0] = 'R'
if got := string(pipeline.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0].Content); got != "reference text" { if got := string(pipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content); got != "reference text" {
t.Fatalf("runner mutated reference set content = %q", got) t.Fatalf("runner mutated reference set content = %q", got)
} }
} }
@@ -965,7 +965,7 @@ func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) {
func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) { func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
resolved := resolvedPipelineWithValidators("configured") resolved := resolvedPipelineWithValidators("configured")
resolved.ArtifactLanes[0].ReferenceSet = contracts.ReferenceSet{ resolved.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{ Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": { "roster": {
Slot: contracts.ReferenceSlot{Name: "roster"}, Slot: contracts.ReferenceSlot{Name: "roster"},
@@ -1003,7 +1003,7 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
t.Fatalf("References = %#v, want one reference provenance entry", manifest.References) t.Fatalf("References = %#v, want one reference provenance entry", manifest.References)
} }
reference := manifest.References[0] reference := manifest.References[0]
if reference.LaneID != "alpha" || reference.SlotName != "roster" || reference.Digest != "sha256:reference" { if reference.Stage != string(StageExtract) || reference.LaneID != "alpha" || reference.SlotName != "roster" || reference.Digest != "sha256:reference" {
t.Fatalf("reference provenance = %#v, want lane slot digest", reference) t.Fatalf("reference provenance = %#v, want lane slot digest", reference)
} }
if reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != int64(len("reference content")) || reference.BindingSource != contracts.ReferenceBindingSourceConfig { if reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != int64(len("reference content")) || reference.BindingSource != contracts.ReferenceBindingSourceConfig {
@@ -1165,12 +1165,15 @@ func resolvedPipeline() ResolvedPipeline {
Digest: "sha256:pipeline", Digest: "sha256:pipeline",
Input: Binding("input"), Input: Binding("input"),
Chunk: Binding("chunk"), Chunk: Binding("chunk"),
ChunkReferences: referenceTarget(StageChunk, "", "chunk", nil),
ArtifactLanes: []ResolvedArtifactLane{ ArtifactLanes: []ResolvedArtifactLane{
{ {
ID: "alpha", ID: "alpha",
Extract: Binding("extract-alpha"), Extract: Binding("extract-alpha"),
Merge: Binding("merge"), Merge: Binding("merge"),
Normalize: Binding("normalize"), Normalize: Binding("normalize"),
ExtractReferences: referenceTarget(StageExtract, "alpha", "extract-alpha", nil),
NormalizeReferences: referenceTarget(StageNormalize, "alpha", "normalize", nil),
}, },
}, },
Output: Binding("output"), Output: Binding("output"),

View File

@@ -1,7 +1,7 @@
{ {
"manifest": { "manifest": {
"pipeline_id": "walking-skeleton", "pipeline_id": "walking-skeleton",
"pipeline_digest": "sha256:5df1e501a2307ef75bbfeb59d315b3710571d52e5466a9c7f8320248740e6fca", "pipeline_digest": "sha256:25084e39a0cadace375c896551d1752413755c1a6772f6b24cfe91c029ea2631",
"validation_status": "approved", "validation_status": "approved",
"artifact_lanes": [ "artifact_lanes": [
{ {

View File

@@ -114,7 +114,7 @@ func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T)
raw := readDNDSpellsFixture(t) raw := readDNDSpellsFixture(t)
expectedDoc := parseDNDSpellsFixture(t, raw) expectedDoc := parseDNDSpellsFixture(t, raw)
resolved := resolveDNDSpellsPipeline(t) resolved := resolveDNDSpellsPipeline(t)
resolved.ResolvedPipeline.ArtifactLanes[0].ReferenceSet = dndSpellsReferenceSet( resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet(
"Aria: party cleric\nBorin: fighter", "Aria: party cleric\nBorin: fighter",
"Fire Bolt: evocation cantrip", "Fire Bolt: evocation cantrip",
) )
@@ -168,7 +168,7 @@ func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T)
func TestRunnerDoesNotExtractSpellMentionedOnlyInRoster(t *testing.T) { func TestRunnerDoesNotExtractSpellMentionedOnlyInRoster(t *testing.T) {
raw := readDNDSpellsFixture(t) raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t) resolved := resolveDNDSpellsPipeline(t)
resolved.ResolvedPipeline.ArtifactLanes[0].ReferenceSet = dndSpellsReferenceSet( resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet(
"Mira: wizard who can cast Lightning Bolt", "Mira: wizard who can cast Lightning Bolt",
"", "",
) )

View File

@@ -148,6 +148,7 @@ func TestEncodeIncludesManifestReferences(t *testing.T) {
RunID: "run-1", RunID: "run-1",
References: []artifacts.ReferenceProvenance{ References: []artifacts.ReferenceProvenance{
{ {
Stage: "extract",
LaneID: "events", LaneID: "events",
SlotName: "roster", SlotName: "roster",
OriginType: "file", OriginType: "file",