eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce target-aware resolved reference storage

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-05 16:12:38 +00:00
parent 39e49d7f77
commit 9278797aa9
17 changed files with 238 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
internal/framework/pipeline/profile.go
View File

@@ -60,23 +60,32 @@ type ReferenceUnbind struct {
SlotName string `json:"slot_name"`
}
type ResolvedArtifactLane struct {
ID string
Extract ModuleBinding
Merge ModuleBinding
Normalize ModuleBinding
Validators []ModuleBinding
References []ReferenceBinding `json:"references,omitempty"`
type ResolvedReferenceTarget struct {
Stage ModuleStage `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
Module string `json:"module"`
Bindings []ReferenceBinding `json:"bindings,omitempty"`
ReferenceSet contracts.ReferenceSet `json:"-"`
}
type ResolvedArtifactLane struct {
ID string
Extract ModuleBinding
Merge ModuleBinding
Normalize ModuleBinding
Validators []ModuleBinding
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
}
type ResolvedPipeline struct {
ID string
Digest string
Input ModuleBinding
Chunk ModuleBinding
ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding
ID string
Digest string
Input ModuleBinding
Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding
}
type ModuleCatalog struct {
@@ -140,10 +149,11 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
}
resolved := ResolvedPipeline{
ID: pipelineID,
Input: input,
Chunk: chunk,
Output: resolveBinding(profile.Output, DefaultOutputModule),
ID: pipelineID,
Input: input,
Chunk: chunk,
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, nil),
Output: resolveBinding(profile.Output, DefaultOutputModule),
}
outputCapabilities := capabilities.clone()
@@ -206,7 +216,7 @@ func resolveArtifactLane(
if err != nil {
return ResolvedArtifactLane{}, nil, err
}
lane.References = references
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
capabilities.add(extractSpec.Provides...)
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
@@ -225,6 +235,7 @@ func resolveArtifactLane(
if missing, ok := capabilities.missing(normalizeSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing)
}
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, nil)
capabilities.add(normalizeSpec.Provides...)
for _, validator := range lane.Validators {
@@ -241,6 +252,15 @@ func resolveArtifactLane(
return lane, capabilities, nil
}
func referenceTarget(stage ModuleStage, laneID string, module string, bindings []ReferenceBinding) ResolvedReferenceTarget {
return ResolvedReferenceTarget{
Stage: stage,
LaneID: strings.TrimSpace(laneID),
Module: strings.TrimSpace(module),
Bindings: append([]ReferenceBinding(nil), bindings...),
}
}
func validatePipelineReferenceDefaults(
pipelineID string,
pipelineReferences map[string]string,
@@ -532,17 +552,19 @@ func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneP
func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
withoutDigest := struct {
ID string
Input ModuleBinding
Chunk ModuleBinding
ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding
ID string
Input ModuleBinding
Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget
ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding
}{
ID: resolved.ID,
Input: resolved.Input,
Chunk: resolved.Chunk,
ArtifactLanes: resolved.ArtifactLanes,
Output: resolved.Output,
ID: resolved.ID,
Input: resolved.Input,
Chunk: resolved.Chunk,
ChunkReferences: resolved.ChunkReferences,
ArtifactLanes: resolved.ArtifactLanes,
Output: resolved.Output,
}
encoded, err := json.Marshal(withoutDigest)
if err != nil {

21
internal/framework/pipeline/profile_test.go
View File

@@ -154,16 +154,25 @@ func TestResolvePipelineAppliesReferenceBindings(t *testing.T) {
}
events := resolvedLane(t, resolved.ArtifactLanes, "events")
if events.ExtractReferences.Stage != StageExtract || events.ExtractReferences.LaneID != "events" || events.ExtractReferences.Module != "event-extractor" {
t.Fatalf("extract reference target = %#v, want event extractor target", events.ExtractReferences)
}
if events.NormalizeReferences.Stage != StageNormalize || events.NormalizeReferences.LaneID != "events" || events.NormalizeReferences.Module != DefaultNormalizeModule {
t.Fatalf("normalize reference target = %#v, want event normalizer target", events.NormalizeReferences)
}
if resolved.ChunkReferences.Stage != StageChunk || resolved.ChunkReferences.Module != DefaultChunkModule {
t.Fatalf("chunk reference target = %#v, want chunk target", resolved.ChunkReferences)
}
want := []ReferenceBinding{
{LaneID: "events", SlotName: "lore", Source: "./lore.md", BindingSource: contracts.ReferenceBindingSourceConfig},
{LaneID: "events", SlotName: "roster", Source: "./lane-roster.yml", BindingSource: contracts.ReferenceBindingSourceConfig},
}
if !reflect.DeepEqual(events.References, want) {
t.Fatalf("events references = %#v, want %#v", events.References, want)
if !reflect.DeepEqual(events.ExtractReferences.Bindings, want) {
t.Fatalf("events references = %#v, want %#v", events.ExtractReferences.Bindings, want)
}
summaries := resolvedLane(t, resolved.ArtifactLanes, "summaries")
if len(summaries.References) != 0 {
t.Fatalf("summaries references = %#v, want none", summaries.References)
if len(summaries.ExtractReferences.Bindings) != 0 {
t.Fatalf("summaries references = %#v, want none", summaries.ExtractReferences.Bindings)
}
}
@@ -184,7 +193,7 @@ func TestResolvePipelineAllowsPipelineReferenceDeclaredOnlyByUnselectedLane(t *t
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if refs := resolved.ArtifactLanes[0].References; len(refs) != 0 {
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("selected lane references = %#v, want none", refs)
}
}
@@ -284,7 +293,7 @@ func TestResolvePipelineUsesReferenceSlotsFromSpecWithoutConstructingExtractor(t
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if got := resolved.ArtifactLanes[0].References[0].Source; got != "./roster.yml" {
if got := resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].Source; got != "./roster.yml" {
t.Fatalf("reference source = %q, want ./roster.yml", got)
}
}

136
internal/framework/pipeline/references.go
View File

@@ -29,6 +29,7 @@ type ReferenceMaterializationOptions struct {
func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, options ReferenceMaterializationOptions) (ResolvedPipeline, []contracts.Warning, error) {
out := resolved
out.ChunkReferences = CloneReferenceTarget(resolved.ChunkReferences)
if len(resolved.ArtifactLanes) == 0 {
return out, nil, nil
}
@@ -37,32 +38,31 @@ func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, opt
out.ArtifactLanes = make([]ResolvedArtifactLane, len(resolved.ArtifactLanes))
for i, lane := range resolved.ArtifactLanes {
materializedLane := lane
referenceSet, laneWarnings, err := materializeLaneReferences(resolved.ID, lane, catalog, options)
materializedLane.ExtractReferences = CloneReferenceTarget(lane.ExtractReferences)
materializedLane.NormalizeReferences = CloneReferenceTarget(lane.NormalizeReferences)
referenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.ExtractReferences, catalog, options)
if err != nil {
return ResolvedPipeline{}, nil, err
}
materializedLane.ReferenceSet = referenceSet
materializedLane.ExtractReferences.ReferenceSet = referenceSet
out.ArtifactLanes[i] = materializedLane
warnings = append(warnings, laneWarnings...)
}
return out, warnings, nil
}
func materializeLaneReferences(
func materializeReferenceTarget(
pipelineID string,
lane ResolvedArtifactLane,
target ResolvedReferenceTarget,
catalog ModuleCatalog,
options ReferenceMaterializationOptions,
) (contracts.ReferenceSet, []contracts.Warning, error) {
if len(lane.References) == 0 {
if len(target.Bindings) == 0 {
return contracts.ReferenceSet{}, nil, nil
}
if catalog.Extractors == nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", pipelineID, lane.ID, lane.Extract.Module, lane.Extract.Module)
}
spec, ok := catalog.Extractors.Spec(lane.Extract.Module)
if !ok {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", pipelineID, lane.ID, lane.Extract.Module, lane.Extract.Module)
spec, err := referenceTargetSpec(target, catalog)
if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s: %w", referenceTargetContext(pipelineID, target), err)
}
slotByName := make(map[string]contracts.ReferenceSlot, len(spec.ReferenceSlots))
@@ -70,38 +70,38 @@ func materializeLaneReferences(
slotByName[slot.Name] = slot
}
set := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(lane.References))}
set := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(target.Bindings))}
var warnings []contracts.Warning
for _, binding := range lane.References {
for _, binding := range target.Bindings {
slotName := strings.TrimSpace(binding.SlotName)
slot, ok := slotByName[slotName]
if !ok {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q is not declared by extractor %q", pipelineID, lane.ID, slotName, lane.Extract.Module)
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q is not declared by %s module %q", referenceTargetContext(pipelineID, target), slotName, target.Stage, target.Module)
}
path, err := referencePath(binding, options)
if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q: %w", pipelineID, lane.ID, slotName, binding.Source, err)
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q: %w", referenceTargetContext(pipelineID, target), slotName, binding.Source, err)
}
content, err := os.ReadFile(path)
if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q read %q: %w", pipelineID, lane.ID, slotName, path, err)
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q read %q: %w", referenceTargetContext(pipelineID, target), slotName, path, err)
}
if !utf8.Valid(content) {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q must be UTF-8 text", pipelineID, lane.ID, slotName, path)
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q must be UTF-8 text", referenceTargetContext(pipelineID, target), slotName, path)
}
mediaType := referenceMediaTypeForPath(path)
if !referenceMediaTypeAccepted(mediaType, slot.AcceptedMediaTypes) {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q media type %q is not accepted", pipelineID, lane.ID, slotName, path, mediaType)
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q media type %q is not accepted", referenceTargetContext(pipelineID, target), slotName, path, mediaType)
}
if slot.MaxBytes > 0 && int64(len(content)) > slot.MaxBytes {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q is %d bytes, limit %d", pipelineID, lane.ID, slotName, path, len(content), slot.MaxBytes)
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q is %d bytes, limit %d", referenceTargetContext(pipelineID, target), slotName, path, len(content), slot.MaxBytes)
}
if len(content) == 0 {
warnings = append(warnings, contracts.Warning{
Scope: fmt.Sprintf("pipeline.%s.lane.%s.reference.%s", pipelineID, lane.ID, slotName),
Scope: referenceWarningScope(pipelineID, target, slotName),
ReasonCode: "empty_reference",
Message: fmt.Sprintf("reference slot %q for lane %q is bound to an empty file", slotName, lane.ID),
Message: fmt.Sprintf("reference slot %q for %s is bound to an empty file", slotName, referenceTargetLabel(target)),
})
}
@@ -122,6 +122,40 @@ func materializeLaneReferences(
return set, warnings, nil
}
func referenceTargetSpec(target ResolvedReferenceTarget, catalog ModuleCatalog) (ModuleSpec, error) {
switch target.Stage {
case StageChunk:
return registrySpec(catalog.Chunkers, target.Module)
case StageExtract:
return registrySpec(catalog.Extractors, target.Module)
case StageNormalize:
return registrySpec(catalog.Normalizers, target.Module)
default:
return ModuleSpec{}, fmt.Errorf("reference target stage %q is not supported", target.Stage)
}
}
func referenceTargetContext(pipelineID string, target ResolvedReferenceTarget) string {
if target.LaneID != "" {
return fmt.Sprintf("pipeline %q lane %q %s module %q", pipelineID, target.LaneID, target.Stage, target.Module)
}
return fmt.Sprintf("pipeline %q %s module %q", pipelineID, target.Stage, target.Module)
}
func referenceTargetLabel(target ResolvedReferenceTarget) string {
if target.LaneID != "" {
return fmt.Sprintf("lane %q %s target", target.LaneID, target.Stage)
}
return fmt.Sprintf("%s target", target.Stage)
}
func referenceWarningScope(pipelineID string, target ResolvedReferenceTarget, slotName string) string {
if target.LaneID != "" {
return fmt.Sprintf("pipeline.%s.lane.%s.%s.reference.%s", pipelineID, target.LaneID, target.Stage, slotName)
}
return fmt.Sprintf("pipeline.%s.%s.reference.%s", pipelineID, target.Stage, slotName)
}
func referenceMediaTypeForPath(path string) string {
extension := strings.ToLower(filepath.Ext(path))
mediaType := mime.TypeByExtension(extension)
@@ -230,31 +264,47 @@ func CloneReferenceSet(in contracts.ReferenceSet) contracts.ReferenceSet {
return out
}
func CloneReferenceTarget(in ResolvedReferenceTarget) ResolvedReferenceTarget {
out := in
out.Bindings = append([]ReferenceBinding(nil), in.Bindings...)
out.ReferenceSet = CloneReferenceSet(in.ReferenceSet)
return out
}
func ReferenceProvenance(resolved ResolvedPipeline) []artifacts.ReferenceProvenance {
provenance := []artifacts.ReferenceProvenance{}
provenance = append(provenance, referenceTargetProvenance(resolved.ChunkReferences)...)
for _, lane := range resolved.ArtifactLanes {
if len(lane.ReferenceSet.Slots) == 0 {
continue
}
slotNames := make([]string, 0, len(lane.ReferenceSet.Slots))
for slotName := range lane.ReferenceSet.Slots {
slotNames = append(slotNames, slotName)
}
sort.Strings(slotNames)
for _, slotName := range slotNames {
slot := lane.ReferenceSet.Slots[slotName]
for _, item := range slot.Items {
provenance = append(provenance, artifacts.ReferenceProvenance{
LaneID: lane.ID,
SlotName: item.SlotName,
OriginType: item.Origin.Type,
OriginURI: item.Origin.URI,
Digest: item.Digest,
MediaType: item.MediaType,
SizeBytes: item.SizeBytes,
BindingSource: item.BindingSource,
})
}
provenance = append(provenance, referenceTargetProvenance(lane.ExtractReferences)...)
provenance = append(provenance, referenceTargetProvenance(lane.NormalizeReferences)...)
}
return provenance
}
func referenceTargetProvenance(target ResolvedReferenceTarget) []artifacts.ReferenceProvenance {
if len(target.ReferenceSet.Slots) == 0 {
return nil
}
provenance := []artifacts.ReferenceProvenance{}
slotNames := make([]string, 0, len(target.ReferenceSet.Slots))
for slotName := range target.ReferenceSet.Slots {
slotNames = append(slotNames, slotName)
}
sort.Strings(slotNames)
for _, slotName := range slotNames {
slot := target.ReferenceSet.Slots[slotName]
for _, item := range slot.Items {
provenance = append(provenance, artifacts.ReferenceProvenance{
Stage: string(target.Stage),
LaneID: target.LaneID,
SlotName: item.SlotName,
OriginType: item.Origin.Type,
OriginURI: item.Origin.URI,
Digest: item.Digest,
MediaType: item.MediaType,
SizeBytes: item.SizeBytes,
BindingSource: item.BindingSource,
})
}
}
return provenance

20
internal/framework/pipeline/references_test.go
View File

@@ -54,12 +54,12 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
t.Fatalf("MaterializeReferences(second) error = %v, want nil", err)
}
referenceSet := first.ArtifactLanes[0].ReferenceSet
referenceSet := first.ArtifactLanes[0].ExtractReferences.ReferenceSet
roster := referenceSet.Slots["roster"].Items[0]
if string(roster.Content) != "config text" {
t.Fatalf("roster content = %q, want config text", roster.Content)
}
if roster.Digest != referenceDigest([]byte("config text")) || roster.Digest != second.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0].Digest {
if roster.Digest != referenceDigest([]byte("config text")) || roster.Digest != second.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Digest {
t.Fatalf("roster digest = %q, want stable digest", roster.Digest)
}
if roster.BindingSource != contracts.ReferenceBindingSourceConfig {
@@ -87,10 +87,10 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
if len(provenance) != 2 {
t.Fatalf("ReferenceProvenance() = %#v, want two entries", provenance)
}
if provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest {
if provenance[0].Stage != string(StageExtract) || provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest {
t.Fatalf("ReferenceProvenance()[0] = %#v, want sorted glossary provenance", provenance[0])
}
if provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest {
if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest {
t.Fatalf("ReferenceProvenance()[1] = %#v, want roster provenance", provenance[1])
}
@@ -129,7 +129,7 @@ func TestMaterializeReferencesAllowsAnyMediaTypeWhenSlotDoesNotRestrictIt(t *tes
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != unknownMediaType {
t.Fatalf("MediaType = %q, want %q", item.MediaType, unknownMediaType)
}
@@ -148,7 +148,7 @@ func TestMaterializeReferencesAcceptsDeclaredMarkdownMediaType(t *testing.T) {
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["glossary"].Items[0]
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["glossary"].Items[0]
if item.MediaType != "text/markdown" {
t.Fatalf("MediaType = %q, want text/markdown", item.MediaType)
}
@@ -167,7 +167,7 @@ func TestMaterializeReferencesAcceptsDeclaredJSONMediaType(t *testing.T) {
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != "application/json" {
t.Fatalf("MediaType = %q, want application/json", item.MediaType)
}
@@ -201,7 +201,7 @@ func TestMaterializeReferencesMatchesAcceptedMediaTypesIgnoringParameters(t *tes
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != referenceMediaType {
t.Fatalf("MediaType = %q, want %q", item.MediaType, referenceMediaType)
}
@@ -222,7 +222,7 @@ func TestMaterializeReferencesWarnsForEmptyFiles(t *testing.T) {
if len(warnings) != 1 || warnings[0].ReasonCode != "empty_reference" {
t.Fatalf("warnings = %#v, want empty reference warning", warnings)
}
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.SizeBytes != 0 || item.Digest != referenceDigest(nil) {
t.Fatalf("empty item = %#v, want zero size and empty digest", item)
}
@@ -255,7 +255,7 @@ func resolvedPipelineWithReference(t *testing.T, slotName, source, bindingSource
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if bindingSource != contracts.ReferenceBindingSourceConfig {
resolved.ArtifactLanes[0].References[0].BindingSource = bindingSource
resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].BindingSource = bindingSource
}
return resolved
}

2
internal/framework/pipeline/runner.go
View File

@@ -186,7 +186,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
result, err := extractor.Extract(ctx, contracts.ExtractionRequest{
Source: doc,
Chunk: &chunk,
References: CloneReferenceSet(lane.ReferenceSet),
References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet),
LLMClient: input.LLMClient,
LLMProfile: lane.Extract.LLMProfile,
Options: cloneOptions(lane.Extract.Options),

27
internal/framework/pipeline/runner_test.go
View File

@@ -566,7 +566,7 @@ func TestRunPassesModuleBindingConfigToStageRequests(t *testing.T) {
func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) {
modules := defaultRunnerModules()
pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].ReferenceSet = contracts.ReferenceSet{
pipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
@@ -596,7 +596,7 @@ func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) {
t.Fatalf("reference content = %q, want reference text", item.Content)
}
item.Content[0] = 'R'
if got := string(pipeline.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0].Content); got != "reference text" {
if got := string(pipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content); got != "reference text" {
t.Fatalf("runner mutated reference set content = %q", got)
}
}
@@ -965,7 +965,7 @@ func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) {
func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
resolved := resolvedPipelineWithValidators("configured")
resolved.ArtifactLanes[0].ReferenceSet = contracts.ReferenceSet{
resolved.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
@@ -1003,7 +1003,7 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
t.Fatalf("References = %#v, want one reference provenance entry", manifest.References)
}
reference := manifest.References[0]
if reference.LaneID != "alpha" || reference.SlotName != "roster" || reference.Digest != "sha256:reference" {
if reference.Stage != string(StageExtract) || reference.LaneID != "alpha" || reference.SlotName != "roster" || reference.Digest != "sha256:reference" {
t.Fatalf("reference provenance = %#v, want lane slot digest", reference)
}
if reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != int64(len("reference content")) || reference.BindingSource != contracts.ReferenceBindingSourceConfig {
@@ -1161,16 +1161,19 @@ func TestRunRejectsNilDefaultValidator(t *testing.T) {
func resolvedPipeline() ResolvedPipeline {
return ResolvedPipeline{
ID: "pipeline-1",
Digest: "sha256:pipeline",
Input: Binding("input"),
Chunk: Binding("chunk"),
ID: "pipeline-1",
Digest: "sha256:pipeline",
Input: Binding("input"),
Chunk: Binding("chunk"),
ChunkReferences: referenceTarget(StageChunk, "", "chunk", nil),
ArtifactLanes: []ResolvedArtifactLane{
{
ID: "alpha",
Extract: Binding("extract-alpha"),
Merge: Binding("merge"),
Normalize: Binding("normalize"),
ID: "alpha",
Extract: Binding("extract-alpha"),
Merge: Binding("merge"),
Normalize: Binding("normalize"),
ExtractReferences: referenceTarget(StageExtract, "alpha", "extract-alpha", nil),
NormalizeReferences: referenceTarget(StageNormalize, "alpha", "normalize", nil),
},
},
Output: Binding("output"),

2
internal/framework/pipeline/testdata/walking_skeleton_output.json vendored
View File

@@ -1,7 +1,7 @@
{
"manifest": {
"pipeline_id": "walking-skeleton",
"pipeline_digest": "sha256:5df1e501a2307ef75bbfeb59d315b3710571d52e5466a9c7f8320248740e6fca",
"pipeline_digest": "sha256:25084e39a0cadace375c896551d1752413755c1a6772f6b24cfe91c029ea2631",
"validation_status": "approved",
"artifact_lanes": [
{