eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce target-aware resolved reference storage

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-05 16:12:38 +00:00
parent 39e49d7f77
commit 9278797aa9
17 changed files with 238 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
docs/integrations/json-output.md
View File

@@ -100,8 +100,9 @@ Fields with empty values may be omitted by JSON encoding.
`source_digests` contains source document digests only. Bound references are
recorded separately under `references`, which contains provenance only:
lane ID, slot name, origin type and URI, digest, media type, byte size, and
binding source. Reference content is not written to durable output.
target stage, lane ID when present, slot name, origin type and URI, digest,
media type, byte size, and binding source. Reference content is not written to
durable output.
When references are bound, the manifest section has this shape:
@@ -109,6 +110,7 @@ When references are bound, the manifest section has this shape:
{
"references": [
{
"stage": "extract",
"lane_id": "spells",
"slot_name": "roster",
"origin_type": "file",

9
docs/internal/pipeline.md
View File

@@ -37,8 +37,9 @@ bindings override or add lane bindings, runtime `--reference` requests override
config bindings, and runtime unbinds remove optional bindings. Flat runtime slot
names are resolved only when exactly one selected lane declares the slot;
otherwise the CLI requires `lane.slot`. Resolution validates bindings against
extractor specs and records lane-scoped binding metadata. It does not read
reference files or include reference bytes in source digests.
extractor specs and stores the bindings in target-aware resolved reference
holders. It does not read reference files or include reference bytes in source
digests.
During run preparation, resolved file references are materialized before any
LLM-backed pipeline work. Config bindings resolve relative to the config file,
@@ -51,8 +52,8 @@ bound files. Media-type acceptance is checked only when a slot declares
`AcceptedMediaTypes`; unknown extensions are recorded as
`application/octet-stream`. Reference content is omitted from diagnostics and
manifests. The CLI writes provenance-only resolved reference diagnostics, and
the run manifest records lane-scoped reference provenance separately from source
digests.
the run manifest records target-stage reference provenance separately from
source digests.
Prompt bundles can declare reference slots and use `reference` and
`hasreference` template functions. Bundle loading validates string-literal slot

6
docs/operations.md
View File

@@ -63,9 +63,9 @@ Implemented diagnostics artifacts:
path, selected lanes, run ID, and pipeline digest when available.
- `effective-config.json`: resolved config with API keys redacted.
- `resolved-pipeline.json`: resolved module bindings and pipeline digest.
- `resolved-references.json`: lane-scoped resolved reference provenance,
including origin, digest, media type, byte size, and binding source, without
reference content.
- `resolved-references.json`: resolved reference provenance, including target
stage, lane ID when present, origin, digest, media type, byte size, and
binding source, without reference content.
- `run-manifest.json`: the same run manifest written to durable output when it
is available, including top-level module metadata when present.
- `warnings.json`: warning list.

12
internal/cli/run_test.go
View File

@@ -769,7 +769,7 @@ func TestRunPipelineReferenceFlagBindsUnambiguousSlot(t *testing.T) {
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
}
resolved := readResolvedPipeline(t, diagnosticsDir)
refs := resolved.ArtifactLanes[0].References
refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings
want := []pipeline.ReferenceBinding{
{LaneID: "events", SlotName: "roster", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI},
}
@@ -810,12 +810,12 @@ func TestRunPipelineReferenceFlagBindsLaneQualifiedSlot(t *testing.T) {
}
resolved := readResolvedPipeline(t, diagnosticsDir)
events := resolvedArtifactLane(t, resolved, "events")
if len(events.References) != 0 {
t.Fatalf("events references = %#v, want none", events.References)
if len(events.ExtractReferences.Bindings) != 0 {
t.Fatalf("events references = %#v, want none", events.ExtractReferences.Bindings)
}
notes := resolvedArtifactLane(t, resolved, "notes")
if len(notes.References) != 1 || notes.References[0].Source != referencePath {
t.Fatalf("notes references = %#v, want lane-qualified binding", notes.References)
if len(notes.ExtractReferences.Bindings) != 1 || notes.ExtractReferences.Bindings[0].Source != referencePath {
t.Fatalf("notes references = %#v, want lane-qualified binding", notes.ExtractReferences.Bindings)
}
}
@@ -914,7 +914,7 @@ func TestRunPipelineWithoutReferenceRemovesOptionalConfigBinding(t *testing.T) {
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
}
resolved := readResolvedPipeline(t, diagnosticsDir)
if refs := resolved.ArtifactLanes[0].References; len(refs) != 0 {
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("references = %#v, want unbound optional slot", refs)
}
}

3
internal/core/artifacts/artifacts.go
View File

@@ -49,7 +49,8 @@ type LLMProfileManifest struct {
}
type ReferenceProvenance struct {
LaneID string `json:"lane_id"`
Stage string `json:"stage,omitempty"`
LaneID string `json:"lane_id,omitempty"`
SlotName string `json:"slot_name"`
OriginType string `json:"origin_type"`
OriginURI string `json:"origin_uri,omitempty"`

3
internal/core/artifacts/artifacts_test.go
View File

@@ -187,6 +187,7 @@ func TestRunManifestIncludesReferenceProvenance(t *testing.T) {
manifest := RunManifest{
References: []ReferenceProvenance{
{
Stage: "extract",
LaneID: "events",
SlotName: "roster",
OriginType: "file",
@@ -212,7 +213,7 @@ func TestRunManifestIncludesReferenceProvenance(t *testing.T) {
t.Fatalf("len(References) = %d, want 1", len(got.References))
}
reference := got.References[0]
if reference.LaneID != "events" || reference.SlotName != "roster" || reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" {
if reference.Stage != "extract" || reference.LaneID != "events" || reference.SlotName != "roster" || reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" {
t.Fatalf("reference provenance = %#v, want lane-scoped origin details", reference)
}
if reference.Digest != "sha256:reference" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != 12 || reference.BindingSource != "config" {

5
internal/core/config/redaction.go
View File

@@ -34,6 +34,7 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli
out := in
out.Input = cloneModuleBinding(in.Input)
out.Chunk = cloneModuleBinding(in.Chunk)
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)
out.Output = cloneModuleBinding(in.Output)
if len(in.ArtifactLanes) > 0 {
out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes))
@@ -49,8 +50,8 @@ func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.Resolv
out.Extract = cloneModuleBinding(in.Extract)
out.Merge = cloneModuleBinding(in.Merge)
out.Normalize = cloneModuleBinding(in.Normalize)
out.References = append([]pipeline.ReferenceBinding(nil), in.References...)
out.ReferenceSet = pipeline.CloneReferenceSet(in.ReferenceSet)
out.ExtractReferences = pipeline.CloneReferenceTarget(in.ExtractReferences)
out.NormalizeReferences = pipeline.CloneReferenceTarget(in.NormalizeReferences)
if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
for i, binding := range in.Validators {

27
internal/core/config/redaction_test.go
View File

@@ -108,8 +108,31 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.Options["temperature"] != 0.2 {
t.Fatalf("expected resolved pipeline options to be copied")
}
payload.ResolvedPipeline.ArtifactLanes[0].References[0].Source = "./changed.yml"
if effective.ResolvedPipeline.ArtifactLanes[0].References[0].Source != "./roster.yml" {
payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source = "./changed.yml"
if effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source != "./roster.yml" {
t.Fatalf("expected resolved pipeline references to be copied")
}
effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{
SlotName: "roster",
Content: []byte("reference content"),
},
},
},
},
}
payload, ok = effective.RedactedDiagnosticsPayload().(EffectiveConfig)
if !ok {
t.Fatalf("expected EffectiveConfig payload, got %T", effective.RedactedDiagnosticsPayload())
}
payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content[0] = 'X'
got := effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content
if string(got) != "reference content" {
t.Fatalf("expected materialized reference content to be copied, got %q", got)
}
}

78
internal/framework/pipeline/profile.go
View File

@@ -60,23 +60,32 @@ type ReferenceUnbind struct {
SlotName string `json:"slot_name"`
}
type ResolvedArtifactLane struct {
ID string
Extract ModuleBinding
Merge ModuleBinding
Normalize ModuleBinding
Validators []ModuleBinding
References []ReferenceBinding `json:"references,omitempty"`
type ResolvedReferenceTarget struct {
Stage ModuleStage `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
Module string `json:"module"`
Bindings []ReferenceBinding `json:"bindings,omitempty"`
ReferenceSet contracts.ReferenceSet `json:"-"`
}
type ResolvedArtifactLane struct {
ID string
Extract ModuleBinding
Merge ModuleBinding
Normalize ModuleBinding
Validators []ModuleBinding
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
}
type ResolvedPipeline struct {
ID string
Digest string
Input ModuleBinding
Chunk ModuleBinding
ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding
ID string
Digest string
Input ModuleBinding
Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding
}
type ModuleCatalog struct {
@@ -140,10 +149,11 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
}
resolved := ResolvedPipeline{
ID: pipelineID,
Input: input,
Chunk: chunk,
Output: resolveBinding(profile.Output, DefaultOutputModule),
ID: pipelineID,
Input: input,
Chunk: chunk,
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, nil),
Output: resolveBinding(profile.Output, DefaultOutputModule),
}
outputCapabilities := capabilities.clone()
@@ -206,7 +216,7 @@ func resolveArtifactLane(
if err != nil {
return ResolvedArtifactLane{}, nil, err
}
lane.References = references
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
capabilities.add(extractSpec.Provides...)
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
@@ -225,6 +235,7 @@ func resolveArtifactLane(
if missing, ok := capabilities.missing(normalizeSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing)
}
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, nil)
capabilities.add(normalizeSpec.Provides...)
for _, validator := range lane.Validators {
@@ -241,6 +252,15 @@ func resolveArtifactLane(
return lane, capabilities, nil
}
func referenceTarget(stage ModuleStage, laneID string, module string, bindings []ReferenceBinding) ResolvedReferenceTarget {
return ResolvedReferenceTarget{
Stage: stage,
LaneID: strings.TrimSpace(laneID),
Module: strings.TrimSpace(module),
Bindings: append([]ReferenceBinding(nil), bindings...),
}
}
func validatePipelineReferenceDefaults(
pipelineID string,
pipelineReferences map[string]string,
@@ -532,17 +552,19 @@ func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneP
func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
withoutDigest := struct {
ID string
Input ModuleBinding
Chunk ModuleBinding
ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding
ID string
Input ModuleBinding
Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget
ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding
}{
ID: resolved.ID,
Input: resolved.Input,
Chunk: resolved.Chunk,
ArtifactLanes: resolved.ArtifactLanes,
Output: resolved.Output,
ID: resolved.ID,
Input: resolved.Input,
Chunk: resolved.Chunk,
ChunkReferences: resolved.ChunkReferences,
ArtifactLanes: resolved.ArtifactLanes,
Output: resolved.Output,
}
encoded, err := json.Marshal(withoutDigest)
if err != nil {

21
internal/framework/pipeline/profile_test.go
View File

@@ -154,16 +154,25 @@ func TestResolvePipelineAppliesReferenceBindings(t *testing.T) {
}
events := resolvedLane(t, resolved.ArtifactLanes, "events")
if events.ExtractReferences.Stage != StageExtract || events.ExtractReferences.LaneID != "events" || events.ExtractReferences.Module != "event-extractor" {
t.Fatalf("extract reference target = %#v, want event extractor target", events.ExtractReferences)
}
if events.NormalizeReferences.Stage != StageNormalize || events.NormalizeReferences.LaneID != "events" || events.NormalizeReferences.Module != DefaultNormalizeModule {
t.Fatalf("normalize reference target = %#v, want event normalizer target", events.NormalizeReferences)
}
if resolved.ChunkReferences.Stage != StageChunk || resolved.ChunkReferences.Module != DefaultChunkModule {
t.Fatalf("chunk reference target = %#v, want chunk target", resolved.ChunkReferences)
}
want := []ReferenceBinding{
{LaneID: "events", SlotName: "lore", Source: "./lore.md", BindingSource: contracts.ReferenceBindingSourceConfig},
{LaneID: "events", SlotName: "roster", Source: "./lane-roster.yml", BindingSource: contracts.ReferenceBindingSourceConfig},
}
if !reflect.DeepEqual(events.References, want) {
t.Fatalf("events references = %#v, want %#v", events.References, want)
if !reflect.DeepEqual(events.ExtractReferences.Bindings, want) {
t.Fatalf("events references = %#v, want %#v", events.ExtractReferences.Bindings, want)
}
summaries := resolvedLane(t, resolved.ArtifactLanes, "summaries")
if len(summaries.References) != 0 {
t.Fatalf("summaries references = %#v, want none", summaries.References)
if len(summaries.ExtractReferences.Bindings) != 0 {
t.Fatalf("summaries references = %#v, want none", summaries.ExtractReferences.Bindings)
}
}
@@ -184,7 +193,7 @@ func TestResolvePipelineAllowsPipelineReferenceDeclaredOnlyByUnselectedLane(t *t
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if refs := resolved.ArtifactLanes[0].References; len(refs) != 0 {
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("selected lane references = %#v, want none", refs)
}
}
@@ -284,7 +293,7 @@ func TestResolvePipelineUsesReferenceSlotsFromSpecWithoutConstructingExtractor(t
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if got := resolved.ArtifactLanes[0].References[0].Source; got != "./roster.yml" {
if got := resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].Source; got != "./roster.yml" {
t.Fatalf("reference source = %q, want ./roster.yml", got)
}
}

136
internal/framework/pipeline/references.go
View File

@@ -29,6 +29,7 @@ type ReferenceMaterializationOptions struct {
func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, options ReferenceMaterializationOptions) (ResolvedPipeline, []contracts.Warning, error) {
out := resolved
out.ChunkReferences = CloneReferenceTarget(resolved.ChunkReferences)
if len(resolved.ArtifactLanes) == 0 {
return out, nil, nil
}
@@ -37,32 +38,31 @@ func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, opt
out.ArtifactLanes = make([]ResolvedArtifactLane, len(resolved.ArtifactLanes))
for i, lane := range resolved.ArtifactLanes {
materializedLane := lane
referenceSet, laneWarnings, err := materializeLaneReferences(resolved.ID, lane, catalog, options)
materializedLane.ExtractReferences = CloneReferenceTarget(lane.ExtractReferences)
materializedLane.NormalizeReferences = CloneReferenceTarget(lane.NormalizeReferences)
referenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.ExtractReferences, catalog, options)
if err != nil {
return ResolvedPipeline{}, nil, err
}
materializedLane.ReferenceSet = referenceSet
materializedLane.ExtractReferences.ReferenceSet = referenceSet
out.ArtifactLanes[i] = materializedLane
warnings = append(warnings, laneWarnings...)
}
return out, warnings, nil
}
func materializeLaneReferences(
func materializeReferenceTarget(
pipelineID string,
lane ResolvedArtifactLane,
target ResolvedReferenceTarget,
catalog ModuleCatalog,
options ReferenceMaterializationOptions,
) (contracts.ReferenceSet, []contracts.Warning, error) {
if len(lane.References) == 0 {
if len(target.Bindings) == 0 {
return contracts.ReferenceSet{}, nil, nil
}
if catalog.Extractors == nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", pipelineID, lane.ID, lane.Extract.Module, lane.Extract.Module)
}
spec, ok := catalog.Extractors.Spec(lane.Extract.Module)
if !ok {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", pipelineID, lane.ID, lane.Extract.Module, lane.Extract.Module)
spec, err := referenceTargetSpec(target, catalog)
if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s: %w", referenceTargetContext(pipelineID, target), err)
}
slotByName := make(map[string]contracts.ReferenceSlot, len(spec.ReferenceSlots))
@@ -70,38 +70,38 @@ func materializeLaneReferences(
slotByName[slot.Name] = slot
}
set := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(lane.References))}
set := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(target.Bindings))}
var warnings []contracts.Warning
for _, binding := range lane.References {
for _, binding := range target.Bindings {
slotName := strings.TrimSpace(binding.SlotName)
slot, ok := slotByName[slotName]
if !ok {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q is not declared by extractor %q", pipelineID, lane.ID, slotName, lane.Extract.Module)
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q is not declared by %s module %q", referenceTargetContext(pipelineID, target), slotName, target.Stage, target.Module)
}
path, err := referencePath(binding, options)
if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q: %w", pipelineID, lane.ID, slotName, binding.Source, err)
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q: %w", referenceTargetContext(pipelineID, target), slotName, binding.Source, err)
}
content, err := os.ReadFile(path)
if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q read %q: %w", pipelineID, lane.ID, slotName, path, err)
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q read %q: %w", referenceTargetContext(pipelineID, target), slotName, path, err)
}
if !utf8.Valid(content) {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q must be UTF-8 text", pipelineID, lane.ID, slotName, path)
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q must be UTF-8 text", referenceTargetContext(pipelineID, target), slotName, path)
}
mediaType := referenceMediaTypeForPath(path)
if !referenceMediaTypeAccepted(mediaType, slot.AcceptedMediaTypes) {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q media type %q is not accepted", pipelineID, lane.ID, slotName, path, mediaType)
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q media type %q is not accepted", referenceTargetContext(pipelineID, target), slotName, path, mediaType)
}
if slot.MaxBytes > 0 && int64(len(content)) > slot.MaxBytes {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q is %d bytes, limit %d", pipelineID, lane.ID, slotName, path, len(content), slot.MaxBytes)
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q is %d bytes, limit %d", referenceTargetContext(pipelineID, target), slotName, path, len(content), slot.MaxBytes)
}
if len(content) == 0 {
warnings = append(warnings, contracts.Warning{
Scope: fmt.Sprintf("pipeline.%s.lane.%s.reference.%s", pipelineID, lane.ID, slotName),
Scope: referenceWarningScope(pipelineID, target, slotName),
ReasonCode: "empty_reference",
Message: fmt.Sprintf("reference slot %q for lane %q is bound to an empty file", slotName, lane.ID),
Message: fmt.Sprintf("reference slot %q for %s is bound to an empty file", slotName, referenceTargetLabel(target)),
})
}
@@ -122,6 +122,40 @@ func materializeLaneReferences(
return set, warnings, nil
}
func referenceTargetSpec(target ResolvedReferenceTarget, catalog ModuleCatalog) (ModuleSpec, error) {
switch target.Stage {
case StageChunk:
return registrySpec(catalog.Chunkers, target.Module)
case StageExtract:
return registrySpec(catalog.Extractors, target.Module)
case StageNormalize:
return registrySpec(catalog.Normalizers, target.Module)
default:
return ModuleSpec{}, fmt.Errorf("reference target stage %q is not supported", target.Stage)
}
}
func referenceTargetContext(pipelineID string, target ResolvedReferenceTarget) string {
if target.LaneID != "" {
return fmt.Sprintf("pipeline %q lane %q %s module %q", pipelineID, target.LaneID, target.Stage, target.Module)
}
return fmt.Sprintf("pipeline %q %s module %q", pipelineID, target.Stage, target.Module)
}
func referenceTargetLabel(target ResolvedReferenceTarget) string {
if target.LaneID != "" {
return fmt.Sprintf("lane %q %s target", target.LaneID, target.Stage)
}
return fmt.Sprintf("%s target", target.Stage)
}
func referenceWarningScope(pipelineID string, target ResolvedReferenceTarget, slotName string) string {
if target.LaneID != "" {
return fmt.Sprintf("pipeline.%s.lane.%s.%s.reference.%s", pipelineID, target.LaneID, target.Stage, slotName)
}
return fmt.Sprintf("pipeline.%s.%s.reference.%s", pipelineID, target.Stage, slotName)
}
func referenceMediaTypeForPath(path string) string {
extension := strings.ToLower(filepath.Ext(path))
mediaType := mime.TypeByExtension(extension)
@@ -230,31 +264,47 @@ func CloneReferenceSet(in contracts.ReferenceSet) contracts.ReferenceSet {
return out
}
func CloneReferenceTarget(in ResolvedReferenceTarget) ResolvedReferenceTarget {
out := in
out.Bindings = append([]ReferenceBinding(nil), in.Bindings...)
out.ReferenceSet = CloneReferenceSet(in.ReferenceSet)
return out
}
func ReferenceProvenance(resolved ResolvedPipeline) []artifacts.ReferenceProvenance {
provenance := []artifacts.ReferenceProvenance{}
provenance = append(provenance, referenceTargetProvenance(resolved.ChunkReferences)...)
for _, lane := range resolved.ArtifactLanes {
if len(lane.ReferenceSet.Slots) == 0 {
continue
}
slotNames := make([]string, 0, len(lane.ReferenceSet.Slots))
for slotName := range lane.ReferenceSet.Slots {
slotNames = append(slotNames, slotName)
}
sort.Strings(slotNames)
for _, slotName := range slotNames {
slot := lane.ReferenceSet.Slots[slotName]
for _, item := range slot.Items {
provenance = append(provenance, artifacts.ReferenceProvenance{
LaneID: lane.ID,
SlotName: item.SlotName,
OriginType: item.Origin.Type,
OriginURI: item.Origin.URI,
Digest: item.Digest,
MediaType: item.MediaType,
SizeBytes: item.SizeBytes,
BindingSource: item.BindingSource,
})
}
provenance = append(provenance, referenceTargetProvenance(lane.ExtractReferences)...)
provenance = append(provenance, referenceTargetProvenance(lane.NormalizeReferences)...)
}
return provenance
}
func referenceTargetProvenance(target ResolvedReferenceTarget) []artifacts.ReferenceProvenance {
if len(target.ReferenceSet.Slots) == 0 {
return nil
}
provenance := []artifacts.ReferenceProvenance{}
slotNames := make([]string, 0, len(target.ReferenceSet.Slots))
for slotName := range target.ReferenceSet.Slots {
slotNames = append(slotNames, slotName)
}
sort.Strings(slotNames)
for _, slotName := range slotNames {
slot := target.ReferenceSet.Slots[slotName]
for _, item := range slot.Items {
provenance = append(provenance, artifacts.ReferenceProvenance{
Stage: string(target.Stage),
LaneID: target.LaneID,
SlotName: item.SlotName,
OriginType: item.Origin.Type,
OriginURI: item.Origin.URI,
Digest: item.Digest,
MediaType: item.MediaType,
SizeBytes: item.SizeBytes,
BindingSource: item.BindingSource,
})
}
}
return provenance

20
internal/framework/pipeline/references_test.go
View File

@@ -54,12 +54,12 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
t.Fatalf("MaterializeReferences(second) error = %v, want nil", err)
}
referenceSet := first.ArtifactLanes[0].ReferenceSet
referenceSet := first.ArtifactLanes[0].ExtractReferences.ReferenceSet
roster := referenceSet.Slots["roster"].Items[0]
if string(roster.Content) != "config text" {
t.Fatalf("roster content = %q, want config text", roster.Content)
}
if roster.Digest != referenceDigest([]byte("config text")) || roster.Digest != second.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0].Digest {
if roster.Digest != referenceDigest([]byte("config text")) || roster.Digest != second.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Digest {
t.Fatalf("roster digest = %q, want stable digest", roster.Digest)
}
if roster.BindingSource != contracts.ReferenceBindingSourceConfig {
@@ -87,10 +87,10 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
if len(provenance) != 2 {
t.Fatalf("ReferenceProvenance() = %#v, want two entries", provenance)
}
if provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest {
if provenance[0].Stage != string(StageExtract) || provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest {
t.Fatalf("ReferenceProvenance()[0] = %#v, want sorted glossary provenance", provenance[0])
}
if provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest {
if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest {
t.Fatalf("ReferenceProvenance()[1] = %#v, want roster provenance", provenance[1])
}
@@ -129,7 +129,7 @@ func TestMaterializeReferencesAllowsAnyMediaTypeWhenSlotDoesNotRestrictIt(t *tes
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != unknownMediaType {
t.Fatalf("MediaType = %q, want %q", item.MediaType, unknownMediaType)
}
@@ -148,7 +148,7 @@ func TestMaterializeReferencesAcceptsDeclaredMarkdownMediaType(t *testing.T) {
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["glossary"].Items[0]
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["glossary"].Items[0]
if item.MediaType != "text/markdown" {
t.Fatalf("MediaType = %q, want text/markdown", item.MediaType)
}
@@ -167,7 +167,7 @@ func TestMaterializeReferencesAcceptsDeclaredJSONMediaType(t *testing.T) {
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != "application/json" {
t.Fatalf("MediaType = %q, want application/json", item.MediaType)
}
@@ -201,7 +201,7 @@ func TestMaterializeReferencesMatchesAcceptedMediaTypesIgnoringParameters(t *tes
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != referenceMediaType {
t.Fatalf("MediaType = %q, want %q", item.MediaType, referenceMediaType)
}
@@ -222,7 +222,7 @@ func TestMaterializeReferencesWarnsForEmptyFiles(t *testing.T) {
if len(warnings) != 1 || warnings[0].ReasonCode != "empty_reference" {
t.Fatalf("warnings = %#v, want empty reference warning", warnings)
}
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.SizeBytes != 0 || item.Digest != referenceDigest(nil) {
t.Fatalf("empty item = %#v, want zero size and empty digest", item)
}
@@ -255,7 +255,7 @@ func resolvedPipelineWithReference(t *testing.T, slotName, source, bindingSource
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if bindingSource != contracts.ReferenceBindingSourceConfig {
resolved.ArtifactLanes[0].References[0].BindingSource = bindingSource
resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].BindingSource = bindingSource
}
return resolved
}

2
internal/framework/pipeline/runner.go
View File

@@ -186,7 +186,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
result, err := extractor.Extract(ctx, contracts.ExtractionRequest{
Source: doc,
Chunk: &chunk,
References: CloneReferenceSet(lane.ReferenceSet),
References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet),
LLMClient: input.LLMClient,
LLMProfile: lane.Extract.LLMProfile,
Options: cloneOptions(lane.Extract.Options),

27
internal/framework/pipeline/runner_test.go
View File

@@ -566,7 +566,7 @@ func TestRunPassesModuleBindingConfigToStageRequests(t *testing.T) {
func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) {
modules := defaultRunnerModules()
pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].ReferenceSet = contracts.ReferenceSet{
pipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
@@ -596,7 +596,7 @@ func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) {
t.Fatalf("reference content = %q, want reference text", item.Content)
}
item.Content[0] = 'R'
if got := string(pipeline.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0].Content); got != "reference text" {
if got := string(pipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content); got != "reference text" {
t.Fatalf("runner mutated reference set content = %q", got)
}
}
@@ -965,7 +965,7 @@ func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) {
func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
resolved := resolvedPipelineWithValidators("configured")
resolved.ArtifactLanes[0].ReferenceSet = contracts.ReferenceSet{
resolved.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
@@ -1003,7 +1003,7 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
t.Fatalf("References = %#v, want one reference provenance entry", manifest.References)
}
reference := manifest.References[0]
if reference.LaneID != "alpha" || reference.SlotName != "roster" || reference.Digest != "sha256:reference" {
if reference.Stage != string(StageExtract) || reference.LaneID != "alpha" || reference.SlotName != "roster" || reference.Digest != "sha256:reference" {
t.Fatalf("reference provenance = %#v, want lane slot digest", reference)
}
if reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != int64(len("reference content")) || reference.BindingSource != contracts.ReferenceBindingSourceConfig {
@@ -1161,16 +1161,19 @@ func TestRunRejectsNilDefaultValidator(t *testing.T) {
func resolvedPipeline() ResolvedPipeline {
return ResolvedPipeline{
ID: "pipeline-1",
Digest: "sha256:pipeline",
Input: Binding("input"),
Chunk: Binding("chunk"),
ID: "pipeline-1",
Digest: "sha256:pipeline",
Input: Binding("input"),
Chunk: Binding("chunk"),
ChunkReferences: referenceTarget(StageChunk, "", "chunk", nil),
ArtifactLanes: []ResolvedArtifactLane{
{
ID: "alpha",
Extract: Binding("extract-alpha"),
Merge: Binding("merge"),
Normalize: Binding("normalize"),
ID: "alpha",
Extract: Binding("extract-alpha"),
Merge: Binding("merge"),
Normalize: Binding("normalize"),
ExtractReferences: referenceTarget(StageExtract, "alpha", "extract-alpha", nil),
NormalizeReferences: referenceTarget(StageNormalize, "alpha", "normalize", nil),
},
},
Output: Binding("output"),

2
internal/framework/pipeline/testdata/walking_skeleton_output.json vendored
View File

@@ -1,7 +1,7 @@
{
"manifest": {
"pipeline_id": "walking-skeleton",
"pipeline_digest": "sha256:5df1e501a2307ef75bbfeb59d315b3710571d52e5466a9c7f8320248740e6fca",
"pipeline_digest": "sha256:25084e39a0cadace375c896551d1752413755c1a6772f6b24cfe91c029ea2631",
"validation_status": "approved",
"artifact_lanes": [
{

4
internal/modules/extract/dnd/spells/runner_test.go
View File

@@ -114,7 +114,7 @@ func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T)
raw := readDNDSpellsFixture(t)
expectedDoc := parseDNDSpellsFixture(t, raw)
resolved := resolveDNDSpellsPipeline(t)
resolved.ResolvedPipeline.ArtifactLanes[0].ReferenceSet = dndSpellsReferenceSet(
resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet(
"Aria: party cleric\nBorin: fighter",
"Fire Bolt: evocation cantrip",
)
@@ -168,7 +168,7 @@ func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T)
func TestRunnerDoesNotExtractSpellMentionedOnlyInRoster(t *testing.T) {
raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t)
resolved.ResolvedPipeline.ArtifactLanes[0].ReferenceSet = dndSpellsReferenceSet(
resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet(
"Mira: wizard who can cast Lightning Bolt",
"",
)

1
internal/modules/output/json/encoder_test.go
View File

@@ -148,6 +148,7 @@ func TestEncodeIncludesManifestReferences(t *testing.T) {
RunID: "run-1",
References: []artifacts.ReferenceProvenance{
{
Stage: "extract",
LaneID: "events",
SlotName: "roster",
OriginType: "file",