Add merge references and retry config

This commit is contained in:
2026-07-07 19:10:55 +00:00
parent c05ecb58d8
commit bcedf19a08
25 changed files with 466 additions and 95 deletions

View File

@@ -49,8 +49,8 @@ Flags:
use one Scriptorium profile ID. use one Scriptorium profile ID.
- `--session-id id`: pass a stable prompt session identifier through LLM-backed - `--session-id id`: pass a stable prompt session identifier through LLM-backed
module calls. module calls.
- `--reference selector=path`: bind a reference path to a chunk, extractor, or - `--reference selector=path`: bind a reference path to a chunk, extractor,
normalizer reference slot. Repeatable. merger, or normalizer reference slot. Repeatable.
- `--without-reference selector`: remove a configured optional reference binding. - `--without-reference selector`: remove a configured optional reference binding.
Repeatable. It accepts the same selector forms as `--reference`, without Repeatable. It accepts the same selector forms as `--reference`, without
`=path`. `=path`.
@@ -59,12 +59,12 @@ On success, the command prints the completed pipeline ID, normalized output and
rejected output counts, and the output directory. If the run completes with warnings, rejected output counts, and the output directory. If the run completes with warnings,
the warning count is printed to stderr. the warning count is printed to stderr.
Reference flags are resolved against selected chunk, extractor, and normalizer Reference flags are resolved against selected chunk, extractor, merger, and normalizer
targets before the run starts. Flat slot names are accepted only when exactly targets before the run starts. Flat slot names are accepted only when exactly
one selected target declares that slot. Bound reference files are read before one selected target declares that slot. Bound reference files are read before
pipeline work starts, validated as UTF-8 text, and recorded as provenance for pipeline work starts, validated as UTF-8 text, and recorded as provenance for
the target that declares the slot. Runtime reference content is passed to the the target that declares the slot. Runtime reference content is passed to the
chunker, extractor, or normalizer target that declares the slot. Notarius infers chunker, extractor, merger, or normalizer target that declares the slot. Notarius infers
reference media types from file extensions for provenance and for optional slot reference media types from file extensions for provenance and for optional slot
checks. Reference content is not written to diagnostics, logs, errors, or checks. Reference content is not written to diagnostics, logs, errors, or
manifests. manifests.
@@ -81,9 +81,11 @@ Reference binding precedence is:
- `slot=path`: valid when exactly one selected target declares `slot`; - `slot=path`: valid when exactly one selected target declares `slot`;
- `chunk.slot=path`: target the chunker; - `chunk.slot=path`: target the chunker;
- `lane.slot=path`: valid when exactly one selected extractor or normalizer in - `merge.slot=path`: valid when exactly one selected merger declares `slot`;
that lane declares `slot`; - `lane.slot=path`: valid when exactly one selected extractor, merger, or
normalizer in that lane declares `slot`;
- `lane.extract.slot=path`: target a lane extractor; - `lane.extract.slot=path`: target a lane extractor;
- `lane.merge.slot=path`: target a lane merger;
- `lane.normalize.slot=path`: target a lane normalizer. - `lane.normalize.slot=path`: target a lane normalizer.
Use `slot=path` when the selected targets declare the slot unambiguously: Use `slot=path` when the selected targets declare the slot unambiguously:
@@ -105,7 +107,7 @@ go run ./cmd/notarius run dnd-session \
--reference spells.extract.glossary=./campaign-glossary.txt --reference spells.extract.glossary=./campaign-glossary.txt
``` ```
The same grammar can target chunk and normalize slots when the configured The same grammar can target chunk, merge, and normalize slots when the configured
modules declare them: modules declare them:
```sh ```sh
@@ -113,6 +115,7 @@ go run ./cmd/notarius run dnd-session \
--config path/to/config.yml \ --config path/to/config.yml \
--input examples/seriatim-minimal-transcript.json \ --input examples/seriatim-minimal-transcript.json \
--reference chunk.scene_guide=./campaign-scenes.txt \ --reference chunk.scene_guide=./campaign-scenes.txt \
--reference spells.merge.merge_notes=./merge-notes.txt \
--reference spells.normalize.normalization_notes=./normalization-notes.txt --reference spells.normalize.normalization_notes=./normalization-notes.txt
``` ```

View File

@@ -141,13 +141,13 @@ against the production module catalog and fail fast for unknown or incompatible
module keys. module keys.
Reference bindings are validated against reference slots declared by eligible Reference bindings are validated against reference slots declared by eligible
chunk, extract, and normalize targets during pipeline resolution. Required slots chunk, extract, merge, and normalize targets during pipeline resolution. Required slots
must be bound after config defaults, target-local references, lane-level must be bound after config defaults, target-local references, lane-level
compatibility bindings, and run-time `--reference` or `--without-reference` compatibility bindings, and run-time `--reference` or `--without-reference`
overrides are applied. Config-relative paths are resolved relative to the config overrides are applied. Config-relative paths are resolved relative to the config
file; CLI reference paths are resolved relative to the current working file; CLI reference paths are resolved relative to the current working
directory. Materialized bound files must be UTF-8 text. Materialized reference directory. Materialized bound files must be UTF-8 text. Materialized reference
provenance is recorded for chunk, extractor, and normalizer targets, and runtime provenance is recorded for chunk, extractor, merger, and normalizer targets, and runtime
reference content is passed to the target that declares the slot. Reference reference content is passed to the target that declares the slot. Reference
media types are inferred from file extensions, recorded as canonical base media media types are inferred from file extensions, recorded as canonical base media
types, and checked only when a module declares `AcceptedMediaTypes`; unknown types, and checked only when a module declares `AcceptedMediaTypes`; unknown
@@ -156,8 +156,8 @@ written to diagnostics, logs, errors, or manifests.
Pipeline-level `references` are defaults. They are valid when at least one Pipeline-level `references` are defaults. They are valid when at least one
eligible target in the full configured pipeline declares the slot, including eligible target in the full configured pipeline declares the slot, including
chunk, extractor, and normalizer targets. During a run, they apply only to the chunk, extractor, merger, and normalizer targets. During a run, they apply only
selected targets that declare the slot: to the selected targets that declare the slot:
```yaml ```yaml
pipelines: pipelines:
@@ -192,15 +192,17 @@ pipelines:
party: ./campaign/session-party.txt party: ./campaign/session-party.txt
``` ```
`chunk.references` and `normalize.references` are accepted in object-form `chunk.references`, `merge.references`, and `normalize.references` are accepted
bindings. They override pipeline-level defaults for slots declared by the chunk in object-form bindings. They override pipeline-level defaults for slots
or normalizer module. Extractor-local references apply only to the extractor, declared by that target module. Extractor-local references apply only to the
and normalizer-local references apply only to the normalizer. extractor, merger-local references apply only to the merger, and
normalizer-local references apply only to the normalizer.
Target-local reference fields use the same map shape at: Target-local reference fields use the same map shape at:
- `pipelines.<id>.chunk.references` - `pipelines.<id>.chunk.references`
- `pipelines.<id>.artifacts.<lane>.extract.references` - `pipelines.<id>.artifacts.<lane>.extract.references`
- `pipelines.<id>.artifacts.<lane>.merge.references`
- `pipelines.<id>.artifacts.<lane>.normalize.references` - `pipelines.<id>.artifacts.<lane>.normalize.references`
Each binding is valid only when that target module declares the slot. Each binding is valid only when that target module declares the slot.
@@ -226,13 +228,16 @@ Binding fields:
- `module`: module key. - `module`: module key.
- `llm_profile`: optional Scriptorium profile ID. Empty or omitted lets the - `llm_profile`: optional Scriptorium profile ID. Empty or omitted lets the
Scriptorium prompt default select the profile. Scriptorium prompt default select the profile.
- `retries`: non-negative retry count for runtime stages that support retries.
The current runner preserves this value in resolved config.
- `options`: optional module-specific settings. - `options`: optional module-specific settings.
- `references`: optional reference bindings. Supported only for `chunk`, - `references`: optional reference bindings. Supported only for `chunk`,
`extract`, and `normalize` bindings. `input`, `merge`, validator, and `extract`, `merge`, and `normalize` bindings. `input`, validator, and
`output` bindings reject this field during validation. `output` bindings reject this field during validation.
The `--llm-profile` run flag overrides every effective LLM-capable module The `--llm-profile` run flag overrides every effective LLM-capable module
binding to use one Scriptorium profile ID. binding to use one Scriptorium profile ID: chunk, every selected lane extract,
merge, and normalize binding.
## Implemented Production Modules ## Implemented Production Modules

View File

@@ -41,6 +41,10 @@ The runtime records the actual selected Scriptorium profile, provider, and model
used during execution. Manifest population does not rely on a precomputed used during execution. Manifest population does not rely on a precomputed
profile ID before pipeline execution. profile ID before pipeline execution.
Explicit profile validation and `--llm-profile` overrides apply to LLM-capable
pipeline stages: chunk, extract, merge, and normalize. Input, output, and
validator bindings are not part of the current production LLM profile scope.
## Scriptorium Adapter ## Scriptorium Adapter
`ScriptoriumClient` implements `contracts.StructuredLLMClient` by converting `ScriptoriumClient` implements `contracts.StructuredLLMClient` by converting

View File

@@ -20,7 +20,7 @@ A production module package should provide:
Module specs should describe capabilities accurately. Resolution uses specs to Module specs should describe capabilities accurately. Resolution uses specs to
reject incompatible pipelines before execution. reject incompatible pipelines before execution.
Chunk, extract, and normalize modules that accept auxiliary reference material Chunk, extract, merge, and normalize modules that accept auxiliary reference material
must declare slots through both `ReferenceSlots()` and must declare slots through both `ReferenceSlots()` and
`ModuleSpec().ReferenceSlots`. The runtime slot list and registry metadata `ModuleSpec().ReferenceSlots`. The runtime slot list and registry metadata
should match so config validation can inspect slots without constructing module should match so config validation can inspect slots without constructing module
@@ -31,10 +31,10 @@ must still be UTF-8 text. When a slot declares accepted media types, Notarius
compares the canonical base media type inferred from the file extension, compares the canonical base media type inferred from the file extension,
case-insensitively and without parameters. case-insensitively and without parameters.
The resolver materializes reference content for chunk, extractor, and The resolver materializes reference content for chunk, extractor, merger, and
normalizer targets. Runtime delivery uses `contracts.ChunkRequest.References`, normalizer targets. Runtime delivery uses `contracts.ChunkRequest.References`,
`contracts.ExtractionRequest.References`, and `contracts.ExtractionRequest.References`, `contracts.MergeRequest.References`,
`contracts.NormalizeRequest.References`. Reference material is not source and `contracts.NormalizeRequest.References`. Reference material is not source
evidence and must not be converted into `SourceRef` values. If a module prompt evidence and must not be converted into `SourceRef` values. If a module prompt
uses references, pass them as prompt input materials through the structured LLM uses references, pass them as prompt input materials through the structured LLM
request. Prompt metadata hashes remain based on prompt asset source, not request. Prompt metadata hashes remain based on prompt asset source, not
@@ -48,7 +48,7 @@ Common D&D prompt fragments, reference slot helpers, prompt input assembly, and
reference rendering live under `internal/modules/sharedassets/dnd`. Module reference rendering live under `internal/modules/sharedassets/dnd`. Module
contracts should expose prompt IDs, versions, input material names, and contracts should expose prompt IDs, versions, input material names, and
non-secret prompt/schema hashes through manifest metadata; they should not non-secret prompt/schema hashes through manifest metadata; they should not
expose Scriptorium public types through chunk, extract, or normalize contracts. expose Scriptorium public types through chunk, extract, merge, or normalize contracts.
Chunk modules receive the structured LLM client, configured Scriptorium profile Chunk modules receive the structured LLM client, configured Scriptorium profile
ID, prompt session ID, and raw source input material through ID, prompt session ID, and raw source input material through
@@ -60,6 +60,10 @@ Normalize modules receive the structured LLM client, configured Scriptorium
profile ID, prompt session ID, and reference material through profile ID, prompt session ID, and reference material through
`contracts.NormalizeRequest` when they need model-backed reconciliation. `contracts.NormalizeRequest` when they need model-backed reconciliation.
Merge modules receive the structured LLM client, configured Scriptorium profile
ID, prompt session ID, raw source input material, and reference material through
`contracts.MergeRequest` when they need model-backed merge behavior.
## `seriatim` Input ## `seriatim` Input
Package: `internal/modules/input/seriatim` Package: `internal/modules/input/seriatim`

View File

@@ -33,18 +33,19 @@ The CLI writes the resolved pipeline and digest to diagnostics.
Pipeline profiles and artifact lanes may include reference binding maps keyed by Pipeline profiles and artifact lanes may include reference binding maps keyed by
reference slot name. During resolution, pipeline-level bindings act as defaults reference slot name. During resolution, pipeline-level bindings act as defaults
for selected chunk, extractor, and normalizer targets that declare the slot; for selected chunk, extractor, merger, and normalizer targets that declare the
target-local bindings override or add bindings for that target. Runtime slot; target-local bindings override or add bindings for that target. Runtime
`--reference` requests override target config bindings, and runtime unbinds `--reference` requests override target config bindings, and runtime unbinds
remove optional target bindings. Flat runtime slot names are resolved only when remove optional target bindings. Flat runtime slot names are resolved only when
exactly one selected target declares the slot; otherwise the CLI requires a more exactly one selected target declares the slot; otherwise the CLI requires a more
specific selector such as `chunk.slot`, `lane.extract.slot`, or specific selector such as `chunk.slot`, `lane.extract.slot`,
`lane.normalize.slot`. Resolution validates bindings against the declaring `lane.merge.slot`, or `lane.normalize.slot`. Resolution validates bindings
target specs and stores the bindings in target-aware resolved reference holders. against the declaring target specs and stores the bindings in target-aware
It does not read reference files or include reference bytes in source digests. resolved reference holders. It does not read reference files or include
reference bytes in source digests.
During run preparation, resolved file references for chunk, extractor, and During run preparation, resolved file references for chunk, extractor, merger,
normalizer targets are materialized before any LLM-backed pipeline work. Config and normalizer targets are materialized before any LLM-backed pipeline work. Config
bindings resolve relative to the config file, and CLI bindings resolve relative bindings resolve relative to the config file, and CLI bindings resolve relative
to the current working directory. Materialization accepts UTF-8 text files, to the current working directory. Materialization accepts UTF-8 text files,
computes `sha256:` content digests, records file origins, infers canonical base computes `sha256:` content digests, records file origins, infers canonical base
@@ -55,8 +56,8 @@ empty bound files. Media-type acceptance is checked only when a slot declares
manifests. The CLI writes provenance-only resolved reference diagnostics, and manifests. The CLI writes provenance-only resolved reference diagnostics, and
the run manifest records target-stage reference provenance separately from the run manifest records target-stage reference provenance separately from
source digests. Runtime reference content is passed to the matching chunker, source digests. Runtime reference content is passed to the matching chunker,
extractor, or normalizer request. LLM-backed modules pass that material onward extractor, merger, or normalizer request. LLM-backed modules pass that material
as named Scriptorium prompt inputs. onward as named Scriptorium prompt inputs.
The CLI carries raw input bytes into `pipeline.RunInput`. Input adapters parse The CLI carries raw input bytes into `pipeline.RunInput`. Input adapters parse
those bytes into the source document, while LLM-backed modules that need the those bytes into the source document, while LLM-backed modules that need the
@@ -65,9 +66,9 @@ origin metadata. The raw input payload is not written to manifests or default
diagnostics. diagnostics.
The CLI also carries an optional run `session_id`. The runner makes it available The CLI also carries an optional run `session_id`. The runner makes it available
to chunk, extract, and normalize requests; LLM-backed modules forward it through to chunk, extract, merge, and normalize requests; LLM-backed modules forward it
their structured completion requests so Scriptorium can include it in prompt through their structured completion requests so Scriptorium can include it in
execution metadata. prompt execution metadata.
## Registries And Module Specs ## Registries And Module Specs
@@ -83,10 +84,9 @@ Every production module registers a `ModuleSpec` with:
- `Provides`: capabilities added after that module runs; - `Provides`: capabilities added after that module runs;
- `Requires`: capabilities that must already be available. - `Requires`: capabilities that must already be available.
Chunk, extract, and normalize specs may also declare reference slots. Slot Chunk, extract, merge, and normalize specs may also declare reference slots. Slot
declarations are available from registry metadata without constructing module declarations are available from registry metadata without constructing module
instances. Input, merge, validate, and output specs must not declare reference instances. Input, validate, and output specs must not declare reference slots.
slots.
Capability checks prevent incompatible pipeline composition before a run starts. Capability checks prevent incompatible pipeline composition before a run starts.

View File

@@ -118,16 +118,18 @@ Symptoms include:
Fix: Fix:
- Confirm the selected chunker, extractor, or normalizer declares the slot. The - Confirm the selected chunker, extractor, merger, or normalizer declares the slot. The
implemented `dnd/scenes` chunker and `dnd/spells` extractor declare optional implemented `dnd/scenes` chunker and `dnd/spells` extractor declare optional
`roster` and `glossary` slots. `roster` and `glossary` slots.
- Use a specific selector when more than one selected target declares the same - Use a specific selector when more than one selected target declares the same
slot. Examples include `chunk.context=./context.txt`, slot: `chunk.context=./context.txt`,
`spells.extract.context=./extract-context.txt`, and `spells.extract.context=./extract-context.txt`,
`spells.merge.context=./merge-context.txt`, or
`spells.normalize.context=./normalize-context.txt`. `spells.normalize.context=./normalize-context.txt`.
- `lane.slot=path` is valid only when exactly one selected extractor or - `lane.slot=path` is valid only when exactly one selected extractor, merger,
normalizer in that lane declares the slot. If both do, use or normalizer in that lane declares the slot. If more than one does, use
`lane.extract.slot=path` or `lane.normalize.slot=path`. `lane.extract.slot=path`, `lane.merge.slot=path`, or
`lane.normalize.slot=path`.
- Use `--without-reference selector` to remove optional config bindings; do not - Use `--without-reference selector` to remove optional config bindings; do not
pass an empty `--reference selector=`. pass an empty `--reference selector=`.
- Check whether a path came from config or CLI. Config paths are relative to - Check whether a path came from config or CLI. Config paths are relative to

View File

@@ -99,7 +99,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
referenceFlags := stringListFlag{} referenceFlags := stringListFlag{}
withoutReferenceFlags := stringListFlag{} withoutReferenceFlags := stringListFlag{}
fs.Var(&sessionID, "session-id", "prompt session identifier") fs.Var(&sessionID, "session-id", "prompt session identifier")
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, lane.slot=path, lane.extract.slot=path, or lane.normalize.slot=path") fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, merge.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference") fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
if err := validateRunFlagValues(args); err != nil { if err := validateRunFlagValues(args); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err) fmt.Fprintf(stderr, "notarius: %v\n", err)
@@ -492,6 +492,7 @@ func effectiveLLMProfileIDs(resolved pipeline.ResolvedPipeline) []string {
add(resolved.Chunk) add(resolved.Chunk)
for _, lane := range resolved.ArtifactLanes { for _, lane := range resolved.ArtifactLanes {
add(lane.Extract) add(lane.Extract)
add(lane.Merge)
add(lane.Normalize) add(lane.Normalize)
} }
ids := make([]string, 0, len(seen)) ids := make([]string, 0, len(seen))
@@ -834,17 +835,20 @@ func parseReferenceSelector(raw string, flagName string) (cliReferenceSelector,
if first == string(pipeline.StageChunk) { if first == string(pipeline.StageChunk) {
return cliReferenceSelector{Stage: pipeline.StageChunk, SlotName: slotName}, nil return cliReferenceSelector{Stage: pipeline.StageChunk, SlotName: slotName}, nil
} }
if first == string(pipeline.StageMerge) {
return cliReferenceSelector{Stage: pipeline.StageMerge, SlotName: slotName}, nil
}
return cliReferenceSelector{LaneID: first, SlotName: slotName}, nil return cliReferenceSelector{LaneID: first, SlotName: slotName}, nil
case 3: case 3:
laneID := strings.TrimSpace(parts[0]) laneID := strings.TrimSpace(parts[0])
stage := pipeline.ModuleStage(strings.TrimSpace(parts[1])) stage := pipeline.ModuleStage(strings.TrimSpace(parts[1]))
slotName := strings.TrimSpace(parts[2]) slotName := strings.TrimSpace(parts[2])
if stage != pipeline.StageExtract && stage != pipeline.StageNormalize { if stage != pipeline.StageExtract && stage != pipeline.StageMerge && stage != pipeline.StageNormalize {
return cliReferenceSelector{}, fmt.Errorf("%s lane-qualified selector must use lane.extract.slot or lane.normalize.slot", flagName) return cliReferenceSelector{}, fmt.Errorf("%s lane-qualified selector must use lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName)
} }
return cliReferenceSelector{LaneID: laneID, Stage: stage, SlotName: slotName}, nil return cliReferenceSelector{LaneID: laneID, Stage: stage, SlotName: slotName}, nil
default: default:
return cliReferenceSelector{}, fmt.Errorf("%s must use slot, chunk.slot, lane.slot, lane.extract.slot, or lane.normalize.slot", flagName) return cliReferenceSelector{}, fmt.Errorf("%s must use slot, chunk.slot, merge.slot, lane.slot, lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName)
} }
} }
@@ -944,7 +948,7 @@ func selectedReferenceTargets(cfg config.Config, pipelineID string, only []strin
} }
sort.Strings(selectedIDs) sort.Strings(selectedIDs)
targets := make([]selectedReferenceTarget, 0, 1+len(selectedIDs)*2) targets := make([]selectedReferenceTarget, 0, 1+len(selectedIDs)*3)
chunk := pipeline.Binding(profile.Chunk.Module) chunk := pipeline.Binding(profile.Chunk.Module)
chunk.Module = strings.TrimSpace(profile.Chunk.Module) chunk.Module = strings.TrimSpace(profile.Chunk.Module)
if chunk.Module == "" { if chunk.Module == "" {
@@ -977,6 +981,21 @@ func selectedReferenceTargets(cfg config.Config, pipelineID string, only []strin
slots: referenceSlotSet(extractSpec.ReferenceSlots), slots: referenceSlotSet(extractSpec.ReferenceSlots),
}) })
mergeModule := strings.TrimSpace(lane.Merge.Module)
if mergeModule == "" {
mergeModule = pipeline.DefaultMergeModule
}
mergeSpec, err := cliReferenceMergerSpec(catalog, mergeModule)
if err != nil {
return nil, fmt.Errorf("pipeline %q lane %q merge module %q: %w", strings.TrimSpace(pipelineID), laneID, mergeModule, err)
}
targets = append(targets, selectedReferenceTarget{
laneID: laneID,
stage: pipeline.StageMerge,
module: mergeModule,
slots: referenceSlotSet(mergeSpec.ReferenceSlots),
})
normalizeModule := strings.TrimSpace(lane.Normalize.Module) normalizeModule := strings.TrimSpace(lane.Normalize.Module)
if normalizeModule == "" { if normalizeModule == "" {
normalizeModule = pipeline.DefaultNormalizeModule normalizeModule = pipeline.DefaultNormalizeModule
@@ -1027,6 +1046,17 @@ func cliReferenceExtractorSpec(catalog pipeline.ModuleCatalog, module string) (p
return spec, nil return spec, nil
} }
func cliReferenceMergerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
if catalog.Mergers == nil {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
spec, ok := catalog.Mergers.Spec(module)
if !ok {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
return spec, nil
}
func cliReferenceNormalizerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) { func cliReferenceNormalizerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
if catalog.Normalizers == nil { if catalog.Normalizers == nil {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module) return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
@@ -1063,7 +1093,10 @@ func resolveCLIReferenceTarget(targets []selectedReferenceTarget, selector cliRe
} }
return selectedReferenceTarget{}, fmt.Errorf("reference chunk target is not selected") return selectedReferenceTarget{}, fmt.Errorf("reference chunk target is not selected")
} }
if selector.Stage == pipeline.StageExtract || selector.Stage == pipeline.StageNormalize { if selector.Stage == pipeline.StageExtract || selector.Stage == pipeline.StageMerge || selector.Stage == pipeline.StageNormalize {
if selector.LaneID == "" && selector.Stage == pipeline.StageMerge {
return resolveCLIReferenceStageTarget(targets, selector.Stage, slotName)
}
for _, target := range targets { for _, target := range targets {
if target.laneID == selector.LaneID && target.stage == selector.Stage { if target.laneID == selector.LaneID && target.stage == selector.Stage {
if _, ok := target.slots[slotName]; !ok { if _, ok := target.slots[slotName]; !ok {
@@ -1080,6 +1113,26 @@ func resolveCLIReferenceTarget(targets []selectedReferenceTarget, selector cliRe
return resolveCLIReferenceFlatTarget(targets, slotName) return resolveCLIReferenceFlatTarget(targets, slotName)
} }
func resolveCLIReferenceStageTarget(targets []selectedReferenceTarget, stage pipeline.ModuleStage, slotName string) (selectedReferenceTarget, error) {
matches := make([]selectedReferenceTarget, 0, 2)
for _, target := range targets {
if target.stage != stage {
continue
}
if _, ok := target.slots[slotName]; ok {
matches = append(matches, target)
}
}
switch len(matches) {
case 0:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected %s target", slotName, stage)
case 1:
return matches[0], nil
default:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected %s targets (%s); use a more specific selector such as %s", slotName, stage, targetList(matches), selectorSuggestions(matches, slotName))
}
}
func resolveCLIReferenceLaneTarget(targets []selectedReferenceTarget, laneID string, slotName string) (selectedReferenceTarget, error) { func resolveCLIReferenceLaneTarget(targets []selectedReferenceTarget, laneID string, slotName string) (selectedReferenceTarget, error) {
laneSelected := false laneSelected := false
matches := make([]selectedReferenceTarget, 0, 2) matches := make([]selectedReferenceTarget, 0, 2)
@@ -1101,7 +1154,7 @@ func resolveCLIReferenceLaneTarget(targets []selectedReferenceTarget, laneID str
case 1: case 1:
return matches[0], nil return matches[0], nil
default: default:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets in lane %q (%s); use %s.extract.%s or %s.normalize.%s", slotName, laneID, targetList(matches), laneID, slotName, laneID, slotName) return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets in lane %q (%s); use a more specific selector such as %s", slotName, laneID, targetList(matches), selectorSuggestions(matches, slotName))
} }
} }

View File

@@ -816,7 +816,7 @@ pipelines:
} }
} }
func TestRunConfigValidateIgnoresNonLLMStageScriptoriumProfiles(t *testing.T) { func TestRunConfigValidateIncludesMergeAndIgnoresNonLLMStageScriptoriumProfiles(t *testing.T) {
profilePath := writeScriptoriumProfileFile(t, "known", "http://profile.test/v1", "test-model") profilePath := writeScriptoriumProfileFile(t, "known", "http://profile.test/v1", "test-model")
configPath := writeTestConfig(t, `version: 2 configPath := writeTestConfig(t, `version: 2
scriptorium: scriptorium:
@@ -843,11 +843,11 @@ pipelines:
Catalog: fakeCatalog(t), Catalog: fakeCatalog(t),
}) })
if code != 0 { if code != 1 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q, want success", code, stderr.String()) t.Fatalf("RunWithOptions() code = %d, want failure for missing merge profile", code)
} }
if strings.Contains(stderr.String(), "missing-") { if !strings.Contains(stderr.String(), "Scriptorium profile") || !strings.Contains(stderr.String(), "missing-merge") {
t.Fatalf("stderr = %q, want non-LLM stage profiles ignored", stderr.String()) t.Fatalf("stderr = %q, want missing merge profile error", stderr.String())
} }
} }
@@ -867,7 +867,7 @@ func TestEffectiveLLMProfileIDsUsesLLMCapableStagesOnly(t *testing.T) {
} }
got := effectiveLLMProfileIDs(resolved) got := effectiveLLMProfileIDs(resolved)
want := []string{"chunk-profile", "extract-profile", "normalize-profile"} want := []string{"chunk-profile", "extract-profile", "merge-profile", "normalize-profile"}
if !reflect.DeepEqual(got, want) { if !reflect.DeepEqual(got, want) {
t.Fatalf("effectiveLLMProfileIDs() = %#v, want %#v", got, want) t.Fatalf("effectiveLLMProfileIDs() = %#v, want %#v", got, want)
} }
@@ -1166,6 +1166,81 @@ func TestRunPipelineReferenceFlagBindsExplicitNormalizeSlot(t *testing.T) {
} }
} }
func TestRunPipelineReferenceFlagBindsExplicitMergeSlot(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAML("example", "events"))
inputPath := filepath.Join(t.TempDir(), "missing.json")
referencePath := writeFile(t, "merge.md", "Merge notes\n")
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "example",
"--config", configPath,
"--input", inputPath,
"--diagnostics-dir", diagnosticsDir,
"--reference", "events.merge.notes=" + referencePath,
}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "appendorder",
Stage: pipeline.StageMerge,
Requires: []string{"artifact"},
Provides: []string{"merged"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "notes"},
},
}),
})
if code != 1 || !strings.Contains(stderr.String(), "read input") {
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
}
resolved := readResolvedPipeline(t, diagnosticsDir)
refs := resolved.ArtifactLanes[0].MergeReferences.Bindings
want := []pipeline.ReferenceBinding{
{LaneID: "events", SlotName: "notes", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI},
}
if !reflect.DeepEqual(refs, want) {
t.Fatalf("merge references = %#v, want %#v", refs, want)
}
}
func TestRunPipelineReferenceFlagBindsUnambiguousMergeSlot(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAML("example", "events"))
inputPath := filepath.Join(t.TempDir(), "missing.json")
referencePath := writeFile(t, "merge.md", "Merge notes\n")
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "example",
"--config", configPath,
"--input", inputPath,
"--diagnostics-dir", diagnosticsDir,
"--reference", "merge.notes=" + referencePath,
}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "appendorder",
Stage: pipeline.StageMerge,
Requires: []string{"artifact"},
Provides: []string{"merged"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "notes"},
},
}),
})
if code != 1 || !strings.Contains(stderr.String(), "read input") {
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
}
resolved := readResolvedPipeline(t, diagnosticsDir)
refs := resolved.ArtifactLanes[0].MergeReferences.Bindings
if len(refs) != 1 || refs[0].Source != referencePath || refs[0].LaneID != "events" {
t.Fatalf("merge references = %#v, want unambiguous merge binding", refs)
}
}
func TestRunPipelineReferenceFlagBindsFlatSlotAcrossOneTarget(t *testing.T) { func TestRunPipelineReferenceFlagBindsFlatSlotAcrossOneTarget(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAML("example", "events")) configPath := writeTestConfig(t, testConfigYAML("example", "events"))
inputPath := filepath.Join(t.TempDir(), "missing.json") inputPath := filepath.Join(t.TempDir(), "missing.json")
@@ -1366,8 +1441,8 @@ func TestRunPipelineReferenceFlagsRejectMalformedValues(t *testing.T) {
{name: "missing equals", args: []string{"--reference", "roster"}, want: "slot=path"}, {name: "missing equals", args: []string{"--reference", "roster"}, want: "slot=path"},
{name: "empty path", args: []string{"--reference", "roster="}, want: "path must not be empty"}, {name: "empty path", args: []string{"--reference", "roster="}, want: "path must not be empty"},
{name: "empty slot", args: []string{"--reference", "=./roster.yml"}, want: "slot must not be empty"}, {name: "empty slot", args: []string{"--reference", "=./roster.yml"}, want: "slot must not be empty"},
{name: "unsupported explicit stage", args: []string{"--reference", "a.b.c=./roster.yml"}, want: "lane.extract.slot or lane.normalize.slot"}, {name: "unsupported explicit stage", args: []string{"--reference", "a.b.c=./roster.yml"}, want: "lane.extract.slot, lane.merge.slot, or lane.normalize.slot"},
{name: "too many selector parts", args: []string{"--reference", "a.b.c.d=./roster.yml"}, want: "slot, chunk.slot, lane.slot, lane.extract.slot, or lane.normalize.slot"}, {name: "too many selector parts", args: []string{"--reference", "a.b.c.d=./roster.yml"}, want: "slot, chunk.slot, merge.slot, lane.slot, lane.extract.slot, lane.merge.slot, or lane.normalize.slot"},
{name: "unbind with equals", args: []string{"--without-reference", "roster=./roster.yml"}, want: "without =path"}, {name: "unbind with equals", args: []string{"--without-reference", "roster=./roster.yml"}, want: "without =path"},
} }
@@ -1393,9 +1468,10 @@ func TestRunPipelineReferenceFlagsRejectMalformedValues(t *testing.T) {
func TestRunPipelineWithoutReferenceRemovesConfigBindingsForEligibleTargets(t *testing.T) { func TestRunPipelineWithoutReferenceRemovesConfigBindingsForEligibleTargets(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAMLWithPipelineReferences("example", "events", map[string]string{ configPath := writeTestConfig(t, testConfigYAMLWithPipelineReferences("example", "events", map[string]string{
"context": "./config-context.md", "context": "./config-context.md",
"notes": "./config-notes.md", "merge_notes": "./config-merge.md",
"roster": "./config-roster.yml", "notes": "./config-notes.md",
"roster": "./config-roster.yml",
})) }))
inputPath := filepath.Join(t.TempDir(), "missing.json") inputPath := filepath.Join(t.TempDir(), "missing.json")
diagnosticsDir := t.TempDir() diagnosticsDir := t.TempDir()
@@ -1409,6 +1485,7 @@ func TestRunPipelineWithoutReferenceRemovesConfigBindingsForEligibleTargets(t *t
"--diagnostics-dir", diagnosticsDir, "--diagnostics-dir", diagnosticsDir,
"--without-reference", "chunk.context", "--without-reference", "chunk.context",
"--without-reference", "events.extract.roster", "--without-reference", "events.extract.roster",
"--without-reference", "events.merge.merge_notes",
"--without-reference", "events.normalize.notes", "--without-reference", "events.normalize.notes",
}, &stdout, &stderr, Options{ }, &stdout, &stderr, Options{
Catalog: fakeCatalog(t, Catalog: fakeCatalog(t,
@@ -1421,6 +1498,15 @@ func TestRunPipelineWithoutReferenceRemovesConfigBindingsForEligibleTargets(t *t
{Name: "context"}, {Name: "context"},
}, },
}, },
pipeline.ModuleSpec{
Key: "appendorder",
Stage: pipeline.StageMerge,
Requires: []string{"artifact"},
Provides: []string{"merged"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "merge_notes"},
},
},
pipeline.ModuleSpec{ pipeline.ModuleSpec{
Key: "fake/extract", Key: "fake/extract",
Stage: pipeline.StageExtract, Stage: pipeline.StageExtract,
@@ -1452,6 +1538,9 @@ func TestRunPipelineWithoutReferenceRemovesConfigBindingsForEligibleTargets(t *t
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 { if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("extract references = %#v, want none", refs) t.Fatalf("extract references = %#v, want none", refs)
} }
if refs := resolved.ArtifactLanes[0].MergeReferences.Bindings; len(refs) != 0 {
t.Fatalf("merge references = %#v, want none", refs)
}
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 { if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
t.Fatalf("normalize references = %#v, want none", refs) t.Fatalf("normalize references = %#v, want none", refs)
} }

View File

@@ -68,6 +68,7 @@ func applyLLMProfileOverride(profile *pipeline.PipelineProfile, profileID string
profile.Chunk.LLMProfile = profileID profile.Chunk.LLMProfile = profileID
for laneID, lane := range profile.Artifacts { for laneID, lane := range profile.Artifacts {
lane.Extract.LLMProfile = profileID lane.Extract.LLMProfile = profileID
lane.Merge.LLMProfile = profileID
lane.Normalize.LLMProfile = profileID lane.Normalize.LLMProfile = profileID
profile.Artifacts[laneID] = lane profile.Artifacts[laneID] = lane
} }

View File

@@ -193,8 +193,8 @@ func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
t.Fatalf("output profile = %q, want original output-profile", effective.ResolvedPipeline.Output.LLMProfile) t.Fatalf("output profile = %q, want original output-profile", effective.ResolvedPipeline.Output.LLMProfile)
} }
eventLane := effective.ResolvedPipeline.ArtifactLanes[0] eventLane := effective.ResolvedPipeline.ArtifactLanes[0]
if eventLane.Merge.LLMProfile != "merge-profile" { if eventLane.Merge.LLMProfile != "runtime" {
t.Fatalf("merge profile = %q, want original merge-profile", eventLane.Merge.LLMProfile) t.Fatalf("merge profile = %q, want runtime", eventLane.Merge.LLMProfile)
} }
if len(eventLane.Validators) != 1 || eventLane.Validators[0].LLMProfile != "validator-profile" { if len(eventLane.Validators) != 1 || eventLane.Validators[0].LLMProfile != "validator-profile" {
t.Fatalf("validator profiles = %#v, want original validator-profile", eventLane.Validators) t.Fatalf("validator profiles = %#v, want original validator-profile", eventLane.Validators)
@@ -204,7 +204,7 @@ func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
func llmCapableBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding { func llmCapableBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding {
bindings := []pipeline.ModuleBinding{resolved.Chunk} bindings := []pipeline.ModuleBinding{resolved.Chunk}
for _, lane := range resolved.ArtifactLanes { for _, lane := range resolved.ArtifactLanes {
bindings = append(bindings, lane.Extract, lane.Normalize) bindings = append(bindings, lane.Extract, lane.Merge, lane.Normalize)
} }
return bindings return bindings
} }

View File

@@ -53,6 +53,7 @@ type FileDiagnosticsConfig struct {
type fileModuleBinding struct { type fileModuleBinding struct {
Module string Module string
LLMProfile string LLMProfile string
Retries int
Options map[string]any Options map[string]any
References map[string]string References map[string]string
} }
@@ -83,6 +84,12 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
return err return err
} }
b.LLMProfile = strings.TrimSpace(llmProfile) b.LLMProfile = strings.TrimSpace(llmProfile)
case "retries":
var retries int
if err := valueNode.Decode(&retries); err != nil {
return err
}
b.Retries = retries
case "options": case "options":
var options map[string]any var options map[string]any
if err := valueNode.Decode(&options); err != nil { if err := valueNode.Decode(&options); err != nil {
@@ -109,6 +116,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
return pipeline.ModuleBinding{ return pipeline.ModuleBinding{
Module: strings.TrimSpace(b.Module), Module: strings.TrimSpace(b.Module),
LLMProfile: strings.TrimSpace(b.LLMProfile), LLMProfile: strings.TrimSpace(b.LLMProfile),
Retries: b.Retries,
Options: cloneOptions(b.Options), Options: cloneOptions(b.Options),
References: normalizedStringMap(b.References), References: normalizedStringMap(b.References),
} }

View File

@@ -127,6 +127,7 @@ pipelines:
input: fake/input input: fake/input
chunk: chunk:
module: generic module: generic
retries: 2
options: options:
size: 10 size: 10
flags: flags:
@@ -138,9 +139,12 @@ pipelines:
extract: extract:
module: fake/extract module: fake/extract
llm_profile: fast llm_profile: fast
retries: 3
options: options:
temperature: 0 temperature: 0
merge: appendorder merge:
module: appendorder
retries: 1
normalize: normalize:
module: noop module: noop
output: json output: json
@@ -153,6 +157,9 @@ pipelines:
if profile.Chunk.Module != "generic" { if profile.Chunk.Module != "generic" {
t.Fatalf("unexpected chunk binding: %+v", profile.Chunk) t.Fatalf("unexpected chunk binding: %+v", profile.Chunk)
} }
if profile.Chunk.Retries != 2 {
t.Fatalf("chunk retries = %d, want 2", profile.Chunk.Retries)
}
if profile.Chunk.Options["size"] != 10 { if profile.Chunk.Options["size"] != 10 {
t.Fatalf("expected chunk options to preserve scalar, got %#v", profile.Chunk.Options) t.Fatalf("expected chunk options to preserve scalar, got %#v", profile.Chunk.Options)
} }
@@ -168,6 +175,9 @@ pipelines:
if lane.Extract.Module != "fake/extract" || lane.Extract.LLMProfile != "fast" { if lane.Extract.Module != "fake/extract" || lane.Extract.LLMProfile != "fast" {
t.Fatalf("unexpected extract binding: %+v", lane.Extract) t.Fatalf("unexpected extract binding: %+v", lane.Extract)
} }
if lane.Extract.Retries != 3 || lane.Merge.Retries != 1 {
t.Fatalf("unexpected retries: extract=%d merge=%d", lane.Extract.Retries, lane.Merge.Retries)
}
if lane.Extract.Options["temperature"] != 0 { if lane.Extract.Options["temperature"] != 0 {
t.Fatalf("expected object options, got %#v", lane.Extract.Options) t.Fatalf("expected object options, got %#v", lane.Extract.Options)
} }
@@ -224,6 +234,10 @@ pipelines:
references: references:
roster: ./legacy-roster.yml roster: ./legacy-roster.yml
lore: ./lore.md lore: ./lore.md
merge:
module: appendorder
references:
" merge_notes ": " ./merge.md "
normalize: normalize:
module: noop module: noop
references: references:
@@ -246,6 +260,9 @@ pipelines:
if !reflect.DeepEqual(lane.Extract.References, wantExtract) { if !reflect.DeepEqual(lane.Extract.References, wantExtract) {
t.Fatalf("extract references = %#v, want legacy merged with extract override %#v", lane.Extract.References, wantExtract) t.Fatalf("extract references = %#v, want legacy merged with extract override %#v", lane.Extract.References, wantExtract)
} }
if !reflect.DeepEqual(lane.Merge.References, map[string]string{"merge_notes": "./merge.md"}) {
t.Fatalf("merge references = %#v, want trimmed map", lane.Merge.References)
}
if !reflect.DeepEqual(lane.Normalize.References, map[string]string{"normalization_notes": "./normalization.md"}) { if !reflect.DeepEqual(lane.Normalize.References, map[string]string{"normalization_notes": "./normalization.md"}) {
t.Fatalf("normalize references = %#v, want trimmed map", lane.Normalize.References) t.Fatalf("normalize references = %#v, want trimmed map", lane.Normalize.References)
} }

View File

@@ -42,6 +42,7 @@ func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.Resolv
out.Merge = cloneModuleBinding(in.Merge) out.Merge = cloneModuleBinding(in.Merge)
out.Normalize = cloneModuleBinding(in.Normalize) out.Normalize = cloneModuleBinding(in.Normalize)
out.ExtractReferences = pipeline.CloneReferenceTarget(in.ExtractReferences) out.ExtractReferences = pipeline.CloneReferenceTarget(in.ExtractReferences)
out.MergeReferences = pipeline.CloneReferenceTarget(in.MergeReferences)
out.NormalizeReferences = pipeline.CloneReferenceTarget(in.NormalizeReferences) out.NormalizeReferences = pipeline.CloneReferenceTarget(in.NormalizeReferences)
if len(in.Validators) > 0 { if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators)) out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))

View File

@@ -78,7 +78,7 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) erro
if err := validateBinding(id, laneID, "extract", lane.Extract, true); err != nil { if err := validateBinding(id, laneID, "extract", lane.Extract, true); err != nil {
return err return err
} }
if err := validateBinding(id, laneID, "merge", lane.Merge, false); err != nil { if err := validateBinding(id, laneID, "merge", lane.Merge, true); err != nil {
return err return err
} }
if err := validateBinding(id, laneID, "normalize", lane.Normalize, true); err != nil { if err := validateBinding(id, laneID, "normalize", lane.Normalize, true); err != nil {
@@ -104,6 +104,12 @@ func validateBinding(
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding); err != nil { if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding); err != nil {
return err return err
} }
if binding.Retries < 0 {
if laneID != "" {
return fmt.Errorf("pipeline %q lane %q %s retries must be greater than or equal to zero", pipelineID, laneID, slot)
}
return fmt.Errorf("pipeline %q %s retries must be greater than or equal to zero", pipelineID, slot)
}
if len(binding.References) == 0 { if len(binding.References) == 0 {
return nil return nil
} }

View File

@@ -54,6 +54,18 @@ func TestValidateRejectsInvalidNumericFields(t *testing.T) {
}, },
want: "total LLM concurrency", want: "total LLM concurrency",
}, },
{
name: "negative retries",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Merge.Retries = -1
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: "retries",
},
} }
for _, tc := range tests { for _, tc := range tests {
@@ -238,18 +250,6 @@ func TestValidateRejectsReferencesOnUnsupportedBindings(t *testing.T) {
}, },
want: []string{"example", "input", "references", "not supported"}, want: []string{"example", "input", "references", "not supported"},
}, },
{
name: "merge",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Merge.References = map[string]string{"roster": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "merge", "references", "not supported"},
},
{ {
name: "validator", name: "validator",
mutate: func(cfg Config) Config { mutate: func(cfg Config) Config {

View File

@@ -223,6 +223,10 @@ type MergeRequest struct {
Source *source.SourceDocument `json:"-"` Source *source.SourceDocument `json:"-"`
LaneID string `json:"lane_id"` LaneID string `json:"lane_id"`
ExtractOutputs []ExtractOutput `json:"extract_outputs"` ExtractOutputs []ExtractOutput `json:"extract_outputs"`
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
SessionID string `json:"session_id,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"` Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"` Metadata map[string]any `json:"metadata,omitempty"`

View File

@@ -97,7 +97,7 @@ func validateModuleSpec(kind string, expectedStage ModuleStage, spec ModuleSpec)
} }
func referenceSlotStage(stage ModuleStage) bool { func referenceSlotStage(stage ModuleStage) bool {
return stage == StageChunk || stage == StageExtract || stage == StageNormalize return stage == StageChunk || stage == StageExtract || stage == StageMerge || stage == StageNormalize
} }
func sortedRegistryKeys[C any](constructors map[string]C) []string { func sortedRegistryKeys[C any](constructors map[string]C) []string {

View File

@@ -15,6 +15,7 @@ func TestValidateModuleSpecAllowsReferenceSlotsForEligibleStages(t *testing.T) {
}{ }{
{name: "chunker", kind: "chunker", stage: StageChunk}, {name: "chunker", kind: "chunker", stage: StageChunk},
{name: "extractor", kind: "extractor", stage: StageExtract}, {name: "extractor", kind: "extractor", stage: StageExtract},
{name: "merger", kind: "merger", stage: StageMerge},
{name: "normalizer", kind: "normalizer", stage: StageNormalize}, {name: "normalizer", kind: "normalizer", stage: StageNormalize},
} }
@@ -42,7 +43,6 @@ func TestValidateModuleSpecRejectsReferenceSlotsForIneligibleStages(t *testing.T
stage ModuleStage stage ModuleStage
}{ }{
{name: "input", kind: "input adapter", stage: StageInput}, {name: "input", kind: "input adapter", stage: StageInput},
{name: "merge", kind: "merger", stage: StageMerge},
{name: "validate", kind: "validator", stage: StageValidate}, {name: "validate", kind: "validator", stage: StageValidate},
{name: "output", kind: "output encoder", stage: StageOutput}, {name: "output", kind: "output encoder", stage: StageOutput},
} }
@@ -99,6 +99,7 @@ func TestValidateModuleSpecRejectsInvalidReferenceSlotsForEligibleStages(t *test
}{ }{
{name: "chunk", kind: "chunker", stage: StageChunk}, {name: "chunk", kind: "chunker", stage: StageChunk},
{name: "extract", kind: "extractor", stage: StageExtract}, {name: "extract", kind: "extractor", stage: StageExtract},
{name: "merge", kind: "merger", stage: StageMerge},
{name: "normalize", kind: "normalizer", stage: StageNormalize}, {name: "normalize", kind: "normalizer", stage: StageNormalize},
} }

View File

@@ -22,6 +22,7 @@ const (
type ModuleBinding struct { type ModuleBinding struct {
Module string `json:"module"` Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"` Options map[string]any `json:"options,omitempty"`
References map[string]string `json:"references,omitempty"` References map[string]string `json:"references,omitempty"`
} }
@@ -78,6 +79,7 @@ type ResolvedArtifactLane struct {
Normalize ModuleBinding Normalize ModuleBinding
Validators []ModuleBinding Validators []ModuleBinding
ExtractReferences ResolvedReferenceTarget `json:"extract_references"` ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
MergeReferences ResolvedReferenceTarget `json:"merge_references"`
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"` NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
} }
@@ -251,6 +253,20 @@ func resolveArtifactLane(
if missing, ok := capabilities.missing(mergeSpec.Requires); ok { if missing, ok := capabilities.missing(mergeSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageMerge, lane.Merge.Module, missing) return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageMerge, lane.Merge.Module, missing)
} }
mergeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID,
LaneID: laneID,
Stage: StageMerge,
Module: lane.Merge.Module,
Slots: mergeSpec.ReferenceSlots,
PipelineReferences: pipelineReferences,
LocalReferences: lane.Merge.References,
Options: options,
})
if err != nil {
return ResolvedArtifactLane{}, nil, err
}
lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences)
capabilities.add(mergeSpec.Provides...) capabilities.add(mergeSpec.Provides...)
normalizeSpec, err := normalizerSpec(catalog, lane.Normalize.Module) normalizeSpec, err := normalizerSpec(catalog, lane.Normalize.Module)
@@ -346,6 +362,15 @@ func validatePipelineReferenceDefaults(
declaredByAnyTarget[slot.Name] = struct{}{} declaredByAnyTarget[slot.Name] = struct{}{}
} }
merge := resolveBinding(laneProfile.Merge, DefaultMergeModule)
mergeSpec, err := mergerSpec(catalog, merge.Module)
if err != nil {
return moduleLookupError(pipelineID, laneID, StageMerge, merge.Module, err)
}
for _, slot := range mergeSpec.ReferenceSlots {
declaredByAnyTarget[slot.Name] = struct{}{}
}
normalize := resolveBinding(laneProfile.Normalize, DefaultNormalizeModule) normalize := resolveBinding(laneProfile.Normalize, DefaultNormalizeModule)
normalizeSpec, err := normalizerSpec(catalog, normalize.Module) normalizeSpec, err := normalizerSpec(catalog, normalize.Module)
if err != nil { if err != nil {
@@ -500,7 +525,7 @@ func normalizeReferenceOptionTarget(pipelineID string, operation string, stage M
if laneID != "" { if laneID != "" {
return "", "", fmt.Errorf("pipeline %q reference %s for chunk must not include a lane id", pipelineID, operation) return "", "", fmt.Errorf("pipeline %q reference %s for chunk must not include a lane id", pipelineID, operation)
} }
case StageExtract, StageNormalize: case StageExtract, StageMerge, StageNormalize:
if laneID == "" { if laneID == "" {
return "", "", fmt.Errorf("pipeline %q reference %s lane id must not be empty", pipelineID, operation) return "", "", fmt.Errorf("pipeline %q reference %s lane id must not be empty", pipelineID, operation)
} }
@@ -600,6 +625,7 @@ func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
return ModuleBinding{ return ModuleBinding{
Module: module, Module: module,
LLMProfile: llmProfile, LLMProfile: llmProfile,
Retries: binding.Retries,
Options: cloneOptions(binding.Options), Options: cloneOptions(binding.Options),
References: normalizeReferenceMap(binding.References), References: normalizeReferenceMap(binding.References),
} }

View File

@@ -160,6 +160,9 @@ func TestResolvePipelineAppliesReferenceBindings(t *testing.T) {
if events.NormalizeReferences.Stage != StageNormalize || events.NormalizeReferences.LaneID != "events" || events.NormalizeReferences.Module != DefaultNormalizeModule { if events.NormalizeReferences.Stage != StageNormalize || events.NormalizeReferences.LaneID != "events" || events.NormalizeReferences.Module != DefaultNormalizeModule {
t.Fatalf("normalize reference target = %#v, want event normalizer target", events.NormalizeReferences) t.Fatalf("normalize reference target = %#v, want event normalizer target", events.NormalizeReferences)
} }
if events.MergeReferences.Stage != StageMerge || events.MergeReferences.LaneID != "events" || events.MergeReferences.Module != DefaultMergeModule {
t.Fatalf("merge reference target = %#v, want event merger target", events.MergeReferences)
}
if resolved.ChunkReferences.Stage != StageChunk || resolved.ChunkReferences.Module != DefaultChunkModule { if resolved.ChunkReferences.Stage != StageChunk || resolved.ChunkReferences.Module != DefaultChunkModule {
t.Fatalf("chunk reference target = %#v, want chunk target", resolved.ChunkReferences) t.Fatalf("chunk reference target = %#v, want chunk target", resolved.ChunkReferences)
} }
@@ -260,12 +263,44 @@ func TestResolvePipelineAppliesPipelineReferenceDefaultToNormalizerTarget(t *tes
} }
} }
func TestResolvePipelineAppliesPipelineReferenceDefaultToMergeTarget(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"merge_notes": "./merge.md"}
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "appendorder",
Stage: StageMerge,
Requires: []string{"candidate"},
Provides: []string{"merged"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "merge_notes"}},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
want := []ReferenceBinding{{LaneID: "events", SlotName: "merge_notes", Source: "./merge.md", BindingSource: contracts.ReferenceBindingSourceConfig}}
if !reflect.DeepEqual(resolved.ArtifactLanes[0].MergeReferences.Bindings, want) {
t.Fatalf("merge references = %#v, want %#v", resolved.ArtifactLanes[0].MergeReferences.Bindings, want)
}
if refs := resolved.ChunkReferences.Bindings; len(refs) != 0 {
t.Fatalf("chunk references = %#v, want none", refs)
}
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("extract references = %#v, want none", refs)
}
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
t.Fatalf("normalize references = %#v, want none", refs)
}
}
func TestResolvePipelineAppliesOnePipelineReferenceDefaultToMultipleTargets(t *testing.T) { func TestResolvePipelineAppliesOnePipelineReferenceDefaultToMultipleTargets(t *testing.T) {
profile := baselineProfile() profile := baselineProfile()
profile.References = map[string]string{"context": "./context.md"} profile.References = map[string]string{"context": "./context.md"}
catalog := newProfileCatalogWithOverrides(t, catalog := newProfileCatalogWithOverrides(t,
ModuleSpec{Key: "generic", Stage: StageChunk, Requires: []string{"source"}, Provides: []string{"chunk"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}}, ModuleSpec{Key: "generic", Stage: StageChunk, Requires: []string{"source"}, Provides: []string{"chunk"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
ModuleSpec{Key: "event-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}}, ModuleSpec{Key: "event-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
ModuleSpec{Key: "appendorder", Stage: StageMerge, Requires: []string{"candidate"}, Provides: []string{"merged"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
ModuleSpec{Key: "noop", Stage: StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}}, ModuleSpec{Key: "noop", Stage: StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
) )
@@ -276,6 +311,7 @@ func TestResolvePipelineAppliesOnePipelineReferenceDefaultToMultipleTargets(t *t
assertBindingSource(t, resolved.ChunkReferences.Bindings, "context", "./context.md") assertBindingSource(t, resolved.ChunkReferences.Bindings, "context", "./context.md")
assertBindingSource(t, resolved.ArtifactLanes[0].ExtractReferences.Bindings, "context", "./context.md") assertBindingSource(t, resolved.ArtifactLanes[0].ExtractReferences.Bindings, "context", "./context.md")
assertBindingSource(t, resolved.ArtifactLanes[0].MergeReferences.Bindings, "context", "./context.md")
assertBindingSource(t, resolved.ArtifactLanes[0].NormalizeReferences.Bindings, "context", "./context.md") assertBindingSource(t, resolved.ArtifactLanes[0].NormalizeReferences.Bindings, "context", "./context.md")
} }
@@ -373,6 +409,28 @@ func TestResolvePipelineRejectsExtractLocalReferenceDeclaredOnlyByNormalizer(t *
assertErrorContains(t, err, "baseline", "events", "extract", "event-extractor", "normalization_notes", "not declared") assertErrorContains(t, err, "baseline", "events", "extract", "event-extractor", "normalization_notes", "not declared")
} }
func TestResolvePipelineRejectsMergeLocalReferenceDeclaredOnlyByNormalizer(t *testing.T) {
profile := baselineProfile()
lane := profile.Artifacts["events"]
lane.Merge.References = map[string]string{"normalization_notes": "./normalize.md"}
profile.Artifacts["events"] = lane
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "noop",
Stage: StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "normalization_notes"},
},
})
_, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "merge", "normalization_notes", "not declared")
}
func TestResolvePipelineRejectsNormalizeLocalReferenceDeclaredOnlyByExtractor(t *testing.T) { func TestResolvePipelineRejectsNormalizeLocalReferenceDeclaredOnlyByExtractor(t *testing.T) {
profile := baselineProfile() profile := baselineProfile()
lane := profile.Artifacts["events"] lane := profile.Artifacts["events"]

View File

@@ -44,6 +44,7 @@ func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, opt
for i, lane := range resolved.ArtifactLanes { for i, lane := range resolved.ArtifactLanes {
materializedLane := lane materializedLane := lane
materializedLane.ExtractReferences = CloneReferenceTarget(lane.ExtractReferences) materializedLane.ExtractReferences = CloneReferenceTarget(lane.ExtractReferences)
materializedLane.MergeReferences = CloneReferenceTarget(lane.MergeReferences)
materializedLane.NormalizeReferences = CloneReferenceTarget(lane.NormalizeReferences) materializedLane.NormalizeReferences = CloneReferenceTarget(lane.NormalizeReferences)
extractReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.ExtractReferences, catalog, options) extractReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.ExtractReferences, catalog, options)
if err != nil { if err != nil {
@@ -52,6 +53,13 @@ func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, opt
materializedLane.ExtractReferences.ReferenceSet = extractReferenceSet materializedLane.ExtractReferences.ReferenceSet = extractReferenceSet
warnings = append(warnings, laneWarnings...) warnings = append(warnings, laneWarnings...)
mergeReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.MergeReferences, catalog, options)
if err != nil {
return ResolvedPipeline{}, nil, err
}
materializedLane.MergeReferences.ReferenceSet = mergeReferenceSet
warnings = append(warnings, laneWarnings...)
normalizeReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.NormalizeReferences, catalog, options) normalizeReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.NormalizeReferences, catalog, options)
if err != nil { if err != nil {
return ResolvedPipeline{}, nil, err return ResolvedPipeline{}, nil, err
@@ -140,6 +148,8 @@ func referenceTargetSpec(target ResolvedReferenceTarget, catalog ModuleCatalog)
return registrySpec(catalog.Chunkers, target.Module) return registrySpec(catalog.Chunkers, target.Module)
case StageExtract: case StageExtract:
return registrySpec(catalog.Extractors, target.Module) return registrySpec(catalog.Extractors, target.Module)
case StageMerge:
return registrySpec(catalog.Mergers, target.Module)
case StageNormalize: case StageNormalize:
return registrySpec(catalog.Normalizers, target.Module) return registrySpec(catalog.Normalizers, target.Module)
default: default:
@@ -291,6 +301,7 @@ func ReferenceProvenance(resolved ResolvedPipeline) []artifacts.ReferenceProvena
provenance = append(provenance, referenceTargetProvenance(resolved.ChunkReferences)...) provenance = append(provenance, referenceTargetProvenance(resolved.ChunkReferences)...)
for _, lane := range resolved.ArtifactLanes { for _, lane := range resolved.ArtifactLanes {
provenance = append(provenance, referenceTargetProvenance(lane.ExtractReferences)...) provenance = append(provenance, referenceTargetProvenance(lane.ExtractReferences)...)
provenance = append(provenance, referenceTargetProvenance(lane.MergeReferences)...)
provenance = append(provenance, referenceTargetProvenance(lane.NormalizeReferences)...) provenance = append(provenance, referenceTargetProvenance(lane.NormalizeReferences)...)
} }
return provenance return provenance

View File

@@ -109,17 +109,20 @@ func TestMaterializeReferencesStoresSetsAndProvenanceForAllTargets(t *testing.T)
configDir := t.TempDir() configDir := t.TempDir()
writeReferenceFile(t, filepath.Join(configDir, "chunk.txt"), []byte("chunk text")) writeReferenceFile(t, filepath.Join(configDir, "chunk.txt"), []byte("chunk text"))
writeReferenceFile(t, filepath.Join(configDir, "extract.txt"), []byte("extract text")) writeReferenceFile(t, filepath.Join(configDir, "extract.txt"), []byte("extract text"))
writeReferenceFile(t, filepath.Join(configDir, "merge.txt"), []byte("merge text"))
writeReferenceFile(t, filepath.Join(configDir, "normalize.txt"), []byte("normalize text")) writeReferenceFile(t, filepath.Join(configDir, "normalize.txt"), []byte("normalize text"))
profile := baselineProfile() profile := baselineProfile()
profile.References = map[string]string{ profile.References = map[string]string{
"scene_guide": "chunk.txt", "scene_guide": "chunk.txt",
"roster": "extract.txt", "roster": "extract.txt",
"merge_notes": "merge.txt",
"normalization_notes": "normalize.txt", "normalization_notes": "normalize.txt",
} }
catalog := referenceCatalogForTargets(t, catalog := referenceCatalogForTargets(t,
[]contracts.ReferenceSlot{{Name: "scene_guide"}}, []contracts.ReferenceSlot{{Name: "scene_guide"}},
[]contracts.ReferenceSlot{{Name: "roster"}}, []contracts.ReferenceSlot{{Name: "roster"}},
[]contracts.ReferenceSlot{{Name: "merge_notes"}},
[]contracts.ReferenceSlot{{Name: "normalization_notes"}}, []contracts.ReferenceSlot{{Name: "normalization_notes"}},
) )
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog) resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
@@ -145,14 +148,18 @@ func TestMaterializeReferencesStoresSetsAndProvenanceForAllTargets(t *testing.T)
if string(extractItem.Content) != "extract text" { if string(extractItem.Content) != "extract text" {
t.Fatalf("extract content = %q, want extract text", extractItem.Content) t.Fatalf("extract content = %q, want extract text", extractItem.Content)
} }
mergeItem := materialized.ArtifactLanes[0].MergeReferences.ReferenceSet.Slots["merge_notes"].Items[0]
if string(mergeItem.Content) != "merge text" {
t.Fatalf("merge content = %q, want merge text", mergeItem.Content)
}
normalizeItem := materialized.ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["normalization_notes"].Items[0] normalizeItem := materialized.ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["normalization_notes"].Items[0]
if string(normalizeItem.Content) != "normalize text" { if string(normalizeItem.Content) != "normalize text" {
t.Fatalf("normalize content = %q, want normalize text", normalizeItem.Content) t.Fatalf("normalize content = %q, want normalize text", normalizeItem.Content)
} }
provenance := ReferenceProvenance(materialized) provenance := ReferenceProvenance(materialized)
if len(provenance) != 3 { if len(provenance) != 4 {
t.Fatalf("ReferenceProvenance() = %#v, want three entries", provenance) t.Fatalf("ReferenceProvenance() = %#v, want four entries", provenance)
} }
if provenance[0].Stage != string(StageChunk) || provenance[0].LaneID != "" || provenance[0].SlotName != "scene_guide" || provenance[0].Digest != chunkItem.Digest { if provenance[0].Stage != string(StageChunk) || provenance[0].LaneID != "" || provenance[0].SlotName != "scene_guide" || provenance[0].Digest != chunkItem.Digest {
t.Fatalf("ReferenceProvenance()[0] = %#v, want chunk scene guide provenance", provenance[0]) t.Fatalf("ReferenceProvenance()[0] = %#v, want chunk scene guide provenance", provenance[0])
@@ -160,7 +167,10 @@ func TestMaterializeReferencesStoresSetsAndProvenanceForAllTargets(t *testing.T)
if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != extractItem.Digest { if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != extractItem.Digest {
t.Fatalf("ReferenceProvenance()[1] = %#v, want extract roster provenance", provenance[1]) t.Fatalf("ReferenceProvenance()[1] = %#v, want extract roster provenance", provenance[1])
} }
if provenance[2].Stage != string(StageNormalize) || provenance[2].LaneID != "events" || provenance[2].SlotName != "normalization_notes" || provenance[2].Digest != normalizeItem.Digest { if provenance[2].Stage != string(StageMerge) || provenance[2].LaneID != "events" || provenance[2].SlotName != "merge_notes" || provenance[2].Digest != mergeItem.Digest {
t.Fatalf("ReferenceProvenance()[2] = %#v, want merge notes provenance", provenance[2])
}
if provenance[3].Stage != string(StageNormalize) || provenance[3].LaneID != "events" || provenance[3].SlotName != "normalization_notes" || provenance[3].Digest != normalizeItem.Digest {
t.Fatalf("ReferenceProvenance()[2] = %#v, want normalize notes provenance", provenance[2]) t.Fatalf("ReferenceProvenance()[2] = %#v, want normalize notes provenance", provenance[2])
} }
} }
@@ -185,7 +195,7 @@ func TestMaterializeReferencesRejectsNonUTF8ContentForChunkTarget(t *testing.T)
writeReferenceFile(t, path, []byte{0xff, 0xfe}) writeReferenceFile(t, path, []byte{0xff, 0xfe})
resolved := resolvedPipelineWithTargetReference(t, StageChunk, "", "scene_guide", "bad.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "scene_guide"}) resolved := resolvedPipelineWithTargetReference(t, StageChunk, "", "scene_guide", "bad.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "scene_guide"})
_, _, err := MaterializeReferences(resolved, referenceCatalogForTargets(t, []contracts.ReferenceSlot{{Name: "scene_guide"}}, nil, nil), ReferenceMaterializationOptions{ _, _, err := MaterializeReferences(resolved, referenceCatalogForTargets(t, []contracts.ReferenceSlot{{Name: "scene_guide"}}, nil, nil, nil), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"), ConfigPath: filepath.Join(configDir, "config.yml"),
}) })
if err == nil || !strings.Contains(err.Error(), "chunk") || !strings.Contains(err.Error(), "UTF-8") || !strings.Contains(err.Error(), "scene_guide") || !strings.Contains(err.Error(), path) { if err == nil || !strings.Contains(err.Error(), "chunk") || !strings.Contains(err.Error(), "UTF-8") || !strings.Contains(err.Error(), "scene_guide") || !strings.Contains(err.Error(), path) {
@@ -309,7 +319,7 @@ func TestMaterializeReferencesRejectsUnacceptedMediaTypeForNormalizeTarget(t *te
slot := contracts.ReferenceSlot{Name: "normalization_notes", AcceptedMediaTypes: []string{"text/markdown"}} slot := contracts.ReferenceSlot{Name: "normalization_notes", AcceptedMediaTypes: []string{"text/markdown"}}
resolved := resolvedPipelineWithTargetReference(t, StageNormalize, "events", "normalization_notes", "notes.json", contracts.ReferenceBindingSourceConfig, slot) resolved := resolvedPipelineWithTargetReference(t, StageNormalize, "events", "normalization_notes", "notes.json", contracts.ReferenceBindingSourceConfig, slot)
_, _, err := MaterializeReferences(resolved, referenceCatalogForTargets(t, nil, nil, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{ _, _, err := MaterializeReferences(resolved, referenceCatalogForTargets(t, nil, nil, nil, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"), ConfigPath: filepath.Join(configDir, "config.yml"),
}) })
if err == nil || !strings.Contains(err.Error(), "normalize") || !strings.Contains(err.Error(), "media type") || !strings.Contains(err.Error(), "application/json") || !strings.Contains(err.Error(), "normalization_notes") { if err == nil || !strings.Contains(err.Error(), "normalize") || !strings.Contains(err.Error(), "media type") || !strings.Contains(err.Error(), "application/json") || !strings.Contains(err.Error(), "normalization_notes") {
@@ -353,6 +363,7 @@ func TestMaterializeReferencesWarningScopesIncludeTargetContext(t *testing.T) {
catalog := referenceCatalogForTargets(t, catalog := referenceCatalogForTargets(t,
[]contracts.ReferenceSlot{{Name: "scene_guide"}}, []contracts.ReferenceSlot{{Name: "scene_guide"}},
[]contracts.ReferenceSlot{{Name: "roster"}}, []contracts.ReferenceSlot{{Name: "roster"}},
nil,
[]contracts.ReferenceSlot{{Name: "normalization_notes"}}, []contracts.ReferenceSlot{{Name: "normalization_notes"}},
) )
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog) resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
@@ -407,6 +418,10 @@ func resolvedPipelineWithTargetReference(t *testing.T, stage ModuleStage, laneID
lane := profile.Artifacts[laneID] lane := profile.Artifacts[laneID]
lane.References = map[string]string{slotName: source} lane.References = map[string]string{slotName: source}
profile.Artifacts[laneID] = lane profile.Artifacts[laneID] = lane
case StageMerge:
lane := profile.Artifacts[laneID]
lane.Merge.References = map[string]string{slotName: source}
profile.Artifacts[laneID] = lane
case StageNormalize: case StageNormalize:
lane := profile.Artifacts[laneID] lane := profile.Artifacts[laneID]
lane.Normalize.References = map[string]string{slotName: source} lane.Normalize.References = map[string]string{slotName: source}
@@ -425,6 +440,8 @@ func resolvedPipelineWithTargetReference(t *testing.T, stage ModuleStage, laneID
resolved.ChunkReferences.Bindings[0].BindingSource = bindingSource resolved.ChunkReferences.Bindings[0].BindingSource = bindingSource
case StageExtract: case StageExtract:
resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].BindingSource = bindingSource resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].BindingSource = bindingSource
case StageMerge:
resolved.ArtifactLanes[0].MergeReferences.Bindings[0].BindingSource = bindingSource
case StageNormalize: case StageNormalize:
resolved.ArtifactLanes[0].NormalizeReferences.Bindings[0].BindingSource = bindingSource resolved.ArtifactLanes[0].NormalizeReferences.Bindings[0].BindingSource = bindingSource
} }
@@ -441,18 +458,20 @@ func referenceCatalogForStage(t *testing.T, stage ModuleStage, slots []contracts
t.Helper() t.Helper()
switch stage { switch stage {
case StageChunk: case StageChunk:
return referenceCatalogForTargets(t, slots, nil, nil) return referenceCatalogForTargets(t, slots, nil, nil, nil)
case StageExtract: case StageExtract:
return referenceCatalogForTargets(t, nil, slots, nil) return referenceCatalogForTargets(t, nil, slots, nil, nil)
case StageMerge:
return referenceCatalogForTargets(t, nil, nil, slots, nil)
case StageNormalize: case StageNormalize:
return referenceCatalogForTargets(t, nil, nil, slots) return referenceCatalogForTargets(t, nil, nil, nil, slots)
default: default:
t.Fatalf("unsupported reference target stage %q", stage) t.Fatalf("unsupported reference target stage %q", stage)
return ModuleCatalog{} return ModuleCatalog{}
} }
} }
func referenceCatalogForTargets(t *testing.T, chunkSlots, extractSlots, normalizeSlots []contracts.ReferenceSlot) ModuleCatalog { func referenceCatalogForTargets(t *testing.T, chunkSlots, extractSlots, mergeSlots, normalizeSlots []contracts.ReferenceSlot) ModuleCatalog {
t.Helper() t.Helper()
return newProfileCatalogWithOverrides(t, return newProfileCatalogWithOverrides(t,
ModuleSpec{ ModuleSpec{
@@ -469,6 +488,13 @@ func referenceCatalogForTargets(t *testing.T, chunkSlots, extractSlots, normaliz
Provides: []string{"candidate"}, Provides: []string{"candidate"},
ReferenceSlots: extractSlots, ReferenceSlots: extractSlots,
}, },
ModuleSpec{
Key: "appendorder",
Stage: StageMerge,
Requires: []string{"candidate"},
Provides: []string{"merged"},
ReferenceSlots: mergeSlots,
},
ModuleSpec{ ModuleSpec{
Key: "noop", Key: "noop",
Stage: StageNormalize, Stage: StageNormalize,

View File

@@ -212,6 +212,10 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
Source: doc, Source: doc,
LaneID: lane.ID, LaneID: lane.ID,
ExtractOutputs: cloneExtractOutputs(extractOutputs), ExtractOutputs: cloneExtractOutputs(extractOutputs),
SourceInput: sourceInput.Clone(),
SessionID: sessionID,
References: CloneReferenceSet(lane.MergeReferences.ReferenceSet),
LLMClient: input.LLMClient,
LLMProfile: lane.Merge.LLMProfile, LLMProfile: lane.Merge.LLMProfile,
Options: cloneOptions(lane.Merge.Options), Options: cloneOptions(lane.Merge.Options),
Metadata: input.Metadata, Metadata: input.Metadata,

View File

@@ -500,6 +500,9 @@ func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) {
if len(normalizer.requests) != 1 || normalizer.requests[0].LLMClient == nil { if len(normalizer.requests) != 1 || normalizer.requests[0].LLMClient == nil {
t.Fatalf("normalizer LLM client = %#v, want client on normalize request", normalizer.requests) t.Fatalf("normalizer LLM client = %#v, want client on normalize request", normalizer.requests)
} }
if len(modules.mergers["merge"].requests) != 1 || modules.mergers["merge"].requests[0].LLMClient == nil {
t.Fatalf("merger LLM client = %#v, want client on merge request", modules.mergers["merge"].requests)
}
if extractor.seenMetadata[0]["request"] != "test" { if extractor.seenMetadata[0]["request"] != "test" {
t.Fatalf("seen metadata = %#v, want request metadata", extractor.seenMetadata) t.Fatalf("seen metadata = %#v, want request metadata", extractor.seenMetadata)
} }
@@ -534,6 +537,7 @@ func TestRunPassesSourceInputAndSessionIDToPromptCapableStages(t *testing.T) {
{name: "chunk", material: modules.chunker.requests[0].SourceInput, sessionID: modules.chunker.requests[0].SessionID}, {name: "chunk", material: modules.chunker.requests[0].SourceInput, sessionID: modules.chunker.requests[0].SessionID},
{name: "extract first", material: modules.extractors["extract-alpha"].requests[0].SourceInput, sessionID: modules.extractors["extract-alpha"].requests[0].SessionID}, {name: "extract first", material: modules.extractors["extract-alpha"].requests[0].SourceInput, sessionID: modules.extractors["extract-alpha"].requests[0].SessionID},
{name: "extract second", material: modules.extractors["extract-alpha"].requests[1].SourceInput, sessionID: modules.extractors["extract-alpha"].requests[1].SessionID}, {name: "extract second", material: modules.extractors["extract-alpha"].requests[1].SourceInput, sessionID: modules.extractors["extract-alpha"].requests[1].SessionID},
{name: "merge", material: modules.mergers["merge"].requests[0].SourceInput, sessionID: modules.mergers["merge"].requests[0].SessionID},
{name: "normalize", material: modules.normalizers["normalize"].requests[0].SourceInput, sessionID: modules.normalizers["normalize"].requests[0].SessionID}, {name: "normalize", material: modules.normalizers["normalize"].requests[0].SourceInput, sessionID: modules.normalizers["normalize"].requests[0].SessionID},
} }
for _, req := range requests { for _, req := range requests {
@@ -688,6 +692,27 @@ func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) {
} }
} }
func TestRunPassesMergeReferencesToMergerRequest(t *testing.T) {
modules := defaultRunnerModules()
pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].MergeReferences.ReferenceSet = testReferenceSet("merge_notes", "merge reference text")
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
req := modules.mergers["merge"].requests[0]
item := req.References.Slots["merge_notes"].Items[0]
if string(item.Content) != "merge reference text" {
t.Fatalf("merge reference content = %q, want merge reference text", item.Content)
}
item.Content[0] = 'M'
if got := string(pipeline.ArtifactLanes[0].MergeReferences.ReferenceSet.Slots["merge_notes"].Items[0].Content); got != "merge reference text" {
t.Fatalf("runner mutated merge reference set content = %q", got)
}
}
func TestRunPassesChunkReferencesToChunkerRequest(t *testing.T) { func TestRunPassesChunkReferencesToChunkerRequest(t *testing.T) {
modules := defaultRunnerModules() modules := defaultRunnerModules()
pipeline := resolvedPipeline() pipeline := resolvedPipeline()
@@ -984,6 +1009,24 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
}, },
}, },
} }
resolved.ArtifactLanes[0].MergeReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"merge_notes": {
Slot: contracts.ReferenceSlot{Name: "merge_notes"},
Items: []contracts.ReferenceItem{
{
SlotName: "merge_notes",
MediaType: "text/plain; charset=utf-8",
Content: []byte("merge reference content"),
Digest: "sha256:merge-reference",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/merge.txt"},
SizeBytes: int64(len("merge reference content")),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
},
},
}
resolved.ArtifactLanes[0].NormalizeReferences.ReferenceSet = contracts.ReferenceSet{ resolved.ArtifactLanes[0].NormalizeReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{ Slots: map[string]contracts.ResolvedReferenceSlot{
"normalization_notes": { "normalization_notes": {
@@ -1018,8 +1061,8 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:source"}) { if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:source"}) {
t.Fatalf("SourceDigests = %#v, want source digest", manifest.SourceDigests) t.Fatalf("SourceDigests = %#v, want source digest", manifest.SourceDigests)
} }
if len(manifest.References) != 3 { if len(manifest.References) != 4 {
t.Fatalf("References = %#v, want three reference provenance entries", manifest.References) t.Fatalf("References = %#v, want four reference provenance entries", manifest.References)
} }
chunkReference := manifest.References[0] chunkReference := manifest.References[0]
if chunkReference.Stage != string(StageChunk) || chunkReference.LaneID != "" || chunkReference.SlotName != "scene_guide" || chunkReference.Digest != "sha256:chunk-reference" { if chunkReference.Stage != string(StageChunk) || chunkReference.LaneID != "" || chunkReference.SlotName != "scene_guide" || chunkReference.Digest != "sha256:chunk-reference" {
@@ -1035,7 +1078,11 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
if extractReference.OriginType != "file" || extractReference.OriginURI != "file:///tmp/roster.txt" || extractReference.MediaType != "text/plain; charset=utf-8" || extractReference.SizeBytes != int64(len("reference content")) || extractReference.BindingSource != contracts.ReferenceBindingSourceConfig { if extractReference.OriginType != "file" || extractReference.OriginURI != "file:///tmp/roster.txt" || extractReference.MediaType != "text/plain; charset=utf-8" || extractReference.SizeBytes != int64(len("reference content")) || extractReference.BindingSource != contracts.ReferenceBindingSourceConfig {
t.Fatalf("extract reference provenance = %#v, want origin/media/size/source", extractReference) t.Fatalf("extract reference provenance = %#v, want origin/media/size/source", extractReference)
} }
normalizeReference := manifest.References[2] mergeReference := manifest.References[2]
if mergeReference.Stage != string(StageMerge) || mergeReference.LaneID != "alpha" || mergeReference.SlotName != "merge_notes" || mergeReference.Digest != "sha256:merge-reference" {
t.Fatalf("merge reference provenance = %#v, want lane slot digest", mergeReference)
}
normalizeReference := manifest.References[3]
if normalizeReference.Stage != string(StageNormalize) || normalizeReference.LaneID != "alpha" || normalizeReference.SlotName != "normalization_notes" || normalizeReference.Digest != "sha256:normalize-reference" { if normalizeReference.Stage != string(StageNormalize) || normalizeReference.LaneID != "alpha" || normalizeReference.SlotName != "normalization_notes" || normalizeReference.Digest != "sha256:normalize-reference" {
t.Fatalf("normalize reference provenance = %#v, want lane slot digest", normalizeReference) t.Fatalf("normalize reference provenance = %#v, want lane slot digest", normalizeReference)
} }
@@ -1187,6 +1234,7 @@ func resolvedPipeline() ResolvedPipeline {
Merge: Binding("merge"), Merge: Binding("merge"),
Normalize: Binding("normalize"), Normalize: Binding("normalize"),
ExtractReferences: referenceTarget(StageExtract, "alpha", "extract-alpha", nil), ExtractReferences: referenceTarget(StageExtract, "alpha", "extract-alpha", nil),
MergeReferences: referenceTarget(StageMerge, "alpha", "merge", nil),
NormalizeReferences: referenceTarget(StageNormalize, "alpha", "normalize", nil), NormalizeReferences: referenceTarget(StageNormalize, "alpha", "normalize", nil),
}, },
}, },

View File

@@ -1,7 +1,7 @@
{ {
"manifest": { "manifest": {
"pipeline_id": "walking-skeleton", "pipeline_id": "walking-skeleton",
"pipeline_digest": "sha256:437d7ff486c336ddba38654ea00b7ef9dd6e49ce09f468698202ad8fc459bdcd", "pipeline_digest": "sha256:75c1f6d64d86666734ccf175cdb82ae90e0ba5be3194be952850d7c00c66e615",
"validation_status": "approved", "validation_status": "approved",
"artifact_lanes": [ "artifact_lanes": [
{ {