Compare commits

...

9 Commits

48 changed files with 2583 additions and 878 deletions

View File

@@ -20,7 +20,7 @@ a bearer token.
```text
notarius help
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--reference slot=path] [--without-reference slot]
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--reference selector=path] [--without-reference selector]
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list --config path/to/config.yml [--json]
```
@@ -46,34 +46,44 @@ Flags:
invocation.
- `--llm-profile id`: override every effective module binding to use one LLM
profile.
- `--reference slot=path`: bind a reference path to an extractor reference
slot. Repeatable. Use `lane.slot=path` when multiple selected lanes declare
the same slot.
- `--without-reference slot`: remove a configured optional reference binding.
Repeatable. Use `lane.slot` when multiple selected lanes declare the same
slot.
- `--reference selector=path`: bind a reference path to a chunk, extractor, or
normalizer reference slot. Repeatable.
- `--without-reference selector`: remove a configured optional reference binding.
Repeatable. It accepts the same selector forms as `--reference`, without
`=path`.
On success, the command prints the completed pipeline ID, approved and rejected
artifact counts, and the output directory. If the run completes with warnings,
the warning count is printed to stderr.
Reference flags are resolved against selected artifact lanes before the run
starts. Flat slot names are accepted only when exactly one selected lane
declares that slot. Bound reference files are read before extraction, validated
as UTF-8 text, and passed only to the lane extractor that declares the slot.
Notarius infers reference media types from file extensions for provenance and
for optional slot checks. Reference content is not written to diagnostics, logs,
errors, or manifests.
Reference flags are resolved against selected chunk, extractor, and normalizer
targets before the run starts. Flat slot names are accepted only when exactly
one selected target declares that slot. Bound reference files are read before
pipeline work starts, validated as UTF-8 text, and recorded as provenance for
the target that declares the slot. Runtime reference content is passed to the
chunker, extractor, or normalizer target that declares the slot. Notarius infers
reference media types from file extensions for provenance and for optional slot
checks. Reference content is not written to diagnostics, logs, errors, or
manifests.
Reference binding precedence is:
1. pipeline-level config `references`;
2. lane-level config `references`;
2. target-local config references, including legacy lane-level extractor
`references`;
3. `--reference` run flags;
4. `--without-reference` run flags.
`--reference` binds or replaces one slot for one selected lane. Use
`slot=path` when the selected lanes declare the slot unambiguously:
`--reference` binds or replaces one slot for one selected target. Selectors are:
- `slot=path`: valid when exactly one selected target declares `slot`;
- `chunk.slot=path`: target the chunker;
- `lane.slot=path`: valid when exactly one selected extractor or normalizer in
that lane declares `slot`;
- `lane.extract.slot=path`: target a lane extractor;
- `lane.normalize.slot=path`: target a lane normalizer.
Use `slot=path` when the selected targets declare the slot unambiguously:
```sh
go run ./cmd/notarius run dnd-session \
@@ -82,14 +92,25 @@ go run ./cmd/notarius run dnd-session \
--reference roster=./campaign-roster.txt
```
Use `lane.slot=path` when multiple selected lanes declare the same slot or when
you want to target a specific lane:
Use an explicit selector when multiple selected targets declare the same slot or
when you want to target a specific target:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--reference spells.glossary=./campaign-glossary.txt
--reference spells.extract.glossary=./campaign-glossary.txt
```
The same grammar can target chunk and normalize slots when the configured
modules declare them:
```sh
go run ./cmd/notarius run dnd-session \
--config path/to/config.yml \
--input examples/seriatim-minimal-transcript.json \
--reference chunk.scene_guide=./campaign-scenes.txt \
--reference spells.normalize.normalization_notes=./normalization-notes.txt
```
Use `--without-reference` to remove a configured optional binding for a run:

View File

@@ -124,9 +124,9 @@ Pipeline fields:
- `artifacts`: required for pipeline resolution. It maps artifact lane IDs to
lane definitions.
- `output`: optional module binding. Default module is `json`.
- `references`: optional map of extractor reference slot names to reference
paths. These bindings are defaults for artifact lanes whose extractor declares
the matching slot.
- `references`: optional map of reference slot names to reference paths. These
bindings are defaults for eligible pipeline targets that declare the matching
slot.
Artifact lane fields:
@@ -135,28 +135,32 @@ Artifact lane fields:
- `normalize`: optional module binding. Default module is `noop`.
- `validators`: optional list of module bindings. The production CLI currently
does not register validator modules.
- `references`: optional map of extractor reference slot names to reference
paths. Lane bindings override pipeline-level bindings for the same slot.
- `references`: optional compatibility alias for extractor reference bindings.
Lane bindings override pipeline-level bindings for the same slot.
`notarius run` and `notarius config validate --pipeline` resolve the pipeline
against the production module catalog and fail fast for unknown or incompatible
module keys.
Reference bindings are validated against extractor-declared slots during
pipeline resolution. Required slots must be bound after config defaults,
lane-level bindings, and run-time `--reference` or `--without-reference`
overrides are applied. Config-relative paths are resolved relative to the
config file; CLI reference paths are resolved relative to the current working
directory. Bound files must be UTF-8 text and are passed only to lane
extractors that declare the slot. Reference media types are inferred from file
extensions, recorded as canonical base media types, and checked only when a
module declares `AcceptedMediaTypes`; unknown extensions are recorded as
Reference bindings are validated against reference slots declared by eligible
chunk, extract, and normalize targets during pipeline resolution. Required slots
must be bound after config defaults, target-local references, lane-level
compatibility bindings, and run-time `--reference` or `--without-reference`
overrides are applied. Config-relative paths are resolved relative to the config
file; CLI reference paths are resolved relative to the current working
directory. Materialized bound files must be UTF-8 text. Materialized reference
provenance is recorded for chunk, extractor, and normalizer targets, and runtime
reference content is passed to the target that declares the slot. Reference
media types are inferred from file extensions, recorded as canonical base media
types, and checked only when a module declares
`AcceptedMediaTypes`; unknown extensions are recorded as
`application/octet-stream`. Reference content is not written to diagnostics,
logs, errors, or manifests.
Pipeline-level `references` are defaults. They are valid when at least one
declared lane in the pipeline has an extractor that declares the slot. During a
run, they apply only to selected lanes whose extractor declares the slot:
eligible target in the full configured pipeline declares the slot, including
chunk, extractor, and normalizer targets. During a run, they apply only to the
selected targets that declare the slot:
```yaml
pipelines:
@@ -170,7 +174,9 @@ pipelines:
extract: dnd/spells
```
Lane-level `references` override or add bindings for one lane:
Extractor binding `references` are the canonical lane-local location. The
legacy lane-level `references` field remains supported as an alias; when both
bind the same slot, `extract.references` wins:
```yaml
pipelines:
@@ -180,11 +186,27 @@ pipelines:
glossary: ./campaign/glossary.txt
artifacts:
spells:
extract: dnd/spells
references:
roster: ./campaign/session-roster.txt
roster: ./campaign/legacy-roster.txt
extract:
module: dnd/spells
references:
roster: ./campaign/session-roster.txt
```
`chunk.references` and `normalize.references` are accepted in object-form
bindings. They override pipeline-level defaults for slots declared by the chunk
or normalizer module. Extractor-local references apply only to the extractor,
and normalizer-local references apply only to the normalizer.
Stage-local reference fields use the same map shape at:
- `pipelines.<id>.chunk.references`
- `pipelines.<id>.artifacts.<lane>.extract.references`
- `pipelines.<id>.artifacts.<lane>.normalize.references`
Each binding is valid only when that target module declares the slot.
## Module Bindings
Every module binding may use shorthand:
@@ -208,6 +230,9 @@ Binding fields:
- `module`: module key.
- `llm_profile`: optional LLM profile ID. Empty means `default`.
- `options`: optional module-specific settings.
- `references`: optional reference bindings. Supported only for `chunk`,
`extract`, and `normalize` bindings. `input`, `merge`, validator, and
`output` bindings reject this field during validation.
The `--llm-profile` run flag overrides every effective module binding to use
one configured profile.

View File

@@ -100,8 +100,12 @@ Fields with empty values may be omitted by JSON encoding.
`source_digests` contains source document digests only. Bound references are
recorded separately under `references`, which contains provenance only:
lane ID, slot name, origin type and URI, digest, media type, byte size, and
binding source. Reference content is not written to durable output.
target stage, lane ID when present, slot name, origin type and URI, digest,
media type, byte size, and binding source. Reference content is not written to
durable output.
Reference `stage` is `chunk`, `extract`, or `normalize`. `lane_id` is omitted
for chunk references and present for extract and normalize references.
When references are bound, the manifest section has this shape:
@@ -109,6 +113,7 @@ When references are bound, the manifest section has this shape:
{
"references": [
{
"stage": "extract",
"lane_id": "spells",
"slot_name": "roster",
"origin_type": "file",

View File

@@ -17,8 +17,8 @@ The request contains messages, optional model override, response schema name,
and response schema JSON. The caller supplies a pointer target for decoded
structured output.
Extractors own prompts and schemas. Provider adapters should not contain
domain-specific prompt logic.
Modules that call the LLM own their prompts and schemas. Provider adapters
should not contain domain-specific prompt logic.
## Production Client Construction

View File

@@ -20,21 +20,24 @@ A production module package should provide:
Module specs should describe capabilities accurately. Resolution uses specs to
reject incompatible pipelines before execution.
Extractor modules that accept auxiliary reference material must declare slots
through both `ReferenceSlots()` and `ModuleSpec().ReferenceSlots`. The runtime
slot list and registry metadata should match so config validation can inspect
slots without constructing extractor instances. A slot declaration names the
slot, whether it is required, accepted media types, whether multiple items are
allowed, and any byte limit. Empty `AcceptedMediaTypes` means any inferred
media type is accepted, though the file must still be UTF-8 text. When a slot
declares accepted media types, Notarius compares the canonical base media type
inferred from the file extension, case-insensitively and without parameters.
Chunk, extract, and normalize modules that accept auxiliary reference material
must declare slots through both `ReferenceSlots()` and
`ModuleSpec().ReferenceSlots`. The runtime slot list and registry metadata
should match so config validation can inspect slots without constructing module
instances. A slot declaration names the slot, whether it is required, accepted
media types, whether multiple items are allowed, and any byte limit. Empty
`AcceptedMediaTypes` means any inferred media type is accepted, though the file
must still be UTF-8 text. When a slot declares accepted media types, Notarius
compares the canonical base media type inferred from the file extension,
case-insensitively and without parameters.
Reference content is delivered only to the lane extractor through
`contracts.ExtractionRequest.References`. It is not source evidence and must not
be converted into `SourceRef` values. If a module prompt uses references, load
the prompt bundle with the same declared slots and render with
`RenderUserSystemWithReferences`. Prompt templates may use the `reference`
The resolver materializes reference content for chunk, extractor, and
normalizer targets. Runtime delivery uses `contracts.ChunkRequest.References`,
`contracts.ExtractionRequest.References`, and
`contracts.NormalizeRequest.References`. Reference material is not source
evidence and must not be converted into `SourceRef` values. If a module prompt
uses references, load the prompt bundle with the same declared slots and render
with `RenderUserSystemWithReferences`. Prompt templates may use the `reference`
function for content and the `hasreference` function for conditional sections.
Prompt metadata hashes remain based on template source, not rendered reference
bytes.
@@ -44,6 +47,9 @@ when they need model-backed chunking. The pipeline runner validates generic
chunk result invariants before extraction; module-owned policies may be stricter
but must stay within the module package.
Normalize modules receive the structured LLM client through
`contracts.NormalizeRequest` when they need model-backed reconciliation.
## `seriatim` Input
Package: `internal/modules/input/seriatim`

View File

@@ -31,32 +31,34 @@ before execution:
The CLI writes the resolved pipeline and digest to diagnostics.
Pipeline profiles and artifact lanes may include reference binding maps keyed by
extractor reference slot name. During resolution, pipeline-level bindings act as
defaults for selected lanes whose extractor declares the slot, lane-level
bindings override or add lane bindings, runtime `--reference` requests override
config bindings, and runtime unbinds remove optional bindings. Flat runtime slot
names are resolved only when exactly one selected lane declares the slot;
otherwise the CLI requires `lane.slot`. Resolution validates bindings against
extractor specs and records lane-scoped binding metadata. It does not read
reference files or include reference bytes in source digests.
reference slot name. During resolution, pipeline-level bindings act as defaults
for selected chunk, extractor, and normalizer targets that declare the slot;
target-local bindings override or add bindings for that target. Runtime
`--reference` requests override target config bindings, and runtime unbinds
remove optional target bindings. Flat runtime slot names are resolved only when
exactly one selected target declares the slot; otherwise the CLI requires a more
specific selector such as `chunk.slot`, `lane.extract.slot`, or
`lane.normalize.slot`. Resolution validates bindings against the declaring
target specs and stores the bindings in target-aware resolved reference holders.
It does not read reference files or include reference bytes in source digests.
During run preparation, resolved file references are materialized before any
LLM-backed pipeline work. Config bindings resolve relative to the config file,
CLI bindings resolve relative to the current working directory, and materialized
reference content is passed only to the matching lane extractor through
`ExtractionRequest`. Materialization accepts UTF-8 text files, computes
`sha256:` content digests, records file origins, infers canonical base media
types from file extensions, enforces declared byte limits, and warns for empty
bound files. Media-type acceptance is checked only when a slot declares
During run preparation, resolved file references for chunk, extractor, and
normalizer targets are materialized before any LLM-backed pipeline work. Config
bindings resolve relative to the config file, and CLI bindings resolve relative
to the current working directory. Materialization accepts UTF-8 text files,
computes `sha256:` content digests, records file origins, infers canonical base
media types from file extensions, enforces declared byte limits, and warns for
empty bound files. Media-type acceptance is checked only when a slot declares
`AcceptedMediaTypes`; unknown extensions are recorded as
`application/octet-stream`. Reference content is omitted from diagnostics and
manifests. The CLI writes provenance-only resolved reference diagnostics, and
the run manifest records lane-scoped reference provenance separately from source
digests.
the run manifest records target-stage reference provenance separately from
source digests. Runtime reference content is passed to the matching chunker,
extractor, or normalizer request.
Prompt bundles can declare reference slots and use `reference` and
`hasreference` template functions. Bundle loading validates string-literal slot
names against the declaration. Rendering receives a lane reference set from the
names against the declaration. Rendering receives a target reference set from the
caller; unbound optional slots render as empty strings, and `hasreference`
returns true only when at least one bound item has content.
@@ -74,9 +76,10 @@ Every production module registers a `ModuleSpec` with:
- `Provides`: capabilities added after that module runs;
- `Requires`: capabilities that must already be available.
Extractor specs may also declare reference slots. Slot declarations are
available from registry metadata without constructing extractor instances.
Non-extractor module specs must not declare reference slots.
Chunk, extract, and normalize specs may also declare reference slots. Slot
declarations are available from registry metadata without constructing module
instances. Input, merge, validate, and output specs must not declare reference
slots.
Capability checks prevent incompatible pipeline composition before a run starts.
@@ -114,10 +117,10 @@ The runner:
## Chunk Results
Chunkers implement `contracts.Chunker` and receive a `contracts.ChunkRequest`
with the validated source document, the structured LLM client, the configured
LLM profile, module options, and run metadata. Deterministic and LLM-backed
chunkers use the same contract; provider construction stays outside chunk
modules.
with the validated source document, reference set, structured LLM client, the
configured LLM profile, module options, and run metadata. Deterministic and
LLM-backed chunkers use the same contract; provider construction stays outside
chunk modules.
After `Chunk` returns, the runner appends chunker warnings before returning any
chunker error. When chunking succeeds, the runner validates generic chunk

View File

@@ -63,9 +63,9 @@ Implemented diagnostics artifacts:
path, selected lanes, run ID, and pipeline digest when available.
- `effective-config.json`: resolved config with API keys redacted.
- `resolved-pipeline.json`: resolved module bindings and pipeline digest.
- `resolved-references.json`: lane-scoped resolved reference provenance,
including origin, digest, media type, byte size, and binding source, without
reference content.
- `resolved-references.json`: resolved reference provenance, including target
stage, lane ID when present, origin, digest, media type, byte size, and
binding source, without reference content.
- `run-manifest.json`: the same run manifest written to durable output when it
is available, including top-level module metadata when present.
- `warnings.json`: warning list.

View File

@@ -8,11 +8,6 @@ future work only.
- Additional input adapters, such as Markdown or note-export formats.
- Additional D&D extractors beyond spell casts.
- First-class LLM access for all LLM-eligible pipeline stages: `chunk`,
`extract`, and `normalize`.
- Expand reference support beyond extractors to `chunk` and `normalize`
modules, using the same declared-slot model and preserving stage-specific
ownership boundaries.
- Add non-file reference producers, such as prior-run artifacts, derived
summaries, or entity registries, without changing module-facing reference
item contracts.

View File

@@ -1,463 +1,21 @@
# LLM And Stage Reference Expansion Implementation Plan
# LLM And Reference Expansion Completed
## Purpose
This roadmap item is implemented.
Implement these future roadmap items:
Notarius now supports first-class LLM access for chunk, extract, and normalize
modules, plus target-aware reference declaration, binding, CLI override,
materialization, provenance, and runtime delivery for chunker, extractor, and
normalizer targets.
- First-class LLM access for all LLM-eligible pipeline stages: `chunk`,
`extract`, and `normalize`.
- Expand reference support beyond extractors to `chunk` and `normalize`
modules, using the same declared-slot model and preserving stage-specific
ownership boundaries.
Current behavior is documented in:
This is a decision-complete implementation plan for an LLM coding agent. Follow
the stages in order. Keep current-behavior docs unchanged until the
corresponding behavior exists. Do not implement token budgeting, non-file
reference producers, structured references, retrieval, or multiple effective LLM
profiles in this pass.
- [CLI Reference](../cli.md)
- [Configuration](../config.md)
- [Pipeline Internals](../internal/pipeline.md)
- [Module Internals](../internal/modules.md)
- [JSON Output](../integrations/json-output.md)
- [Troubleshooting](../troubleshooting.md)
## Target Behavior
LLM access:
- `contracts.ChunkRequest`, `contracts.ExtractionRequest`, and
`contracts.NormalizeRequest` all carry `LLMClient`, `LLMProfile`, `Options`,
and `Metadata`.
- The runner passes the configured structured LLM client to chunkers,
extractors, and normalizers.
- The existing single-effective-LLM-profile CLI restriction remains in place.
References:
- `chunk`, `extract`, and `normalize` modules can declare reference slots.
- The framework treats references as opaque UTF-8 text plus provenance. Stage
modules own slot names and semantics.
- Reference slots are declared in both runtime module contracts and
`ModuleSpec.ReferenceSlots`, so config and pipeline validation can inspect
slots without constructing modules.
- `ModuleSpec.ReferenceSlots` is valid only for `chunk`, `extract`, and
`normalize` modules. `input`, `merge`, `validate`, and `output` modules must
not declare reference slots.
- Pipeline-level `references` remain defaults. A pipeline-level binding is
valid when any declared eligible module in the pipeline declares the slot. In
a run, it is applied only to selected targets that declare the slot.
- Stage-local reference bindings override pipeline-level defaults for that
target.
- CLI `--reference` and `--without-reference` can target chunk, extract, and
normalize references explicitly, while preserving existing unambiguous flat
forms.
- Reference materialization still accepts file producers only, validates UTF-8,
infers media types from extensions, enforces optional `AcceptedMediaTypes` and
`MaxBytes`, warns for empty files, and never writes content to diagnostics,
logs, errors, or manifests.
- Manifest and diagnostics provenance identifies the reference target stage and,
for lane-owned stages, the lane ID.
## Target Config Shape
Keep existing config compatibility and add stage-specific binding locations:
```yaml
pipelines:
dnd-session:
input: seriatim
references:
campaign_context: ./campaign/context.md
chunk:
module: dnd/scenes
references:
scene_guide: ./campaign/scenes.md
artifacts:
spells:
extract: dnd/spells
references:
glossary: ./campaign/glossary.md
normalize:
module: noop
references:
normalization_notes: ./campaign/normalization.md
```
Compatibility rule:
- Existing `artifacts.<lane>.references` remains supported as an alias for
extractor references.
- New `artifacts.<lane>.extract.references` is the canonical extract-stage
location.
- If both the legacy lane-level `references` map and
`extract.references` bind the same slot, `extract.references` wins. This keeps
migration deterministic and avoids breaking existing configs.
`chunk.references` and `normalize.references` are supported only when their
module binding uses object form. Scalar bindings continue to work when no
stage-local references are needed.
## Target CLI Reference Selectors
Preserve existing valid forms and add explicit stage selectors:
- `slot=path`: valid only when exactly one selected reference target across
chunk, selected extractors, and selected normalizers declares `slot`.
- `chunk.slot=path`: targets the singleton chunk module.
- `lane.slot=path`: valid only when exactly one selected module in that lane,
extract or normalize, declares `slot`.
- `lane.extract.slot=path`: targets a lane extractor.
- `lane.normalize.slot=path`: targets a lane normalizer.
`--without-reference` accepts the same selector forms without `=path`.
Ambiguous selectors must fail before any LLM call and should name the ambiguous
targets and the more specific selector forms.
## Stage 1: Contract And Registry Expansion
Add reference declaration support to chunkers and normalizers and LLM delivery
to normalizers. This stage should not change config shape or runtime reference
resolution yet.
Implementation steps:
- Add `References contracts.ReferenceSet` to `contracts.ChunkRequest`.
- Add `LLMClient contracts.StructuredLLMClient` and
`References contracts.ReferenceSet` to `contracts.NormalizeRequest`.
- Add `ReferenceSlots() []ReferenceSlot` to `contracts.Chunker` and
`contracts.Normalizer`.
- Update every concrete chunker and normalizer plus all test fakes to implement
`ReferenceSlots()`. Existing modules without references should return `nil`.
- Change `validateModuleSpec` so `ModuleSpec.ReferenceSlots` is allowed for
`StageChunk`, `StageExtract`, and `StageNormalize`, and rejected for all other
stages.
- Add registry/module tests proving:
- chunker specs may declare valid reference slots;
- normalizer specs may declare valid reference slots;
- extractor specs still may declare valid reference slots;
- input, merge, validate, and output specs reject reference slots;
- invalid slot names, duplicate slots, and negative `MaxBytes` are rejected
for all eligible stages.
- Add production registration tests comparing runtime `ReferenceSlots()` with
registered `ModuleSpec.ReferenceSlots` for chunkers, extractors, and
normalizers that declare slots. Modules with no slots should be covered by
simple nil/empty assertions.
Verification:
- `go test ./internal/framework/contracts`
- `go test ./internal/framework/pipeline`
- `go test ./internal/modules/chunk/...`
- `go test ./internal/modules/normalize/...`
- `go test ./...`
## Stage 2: Resolved Reference Target Model
Replace extractor-only resolved reference storage with a target-aware model.
Keep external config and CLI behavior unchanged in this stage where possible.
Implementation steps:
- Introduce a target model in `internal/framework/pipeline`, for example:
- target stage: `chunk`, `extract`, or `normalize`;
- optional lane ID for lane-owned targets;
- module key;
- reference bindings;
- materialized `ReferenceSet`.
- Add a resolved chunk reference holder to `ResolvedPipeline`.
- Split lane reference storage into extract and normalize holders on
`ResolvedArtifactLane`.
- Keep JSON field names deterministic and redaction-friendly. Do not expose
reference content through resolved pipeline JSON.
- Update `CloneReferenceSet`, `ReferenceProvenance`, redacted effective config,
resolved-pipeline diagnostics, and any affected tests for the target-aware
model.
- Preserve existing extractor behavior by mapping old lane extractor references
into the new extract target.
- Keep the pipeline digest deterministic and based on resolved reference
bindings, not materialized reference bytes. Materialized reference digests
remain manifest provenance.
Verification:
- `go test ./internal/framework/pipeline`
- `go test ./internal/core/config`
- `go test ./internal/core/diagnostics`
- `go test ./...`
## Stage 3: Config Parsing And Validation
Add stage-local reference fields while preserving existing configs.
Implementation steps:
- Extend the file config module binding object to accept `references`.
- Add `References map[string]string` to `pipeline.ModuleBinding`, or use a
narrower equivalent if the implementation can do so without duplicating
binding parsing. If added to `ModuleBinding`, validation must reject
references on non-eligible stages.
- Apply file config references from:
- `pipelines.<id>.chunk.references`;
- `pipelines.<id>.artifacts.<lane>.extract.references`;
- `pipelines.<id>.artifacts.<lane>.normalize.references`;
- existing `pipelines.<id>.artifacts.<lane>.references` as an extract-stage
compatibility alias.
- Validate reference maps for empty names, duplicate-after-trim names, and empty
sources at each new location.
- Reject references on `input`, `merge`, validators, and `output` bindings with
clear config errors.
- Preserve existing `llm_profile` validation for chunk, extract, and normalize
bindings. No new LLM profile semantics are introduced in this pass.
- Add config tests for:
- parsing `chunk.references`;
- parsing `extract.references`;
- parsing `normalize.references`;
- legacy lane `references` compatibility;
- duplicate conflicts where `extract.references` overrides legacy lane
references for the same slot;
- rejection of references on unsupported module binding locations;
- redaction/effective config cloning for all new reference maps.
Verification:
- `go test ./internal/core/config`
- `go test ./internal/framework/pipeline`
- `go test ./...`
## Stage 4: Pipeline Resolution
Resolve reference bindings against all eligible stage targets and selected lanes
without constructing modules.
Implementation steps:
- During `ResolvePipeline`, collect declared reference slots from:
- the configured chunker spec;
- every declared artifact lane extractor spec;
- every declared artifact lane normalizer spec.
- Keep pipeline-level reference validation broad: a pipeline-level binding is
valid if any declared eligible target in the full pipeline declares that slot,
even when `--only` selects lanes that do not consume it.
- During a selected run, apply pipeline-level defaults only to:
- the singleton chunk target if its chunker declares the slot;
- selected extractor targets that declare the slot;
- selected normalizer targets that declare the slot.
- Apply stage-local references after pipeline defaults:
- chunk-local references override chunk pipeline defaults;
- extract-local references override extract pipeline defaults;
- normalize-local references override normalize pipeline defaults.
- Enforce required slots independently per selected target after defaults,
local references, CLI overrides, and CLI unbinds.
- Keep lane-local strictness: extract-local references must be declared by that
lane's extractor, and normalize-local references must be declared by that
lane's normalizer.
- Return clear errors that name pipeline ID, stage, lane ID when present, slot,
and module key.
- Add pipeline tests for:
- pipeline default consumed by chunk only;
- pipeline default consumed by extract only;
- pipeline default consumed by normalize only;
- one pipeline default consumed by several eligible targets;
- valid pipeline default declared only by an unselected lane target;
- unknown pipeline-level slot rejected when no declared target has it;
- extract-local reference rejected by normalizer-only slot;
- normalize-local reference rejected by extractor-only slot;
- required chunk reference missing;
- required normalize reference missing;
- local overrides over pipeline defaults for each eligible stage.
Verification:
- `go test ./internal/framework/pipeline`
- `go test ./internal/core/config`
- `go test ./...`
## Stage 5: CLI Reference Selectors
Extend CLI reference override and unbind resolution to target chunk, extract,
and normalize references.
Implementation steps:
- Replace extractor-only CLI reference target lookup with target-aware lookup
over the resolved eligible targets for the selected run.
- Support selector forms listed in "Target CLI Reference Selectors".
- Preserve existing behavior for unambiguous `slot` and `lane.slot` forms.
- Reject ambiguous `slot` and `lane.slot` forms with a message naming matching
targets and suggesting `chunk.slot`, `lane.extract.slot`, or
`lane.normalize.slot` as appropriate.
- Reject selectors that target an unselected lane during `--only` runs.
- Reject selectors for stages that do not declare the slot.
- Keep `--reference` empty-path rejection and `--without-reference` optional
unbinding semantics.
- Add CLI tests for:
- `chunk.slot=path`;
- `lane.extract.slot=path`;
- `lane.normalize.slot=path`;
- flat slot resolution across exactly one target;
- lane slot resolution across exactly one lane-local target;
- ambiguous flat slot across chunk and extract;
- ambiguous lane slot across extract and normalize;
- unbind forms for chunk, extract, and normalize;
- unbind leaving required chunk or normalize slot missing;
- `--only` with an unselected lane selector.
Verification:
- `go test ./internal/cli`
- `go test ./internal/framework/pipeline`
- `go test ./...`
## Stage 6: Materialization And Provenance
Materialize references for chunk, extract, and normalize targets with one shared
implementation.
Implementation steps:
- Refactor materialization so it accepts a resolved target plus that target's
declared slots, rather than assuming an extractor lane.
- Reuse the existing file behavior:
- config paths relative to the config file;
- CLI paths relative to the current working directory;
- UTF-8 validation for all files;
- extension-based media type inference;
- optional `AcceptedMediaTypes` enforcement;
- optional `MaxBytes` enforcement;
- empty-file warning;
- `sha256:` content digest.
- Store materialized reference sets on the resolved chunk target, extract
targets, and normalize targets.
- Update warnings to include target context in `Scope`, such as:
- `pipeline.<id>.chunk.reference.<slot>`;
- `pipeline.<id>.lane.<lane>.extract.reference.<slot>`;
- `pipeline.<id>.lane.<lane>.normalize.reference.<slot>`.
- Update manifest and diagnostics provenance to include:
- `stage`: `chunk`, `extract`, or `normalize`;
- `lane_id`: omitted for chunk and present for extract/normalize;
- `slot_name`;
- origin type and URI;
- digest;
- media type;
- size bytes;
- binding source.
- Keep reference content out of manifests, diagnostics, logs, and errors.
- Add tests for materialization and provenance for all three eligible stages,
including media-type rejection and UTF-8 rejection on non-extractor targets.
Verification:
- `go test ./internal/framework/pipeline`
- `go test ./internal/core/artifacts`
- `go test ./internal/core/diagnostics`
- `go test ./internal/modules/output/json`
- `go test ./internal/cli`
- `go test ./...`
## Stage 7: Runner Delivery
Deliver LLM clients and references to eligible stage modules at runtime.
Implementation steps:
- Pass `CloneReferenceSet` of the materialized chunk reference set into
`contracts.ChunkRequest.References`.
- Continue passing `LLMClient` into `contracts.ChunkRequest`.
- Continue passing extract references and `LLMClient` into
`contracts.ExtractionRequest`.
- Pass `LLMClient` and `CloneReferenceSet` of the materialized normalize
reference set into `contracts.NormalizeRequest`.
- Ensure normalizers receive their configured `LLMProfile`, options, and
metadata as they do today.
- Add runner tests proving:
- chunkers receive LLM client and chunk references;
- extractors still receive LLM client and extract references;
- normalizers receive LLM client and normalize references;
- reference sets are cloned before delivery so modules cannot mutate resolved
pipeline state;
- missing LLM client still fails only when a module actually uses it, matching
existing chunk/extract behavior.
- Update the walking skeleton to exercise LLM-backed normalize access if a small
fake normalizer can do so without making the fixture noisy.
Verification:
- `go test ./internal/framework/pipeline`
- `go test ./...`
## Stage 8: Prompt Helpers And Module Usage
Make prompt reference helpers usable by chunk and normalize modules, then update
production modules only where needed.
Implementation steps:
- Confirm `internal/framework/prompt` reference rendering is contract-based and
not extractor-specific. If any naming or error text says extractor-only,
generalize it to module/stage language.
- Add prompt tests rendering references for a fake chunker or normalizer prompt
bundle if existing tests cover only extractor usage.
- Update production chunkers and normalizers:
- `generic` chunker: no reference slots.
- `dnd/scenes` chunker: no reference slots in this pass unless a clear
module-owned slot already exists.
- `noop` normalizer: no reference slots.
- Do not invent a new production normalize module in this pass. The goal is
framework support, not a domain normalizer.
Verification:
- `go test ./internal/framework/prompt`
- `go test ./internal/modules/chunk/...`
- `go test ./internal/modules/normalize/...`
- `go test ./...`
## Stage 9: Documentation And Examples
After behavior is implemented and tested, update canonical docs and examples.
Implementation steps:
- Update `docs/cli.md` with the expanded `--reference` and
`--without-reference` selector grammar and examples.
- Update `docs/config.md` with `chunk.references`,
`artifacts.<lane>.extract.references`, `artifacts.<lane>.normalize.references`,
and the legacy `artifacts.<lane>.references` compatibility rule.
- Update `docs/internal/modules.md` with module-author guidance for
`ReferenceSlots()` on chunkers, extractors, and normalizers, plus LLM access
expectations for chunk, extract, and normalize.
- Update `docs/internal/pipeline.md` with target-aware reference resolution,
materialization, provenance, and runtime delivery.
- Update `docs/integrations/json-output.md` with the new reference provenance
shape including `stage` and optional `lane_id`.
- Update `docs/troubleshooting.md` with ambiguous selector examples for chunk,
extract, and normalize targets.
- Update maintained examples only if they can remain small and validated. Do
not add speculative reference files for chunk or normalize modules without a
production module that actually declares those slots.
- Update `docs/roadmap/future.md` by removing or revising the two completed
items while leaving still-deferred work such as token budgeting and non-file
producers.
- Replace this implementation plan with a concise completed-note document after
implementation is finished.
Verification:
- `rg -n "chunk.references|normalize.references|lane.extract.slot|lane.normalize.slot" docs/cli.md docs/config.md docs/internal docs/integrations docs/troubleshooting.md`
- `go test ./...`
- `go vet ./...`
- `go build ./cmd/notarius`
## Final Acceptance Criteria
The feature is complete when:
- chunkers, extractors, and normalizers can declare reference slots through
runtime contracts and registry specs;
- normalizers receive the structured LLM client at runtime;
- config can bind references at pipeline, chunk, extract, and normalize levels;
- CLI reference overrides and unbinds can target chunk, extract, and normalize
references without ambiguous selector behavior;
- pipeline resolution validates all reference bindings before any LLM call;
- materialization and provenance work uniformly for chunk, extract, and
normalize references;
- reference manifest entries identify stage and lane context;
- existing extractor reference behavior remains backward compatible;
- canonical docs describe only implemented behavior;
- `go test ./...`, `go vet ./...`, and `go build ./cmd/notarius` pass.
Deferred reference work remains tracked in [Future Work](future.md), including
non-file reference producers, token budgeting, structured references, and
reference preprocessing or retrieval.

View File

@@ -107,10 +107,10 @@ go run ./cmd/notarius run dnd-session \
Symptoms include:
- `reference slot "..." is not declared`
- `reference slot "..." is declared by multiple selected lanes`
- `reference slot "..." is declared by multiple selected targets`
- `required reference slot "..." is not bound`
- `--reference must use slot=path or lane.slot=path`
- `--without-reference must use slot or lane.slot`
- `--reference must use slot=path`
- `--without-reference must use a reference selector without =path`
- `read "...": no such file`
- `must be UTF-8 text`
- `media type "..." is not accepted`
@@ -118,17 +118,24 @@ Symptoms include:
Fix:
- Confirm the selected extractor declares the slot. The implemented
`dnd/spells` extractor declares optional `roster` and `glossary` slots.
- Use `lane.slot=path` when more than one selected lane declares the same slot.
- Use `--without-reference slot` to remove optional config bindings; do not pass
an empty `--reference slot=`.
- Confirm the selected chunker, extractor, or normalizer declares the slot. The
implemented `dnd/spells` extractor declares optional `roster` and `glossary`
slots.
- Use a specific selector when more than one selected target declares the same
slot. Examples include `chunk.context=./context.txt`,
`spells.extract.context=./extract-context.txt`, and
`spells.normalize.context=./normalize-context.txt`.
- `lane.slot=path` is valid only when exactly one selected extractor or
normalizer in that lane declares the slot. If both do, use
`lane.extract.slot=path` or `lane.normalize.slot=path`.
- Use `--without-reference selector` to remove optional config bindings; do not
pass an empty `--reference selector=`.
- Check whether a path came from config or CLI. Config paths are relative to
the config file. CLI reference paths are relative to the current working
directory.
- Ensure the file is readable UTF-8 text and within any byte limit declared by
the extractor.
- If the extractor narrows accepted media types, use a file extension that
the declaring module.
- If the declaring module narrows accepted media types, use a file extension that
infers an accepted type such as `text/markdown` or `application/json`.
Unknown extensions infer `application/octet-stream`.
- If diagnostics are retained, inspect `resolved-pipeline.json`,

View File

@@ -25,7 +25,7 @@ const defaultOutputRoot = "./notarius-output"
const usage = `Usage:
notarius help
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--reference slot=path] [--without-reference slot]
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--reference selector=path] [--without-reference selector]
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list --config path/to/config.yml [--json]
`
@@ -97,8 +97,8 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
llmProfile := fs.String("llm-profile", "", "LLM profile override")
referenceFlags := stringListFlag{}
withoutReferenceFlags := stringListFlag{}
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path or lane.slot=path")
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, as slot or lane.slot")
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, lane.slot=path, lane.extract.slot=path, or lane.normalize.slot=path")
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
if err := fs.Parse(reorderRunArgs(args)); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
@@ -712,13 +712,17 @@ func (flag *stringListFlag) Set(value string) error {
}
type cliReferenceRequest struct {
LaneID string
SlotName string
Selector cliReferenceSelector
Source string
}
type cliReferenceUnbindRequest struct {
Selector cliReferenceSelector
}
type cliReferenceSelector struct {
LaneID string
Stage pipeline.ModuleStage
SlotName string
}
@@ -735,13 +739,12 @@ func parseReferenceFlags(values []string) ([]cliReferenceRequest, error) {
if strings.TrimSpace(source) == "" {
return nil, fmt.Errorf("--reference path must not be empty; use --without-reference to unbind")
}
laneID, slotName, err := parseReferenceSelector(name, "--reference")
selector, err := parseReferenceSelector(name, "--reference")
if err != nil {
return nil, err
}
requests = append(requests, cliReferenceRequest{
LaneID: laneID,
SlotName: slotName,
Selector: selector,
Source: strings.TrimSpace(source),
})
}
@@ -755,39 +758,51 @@ func parseReferenceUnbindFlags(values []string) ([]cliReferenceUnbindRequest, er
requests := make([]cliReferenceUnbindRequest, 0, len(values))
for _, raw := range values {
if strings.Contains(raw, "=") {
return nil, fmt.Errorf("--without-reference must use slot or lane.slot")
return nil, fmt.Errorf("--without-reference must use a reference selector without =path")
}
laneID, slotName, err := parseReferenceSelector(raw, "--without-reference")
selector, err := parseReferenceSelector(raw, "--without-reference")
if err != nil {
return nil, err
}
requests = append(requests, cliReferenceUnbindRequest{
LaneID: laneID,
SlotName: slotName,
Selector: selector,
})
}
return requests, nil
}
func parseReferenceSelector(raw string, flagName string) (string, string, error) {
func parseReferenceSelector(raw string, flagName string) (cliReferenceSelector, error) {
selector := strings.TrimSpace(raw)
if selector == "" {
return "", "", fmt.Errorf("%s reference slot must not be empty", flagName)
return cliReferenceSelector{}, fmt.Errorf("%s reference slot must not be empty", flagName)
}
if strings.Count(selector, ".") > 1 {
return "", "", fmt.Errorf("%s must use slot or lane.slot", flagName)
}
laneID := ""
slotName := selector
if strings.Contains(selector, ".") {
before, after, _ := strings.Cut(selector, ".")
laneID = strings.TrimSpace(before)
slotName = strings.TrimSpace(after)
if laneID == "" || slotName == "" {
return "", "", fmt.Errorf("%s must use non-empty lane.slot values", flagName)
parts := strings.Split(selector, ".")
for _, part := range parts {
if strings.TrimSpace(part) == "" {
return cliReferenceSelector{}, fmt.Errorf("%s must use non-empty reference selector values", flagName)
}
}
return laneID, slotName, nil
switch len(parts) {
case 1:
return cliReferenceSelector{SlotName: strings.TrimSpace(parts[0])}, nil
case 2:
first := strings.TrimSpace(parts[0])
slotName := strings.TrimSpace(parts[1])
if first == string(pipeline.StageChunk) {
return cliReferenceSelector{Stage: pipeline.StageChunk, SlotName: slotName}, nil
}
return cliReferenceSelector{LaneID: first, SlotName: slotName}, nil
case 3:
laneID := strings.TrimSpace(parts[0])
stage := pipeline.ModuleStage(strings.TrimSpace(parts[1]))
slotName := strings.TrimSpace(parts[2])
if stage != pipeline.StageExtract && stage != pipeline.StageNormalize {
return cliReferenceSelector{}, fmt.Errorf("%s lane-qualified selector must use lane.extract.slot or lane.normalize.slot", flagName)
}
return cliReferenceSelector{LaneID: laneID, Stage: stage, SlotName: slotName}, nil
default:
return cliReferenceSelector{}, fmt.Errorf("%s must use slot, chunk.slot, lane.slot, lane.extract.slot, or lane.normalize.slot", flagName)
}
}
func resolveCLIReferenceRequests(
@@ -802,20 +817,21 @@ func resolveCLIReferenceRequests(
return nil, nil, nil
}
selected, err := selectedReferenceLanes(cfg, pipelineID, only, catalog)
targets, err := selectedReferenceTargets(cfg, pipelineID, only, catalog)
if err != nil {
return nil, nil, err
}
overrides := make([]pipeline.ReferenceBinding, 0, len(referenceRequests))
for _, request := range referenceRequests {
laneID, err := resolveCLIReferenceLane(selected, request.LaneID, request.SlotName)
target, err := resolveCLIReferenceTarget(targets, request.Selector)
if err != nil {
return nil, nil, err
}
overrides = append(overrides, pipeline.ReferenceBinding{
LaneID: laneID,
SlotName: request.SlotName,
Stage: target.stage,
LaneID: target.laneID,
SlotName: request.Selector.SlotName,
Source: request.Source,
BindingSource: contracts.ReferenceBindingSourceCLI,
})
@@ -823,25 +839,28 @@ func resolveCLIReferenceRequests(
unbinds := make([]pipeline.ReferenceUnbind, 0, len(unbindRequests))
for _, request := range unbindRequests {
laneID, err := resolveCLIReferenceLane(selected, request.LaneID, request.SlotName)
target, err := resolveCLIReferenceTarget(targets, request.Selector)
if err != nil {
return nil, nil, err
}
unbinds = append(unbinds, pipeline.ReferenceUnbind{
LaneID: laneID,
SlotName: request.SlotName,
Stage: target.stage,
LaneID: target.laneID,
SlotName: request.Selector.SlotName,
})
}
return overrides, unbinds, nil
}
type selectedReferenceLane struct {
id string
slots map[string]struct{}
type selectedReferenceTarget struct {
laneID string
stage pipeline.ModuleStage
module string
slots map[string]struct{}
}
func selectedReferenceLanes(cfg config.Config, pipelineID string, only []string, catalog pipeline.ModuleCatalog) ([]selectedReferenceLane, error) {
func selectedReferenceTargets(cfg config.Config, pipelineID string, only []string, catalog pipeline.ModuleCatalog) ([]selectedReferenceTarget, error) {
profile, ok := lookupCLIReferencePipeline(cfg.Pipelines, pipelineID)
if !ok {
return nil, fmt.Errorf("pipeline %q is not configured", strings.TrimSpace(pipelineID))
@@ -882,27 +901,55 @@ func selectedReferenceLanes(cfg config.Config, pipelineID string, only []string,
}
sort.Strings(selectedIDs)
selected := make([]selectedReferenceLane, 0, len(selectedIDs))
targets := make([]selectedReferenceTarget, 0, 1+len(selectedIDs)*2)
chunk := pipeline.Binding(profile.Chunk.Module)
chunk.Module = strings.TrimSpace(profile.Chunk.Module)
if chunk.Module == "" {
chunk.Module = pipeline.DefaultChunkModule
}
chunkSpec, err := cliReferenceChunkerSpec(catalog, chunk.Module)
if err != nil {
return nil, fmt.Errorf("pipeline %q chunk module %q: %w", strings.TrimSpace(pipelineID), chunk.Module, err)
}
targets = append(targets, selectedReferenceTarget{
stage: pipeline.StageChunk,
module: chunk.Module,
slots: referenceSlotSet(chunkSpec.ReferenceSlots),
})
for _, laneID := range selectedIDs {
lane := lanesByID[laneID]
extractModule := strings.TrimSpace(lane.Extract.Module)
if extractModule == "" {
return nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", strings.TrimSpace(pipelineID), laneID)
}
if catalog.Extractors == nil {
return nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", strings.TrimSpace(pipelineID), laneID, extractModule, extractModule)
extractSpec, err := cliReferenceExtractorSpec(catalog, extractModule)
if err != nil {
return nil, fmt.Errorf("pipeline %q lane %q extract module %q: %w", strings.TrimSpace(pipelineID), laneID, extractModule, err)
}
spec, ok := catalog.Extractors.Spec(extractModule)
if !ok {
return nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", strings.TrimSpace(pipelineID), laneID, extractModule, extractModule)
targets = append(targets, selectedReferenceTarget{
laneID: laneID,
stage: pipeline.StageExtract,
module: extractModule,
slots: referenceSlotSet(extractSpec.ReferenceSlots),
})
normalizeModule := strings.TrimSpace(lane.Normalize.Module)
if normalizeModule == "" {
normalizeModule = pipeline.DefaultNormalizeModule
}
slotSet := make(map[string]struct{}, len(spec.ReferenceSlots))
for _, slot := range spec.ReferenceSlots {
slotSet[slot.Name] = struct{}{}
normalizeSpec, err := cliReferenceNormalizerSpec(catalog, normalizeModule)
if err != nil {
return nil, fmt.Errorf("pipeline %q lane %q normalize module %q: %w", strings.TrimSpace(pipelineID), laneID, normalizeModule, err)
}
selected = append(selected, selectedReferenceLane{id: laneID, slots: slotSet})
targets = append(targets, selectedReferenceTarget{
laneID: laneID,
stage: pipeline.StageNormalize,
module: normalizeModule,
slots: referenceSlotSet(normalizeSpec.ReferenceSlots),
})
}
return selected, nil
return targets, nil
}
func lookupCLIReferencePipeline(profiles map[string]pipeline.PipelineProfile, pipelineID string) (pipeline.PipelineProfile, bool) {
@@ -915,37 +962,152 @@ func lookupCLIReferencePipeline(profiles map[string]pipeline.PipelineProfile, pi
return pipeline.PipelineProfile{}, false
}
func resolveCLIReferenceLane(selected []selectedReferenceLane, requestedLaneID string, slotName string) (string, error) {
slotName = strings.TrimSpace(slotName)
requestedLaneID = strings.TrimSpace(requestedLaneID)
if requestedLaneID != "" {
for _, lane := range selected {
if lane.id == requestedLaneID {
if _, ok := lane.slots[slotName]; !ok {
return "", fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, requestedLaneID)
func cliReferenceChunkerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
if catalog.Chunkers == nil {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
spec, ok := catalog.Chunkers.Spec(module)
if !ok {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
return spec, nil
}
func cliReferenceExtractorSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
if catalog.Extractors == nil {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
spec, ok := catalog.Extractors.Spec(module)
if !ok {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
return spec, nil
}
func cliReferenceNormalizerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
if catalog.Normalizers == nil {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
spec, ok := catalog.Normalizers.Spec(module)
if !ok {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
return spec, nil
}
func referenceSlotSet(slots []contracts.ReferenceSlot) map[string]struct{} {
slotSet := make(map[string]struct{}, len(slots))
for _, slot := range slots {
slotSet[slot.Name] = struct{}{}
}
return slotSet
}
func resolveCLIReferenceTarget(targets []selectedReferenceTarget, selector cliReferenceSelector) (selectedReferenceTarget, error) {
slotName := strings.TrimSpace(selector.SlotName)
if slotName == "" {
return selectedReferenceTarget{}, fmt.Errorf("reference slot must not be empty")
}
if selector.Stage == pipeline.StageChunk {
for _, target := range targets {
if target.stage != pipeline.StageChunk {
continue
}
if _, ok := target.slots[slotName]; !ok {
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by chunk module %q", slotName, target.module)
}
return target, nil
}
return selectedReferenceTarget{}, fmt.Errorf("reference chunk target is not selected")
}
if selector.Stage == pipeline.StageExtract || selector.Stage == pipeline.StageNormalize {
for _, target := range targets {
if target.laneID == selector.LaneID && target.stage == selector.Stage {
if _, ok := target.slots[slotName]; !ok {
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected %s target %q", slotName, selector.Stage, targetLabel(target))
}
return requestedLaneID, nil
return target, nil
}
}
return "", fmt.Errorf("reference lane %q is not selected", requestedLaneID)
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", selector.LaneID)
}
if strings.TrimSpace(selector.LaneID) != "" {
return resolveCLIReferenceLaneTarget(targets, strings.TrimSpace(selector.LaneID), slotName)
}
return resolveCLIReferenceFlatTarget(targets, slotName)
}
matches := make([]string, 0, 1)
for _, lane := range selected {
if _, ok := lane.slots[slotName]; ok {
matches = append(matches, lane.id)
func resolveCLIReferenceLaneTarget(targets []selectedReferenceTarget, laneID string, slotName string) (selectedReferenceTarget, error) {
laneSelected := false
matches := make([]selectedReferenceTarget, 0, 2)
for _, target := range targets {
if target.laneID != laneID {
continue
}
laneSelected = true
if _, ok := target.slots[slotName]; ok {
matches = append(matches, target)
}
}
if !laneSelected {
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", laneID)
}
switch len(matches) {
case 0:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, laneID)
case 1:
return matches[0], nil
default:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets in lane %q (%s); use %s.extract.%s or %s.normalize.%s", slotName, laneID, targetList(matches), laneID, slotName, laneID, slotName)
}
}
func resolveCLIReferenceFlatTarget(targets []selectedReferenceTarget, slotName string) (selectedReferenceTarget, error) {
matches := make([]selectedReferenceTarget, 0, 2)
for _, target := range targets {
if _, ok := target.slots[slotName]; ok {
matches = append(matches, target)
}
}
switch len(matches) {
case 0:
return "", fmt.Errorf("reference slot %q is not declared by any selected lane", slotName)
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected reference target", slotName)
case 1:
return matches[0], nil
default:
return "", fmt.Errorf("reference slot %q is declared by multiple selected lanes (%s); use lane.slot", slotName, strings.Join(matches, ", "))
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets (%s); use a more specific selector such as %s", slotName, targetList(matches), selectorSuggestions(matches, slotName))
}
}
func targetList(targets []selectedReferenceTarget) string {
labels := make([]string, 0, len(targets))
for _, target := range targets {
labels = append(labels, targetLabel(target))
}
sort.Strings(labels)
return strings.Join(labels, ", ")
}
func targetLabel(target selectedReferenceTarget) string {
if target.stage == pipeline.StageChunk {
return "chunk"
}
return target.laneID + "." + string(target.stage)
}
func selectorSuggestions(targets []selectedReferenceTarget, slotName string) string {
suggestions := make([]string, 0, len(targets))
for _, target := range targets {
if target.stage == pipeline.StageChunk {
suggestions = append(suggestions, "chunk."+slotName)
continue
}
suggestions = append(suggestions, target.laneID+"."+string(target.stage)+"."+slotName)
}
sort.Strings(suggestions)
return strings.Join(suggestions, " or ")
}
func sortedPipelineIDs(cfg config.Config) []string {
ids := make([]string, 0, len(cfg.Pipelines))
for id := range cfg.Pipelines {

View File

@@ -769,7 +769,7 @@ func TestRunPipelineReferenceFlagBindsUnambiguousSlot(t *testing.T) {
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
}
resolved := readResolvedPipeline(t, diagnosticsDir)
refs := resolved.ArtifactLanes[0].References
refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings
want := []pipeline.ReferenceBinding{
{LaneID: "events", SlotName: "roster", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI},
}
@@ -810,12 +810,199 @@ func TestRunPipelineReferenceFlagBindsLaneQualifiedSlot(t *testing.T) {
}
resolved := readResolvedPipeline(t, diagnosticsDir)
events := resolvedArtifactLane(t, resolved, "events")
if len(events.References) != 0 {
t.Fatalf("events references = %#v, want none", events.References)
if len(events.ExtractReferences.Bindings) != 0 {
t.Fatalf("events references = %#v, want none", events.ExtractReferences.Bindings)
}
notes := resolvedArtifactLane(t, resolved, "notes")
if len(notes.References) != 1 || notes.References[0].Source != referencePath {
t.Fatalf("notes references = %#v, want lane-qualified binding", notes.References)
if len(notes.ExtractReferences.Bindings) != 1 || notes.ExtractReferences.Bindings[0].Source != referencePath {
t.Fatalf("notes references = %#v, want lane-qualified binding", notes.ExtractReferences.Bindings)
}
}
func TestRunPipelineReferenceFlagBindsChunkQualifiedSlot(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAML("example", "events"))
inputPath := filepath.Join(t.TempDir(), "missing.json")
referencePath := writeFile(t, "scenes.md", "Scenes\n")
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "example",
"--config", configPath,
"--input", inputPath,
"--diagnostics-dir", diagnosticsDir,
"--reference", "chunk.scene_guide=" + referencePath,
}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "generic",
Stage: pipeline.StageChunk,
Requires: []string{"source"},
Provides: []string{"chunks"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "scene_guide"},
},
}),
})
if code != 1 || !strings.Contains(stderr.String(), "read input") {
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
}
resolved := readResolvedPipeline(t, diagnosticsDir)
refs := resolved.ChunkReferences.Bindings
want := []pipeline.ReferenceBinding{
{SlotName: "scene_guide", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI},
}
if !reflect.DeepEqual(refs, want) {
t.Fatalf("chunk references = %#v, want %#v", refs, want)
}
}
func TestRunPipelineReferenceFlagBindsExplicitExtractSlot(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAML("example", "events"))
inputPath := filepath.Join(t.TempDir(), "missing.json")
referencePath := writeFile(t, "roster.yml", "Aria\n")
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "example",
"--config", configPath,
"--input", inputPath,
"--diagnostics-dir", diagnosticsDir,
"--reference", "events.extract.roster=" + referencePath,
}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster"},
},
}),
})
if code != 1 || !strings.Contains(stderr.String(), "read input") {
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
}
resolved := readResolvedPipeline(t, diagnosticsDir)
refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings
want := []pipeline.ReferenceBinding{
{LaneID: "events", SlotName: "roster", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI},
}
if !reflect.DeepEqual(refs, want) {
t.Fatalf("extract references = %#v, want %#v", refs, want)
}
}
func TestRunPipelineReferenceFlagBindsExplicitNormalizeSlot(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAML("example", "events"))
inputPath := filepath.Join(t.TempDir(), "missing.json")
referencePath := writeFile(t, "normalize.md", "Normalize\n")
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "example",
"--config", configPath,
"--input", inputPath,
"--diagnostics-dir", diagnosticsDir,
"--reference", "events.normalize.notes=" + referencePath,
}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "noop",
Stage: pipeline.StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "notes"},
},
}),
})
if code != 1 || !strings.Contains(stderr.String(), "read input") {
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
}
resolved := readResolvedPipeline(t, diagnosticsDir)
refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings
want := []pipeline.ReferenceBinding{
{LaneID: "events", SlotName: "notes", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI},
}
if !reflect.DeepEqual(refs, want) {
t.Fatalf("normalize references = %#v, want %#v", refs, want)
}
}
func TestRunPipelineReferenceFlagBindsFlatSlotAcrossOneTarget(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAML("example", "events"))
inputPath := filepath.Join(t.TempDir(), "missing.json")
referencePath := writeFile(t, "normalize.md", "Normalize\n")
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "example",
"--config", configPath,
"--input", inputPath,
"--diagnostics-dir", diagnosticsDir,
"--reference", "notes=" + referencePath,
}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "noop",
Stage: pipeline.StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "notes"},
},
}),
})
if code != 1 || !strings.Contains(stderr.String(), "read input") {
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
}
resolved := readResolvedPipeline(t, diagnosticsDir)
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 1 || refs[0].Source != referencePath {
t.Fatalf("normalize references = %#v, want flat binding", refs)
}
}
func TestRunPipelineReferenceFlagBindsLaneSlotAcrossOneTarget(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAML("example", "events"))
inputPath := filepath.Join(t.TempDir(), "missing.json")
referencePath := writeFile(t, "normalize.md", "Normalize\n")
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "example",
"--config", configPath,
"--input", inputPath,
"--diagnostics-dir", diagnosticsDir,
"--reference", "events.notes=" + referencePath,
}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "noop",
Stage: pipeline.StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "notes"},
},
}),
})
if code != 1 || !strings.Contains(stderr.String(), "read input") {
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
}
resolved := readResolvedPipeline(t, diagnosticsDir)
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 1 || refs[0].Source != referencePath {
t.Fatalf("normalize references = %#v, want lane-qualified binding", refs)
}
}
@@ -847,11 +1034,99 @@ func TestRunPipelineReferenceFlagRejectsAmbiguousFlatSlot(t *testing.T) {
if code != 1 {
t.Fatalf("RunWithOptions() code = %d, want 1", code)
}
if !strings.Contains(stderr.String(), "multiple selected lanes") || !strings.Contains(stderr.String(), "lane.slot") {
if !strings.Contains(stderr.String(), "multiple selected targets") || !strings.Contains(stderr.String(), "events.extract.roster") || !strings.Contains(stderr.String(), "notes.extract.roster") {
t.Fatalf("stderr = %q, want ambiguous reference error", stderr.String())
}
}
func TestRunPipelineReferenceFlagRejectsAmbiguousFlatSlotAcrossChunkAndExtract(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAML("example", "events"))
inputPath := writeSeriatimInput(t)
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "example",
"--config", configPath,
"--input", inputPath,
"--diagnostics-dir", diagnosticsDir,
"--reference", "context=./context.md",
}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t,
pipeline.ModuleSpec{
Key: "generic",
Stage: pipeline.StageChunk,
Requires: []string{"source"},
Provides: []string{"chunks"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "context"},
},
},
pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "context"},
},
},
),
})
if code != 1 {
t.Fatalf("RunWithOptions() code = %d, want 1", code)
}
if !strings.Contains(stderr.String(), "multiple selected targets") || !strings.Contains(stderr.String(), "chunk.context") || !strings.Contains(stderr.String(), "events.extract.context") {
t.Fatalf("stderr = %q, want ambiguous chunk/extract reference error", stderr.String())
}
}
func TestRunPipelineReferenceFlagRejectsAmbiguousLaneSlotAcrossExtractAndNormalize(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAML("example", "events"))
inputPath := writeSeriatimInput(t)
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "example",
"--config", configPath,
"--input", inputPath,
"--diagnostics-dir", diagnosticsDir,
"--reference", "events.context=./context.md",
}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t,
pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "context"},
},
},
pipeline.ModuleSpec{
Key: "noop",
Stage: pipeline.StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "context"},
},
},
),
})
if code != 1 {
t.Fatalf("RunWithOptions() code = %d, want 1", code)
}
if !strings.Contains(stderr.String(), "multiple selected targets in lane") || !strings.Contains(stderr.String(), "events.extract.context") || !strings.Contains(stderr.String(), "events.normalize.context") {
t.Fatalf("stderr = %q, want ambiguous lane reference error", stderr.String())
}
}
func TestRunPipelineReferenceFlagsRejectMalformedValues(t *testing.T) {
tests := []struct {
name string
@@ -861,8 +1136,9 @@ func TestRunPipelineReferenceFlagsRejectMalformedValues(t *testing.T) {
{name: "missing equals", args: []string{"--reference", "roster"}, want: "slot=path"},
{name: "empty path", args: []string{"--reference", "roster="}, want: "path must not be empty"},
{name: "empty slot", args: []string{"--reference", "=./roster.yml"}, want: "slot must not be empty"},
{name: "too many selector parts", args: []string{"--reference", "a.b.c=./roster.yml"}, want: "lane.slot"},
{name: "unbind with equals", args: []string{"--without-reference", "roster=./roster.yml"}, want: "slot or lane.slot"},
{name: "unsupported explicit stage", args: []string{"--reference", "a.b.c=./roster.yml"}, want: "lane.extract.slot or lane.normalize.slot"},
{name: "too many selector parts", args: []string{"--reference", "a.b.c.d=./roster.yml"}, want: "slot, chunk.slot, lane.slot, lane.extract.slot, or lane.normalize.slot"},
{name: "unbind with equals", args: []string{"--without-reference", "roster=./roster.yml"}, want: "without =path"},
}
for _, test := range tests {
@@ -885,6 +1161,72 @@ func TestRunPipelineReferenceFlagsRejectMalformedValues(t *testing.T) {
}
}
func TestRunPipelineWithoutReferenceRemovesConfigBindingsForEligibleTargets(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAMLWithPipelineReferences("example", "events", map[string]string{
"context": "./config-context.md",
"notes": "./config-notes.md",
"roster": "./config-roster.yml",
}))
inputPath := filepath.Join(t.TempDir(), "missing.json")
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "example",
"--config", configPath,
"--input", inputPath,
"--diagnostics-dir", diagnosticsDir,
"--without-reference", "chunk.context",
"--without-reference", "events.extract.roster",
"--without-reference", "events.normalize.notes",
}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t,
pipeline.ModuleSpec{
Key: "generic",
Stage: pipeline.StageChunk,
Requires: []string{"source"},
Provides: []string{"chunks"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "context"},
},
},
pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster"},
},
},
pipeline.ModuleSpec{
Key: "noop",
Stage: pipeline.StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "notes"},
},
},
),
})
if code != 1 || !strings.Contains(stderr.String(), "read input") {
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
}
resolved := readResolvedPipeline(t, diagnosticsDir)
if refs := resolved.ChunkReferences.Bindings; len(refs) != 0 {
t.Fatalf("chunk references = %#v, want none", refs)
}
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("extract references = %#v, want none", refs)
}
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
t.Fatalf("normalize references = %#v, want none", refs)
}
}
func TestRunPipelineWithoutReferenceRemovesOptionalConfigBinding(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAMLWithReferences("example", "events", map[string]string{"roster": "./config-roster.yml"}))
inputPath := filepath.Join(t.TempDir(), "missing.json")
@@ -914,11 +1256,112 @@ func TestRunPipelineWithoutReferenceRemovesOptionalConfigBinding(t *testing.T) {
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
}
resolved := readResolvedPipeline(t, diagnosticsDir)
if refs := resolved.ArtifactLanes[0].References; len(refs) != 0 {
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("references = %#v, want unbound optional slot", refs)
}
}
func TestRunPipelineWithoutReferenceFailsWhenRequiredChunkSlotWouldBeMissing(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAMLWithPipelineReferences("example", "events", map[string]string{"context": "./config-context.md"}))
inputPath := writeSeriatimInput(t)
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "example",
"--config", configPath,
"--input", inputPath,
"--diagnostics-dir", diagnosticsDir,
"--without-reference", "chunk.context",
}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "generic",
Stage: pipeline.StageChunk,
Requires: []string{"source"},
Provides: []string{"chunks"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "context", Required: true},
},
}),
})
if code != 1 {
t.Fatalf("RunWithOptions() code = %d, want 1", code)
}
if !strings.Contains(stderr.String(), "required reference slot") || !strings.Contains(stderr.String(), "context") {
t.Fatalf("stderr = %q, want required chunk reference error", stderr.String())
}
}
func TestRunPipelineWithoutReferenceFailsWhenRequiredNormalizeSlotWouldBeMissing(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAMLWithPipelineReferences("example", "events", map[string]string{"notes": "./config-notes.md"}))
inputPath := writeSeriatimInput(t)
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "example",
"--config", configPath,
"--input", inputPath,
"--diagnostics-dir", diagnosticsDir,
"--without-reference", "events.normalize.notes",
}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "noop",
Stage: pipeline.StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "notes", Required: true},
},
}),
})
if code != 1 {
t.Fatalf("RunWithOptions() code = %d, want 1", code)
}
if !strings.Contains(stderr.String(), "required reference slot") || !strings.Contains(stderr.String(), "notes") {
t.Fatalf("stderr = %q, want required normalize reference error", stderr.String())
}
}
func TestRunPipelineReferenceFlagRejectsUnselectedLaneSelector(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAML("example", "events", "notes"))
inputPath := writeSeriatimInput(t)
referencePath := writeFile(t, "notes.yml", "Notes\n")
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "example",
"--config", configPath,
"--input", inputPath,
"--only", "events",
"--diagnostics-dir", diagnosticsDir,
"--reference", "notes.extract.roster=" + referencePath,
}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster"},
},
}),
})
if code != 1 {
t.Fatalf("RunWithOptions() code = %d, want 1", code)
}
if !strings.Contains(stderr.String(), `reference lane "notes" is not selected`) {
t.Fatalf("stderr = %q, want unselected lane error", stderr.String())
}
}
func TestRunPipelineWithoutReferenceFailsWhenRequiredSlotWouldBeMissing(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAMLWithReferences("example", "events", map[string]string{"roster": "./config-roster.yml"}))
inputPath := writeSeriatimInput(t)
@@ -1609,6 +2052,27 @@ func testConfigYAMLWithReferences(pipelineID string, laneID string, references m
return b.String()
}
func testConfigYAMLWithPipelineReferences(pipelineID string, laneID string, references map[string]string) string {
var b strings.Builder
b.WriteString("version: 1\n")
b.WriteString("pipelines:\n")
b.WriteString(" " + pipelineID + ":\n")
b.WriteString(" input: fake/input\n")
b.WriteString(" references:\n")
keys := make([]string, 0, len(references))
for key := range references {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
b.WriteString(" " + key + ": " + references[key] + "\n")
}
b.WriteString(" artifacts:\n")
b.WriteString(" " + laneID + ":\n")
b.WriteString(" extract: fake/extract\n")
return b.String()
}
func testConfigYAMLWithReferencesAndDiagnostics(pipelineID string, laneID string, diagnosticsDir string, references map[string]string) string {
var b strings.Builder
b.WriteString("version: 1\n")
@@ -1961,6 +2425,10 @@ func (fakeRunChunker) Key() string {
return "generic"
}
func (fakeRunChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (fakeRunChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{
@@ -2024,6 +2492,10 @@ func (fakeRunNormalizer) Key() string {
return "noop"
}
func (fakeRunNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (fakeRunNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{Candidates: append([]artifacts.ArtifactCandidate(nil), req.Candidates...)}, nil
}

View File

@@ -49,7 +49,8 @@ type LLMProfileManifest struct {
}
type ReferenceProvenance struct {
LaneID string `json:"lane_id"`
Stage string `json:"stage,omitempty"`
LaneID string `json:"lane_id,omitempty"`
SlotName string `json:"slot_name"`
OriginType string `json:"origin_type"`
OriginURI string `json:"origin_uri,omitempty"`

View File

@@ -187,6 +187,7 @@ func TestRunManifestIncludesReferenceProvenance(t *testing.T) {
manifest := RunManifest{
References: []ReferenceProvenance{
{
Stage: "extract",
LaneID: "events",
SlotName: "roster",
OriginType: "file",
@@ -212,7 +213,7 @@ func TestRunManifestIncludesReferenceProvenance(t *testing.T) {
t.Fatalf("len(References) = %d, want 1", len(got.References))
}
reference := got.References[0]
if reference.LaneID != "events" || reference.SlotName != "roster" || reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" {
if reference.Stage != "extract" || reference.LaneID != "events" || reference.SlotName != "roster" || reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" {
t.Fatalf("reference provenance = %#v, want lane-scoped origin details", reference)
}
if reference.Digest != "sha256:reference" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != 12 || reference.BindingSource != "config" {

View File

@@ -114,6 +114,7 @@ func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
if len(in.Options) > 0 {
out.Options = cloneOptions(in.Options)
}
out.References = cloneStringMap(in.References)
return out
}

View File

@@ -99,6 +99,7 @@ type fileModuleBinding struct {
Module string
LLMProfile string
Options map[string]any
References map[string]string
}
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
@@ -133,6 +134,12 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
return err
}
b.Options = normalizeOptions(options)
case "references":
var references map[string]string
if err := valueNode.Decode(&references); err != nil {
return err
}
b.References = references
default:
return fmt.Errorf("field %s not found in module binding", keyNode.Value)
}
@@ -148,6 +155,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
Module: strings.TrimSpace(b.Module),
LLMProfile: strings.TrimSpace(b.LLMProfile),
Options: cloneOptions(b.Options),
References: normalizedStringMap(b.References),
}
}
@@ -217,6 +225,19 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
return err
}
if filePipeline.Chunk != nil {
if _, _, err := normalizedMapKeys(filePipeline.Chunk.References, fmt.Sprintf("pipeline %q chunk reference slot", pipelineID)); err != nil {
return err
}
}
if _, _, err := normalizedMapKeys(filePipeline.Input.References, fmt.Sprintf("pipeline %q input reference slot", pipelineID)); err != nil {
return err
}
if filePipeline.Output != nil {
if _, _, err := normalizedMapKeys(filePipeline.Output.References, fmt.Sprintf("pipeline %q output reference slot", pipelineID)); err != nil {
return err
}
}
for rawLaneID, fileLane := range filePipeline.Artifacts {
laneID := strings.TrimSpace(rawLaneID)
if laneID == "" {
@@ -225,6 +246,24 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil {
return err
}
if _, _, err := normalizedMapKeys(fileLane.Extract.References, fmt.Sprintf("pipeline %q lane %q extract reference slot", pipelineID, laneID)); err != nil {
return err
}
if fileLane.Merge != nil {
if _, _, err := normalizedMapKeys(fileLane.Merge.References, fmt.Sprintf("pipeline %q lane %q merge reference slot", pipelineID, laneID)); err != nil {
return err
}
}
if fileLane.Normalize != nil {
if _, _, err := normalizedMapKeys(fileLane.Normalize.References, fmt.Sprintf("pipeline %q lane %q normalize reference slot", pipelineID, laneID)); err != nil {
return err
}
}
for i, validator := range fileLane.Validators {
if _, _, err := normalizedMapKeys(validator.References, fmt.Sprintf("pipeline %q lane %q validator[%d] reference slot", pipelineID, laneID, i)); err != nil {
return err
}
}
}
}
@@ -280,8 +319,10 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
}
for _, laneID := range laneIDs {
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
extract := fileLane.Extract.toPipelineBinding()
extract.References = mergeStringMaps(normalizedStringMap(fileLane.References), extract.References)
lane := pipeline.ArtifactLaneProfile{
Extract: fileLane.Extract.toPipelineBinding(),
Extract: extract,
References: normalizedStringMap(fileLane.References),
}
if fileLane.Merge != nil {
@@ -353,6 +394,20 @@ func normalizedStringMap(values map[string]string) map[string]string {
return out
}
func mergeStringMaps(base map[string]string, override map[string]string) map[string]string {
if len(base) == 0 && len(override) == 0 {
return nil
}
out := make(map[string]string, len(base)+len(override))
for key, value := range base {
out[key] = value
}
for key, value := range override {
out[key] = value
}
return out
}
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
name := strings.TrimSpace(envName)
if name == "" {

View File

@@ -169,6 +169,53 @@ pipelines:
}
}
func TestParseFileConfigStageLocalReferenceMaps(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 1
pipelines:
example:
input: fake/input
chunk:
module: generic
references:
" scene_guide ": " ./scenes.md "
artifacts:
events:
extract:
module: fake/extract
references:
" glossary ": " ./glossary.md "
" roster ": " ./extract-roster.yml "
references:
roster: ./legacy-roster.yml
lore: ./lore.md
normalize:
module: noop
references:
" normalization_notes ": " ./normalization.md "
`)
profile := cfg.Pipelines["example"]
if !reflect.DeepEqual(profile.Chunk.References, map[string]string{"scene_guide": "./scenes.md"}) {
t.Fatalf("chunk references = %#v, want trimmed map", profile.Chunk.References)
}
lane := profile.Artifacts["events"]
if !reflect.DeepEqual(lane.References, map[string]string{"lore": "./lore.md", "roster": "./legacy-roster.yml"}) {
t.Fatalf("legacy lane references = %#v, want trimmed map", lane.References)
}
wantExtract := map[string]string{
"glossary": "./glossary.md",
"lore": "./lore.md",
"roster": "./extract-roster.yml",
}
if !reflect.DeepEqual(lane.Extract.References, wantExtract) {
t.Fatalf("extract references = %#v, want legacy merged with extract override %#v", lane.Extract.References, wantExtract)
}
if !reflect.DeepEqual(lane.Normalize.References, map[string]string{"normalization_notes": "./normalization.md"}) {
t.Fatalf("normalize references = %#v, want trimmed map", lane.Normalize.References)
}
}
func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 1
@@ -357,6 +404,56 @@ pipelines:
`,
want: `pipeline "example" lane "events" reference slot`,
},
{
name: "chunk",
raw: `
version: 1
pipelines:
example:
input: fake/input
chunk:
module: generic
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" chunk reference slot`,
},
{
name: "extract",
raw: `
version: 1
pipelines:
example:
input: fake/input
artifacts:
events:
extract:
module: fake/extract
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" lane "events" extract reference slot`,
},
{
name: "normalize",
raw: `
version: 1
pipelines:
example:
input: fake/input
artifacts:
events:
extract: fake/extract
normalize:
module: noop
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" lane "events" normalize reference slot`,
},
}
for _, tc := range tests {

View File

@@ -34,6 +34,7 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli
out := in
out.Input = cloneModuleBinding(in.Input)
out.Chunk = cloneModuleBinding(in.Chunk)
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)
out.Output = cloneModuleBinding(in.Output)
if len(in.ArtifactLanes) > 0 {
out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes))
@@ -49,8 +50,8 @@ func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.Resolv
out.Extract = cloneModuleBinding(in.Extract)
out.Merge = cloneModuleBinding(in.Merge)
out.Normalize = cloneModuleBinding(in.Normalize)
out.References = append([]pipeline.ReferenceBinding(nil), in.References...)
out.ReferenceSet = pipeline.CloneReferenceSet(in.ReferenceSet)
out.ExtractReferences = pipeline.CloneReferenceTarget(in.ExtractReferences)
out.NormalizeReferences = pipeline.CloneReferenceTarget(in.NormalizeReferences)
if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
for i, binding := range in.Validators {

View File

@@ -67,20 +67,46 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
lane := cfg.Pipelines["example"].Artifacts["events"]
lane.Extract.Options = map[string]any{"temperature": 0.2}
lane.References = map[string]string{"roster": "./roster.yml"}
lane.Extract.References = map[string]string{"glossary": "./glossary.md"}
lane.Normalize.References = map[string]string{"notes": "./normalize.md"}
cfg.Pipelines["example"].Artifacts["events"] = lane
pipelineProfile := cfg.Pipelines["example"]
pipelineProfile.Chunk.References = map[string]string{"scene_guide": "./scene.md"}
cfg.Pipelines["example"] = pipelineProfile
effective, err := cfg.Resolve(ResolveInput{
PipelineID: "example",
Only: []string{"events"},
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster"},
Catalog: fakeCatalog(t,
pipeline.ModuleSpec{
Key: "generic",
Stage: pipeline.StageChunk,
Requires: []string{"source"},
Provides: []string{"chunks"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "scene_guide"},
},
},
}),
pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "glossary"},
{Name: "roster"},
},
},
pipeline.ModuleSpec{
Key: "noop",
Stage: pipeline.StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "notes"},
},
},
),
})
if err != nil {
t.Fatalf("Resolve: %v", err)
@@ -108,8 +134,52 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.Options["temperature"] != 0.2 {
t.Fatalf("expected resolved pipeline options to be copied")
}
payload.ResolvedPipeline.ArtifactLanes[0].References[0].Source = "./changed.yml"
if effective.ResolvedPipeline.ArtifactLanes[0].References[0].Source != "./roster.yml" {
payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source = "./changed.yml"
if referenceBindingSource(effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings, "roster") != "./roster.yml" {
t.Fatalf("expected resolved pipeline references to be copied")
}
payload.ResolvedPipeline.Chunk.References["scene_guide"] = "./changed-scene.md"
if effective.ResolvedPipeline.Chunk.References["scene_guide"] != "./scene.md" {
t.Fatalf("expected chunk references to be copied")
}
payload.ResolvedPipeline.ArtifactLanes[0].Extract.References["glossary"] = "./changed-glossary.md"
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.References["glossary"] != "./glossary.md" {
t.Fatalf("expected extract references to be copied")
}
payload.ResolvedPipeline.ArtifactLanes[0].Normalize.References["notes"] = "./changed-normalize.md"
if effective.ResolvedPipeline.ArtifactLanes[0].Normalize.References["notes"] != "./normalize.md" {
t.Fatalf("expected normalize references to be copied")
}
effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{
SlotName: "roster",
Content: []byte("reference content"),
},
},
},
},
}
payload, ok = effective.RedactedDiagnosticsPayload().(EffectiveConfig)
if !ok {
t.Fatalf("expected EffectiveConfig payload, got %T", effective.RedactedDiagnosticsPayload())
}
payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content[0] = 'X'
got := effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content
if string(got) != "reference content" {
t.Fatalf("expected materialized reference content to be copied, got %q", got)
}
}
func referenceBindingSource(bindings []pipeline.ReferenceBinding, slotName string) string {
for _, binding := range bindings {
if binding.SlotName == slotName {
return binding.Source
}
}
return ""
}

View File

@@ -89,13 +89,13 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
}
if err := validateBindingLLMProfile(id, "", "input", profile.Input, llmProfiles); err != nil {
if err := validateBinding(id, "", "input", profile.Input, llmProfiles, false); err != nil {
return err
}
if err := validateBindingLLMProfile(id, "", "chunk", profile.Chunk, llmProfiles); err != nil {
if err := validateBinding(id, "", "chunk", profile.Chunk, llmProfiles, true); err != nil {
return err
}
if err := validateBindingLLMProfile(id, "", "output", profile.Output, llmProfiles); err != nil {
if err := validateBinding(id, "", "output", profile.Output, llmProfiles, false); err != nil {
return err
}
if err := validateReferenceMap(id, "", profile.References); err != nil {
@@ -109,17 +109,17 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
if err := validateReferenceMap(id, laneID, lane.References); err != nil {
return err
}
if err := validateBindingLLMProfile(id, laneID, "extract", lane.Extract, llmProfiles); err != nil {
if err := validateBinding(id, laneID, "extract", lane.Extract, llmProfiles, true); err != nil {
return err
}
if err := validateBindingLLMProfile(id, laneID, "merge", lane.Merge, llmProfiles); err != nil {
if err := validateBinding(id, laneID, "merge", lane.Merge, llmProfiles, false); err != nil {
return err
}
if err := validateBindingLLMProfile(id, laneID, "normalize", lane.Normalize, llmProfiles); err != nil {
if err := validateBinding(id, laneID, "normalize", lane.Normalize, llmProfiles, true); err != nil {
return err
}
for i, validator := range lane.Validators {
if err := validateBindingLLMProfile(id, laneID, fmt.Sprintf("validator[%d]", i), validator, llmProfiles); err != nil {
if err := validateBinding(id, laneID, fmt.Sprintf("validator[%d]", i), validator, llmProfiles, false); err != nil {
return err
}
}
@@ -128,33 +128,64 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
return nil
}
func validateBinding(
pipelineID string,
laneID string,
slot string,
binding pipeline.ModuleBinding,
profiles map[string]LLMProfile,
referencesAllowed bool,
) error {
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding, profiles); err != nil {
return err
}
if len(binding.References) == 0 {
return nil
}
if !referencesAllowed {
if laneID != "" {
return fmt.Errorf("pipeline %q lane %q %s references are not supported", pipelineID, laneID, slot)
}
return fmt.Errorf("pipeline %q %s references are not supported", pipelineID, slot)
}
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References)
}
func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error {
return validateReferenceMapForContext(pipelineID, laneID, "", references)
}
func validateReferenceMapForContext(pipelineID string, laneID string, slot string, references map[string]string) error {
seen := make(map[string]struct{}, len(references))
for rawSlotName, rawSource := range references {
slotName := strings.TrimSpace(rawSlotName)
if slotName == "" {
if laneID != "" {
return fmt.Errorf("pipeline %q lane %q reference slot name must not be empty", pipelineID, laneID)
}
return fmt.Errorf("pipeline %q reference slot name must not be empty", pipelineID)
return fmt.Errorf("%s reference slot name must not be empty", referenceContext(pipelineID, laneID, slot))
}
if _, ok := seen[slotName]; ok {
if laneID != "" {
return fmt.Errorf("pipeline %q lane %q reference slot %q is duplicated after trimming", pipelineID, laneID, slotName)
}
return fmt.Errorf("pipeline %q reference slot %q is duplicated after trimming", pipelineID, slotName)
return fmt.Errorf("%s reference slot %q is duplicated after trimming", referenceContext(pipelineID, laneID, slot), slotName)
}
seen[slotName] = struct{}{}
if strings.TrimSpace(rawSource) == "" {
if laneID != "" {
return fmt.Errorf("pipeline %q lane %q reference slot %q source must not be empty", pipelineID, laneID, slotName)
}
return fmt.Errorf("pipeline %q reference slot %q source must not be empty", pipelineID, slotName)
return fmt.Errorf("%s reference slot %q source must not be empty", referenceContext(pipelineID, laneID, slot), slotName)
}
}
return nil
}
func referenceContext(pipelineID string, laneID string, slot string) string {
if laneID != "" && slot != "" {
return fmt.Sprintf("pipeline %q lane %q %s", pipelineID, laneID, slot)
}
if laneID != "" {
return fmt.Sprintf("pipeline %q lane %q", pipelineID, laneID)
}
if slot != "" {
return fmt.Sprintf("pipeline %q %s", pipelineID, slot)
}
return fmt.Sprintf("pipeline %q", pipelineID)
}
func validateBindingLLMProfile(
pipelineID string,
laneID string,

View File

@@ -122,6 +122,74 @@ func TestValidateRejectsInvalidReferenceMaps(t *testing.T) {
mutate func(Config) Config
want []string
}{
{
name: "empty chunk slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Chunk.References = map[string]string{" ": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "chunk", "reference slot", "empty"},
},
{
name: "empty chunk source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Chunk.References = map[string]string{"roster": " "}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "chunk", "roster", "source", "empty"},
},
{
name: "empty extract slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Extract.References = map[string]string{" ": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "extract", "reference slot", "empty"},
},
{
name: "empty extract source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Extract.References = map[string]string{"roster": " "}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "extract", "roster", "source", "empty"},
},
{
name: "empty normalize slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Normalize.References = map[string]string{" ": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "normalize", "reference slot", "empty"},
},
{
name: "empty normalize source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Normalize.References = map[string]string{"roster": " "}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "normalize", "roster", "source", "empty"},
},
{
name: "empty pipeline slot",
mutate: func(cfg Config) Config {
@@ -183,6 +251,76 @@ func TestValidateRejectsInvalidReferenceMaps(t *testing.T) {
}
}
func TestValidateRejectsReferencesOnUnsupportedBindings(t *testing.T) {
tests := []struct {
name string
mutate func(Config) Config
want []string
}{
{
name: "input",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Input.References = map[string]string{"roster": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "input", "references", "not supported"},
},
{
name: "merge",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Merge.References = map[string]string{"roster": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "merge", "references", "not supported"},
},
{
name: "validator",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Validators = []pipeline.ModuleBinding{{
Module: "fake/validator",
References: map[string]string{"roster": "./roster.yml"},
}}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "validator[0]", "references", "not supported"},
},
{
name: "output",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Output.References = map[string]string{"roster": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "output", "references", "not supported"},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
err := tc.mutate(validConfig()).Validate()
if err == nil {
t.Fatal("Validate() error = nil, want error")
}
for _, want := range tc.want {
if !strings.Contains(err.Error(), want) {
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
}
}
})
}
}
func TestValidateRejectsEmptyIDs(t *testing.T) {
tests := []struct {
name string

View File

@@ -162,6 +162,10 @@ func (chunker compositionChunker) Key() string {
return "generic-chunker"
}
func (chunker compositionChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker compositionChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
if req.Source == nil {
return contracts.ChunkResult{}, errors.New("source document is required")
@@ -264,6 +268,10 @@ func (normalizer compositionNormalizer) Key() string {
return "generic-normalizer"
}
func (normalizer compositionNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer compositionNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
}

View File

@@ -58,6 +58,7 @@ type SourceChunk struct {
type ChunkRequest struct {
Source *source.SourceDocument `json:"-"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
@@ -71,6 +72,7 @@ type ChunkResult struct {
type Chunker interface {
Key() string
ReferenceSlots() []ReferenceSlot
Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error)
}
@@ -165,6 +167,8 @@ type NormalizeRequest struct {
Source *source.SourceDocument `json:"-"`
LaneID string `json:"lane_id"`
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
@@ -177,6 +181,7 @@ type NormalizeResult struct {
type Normalizer interface {
Key() string
ReferenceSlots() []ReferenceSlot
Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)
}

View File

@@ -378,6 +378,10 @@ func (chunker fakeChunker) Key() string {
return chunker.key
}
func (chunker fakeChunker) ReferenceSlots() []ReferenceSlot {
return nil
}
func (chunker fakeChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
return ChunkResult{
Chunks: []SourceChunk{
@@ -400,6 +404,10 @@ func (chunker *recordingChunker) Key() string {
return chunker.key
}
func (chunker *recordingChunker) ReferenceSlots() []ReferenceSlot {
return nil
}
func (chunker *recordingChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
chunker.request = req
return fakeChunker{key: chunker.key}.Chunk(ctx, req)
@@ -487,6 +495,10 @@ func (normalizer fakeNormalizer) Key() string {
return normalizer.key
}
func (normalizer fakeNormalizer) ReferenceSlots() []ReferenceSlot {
return nil
}
func (normalizer fakeNormalizer) Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error) {
return NormalizeResult{Candidates: req.Candidates}, nil
}

View File

@@ -330,6 +330,10 @@ func (chunker registryChunker) Key() string {
return chunker.key
}
func (chunker registryChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker registryChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{}, nil
}
@@ -354,6 +358,10 @@ func (normalizer registryNormalizer) Key() string {
return normalizer.key
}
func (normalizer registryNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer registryNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{}, nil
}

View File

@@ -87,7 +87,7 @@ func validateModuleSpec(kind string, expectedStage ModuleStage, spec ModuleSpec)
if spec.Stage != expectedStage {
return fmt.Errorf("%s %q must use %q stage, got %q", kind, spec.Key, expectedStage, spec.Stage)
}
if spec.Stage != StageExtract && len(spec.ReferenceSlots) > 0 {
if !referenceSlotStage(spec.Stage) && len(spec.ReferenceSlots) > 0 {
return fmt.Errorf("%s %q must not declare reference slots", kind, spec.Key)
}
if err := validateReferenceSlots(spec.ReferenceSlots); err != nil {
@@ -96,6 +96,10 @@ func validateModuleSpec(kind string, expectedStage ModuleStage, spec ModuleSpec)
return nil
}
func referenceSlotStage(stage ModuleStage) bool {
return stage == StageChunk || stage == StageExtract || stage == StageNormalize
}
func sortedRegistryKeys[C any](constructors map[string]C) []string {
if len(constructors) == 0 {
return nil

View File

@@ -7,19 +7,117 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestValidateModuleSpecRejectsReferenceSlotsForNonExtractors(t *testing.T) {
err := validateModuleSpec("chunker", StageChunk, ModuleSpec{
Key: "generic",
Stage: StageChunk,
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster"},
},
})
if err == nil {
t.Fatal("validateModuleSpec() error = nil, want error")
func TestValidateModuleSpecAllowsReferenceSlotsForEligibleStages(t *testing.T) {
tests := []struct {
name string
kind string
stage ModuleStage
}{
{name: "chunker", kind: "chunker", stage: StageChunk},
{name: "extractor", kind: "extractor", stage: StageExtract},
{name: "normalizer", kind: "normalizer", stage: StageNormalize},
}
if !strings.Contains(err.Error(), "reference slots") {
t.Fatalf("validateModuleSpec() error = %q, want reference slots context", err.Error())
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
spec := normalizeModuleSpec(ModuleSpec{
Key: "module",
Stage: test.stage,
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Description: "Character roster", MaxBytes: 1024},
},
})
err := validateModuleSpec(test.kind, test.stage, spec)
if err != nil {
t.Fatalf("validateModuleSpec() error = %v, want nil", err)
}
})
}
}
func TestValidateModuleSpecRejectsReferenceSlotsForIneligibleStages(t *testing.T) {
tests := []struct {
name string
kind string
stage ModuleStage
}{
{name: "input", kind: "input adapter", stage: StageInput},
{name: "merge", kind: "merger", stage: StageMerge},
{name: "validate", kind: "validator", stage: StageValidate},
{name: "output", kind: "output encoder", stage: StageOutput},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
spec := normalizeModuleSpec(ModuleSpec{
Key: "module",
Stage: test.stage,
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster"},
},
})
err := validateModuleSpec(test.kind, test.stage, spec)
if err == nil {
t.Fatal("validateModuleSpec() error = nil, want error")
}
if !strings.Contains(err.Error(), "reference slots") {
t.Fatalf("validateModuleSpec() error = %q, want reference slots context", err.Error())
}
})
}
}
func TestValidateModuleSpecRejectsInvalidReferenceSlotsForEligibleStages(t *testing.T) {
invalidSlots := []struct {
name string
slots []contracts.ReferenceSlot
want string
}{
{
name: "empty name",
slots: []contracts.ReferenceSlot{{Name: " "}},
want: "name",
},
{
name: "duplicate name after trim",
slots: []contracts.ReferenceSlot{
{Name: "roster"},
{Name: " roster "},
},
want: "duplicated",
},
{
name: "negative max bytes",
slots: []contracts.ReferenceSlot{{Name: "roster", MaxBytes: -1}},
want: "max_bytes",
},
}
eligibleStages := []struct {
name string
kind string
stage ModuleStage
}{
{name: "chunk", kind: "chunker", stage: StageChunk},
{name: "extract", kind: "extractor", stage: StageExtract},
{name: "normalize", kind: "normalizer", stage: StageNormalize},
}
for _, stage := range eligibleStages {
for _, invalid := range invalidSlots {
t.Run(stage.name+"/"+invalid.name, func(t *testing.T) {
spec := normalizeModuleSpec(ModuleSpec{
Key: "module",
Stage: stage.stage,
ReferenceSlots: invalid.slots,
})
err := validateModuleSpec(stage.kind, stage.stage, spec)
if err == nil {
t.Fatal("validateModuleSpec() error = nil, want error")
}
if !strings.Contains(err.Error(), invalid.want) {
t.Fatalf("validateModuleSpec() error = %q, want %q", err.Error(), invalid.want)
}
})
}
}
}

View File

@@ -20,9 +20,10 @@ const (
)
type ModuleBinding struct {
Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
References map[string]string `json:"references,omitempty"`
}
type ArtifactLaneProfile struct {
@@ -49,34 +50,45 @@ type ResolveOptions struct {
}
type ReferenceBinding struct {
LaneID string `json:"lane_id,omitempty"`
SlotName string `json:"slot_name"`
Source string `json:"source"`
BindingSource string `json:"binding_source,omitempty"`
Stage ModuleStage `json:"stage,omitempty"`
LaneID string `json:"lane_id,omitempty"`
SlotName string `json:"slot_name"`
Source string `json:"source"`
BindingSource string `json:"binding_source,omitempty"`
}
type ReferenceUnbind struct {
LaneID string `json:"lane_id"`
SlotName string `json:"slot_name"`
Stage ModuleStage `json:"stage,omitempty"`
LaneID string `json:"lane_id,omitempty"`
SlotName string `json:"slot_name"`
}
type ResolvedArtifactLane struct {
ID string
Extract ModuleBinding
Merge ModuleBinding
Normalize ModuleBinding
Validators []ModuleBinding
References []ReferenceBinding `json:"references,omitempty"`
type ResolvedReferenceTarget struct {
Stage ModuleStage `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
Module string `json:"module"`
Bindings []ReferenceBinding `json:"bindings,omitempty"`
ReferenceSet contracts.ReferenceSet `json:"-"`
}
type ResolvedArtifactLane struct {
ID string
Extract ModuleBinding
Merge ModuleBinding
Normalize ModuleBinding
Validators []ModuleBinding
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
}
type ResolvedPipeline struct {
ID string
Digest string
Input ModuleBinding
Chunk ModuleBinding
ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding
ID string
Digest string
Input ModuleBinding
Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding
}
type ModuleCatalog struct {
@@ -135,15 +147,28 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
if len(selectedLaneIDs) == 0 {
return ResolvedPipeline{}, fmt.Errorf("pipeline %q must select at least one artifact lane", pipelineID)
}
if err := validatePipelineReferenceDefaults(pipelineID, profile.References, lanesByID, catalog); err != nil {
if err := validatePipelineReferenceDefaults(pipelineID, profile.References, chunkSpec, lanesByID, catalog); err != nil {
return ResolvedPipeline{}, err
}
chunkReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID,
Stage: StageChunk,
Module: chunk.Module,
Slots: chunkSpec.ReferenceSlots,
PipelineReferences: profile.References,
LocalReferences: chunk.References,
Options: options,
})
if err != nil {
return ResolvedPipeline{}, err
}
resolved := ResolvedPipeline{
ID: pipelineID,
Input: input,
Chunk: chunk,
Output: resolveBinding(profile.Output, DefaultOutputModule),
ID: pipelineID,
Input: input,
Chunk: chunk,
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
Output: resolveBinding(profile.Output, DefaultOutputModule),
}
outputCapabilities := capabilities.clone()
@@ -202,11 +227,21 @@ func resolveArtifactLane(
if missing, ok := capabilities.missing(extractSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
}
references, err := resolveReferenceBindings(pipelineID, laneID, lane.Extract.Module, extractSpec.ReferenceSlots, pipelineReferences, profile.References, options)
extractReferences := mergeReferenceMaps(profile.References, lane.Extract.References)
references, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID,
LaneID: laneID,
Stage: StageExtract,
Module: lane.Extract.Module,
Slots: extractSpec.ReferenceSlots,
PipelineReferences: pipelineReferences,
LocalReferences: extractReferences,
Options: options,
})
if err != nil {
return ResolvedArtifactLane{}, nil, err
}
lane.References = references
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
capabilities.add(extractSpec.Provides...)
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
@@ -225,6 +260,20 @@ func resolveArtifactLane(
if missing, ok := capabilities.missing(normalizeSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing)
}
normalizeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID,
LaneID: laneID,
Stage: StageNormalize,
Module: lane.Normalize.Module,
Slots: normalizeSpec.ReferenceSlots,
PipelineReferences: pipelineReferences,
LocalReferences: lane.Normalize.References,
Options: options,
})
if err != nil {
return ResolvedArtifactLane{}, nil, err
}
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
capabilities.add(normalizeSpec.Provides...)
for _, validator := range lane.Validators {
@@ -241,9 +290,33 @@ func resolveArtifactLane(
return lane, capabilities, nil
}
func referenceTarget(stage ModuleStage, laneID string, module string, bindings []ReferenceBinding) ResolvedReferenceTarget {
return ResolvedReferenceTarget{
Stage: stage,
LaneID: strings.TrimSpace(laneID),
Module: strings.TrimSpace(module),
Bindings: append([]ReferenceBinding(nil), bindings...),
}
}
func mergeReferenceMaps(base map[string]string, override map[string]string) map[string]string {
if len(base) == 0 && len(override) == 0 {
return nil
}
out := make(map[string]string, len(base)+len(override))
for key, value := range base {
out[key] = value
}
for key, value := range override {
out[key] = value
}
return out
}
func validatePipelineReferenceDefaults(
pipelineID string,
pipelineReferences map[string]string,
chunkSpec ModuleSpec,
lanesByID map[string]ArtifactLaneProfile,
catalog ModuleCatalog,
) error {
@@ -255,7 +328,10 @@ func validatePipelineReferenceDefaults(
return nil
}
declaredByAnyLane := make(map[string]struct{}, len(normalizedPipelineReferences))
declaredByAnyTarget := make(map[string]struct{}, len(normalizedPipelineReferences))
for _, slot := range chunkSpec.ReferenceSlots {
declaredByAnyTarget[slot.Name] = struct{}{}
}
for _, laneID := range sortedArtifactLaneProfileKeys(lanesByID) {
laneProfile := lanesByID[laneID]
extract := resolveBinding(laneProfile.Extract, "")
@@ -267,29 +343,41 @@ func validatePipelineReferenceDefaults(
return moduleLookupError(pipelineID, laneID, StageExtract, extract.Module, err)
}
for _, slot := range extractSpec.ReferenceSlots {
declaredByAnyLane[slot.Name] = struct{}{}
declaredByAnyTarget[slot.Name] = struct{}{}
}
normalize := resolveBinding(laneProfile.Normalize, DefaultNormalizeModule)
normalizeSpec, err := normalizerSpec(catalog, normalize.Module)
if err != nil {
return moduleLookupError(pipelineID, laneID, StageNormalize, normalize.Module, err)
}
for _, slot := range normalizeSpec.ReferenceSlots {
declaredByAnyTarget[slot.Name] = struct{}{}
}
}
for _, slotName := range sortedStringMapKeys(normalizedPipelineReferences) {
if _, ok := declaredByAnyLane[slotName]; !ok {
return fmt.Errorf("pipeline %q reference slot %q is not declared by any artifact lane", pipelineID, slotName)
if _, ok := declaredByAnyTarget[slotName]; !ok {
return fmt.Errorf("pipeline %q reference slot %q is not declared by any eligible reference target", pipelineID, slotName)
}
}
return nil
}
func resolveReferenceBindings(
pipelineID string,
laneID string,
extractorModule string,
slots []contracts.ReferenceSlot,
pipelineReferences map[string]string,
laneReferences map[string]string,
options ResolveOptions,
) ([]ReferenceBinding, error) {
slotByName := make(map[string]contracts.ReferenceSlot, len(slots))
for _, slot := range slots {
type referenceResolutionTarget struct {
PipelineID string
LaneID string
Stage ModuleStage
Module string
Slots []contracts.ReferenceSlot
PipelineReferences map[string]string
LocalReferences map[string]string
Options ResolveOptions
}
func resolveReferenceTargetBindings(target referenceResolutionTarget) ([]ReferenceBinding, error) {
slotByName := make(map[string]contracts.ReferenceSlot, len(target.Slots))
for _, slot := range target.Slots {
slotByName[slot.Name] = slot
}
@@ -298,16 +386,16 @@ func resolveReferenceBindings(
slotName = strings.TrimSpace(slotName)
source = strings.TrimSpace(source)
if slotName == "" {
return fmt.Errorf("pipeline %q lane %q reference slot name must not be empty", pipelineID, laneID)
return fmt.Errorf("%s reference slot name must not be empty", referenceTargetErrorContext(target))
}
if source == "" {
return fmt.Errorf("pipeline %q lane %q reference slot %q source must not be empty", pipelineID, laneID, slotName)
return fmt.Errorf("%s reference slot %q source must not be empty", referenceTargetErrorContext(target), slotName)
}
if _, ok := slotByName[slotName]; !ok {
return fmt.Errorf("pipeline %q lane %q reference slot %q is not declared by extractor %q", pipelineID, laneID, slotName, extractorModule)
return fmt.Errorf("%s reference slot %q is not declared by %s module %q", referenceTargetErrorContext(target), slotName, target.Stage, target.Module)
}
bindings[slotName] = ReferenceBinding{
LaneID: laneID,
LaneID: target.LaneID,
SlotName: slotName,
Source: source,
BindingSource: bindingSource,
@@ -315,7 +403,7 @@ func resolveReferenceBindings(
return nil
}
normalizedPipelineReferences, err := normalizedReferenceMap(pipelineReferences, fmt.Sprintf("pipeline %q reference slot", pipelineID))
normalizedPipelineReferences, err := normalizedReferenceMap(target.PipelineReferences, fmt.Sprintf("pipeline %q reference slot", target.PipelineID))
if err != nil {
return nil, err
}
@@ -328,22 +416,22 @@ func resolveReferenceBindings(
}
}
normalizedLaneReferences, err := normalizedReferenceMap(laneReferences, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID))
normalizedLocalReferences, err := normalizedReferenceMap(target.LocalReferences, referenceTargetSlotLabel(target))
if err != nil {
return nil, err
}
for _, slotName := range sortedStringMapKeys(normalizedLaneReferences) {
if err := addBinding(slotName, normalizedLaneReferences[slotName], contracts.ReferenceBindingSourceConfig); err != nil {
for _, slotName := range sortedStringMapKeys(normalizedLocalReferences) {
if err := addBinding(slotName, normalizedLocalReferences[slotName], contracts.ReferenceBindingSourceConfig); err != nil {
return nil, err
}
}
for _, override := range options.ReferenceOverrides {
optionLaneID := strings.TrimSpace(override.LaneID)
if optionLaneID == "" {
return nil, fmt.Errorf("pipeline %q reference override lane id must not be empty", pipelineID)
for _, override := range target.Options.ReferenceOverrides {
match, err := referenceOverrideMatchesTarget(target, override)
if err != nil {
return nil, err
}
if optionLaneID != laneID {
if !match {
continue
}
source := override.BindingSource
@@ -355,38 +443,94 @@ func resolveReferenceBindings(
}
}
for _, unbind := range options.ReferenceUnbinds {
optionLaneID := strings.TrimSpace(unbind.LaneID)
if optionLaneID == "" {
return nil, fmt.Errorf("pipeline %q reference unbind lane id must not be empty", pipelineID)
for _, unbind := range target.Options.ReferenceUnbinds {
match, err := referenceUnbindMatchesTarget(target, unbind)
if err != nil {
return nil, err
}
if optionLaneID != laneID {
if !match {
continue
}
slotName := strings.TrimSpace(unbind.SlotName)
if slotName == "" {
return nil, fmt.Errorf("pipeline %q lane %q reference unbind slot name must not be empty", pipelineID, laneID)
return nil, fmt.Errorf("%s reference unbind slot name must not be empty", referenceTargetErrorContext(target))
}
if _, ok := slotByName[slotName]; !ok {
return nil, fmt.Errorf("pipeline %q lane %q reference slot %q is not declared", pipelineID, laneID, slotName)
return nil, fmt.Errorf("%s reference slot %q is not declared", referenceTargetErrorContext(target), slotName)
}
delete(bindings, slotName)
}
for _, slot := range slots {
for _, slot := range target.Slots {
if slot.Required {
if _, ok := bindings[slot.Name]; !ok {
return nil, fmt.Errorf("pipeline %q lane %q required reference slot %q is not bound", pipelineID, laneID, slot.Name)
return nil, fmt.Errorf("%s required reference slot %q is not bound", referenceTargetErrorContext(target), slot.Name)
}
}
}
return sortedReferenceBindings(bindings), nil
}
func referenceOverrideMatchesTarget(target referenceResolutionTarget, override ReferenceBinding) (bool, error) {
stage, laneID, err := normalizeReferenceOptionTarget(target.PipelineID, "override", override.Stage, override.LaneID)
if err != nil {
return false, err
}
return stage == target.Stage && laneID == target.LaneID, nil
}
func referenceUnbindMatchesTarget(target referenceResolutionTarget, unbind ReferenceUnbind) (bool, error) {
stage, laneID, err := normalizeReferenceOptionTarget(target.PipelineID, "unbind", unbind.Stage, unbind.LaneID)
if err != nil {
return false, err
}
return stage == target.Stage && laneID == target.LaneID, nil
}
func normalizeReferenceOptionTarget(pipelineID string, operation string, stage ModuleStage, laneID string) (ModuleStage, string, error) {
stage = ModuleStage(strings.TrimSpace(string(stage)))
if stage == "" {
stage = StageExtract
}
laneID = strings.TrimSpace(laneID)
switch stage {
case StageChunk:
if laneID != "" {
return "", "", fmt.Errorf("pipeline %q reference %s for chunk must not include a lane id", pipelineID, operation)
}
case StageExtract, StageNormalize:
if laneID == "" {
return "", "", fmt.Errorf("pipeline %q reference %s lane id must not be empty", pipelineID, operation)
}
default:
return "", "", fmt.Errorf("pipeline %q reference %s stage %q is not supported", pipelineID, operation, stage)
}
return stage, laneID, nil
}
func sortedReferenceBindings(bindings map[string]ReferenceBinding) []ReferenceBinding {
keys := sortedReferenceBindingKeys(bindings)
resolved := make([]ReferenceBinding, 0, len(keys))
for _, slotName := range keys {
resolved = append(resolved, bindings[slotName])
}
return resolved, nil
return resolved
}
func referenceTargetErrorContext(target referenceResolutionTarget) string {
if target.LaneID != "" {
return fmt.Sprintf("pipeline %q lane %q %s module %q", target.PipelineID, target.LaneID, target.Stage, target.Module)
}
return fmt.Sprintf("pipeline %q %s module %q", target.PipelineID, target.Stage, target.Module)
}
func referenceTargetSlotLabel(target referenceResolutionTarget) string {
if target.LaneID != "" {
return fmt.Sprintf("pipeline %q lane %q %s reference slot", target.PipelineID, target.LaneID, target.Stage)
}
return fmt.Sprintf("pipeline %q %s reference slot", target.PipelineID, target.Stage)
}
func normalizedReferenceMap(values map[string]string, keyName string) (map[string]string, error) {
@@ -460,6 +604,7 @@ func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
Module: module,
LLMProfile: llmProfile,
Options: cloneOptions(binding.Options),
References: normalizeReferenceMap(binding.References),
}
}
@@ -488,6 +633,25 @@ func cloneOptions(options map[string]any) map[string]any {
return copied
}
func normalizeReferenceMap(values map[string]string) map[string]string {
if len(values) == 0 {
return nil
}
out := make(map[string]string, len(values))
keys := make([]string, 0, len(values))
rawByNormalized := make(map[string]string, len(values))
for rawKey := range values {
key := strings.TrimSpace(rawKey)
rawByNormalized[key] = rawKey
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
out[key] = strings.TrimSpace(values[rawByNormalized[key]])
}
return out
}
func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneProfile, options ResolveOptions) (map[string]ArtifactLaneProfile, []string, error) {
lanesByID := make(map[string]ArtifactLaneProfile, len(artifacts))
for rawLaneID, lane := range artifacts {
@@ -532,17 +696,19 @@ func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneP
func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
withoutDigest := struct {
ID string
Input ModuleBinding
Chunk ModuleBinding
ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding
ID string
Input ModuleBinding
Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget
ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding
}{
ID: resolved.ID,
Input: resolved.Input,
Chunk: resolved.Chunk,
ArtifactLanes: resolved.ArtifactLanes,
Output: resolved.Output,
ID: resolved.ID,
Input: resolved.Input,
Chunk: resolved.Chunk,
ChunkReferences: resolved.ChunkReferences,
ArtifactLanes: resolved.ArtifactLanes,
Output: resolved.Output,
}
encoded, err := json.Marshal(withoutDigest)
if err != nil {

View File

@@ -154,19 +154,131 @@ func TestResolvePipelineAppliesReferenceBindings(t *testing.T) {
}
events := resolvedLane(t, resolved.ArtifactLanes, "events")
if events.ExtractReferences.Stage != StageExtract || events.ExtractReferences.LaneID != "events" || events.ExtractReferences.Module != "event-extractor" {
t.Fatalf("extract reference target = %#v, want event extractor target", events.ExtractReferences)
}
if events.NormalizeReferences.Stage != StageNormalize || events.NormalizeReferences.LaneID != "events" || events.NormalizeReferences.Module != DefaultNormalizeModule {
t.Fatalf("normalize reference target = %#v, want event normalizer target", events.NormalizeReferences)
}
if resolved.ChunkReferences.Stage != StageChunk || resolved.ChunkReferences.Module != DefaultChunkModule {
t.Fatalf("chunk reference target = %#v, want chunk target", resolved.ChunkReferences)
}
want := []ReferenceBinding{
{LaneID: "events", SlotName: "lore", Source: "./lore.md", BindingSource: contracts.ReferenceBindingSourceConfig},
{LaneID: "events", SlotName: "roster", Source: "./lane-roster.yml", BindingSource: contracts.ReferenceBindingSourceConfig},
}
if !reflect.DeepEqual(events.References, want) {
t.Fatalf("events references = %#v, want %#v", events.References, want)
if !reflect.DeepEqual(events.ExtractReferences.Bindings, want) {
t.Fatalf("events references = %#v, want %#v", events.ExtractReferences.Bindings, want)
}
summaries := resolvedLane(t, resolved.ArtifactLanes, "summaries")
if len(summaries.References) != 0 {
t.Fatalf("summaries references = %#v, want none", summaries.References)
if len(summaries.ExtractReferences.Bindings) != 0 {
t.Fatalf("summaries references = %#v, want none", summaries.ExtractReferences.Bindings)
}
}
func TestResolvePipelineAppliesPipelineReferenceDefaultToChunkTarget(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"scene_guide": "./scenes.md"}
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "generic",
Stage: StageChunk,
Requires: []string{"source"},
Provides: []string{"chunk"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "scene_guide"}},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
want := []ReferenceBinding{{SlotName: "scene_guide", Source: "./scenes.md", BindingSource: contracts.ReferenceBindingSourceConfig}}
if !reflect.DeepEqual(resolved.ChunkReferences.Bindings, want) {
t.Fatalf("chunk references = %#v, want %#v", resolved.ChunkReferences.Bindings, want)
}
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("extract references = %#v, want none", refs)
}
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
t.Fatalf("normalize references = %#v, want none", refs)
}
}
func TestResolvePipelineAppliesPipelineReferenceDefaultToExtractorTarget(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"roster": "./roster.yml"}
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "roster"}},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
want := []ReferenceBinding{{LaneID: "events", SlotName: "roster", Source: "./roster.yml", BindingSource: contracts.ReferenceBindingSourceConfig}}
if !reflect.DeepEqual(resolved.ArtifactLanes[0].ExtractReferences.Bindings, want) {
t.Fatalf("extract references = %#v, want %#v", resolved.ArtifactLanes[0].ExtractReferences.Bindings, want)
}
if refs := resolved.ChunkReferences.Bindings; len(refs) != 0 {
t.Fatalf("chunk references = %#v, want none", refs)
}
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
t.Fatalf("normalize references = %#v, want none", refs)
}
}
func TestResolvePipelineAppliesPipelineReferenceDefaultToNormalizerTarget(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"normalization_notes": "./normalize.md"}
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "noop",
Stage: StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes"}},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
want := []ReferenceBinding{{LaneID: "events", SlotName: "normalization_notes", Source: "./normalize.md", BindingSource: contracts.ReferenceBindingSourceConfig}}
if !reflect.DeepEqual(resolved.ArtifactLanes[0].NormalizeReferences.Bindings, want) {
t.Fatalf("normalize references = %#v, want %#v", resolved.ArtifactLanes[0].NormalizeReferences.Bindings, want)
}
if refs := resolved.ChunkReferences.Bindings; len(refs) != 0 {
t.Fatalf("chunk references = %#v, want none", refs)
}
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("extract references = %#v, want none", refs)
}
}
func TestResolvePipelineAppliesOnePipelineReferenceDefaultToMultipleTargets(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"context": "./context.md"}
catalog := newProfileCatalogWithOverrides(t,
ModuleSpec{Key: "generic", Stage: StageChunk, Requires: []string{"source"}, Provides: []string{"chunk"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
ModuleSpec{Key: "event-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
ModuleSpec{Key: "noop", Stage: StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
)
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
assertBindingSource(t, resolved.ChunkReferences.Bindings, "context", "./context.md")
assertBindingSource(t, resolved.ArtifactLanes[0].ExtractReferences.Bindings, "context", "./context.md")
assertBindingSource(t, resolved.ArtifactLanes[0].NormalizeReferences.Bindings, "context", "./context.md")
}
func TestResolvePipelineAllowsPipelineReferenceDeclaredOnlyByUnselectedLane(t *testing.T) {
profile := multiLaneProfile()
profile.References = map[string]string{"notes_context": "./notes.md"}
@@ -184,11 +296,39 @@ func TestResolvePipelineAllowsPipelineReferenceDeclaredOnlyByUnselectedLane(t *t
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if refs := resolved.ArtifactLanes[0].References; len(refs) != 0 {
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("selected lane references = %#v, want none", refs)
}
}
func TestResolvePipelineAllowsPipelineReferenceDeclaredOnlyByUnselectedNormalizer(t *testing.T) {
profile := multiLaneProfile()
profile.References = map[string]string{"notes_context": "./notes.md"}
lane := profile.Artifacts["notes"]
lane.Normalize = Binding("note-normalizer")
profile.Artifacts["notes"] = lane
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "note-normalizer",
Stage: StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "notes_context"},
},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{"events"}}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("selected extract references = %#v, want none", refs)
}
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
t.Fatalf("selected normalize references = %#v, want none", refs)
}
}
func TestResolvePipelineRejectsPipelineReferenceNotDeclaredByAnyLane(t *testing.T) {
profile := multiLaneProfile()
profile.References = map[string]string{"missing": "./missing.md"}
@@ -213,6 +353,106 @@ func TestResolvePipelineRejectsUndeclaredReferenceSlot(t *testing.T) {
assertErrorContains(t, err, "events", "missing", "not declared")
}
func TestResolvePipelineRejectsExtractLocalReferenceDeclaredOnlyByNormalizer(t *testing.T) {
profile := baselineProfile()
lane := profile.Artifacts["events"]
lane.Extract.References = map[string]string{"normalization_notes": "./normalize.md"}
profile.Artifacts["events"] = lane
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "noop",
Stage: StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes"}},
})
_, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "baseline", "events", "extract", "event-extractor", "normalization_notes", "not declared")
}
func TestResolvePipelineRejectsNormalizeLocalReferenceDeclaredOnlyByExtractor(t *testing.T) {
profile := baselineProfile()
lane := profile.Artifacts["events"]
lane.Normalize.References = map[string]string{"roster": "./roster.yml"}
profile.Artifacts["events"] = lane
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "roster"}},
})
_, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "baseline", "events", "normalize", "noop", "roster", "not declared")
}
func TestResolvePipelineRequiresBoundChunkReference(t *testing.T) {
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "generic",
Stage: StageChunk,
Requires: []string{"source"},
Provides: []string{"chunk"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "scene_guide", Required: true}},
})
_, err := ResolvePipeline(baselineProfile(), ResolveOptions{}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "baseline", "chunk", "generic", "required", "scene_guide", "not bound")
}
func TestResolvePipelineRequiresBoundNormalizeReference(t *testing.T) {
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "noop",
Stage: StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes", Required: true}},
})
_, err := ResolvePipeline(baselineProfile(), ResolveOptions{}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "baseline", "events", "normalize", "noop", "required", "normalization_notes", "not bound")
}
func TestResolvePipelineLocalReferencesOverridePipelineDefaultsForEligibleTargets(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{
"context": "./shared-context.md",
"roster": "./shared-roster.yml",
"normalization_notes": "./shared-normalize.md",
}
profile.Chunk.References = map[string]string{"context": "./chunk-context.md"}
lane := profile.Artifacts["events"]
lane.Extract.References = map[string]string{"roster": "./extract-roster.yml"}
lane.Normalize.References = map[string]string{"normalization_notes": "./local-normalize.md"}
profile.Artifacts["events"] = lane
catalog := newProfileCatalogWithOverrides(t,
ModuleSpec{Key: "generic", Stage: StageChunk, Requires: []string{"source"}, Provides: []string{"chunk"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
ModuleSpec{Key: "event-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "roster"}}},
ModuleSpec{Key: "noop", Stage: StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes"}}},
)
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
assertBindingSource(t, resolved.ChunkReferences.Bindings, "context", "./chunk-context.md")
assertBindingSource(t, resolved.ArtifactLanes[0].ExtractReferences.Bindings, "roster", "./extract-roster.yml")
assertBindingSource(t, resolved.ArtifactLanes[0].NormalizeReferences.Bindings, "normalization_notes", "./local-normalize.md")
}
func TestResolvePipelineRequiresBoundReferenceSlotsForSelectedLanes(t *testing.T) {
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
Key: "event-extractor",
@@ -284,7 +524,7 @@ func TestResolvePipelineUsesReferenceSlotsFromSpecWithoutConstructingExtractor(t
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if got := resolved.ArtifactLanes[0].References[0].Source; got != "./roster.yml" {
if got := resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].Source; got != "./roster.yml" {
t.Fatalf("reference source = %q, want ./roster.yml", got)
}
}
@@ -649,6 +889,21 @@ func assertErrorContains(t *testing.T, err error, values ...string) {
}
}
func assertBindingSource(t *testing.T, bindings []ReferenceBinding, slotName string, source string) {
t.Helper()
for _, binding := range bindings {
if binding.SlotName != slotName {
continue
}
if binding.Source != source {
t.Fatalf("binding %q source = %q, want %q in %#v", slotName, binding.Source, source, bindings)
}
return
}
t.Fatalf("binding %q not found in %#v", slotName, bindings)
}
func newProfileCatalog(t *testing.T) ModuleCatalog {
t.Helper()
@@ -660,19 +915,29 @@ func newProfileCatalog(t *testing.T) ModuleCatalog {
func newProfileCatalogWithOverride(t *testing.T, override ModuleSpec) ModuleCatalog {
t.Helper()
return newProfileCatalogWithOverrides(t, override)
}
func newProfileCatalogWithOverrides(t *testing.T, overrides ...ModuleSpec) ModuleCatalog {
t.Helper()
specs := defaultProfileSpecs()
for index, spec := range specs {
if spec.Stage == override.Stage && spec.Key == override.Key {
specs[index] = override
catalog := emptyProfileCatalog()
registerProfileSpecs(t, catalog, specs...)
return catalog
for _, override := range overrides {
replaced := false
for index, spec := range specs {
if spec.Stage == override.Stage && spec.Key == override.Key {
specs[index] = override
replaced = true
break
}
}
if !replaced {
specs = append(specs, override)
}
}
catalog := emptyProfileCatalog()
registerProfileSpecs(t, catalog, specs...)
registerProfileSpecs(t, catalog, override)
return catalog
}

View File

@@ -29,40 +29,52 @@ type ReferenceMaterializationOptions struct {
func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, options ReferenceMaterializationOptions) (ResolvedPipeline, []contracts.Warning, error) {
out := resolved
out.ChunkReferences = CloneReferenceTarget(resolved.ChunkReferences)
chunkReferenceSet, chunkWarnings, err := materializeReferenceTarget(resolved.ID, resolved.ChunkReferences, catalog, options)
if err != nil {
return ResolvedPipeline{}, nil, err
}
out.ChunkReferences.ReferenceSet = chunkReferenceSet
warnings := append([]contracts.Warning(nil), chunkWarnings...)
if len(resolved.ArtifactLanes) == 0 {
return out, nil, nil
return out, warnings, nil
}
warnings := []contracts.Warning(nil)
out.ArtifactLanes = make([]ResolvedArtifactLane, len(resolved.ArtifactLanes))
for i, lane := range resolved.ArtifactLanes {
materializedLane := lane
referenceSet, laneWarnings, err := materializeLaneReferences(resolved.ID, lane, catalog, options)
materializedLane.ExtractReferences = CloneReferenceTarget(lane.ExtractReferences)
materializedLane.NormalizeReferences = CloneReferenceTarget(lane.NormalizeReferences)
extractReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.ExtractReferences, catalog, options)
if err != nil {
return ResolvedPipeline{}, nil, err
}
materializedLane.ReferenceSet = referenceSet
out.ArtifactLanes[i] = materializedLane
materializedLane.ExtractReferences.ReferenceSet = extractReferenceSet
warnings = append(warnings, laneWarnings...)
normalizeReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.NormalizeReferences, catalog, options)
if err != nil {
return ResolvedPipeline{}, nil, err
}
materializedLane.NormalizeReferences.ReferenceSet = normalizeReferenceSet
warnings = append(warnings, laneWarnings...)
out.ArtifactLanes[i] = materializedLane
}
return out, warnings, nil
}
func materializeLaneReferences(
func materializeReferenceTarget(
pipelineID string,
lane ResolvedArtifactLane,
target ResolvedReferenceTarget,
catalog ModuleCatalog,
options ReferenceMaterializationOptions,
) (contracts.ReferenceSet, []contracts.Warning, error) {
if len(lane.References) == 0 {
if len(target.Bindings) == 0 {
return contracts.ReferenceSet{}, nil, nil
}
if catalog.Extractors == nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", pipelineID, lane.ID, lane.Extract.Module, lane.Extract.Module)
}
spec, ok := catalog.Extractors.Spec(lane.Extract.Module)
if !ok {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", pipelineID, lane.ID, lane.Extract.Module, lane.Extract.Module)
spec, err := referenceTargetSpec(target, catalog)
if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s: %w", referenceTargetContext(pipelineID, target), err)
}
slotByName := make(map[string]contracts.ReferenceSlot, len(spec.ReferenceSlots))
@@ -70,38 +82,38 @@ func materializeLaneReferences(
slotByName[slot.Name] = slot
}
set := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(lane.References))}
set := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(target.Bindings))}
var warnings []contracts.Warning
for _, binding := range lane.References {
for _, binding := range target.Bindings {
slotName := strings.TrimSpace(binding.SlotName)
slot, ok := slotByName[slotName]
if !ok {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q is not declared by extractor %q", pipelineID, lane.ID, slotName, lane.Extract.Module)
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q is not declared by %s module %q", referenceTargetContext(pipelineID, target), slotName, target.Stage, target.Module)
}
path, err := referencePath(binding, options)
if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q: %w", pipelineID, lane.ID, slotName, binding.Source, err)
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q: %w", referenceTargetContext(pipelineID, target), slotName, binding.Source, err)
}
content, err := os.ReadFile(path)
if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q read %q: %w", pipelineID, lane.ID, slotName, path, err)
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q read %q: %w", referenceTargetContext(pipelineID, target), slotName, path, err)
}
if !utf8.Valid(content) {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q must be UTF-8 text", pipelineID, lane.ID, slotName, path)
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q must be UTF-8 text", referenceTargetContext(pipelineID, target), slotName, path)
}
mediaType := referenceMediaTypeForPath(path)
if !referenceMediaTypeAccepted(mediaType, slot.AcceptedMediaTypes) {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q media type %q is not accepted", pipelineID, lane.ID, slotName, path, mediaType)
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q media type %q is not accepted", referenceTargetContext(pipelineID, target), slotName, path, mediaType)
}
if slot.MaxBytes > 0 && int64(len(content)) > slot.MaxBytes {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q is %d bytes, limit %d", pipelineID, lane.ID, slotName, path, len(content), slot.MaxBytes)
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q is %d bytes, limit %d", referenceTargetContext(pipelineID, target), slotName, path, len(content), slot.MaxBytes)
}
if len(content) == 0 {
warnings = append(warnings, contracts.Warning{
Scope: fmt.Sprintf("pipeline.%s.lane.%s.reference.%s", pipelineID, lane.ID, slotName),
Scope: referenceWarningScope(pipelineID, target, slotName),
ReasonCode: "empty_reference",
Message: fmt.Sprintf("reference slot %q for lane %q is bound to an empty file", slotName, lane.ID),
Message: fmt.Sprintf("reference slot %q for %s is bound to an empty file", slotName, referenceTargetLabel(target)),
})
}
@@ -122,6 +134,40 @@ func materializeLaneReferences(
return set, warnings, nil
}
func referenceTargetSpec(target ResolvedReferenceTarget, catalog ModuleCatalog) (ModuleSpec, error) {
switch target.Stage {
case StageChunk:
return registrySpec(catalog.Chunkers, target.Module)
case StageExtract:
return registrySpec(catalog.Extractors, target.Module)
case StageNormalize:
return registrySpec(catalog.Normalizers, target.Module)
default:
return ModuleSpec{}, fmt.Errorf("reference target stage %q is not supported", target.Stage)
}
}
func referenceTargetContext(pipelineID string, target ResolvedReferenceTarget) string {
if target.LaneID != "" {
return fmt.Sprintf("pipeline %q lane %q %s module %q", pipelineID, target.LaneID, target.Stage, target.Module)
}
return fmt.Sprintf("pipeline %q %s module %q", pipelineID, target.Stage, target.Module)
}
func referenceTargetLabel(target ResolvedReferenceTarget) string {
if target.LaneID != "" {
return fmt.Sprintf("lane %q %s target", target.LaneID, target.Stage)
}
return fmt.Sprintf("%s target", target.Stage)
}
func referenceWarningScope(pipelineID string, target ResolvedReferenceTarget, slotName string) string {
if target.LaneID != "" {
return fmt.Sprintf("pipeline.%s.lane.%s.%s.reference.%s", pipelineID, target.LaneID, target.Stage, slotName)
}
return fmt.Sprintf("pipeline.%s.%s.reference.%s", pipelineID, target.Stage, slotName)
}
func referenceMediaTypeForPath(path string) string {
extension := strings.ToLower(filepath.Ext(path))
mediaType := mime.TypeByExtension(extension)
@@ -230,31 +276,47 @@ func CloneReferenceSet(in contracts.ReferenceSet) contracts.ReferenceSet {
return out
}
func CloneReferenceTarget(in ResolvedReferenceTarget) ResolvedReferenceTarget {
out := in
out.Bindings = append([]ReferenceBinding(nil), in.Bindings...)
out.ReferenceSet = CloneReferenceSet(in.ReferenceSet)
return out
}
func ReferenceProvenance(resolved ResolvedPipeline) []artifacts.ReferenceProvenance {
provenance := []artifacts.ReferenceProvenance{}
provenance = append(provenance, referenceTargetProvenance(resolved.ChunkReferences)...)
for _, lane := range resolved.ArtifactLanes {
if len(lane.ReferenceSet.Slots) == 0 {
continue
}
slotNames := make([]string, 0, len(lane.ReferenceSet.Slots))
for slotName := range lane.ReferenceSet.Slots {
slotNames = append(slotNames, slotName)
}
sort.Strings(slotNames)
for _, slotName := range slotNames {
slot := lane.ReferenceSet.Slots[slotName]
for _, item := range slot.Items {
provenance = append(provenance, artifacts.ReferenceProvenance{
LaneID: lane.ID,
SlotName: item.SlotName,
OriginType: item.Origin.Type,
OriginURI: item.Origin.URI,
Digest: item.Digest,
MediaType: item.MediaType,
SizeBytes: item.SizeBytes,
BindingSource: item.BindingSource,
})
}
provenance = append(provenance, referenceTargetProvenance(lane.ExtractReferences)...)
provenance = append(provenance, referenceTargetProvenance(lane.NormalizeReferences)...)
}
return provenance
}
func referenceTargetProvenance(target ResolvedReferenceTarget) []artifacts.ReferenceProvenance {
if len(target.ReferenceSet.Slots) == 0 {
return nil
}
provenance := []artifacts.ReferenceProvenance{}
slotNames := make([]string, 0, len(target.ReferenceSet.Slots))
for slotName := range target.ReferenceSet.Slots {
slotNames = append(slotNames, slotName)
}
sort.Strings(slotNames)
for _, slotName := range slotNames {
slot := target.ReferenceSet.Slots[slotName]
for _, item := range slot.Items {
provenance = append(provenance, artifacts.ReferenceProvenance{
Stage: string(target.Stage),
LaneID: target.LaneID,
SlotName: item.SlotName,
OriginType: item.Origin.Type,
OriginURI: item.Origin.URI,
Digest: item.Digest,
MediaType: item.MediaType,
SizeBytes: item.SizeBytes,
BindingSource: item.BindingSource,
})
}
}
return provenance

View File

@@ -4,6 +4,8 @@ import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
@@ -54,12 +56,12 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
t.Fatalf("MaterializeReferences(second) error = %v, want nil", err)
}
referenceSet := first.ArtifactLanes[0].ReferenceSet
referenceSet := first.ArtifactLanes[0].ExtractReferences.ReferenceSet
roster := referenceSet.Slots["roster"].Items[0]
if string(roster.Content) != "config text" {
t.Fatalf("roster content = %q, want config text", roster.Content)
}
if roster.Digest != referenceDigest([]byte("config text")) || roster.Digest != second.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0].Digest {
if roster.Digest != referenceDigest([]byte("config text")) || roster.Digest != second.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Digest {
t.Fatalf("roster digest = %q, want stable digest", roster.Digest)
}
if roster.BindingSource != contracts.ReferenceBindingSourceConfig {
@@ -87,10 +89,10 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
if len(provenance) != 2 {
t.Fatalf("ReferenceProvenance() = %#v, want two entries", provenance)
}
if provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest {
if provenance[0].Stage != string(StageExtract) || provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest {
t.Fatalf("ReferenceProvenance()[0] = %#v, want sorted glossary provenance", provenance[0])
}
if provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest {
if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest {
t.Fatalf("ReferenceProvenance()[1] = %#v, want roster provenance", provenance[1])
}
@@ -103,6 +105,66 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
}
}
func TestMaterializeReferencesStoresSetsAndProvenanceForAllTargets(t *testing.T) {
configDir := t.TempDir()
writeReferenceFile(t, filepath.Join(configDir, "chunk.txt"), []byte("chunk text"))
writeReferenceFile(t, filepath.Join(configDir, "extract.txt"), []byte("extract text"))
writeReferenceFile(t, filepath.Join(configDir, "normalize.txt"), []byte("normalize text"))
profile := baselineProfile()
profile.References = map[string]string{
"scene_guide": "chunk.txt",
"roster": "extract.txt",
"normalization_notes": "normalize.txt",
}
catalog := referenceCatalogForTargets(t,
[]contracts.ReferenceSlot{{Name: "scene_guide"}},
[]contracts.ReferenceSlot{{Name: "roster"}},
[]contracts.ReferenceSlot{{Name: "normalization_notes"}},
)
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
materialized, warnings, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
if len(warnings) != 0 {
t.Fatalf("warnings = %#v, want none", warnings)
}
chunkItem := materialized.ChunkReferences.ReferenceSet.Slots["scene_guide"].Items[0]
if string(chunkItem.Content) != "chunk text" {
t.Fatalf("chunk content = %q, want chunk text", chunkItem.Content)
}
extractItem := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if string(extractItem.Content) != "extract text" {
t.Fatalf("extract content = %q, want extract text", extractItem.Content)
}
normalizeItem := materialized.ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["normalization_notes"].Items[0]
if string(normalizeItem.Content) != "normalize text" {
t.Fatalf("normalize content = %q, want normalize text", normalizeItem.Content)
}
provenance := ReferenceProvenance(materialized)
if len(provenance) != 3 {
t.Fatalf("ReferenceProvenance() = %#v, want three entries", provenance)
}
if provenance[0].Stage != string(StageChunk) || provenance[0].LaneID != "" || provenance[0].SlotName != "scene_guide" || provenance[0].Digest != chunkItem.Digest {
t.Fatalf("ReferenceProvenance()[0] = %#v, want chunk scene guide provenance", provenance[0])
}
if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != extractItem.Digest {
t.Fatalf("ReferenceProvenance()[1] = %#v, want extract roster provenance", provenance[1])
}
if provenance[2].Stage != string(StageNormalize) || provenance[2].LaneID != "events" || provenance[2].SlotName != "normalization_notes" || provenance[2].Digest != normalizeItem.Digest {
t.Fatalf("ReferenceProvenance()[2] = %#v, want normalize notes provenance", provenance[2])
}
}
func TestMaterializeReferencesRejectsNonUTF8Content(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "bad.txt")
@@ -117,6 +179,20 @@ func TestMaterializeReferencesRejectsNonUTF8Content(t *testing.T) {
}
}
func TestMaterializeReferencesRejectsNonUTF8ContentForChunkTarget(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "bad.txt")
writeReferenceFile(t, path, []byte{0xff, 0xfe})
resolved := resolvedPipelineWithTargetReference(t, StageChunk, "", "scene_guide", "bad.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "scene_guide"})
_, _, err := MaterializeReferences(resolved, referenceCatalogForTargets(t, []contracts.ReferenceSlot{{Name: "scene_guide"}}, nil, nil), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err == nil || !strings.Contains(err.Error(), "chunk") || !strings.Contains(err.Error(), "UTF-8") || !strings.Contains(err.Error(), "scene_guide") || !strings.Contains(err.Error(), path) {
t.Fatalf("error = %v, want chunk UTF-8 path error", err)
}
}
func TestMaterializeReferencesAllowsAnyMediaTypeWhenSlotDoesNotRestrictIt(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "roster.reference")
@@ -129,7 +205,7 @@ func TestMaterializeReferencesAllowsAnyMediaTypeWhenSlotDoesNotRestrictIt(t *tes
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != unknownMediaType {
t.Fatalf("MediaType = %q, want %q", item.MediaType, unknownMediaType)
}
@@ -148,7 +224,7 @@ func TestMaterializeReferencesAcceptsDeclaredMarkdownMediaType(t *testing.T) {
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["glossary"].Items[0]
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["glossary"].Items[0]
if item.MediaType != "text/markdown" {
t.Fatalf("MediaType = %q, want text/markdown", item.MediaType)
}
@@ -167,7 +243,7 @@ func TestMaterializeReferencesAcceptsDeclaredJSONMediaType(t *testing.T) {
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != "application/json" {
t.Fatalf("MediaType = %q, want application/json", item.MediaType)
}
@@ -201,12 +277,27 @@ func TestMaterializeReferencesMatchesAcceptedMediaTypesIgnoringParameters(t *tes
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != referenceMediaType {
t.Fatalf("MediaType = %q, want %q", item.MediaType, referenceMediaType)
}
}
func TestMaterializeReferencesRejectsUnacceptedMediaTypeForNormalizeTarget(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "notes.json")
writeReferenceFile(t, path, []byte(`{"notes":true}`))
slot := contracts.ReferenceSlot{Name: "normalization_notes", AcceptedMediaTypes: []string{"text/markdown"}}
resolved := resolvedPipelineWithTargetReference(t, StageNormalize, "events", "normalization_notes", "notes.json", contracts.ReferenceBindingSourceConfig, slot)
_, _, err := MaterializeReferences(resolved, referenceCatalogForTargets(t, nil, nil, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err == nil || !strings.Contains(err.Error(), "normalize") || !strings.Contains(err.Error(), "media type") || !strings.Contains(err.Error(), "application/json") || !strings.Contains(err.Error(), "normalization_notes") {
t.Fatalf("error = %v, want normalize media type rejection", err)
}
}
func TestMaterializeReferencesWarnsForEmptyFiles(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "empty.txt")
@@ -222,12 +313,51 @@ func TestMaterializeReferencesWarnsForEmptyFiles(t *testing.T) {
if len(warnings) != 1 || warnings[0].ReasonCode != "empty_reference" {
t.Fatalf("warnings = %#v, want empty reference warning", warnings)
}
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.SizeBytes != 0 || item.Digest != referenceDigest(nil) {
t.Fatalf("empty item = %#v, want zero size and empty digest", item)
}
}
func TestMaterializeReferencesWarningScopesIncludeTargetContext(t *testing.T) {
configDir := t.TempDir()
writeReferenceFile(t, filepath.Join(configDir, "chunk.txt"), nil)
writeReferenceFile(t, filepath.Join(configDir, "extract.txt"), nil)
writeReferenceFile(t, filepath.Join(configDir, "normalize.txt"), nil)
profile := baselineProfile()
profile.References = map[string]string{
"scene_guide": "chunk.txt",
"roster": "extract.txt",
"normalization_notes": "normalize.txt",
}
catalog := referenceCatalogForTargets(t,
[]contracts.ReferenceSlot{{Name: "scene_guide"}},
[]contracts.ReferenceSlot{{Name: "roster"}},
[]contracts.ReferenceSlot{{Name: "normalization_notes"}},
)
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
_, warnings, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
got := warningScopes(warnings)
want := []string{
"pipeline.baseline.chunk.reference.scene_guide",
"pipeline.baseline.lane.events.extract.reference.roster",
"pipeline.baseline.lane.events.normalize.reference.normalization_notes",
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("warning scopes = %#v, want %#v", got, want)
}
}
func TestMaterializeReferencesEnforcesMaxBytes(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "large.txt")
@@ -244,31 +374,99 @@ func TestMaterializeReferencesEnforcesMaxBytes(t *testing.T) {
}
func resolvedPipelineWithReference(t *testing.T, slotName, source, bindingSource string, slot contracts.ReferenceSlot) ResolvedPipeline {
t.Helper()
return resolvedPipelineWithTargetReference(t, StageExtract, "events", slotName, source, bindingSource, slot)
}
func resolvedPipelineWithTargetReference(t *testing.T, stage ModuleStage, laneID string, slotName, source, bindingSource string, slot contracts.ReferenceSlot) ResolvedPipeline {
t.Helper()
profile := baselineProfile()
lane := profile.Artifacts["events"]
lane.References = map[string]string{slotName: source}
profile.Artifacts["events"] = lane
catalog := referenceCatalog(t, []contracts.ReferenceSlot{slot})
switch stage {
case StageChunk:
profile.Chunk.References = map[string]string{slotName: source}
case StageExtract:
lane := profile.Artifacts[laneID]
lane.References = map[string]string{slotName: source}
profile.Artifacts[laneID] = lane
case StageNormalize:
lane := profile.Artifacts[laneID]
lane.Normalize.References = map[string]string{slotName: source}
profile.Artifacts[laneID] = lane
default:
t.Fatalf("unsupported reference target stage %q", stage)
}
catalog := referenceCatalogForStage(t, stage, []contracts.ReferenceSlot{slot})
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if bindingSource != contracts.ReferenceBindingSourceConfig {
resolved.ArtifactLanes[0].References[0].BindingSource = bindingSource
switch stage {
case StageChunk:
resolved.ChunkReferences.Bindings[0].BindingSource = bindingSource
case StageExtract:
resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].BindingSource = bindingSource
case StageNormalize:
resolved.ArtifactLanes[0].NormalizeReferences.Bindings[0].BindingSource = bindingSource
}
}
return resolved
}
func referenceCatalog(t *testing.T, slots []contracts.ReferenceSlot) ModuleCatalog {
t.Helper()
return newProfileCatalogWithOverride(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: slots,
})
return referenceCatalogForStage(t, StageExtract, slots)
}
func referenceCatalogForStage(t *testing.T, stage ModuleStage, slots []contracts.ReferenceSlot) ModuleCatalog {
t.Helper()
switch stage {
case StageChunk:
return referenceCatalogForTargets(t, slots, nil, nil)
case StageExtract:
return referenceCatalogForTargets(t, nil, slots, nil)
case StageNormalize:
return referenceCatalogForTargets(t, nil, nil, slots)
default:
t.Fatalf("unsupported reference target stage %q", stage)
return ModuleCatalog{}
}
}
func referenceCatalogForTargets(t *testing.T, chunkSlots, extractSlots, normalizeSlots []contracts.ReferenceSlot) ModuleCatalog {
t.Helper()
return newProfileCatalogWithOverrides(t,
ModuleSpec{
Key: "generic",
Stage: StageChunk,
Requires: []string{"source"},
Provides: []string{"chunk"},
ReferenceSlots: chunkSlots,
},
ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: extractSlots,
},
ModuleSpec{
Key: "noop",
Stage: StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: normalizeSlots,
},
)
}
func warningScopes(warnings []contracts.Warning) []string {
scopes := make([]string, 0, len(warnings))
for _, warning := range warnings {
scopes = append(scopes, warning.Scope)
}
sort.Strings(scopes)
return scopes
}
func writeReferenceFile(t *testing.T, path string, content []byte) {

View File

@@ -118,6 +118,10 @@ func (chunker integrationChunker) Key() string {
return "chunk"
}
func (chunker integrationChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker integrationChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{
@@ -186,6 +190,10 @@ func (normalizer integrationNormalizer) Key() string {
return "normalize"
}
func (normalizer integrationNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer integrationNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
}

View File

@@ -95,6 +95,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
attachModuleManifestMetadata(&output, "chunker", chunker)
chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
Source: doc,
References: CloneReferenceSet(input.Pipeline.ChunkReferences.ReferenceSet),
LLMClient: input.LLMClient,
LLMProfile: input.Pipeline.Chunk.LLMProfile,
Options: cloneOptions(input.Pipeline.Chunk.Options),
@@ -186,7 +187,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
result, err := extractor.Extract(ctx, contracts.ExtractionRequest{
Source: doc,
Chunk: &chunk,
References: CloneReferenceSet(lane.ReferenceSet),
References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet),
LLMClient: input.LLMClient,
LLMProfile: lane.Extract.LLMProfile,
Options: cloneOptions(lane.Extract.Options),
@@ -224,6 +225,8 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
Source: doc,
LaneID: lane.ID,
Candidates: mergeResult.Candidates,
References: CloneReferenceSet(lane.NormalizeReferences.ReferenceSet),
LLMClient: input.LLMClient,
LLMProfile: lane.Normalize.LLMProfile,
Options: cloneOptions(lane.Normalize.Options),
Metadata: input.Metadata,

View File

@@ -464,6 +464,10 @@ func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) {
if len(extractor.seenLLMClients) != 2 || extractor.seenLLMClients[0] == nil || extractor.seenLLMClients[1] == nil {
t.Fatalf("seen LLM clients = %#v, want client for each chunk", extractor.seenLLMClients)
}
normalizer := modules.normalizers["normalize"]
if len(normalizer.requests) != 1 || normalizer.requests[0].LLMClient == nil {
t.Fatalf("normalizer LLM client = %#v, want client on normalize request", normalizer.requests)
}
if extractor.seenMetadata[0]["request"] != "test" {
t.Fatalf("seen metadata = %#v, want request metadata", extractor.seenMetadata)
}
@@ -562,24 +566,7 @@ func TestRunPassesModuleBindingConfigToStageRequests(t *testing.T) {
func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) {
modules := defaultRunnerModules()
pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{
SlotName: "roster",
MediaType: "text/plain; charset=utf-8",
Content: []byte("reference text"),
Digest: "sha256:test",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/reference.txt"},
SizeBytes: int64(len("reference text")),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
},
},
}
pipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = testReferenceSet("roster", "reference text")
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
if err != nil {
@@ -592,11 +579,60 @@ func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) {
t.Fatalf("reference content = %q, want reference text", item.Content)
}
item.Content[0] = 'R'
if got := string(pipeline.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0].Content); got != "reference text" {
if got := string(pipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content); got != "reference text" {
t.Fatalf("runner mutated reference set content = %q", got)
}
}
func TestRunPassesChunkReferencesToChunkerRequest(t *testing.T) {
modules := defaultRunnerModules()
pipeline := resolvedPipeline()
pipeline.ChunkReferences.ReferenceSet = testReferenceSet("scene_guide", "chunk reference text")
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
req := modules.chunker.requests[0]
item := req.References.Slots["scene_guide"].Items[0]
if string(item.Content) != "chunk reference text" {
t.Fatalf("chunk reference content = %q, want chunk reference text", item.Content)
}
item.Content[0] = 'C'
if got := string(pipeline.ChunkReferences.ReferenceSet.Slots["scene_guide"].Items[0].Content); got != "chunk reference text" {
t.Fatalf("runner mutated chunk reference set content = %q", got)
}
}
func TestRunPassesNormalizeReferencesToNormalizerRequest(t *testing.T) {
modules := defaultRunnerModules()
pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].NormalizeReferences.ReferenceSet = testReferenceSet("normalization_notes", "normalize reference text")
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
req := modules.normalizers["normalize"].requests[0]
item := req.References.Slots["normalization_notes"].Items[0]
if string(item.Content) != "normalize reference text" {
t.Fatalf("normalize reference content = %q, want normalize reference text", item.Content)
}
item.Content[0] = 'N'
if got := string(pipeline.ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["normalization_notes"].Items[0].Content); got != "normalize reference text" {
t.Fatalf("runner mutated normalize reference set content = %q", got)
}
}
func TestRunAllowsNilLLMClientWhenModulesDoNotUseIt(t *testing.T) {
_, err := New(newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil with nil LLM client when modules do not use it", err)
}
}
func TestRunIncludesInputWarnings(t *testing.T) {
modules := defaultRunnerModules()
warning := contracts.Warning{Scope: "reference", ReasonCode: "empty_reference", Message: "empty reference"}
@@ -961,7 +997,25 @@ func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) {
func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
resolved := resolvedPipelineWithValidators("configured")
resolved.ArtifactLanes[0].ReferenceSet = contracts.ReferenceSet{
resolved.ChunkReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"scene_guide": {
Slot: contracts.ReferenceSlot{Name: "scene_guide"},
Items: []contracts.ReferenceItem{
{
SlotName: "scene_guide",
MediaType: "text/plain; charset=utf-8",
Content: []byte("chunk reference content"),
Digest: "sha256:chunk-reference",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/scene-guide.txt"},
SizeBytes: int64(len("chunk reference content")),
BindingSource: contracts.ReferenceBindingSourceCLI,
},
},
},
},
}
resolved.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
@@ -979,6 +1033,24 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
},
},
}
resolved.ArtifactLanes[0].NormalizeReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"normalization_notes": {
Slot: contracts.ReferenceSlot{Name: "normalization_notes"},
Items: []contracts.ReferenceItem{
{
SlotName: "normalization_notes",
MediaType: "text/plain; charset=utf-8",
Content: []byte("normalize reference content"),
Digest: "sha256:normalize-reference",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/normalize.txt"},
SizeBytes: int64(len("normalize reference content")),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
},
},
}
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: resolved})
if err != nil {
@@ -995,15 +1067,26 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:source"}) {
t.Fatalf("SourceDigests = %#v, want source digest", manifest.SourceDigests)
}
if len(manifest.References) != 1 {
t.Fatalf("References = %#v, want one reference provenance entry", manifest.References)
if len(manifest.References) != 3 {
t.Fatalf("References = %#v, want three reference provenance entries", manifest.References)
}
reference := manifest.References[0]
if reference.LaneID != "alpha" || reference.SlotName != "roster" || reference.Digest != "sha256:reference" {
t.Fatalf("reference provenance = %#v, want lane slot digest", reference)
chunkReference := manifest.References[0]
if chunkReference.Stage != string(StageChunk) || chunkReference.LaneID != "" || chunkReference.SlotName != "scene_guide" || chunkReference.Digest != "sha256:chunk-reference" {
t.Fatalf("chunk reference provenance = %#v, want chunk slot digest", chunkReference)
}
if reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != int64(len("reference content")) || reference.BindingSource != contracts.ReferenceBindingSourceConfig {
t.Fatalf("reference provenance = %#v, want origin/media/size/source", reference)
if chunkReference.OriginType != "file" || chunkReference.OriginURI != "file:///tmp/scene-guide.txt" || chunkReference.MediaType != "text/plain; charset=utf-8" || chunkReference.SizeBytes != int64(len("chunk reference content")) || chunkReference.BindingSource != contracts.ReferenceBindingSourceCLI {
t.Fatalf("chunk reference provenance = %#v, want origin/media/size/source", chunkReference)
}
extractReference := manifest.References[1]
if extractReference.Stage != string(StageExtract) || extractReference.LaneID != "alpha" || extractReference.SlotName != "roster" || extractReference.Digest != "sha256:reference" {
t.Fatalf("extract reference provenance = %#v, want lane slot digest", extractReference)
}
if extractReference.OriginType != "file" || extractReference.OriginURI != "file:///tmp/roster.txt" || extractReference.MediaType != "text/plain; charset=utf-8" || extractReference.SizeBytes != int64(len("reference content")) || extractReference.BindingSource != contracts.ReferenceBindingSourceConfig {
t.Fatalf("extract reference provenance = %#v, want origin/media/size/source", extractReference)
}
normalizeReference := manifest.References[2]
if normalizeReference.Stage != string(StageNormalize) || normalizeReference.LaneID != "alpha" || normalizeReference.SlotName != "normalization_notes" || normalizeReference.Digest != "sha256:normalize-reference" {
t.Fatalf("normalize reference provenance = %#v, want lane slot digest", normalizeReference)
}
if manifest.ValidationStatus != "approved" {
t.Fatalf("ValidationStatus = %q, want approved", manifest.ValidationStatus)
@@ -1157,16 +1240,19 @@ func TestRunRejectsNilDefaultValidator(t *testing.T) {
func resolvedPipeline() ResolvedPipeline {
return ResolvedPipeline{
ID: "pipeline-1",
Digest: "sha256:pipeline",
Input: Binding("input"),
Chunk: Binding("chunk"),
ID: "pipeline-1",
Digest: "sha256:pipeline",
Input: Binding("input"),
Chunk: Binding("chunk"),
ChunkReferences: referenceTarget(StageChunk, "", "chunk", nil),
ArtifactLanes: []ResolvedArtifactLane{
{
ID: "alpha",
Extract: Binding("extract-alpha"),
Merge: Binding("merge"),
Normalize: Binding("normalize"),
ID: "alpha",
Extract: Binding("extract-alpha"),
Merge: Binding("merge"),
Normalize: Binding("normalize"),
ExtractReferences: referenceTarget(StageExtract, "alpha", "extract-alpha", nil),
NormalizeReferences: referenceTarget(StageNormalize, "alpha", "normalize", nil),
},
},
Output: Binding("output"),
@@ -1181,6 +1267,27 @@ func resolvedPipelineWithValidators(validators ...string) ResolvedPipeline {
return pipeline
}
func testReferenceSet(slotName string, content string) contracts.ReferenceSet {
return contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
slotName: {
Slot: contracts.ReferenceSlot{Name: slotName},
Items: []contracts.ReferenceItem{
{
SlotName: slotName,
MediaType: "text/plain; charset=utf-8",
Content: []byte(content),
Digest: "sha256:test",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/reference.txt"},
SizeBytes: int64(len(content)),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
},
},
}
}
type runnerModules struct {
input *runnerInputAdapter
chunker *runnerChunker
@@ -1314,6 +1421,10 @@ func (chunker *runnerChunker) Key() string {
return chunker.key
}
func (chunker *runnerChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
chunker.requests = append(chunker.requests, req)
return contracts.ChunkResult{
@@ -1421,6 +1532,10 @@ func (normalizer *runnerNormalizer) Key() string {
return normalizer.key
}
func (normalizer *runnerNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer *runnerNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
normalizer.requests = append(normalizer.requests, req)
candidates := append([]artifacts.ArtifactCandidate(nil), normalizer.result...)

View File

@@ -1,7 +1,7 @@
{
"manifest": {
"pipeline_id": "walking-skeleton",
"pipeline_digest": "sha256:5df1e501a2307ef75bbfeb59d315b3710571d52e5466a9c7f8320248740e6fca",
"pipeline_digest": "sha256:25084e39a0cadace375c896551d1752413755c1a6772f6b24cfe91c029ea2631",
"validation_status": "approved",
"artifact_lanes": [
{

View File

@@ -43,8 +43,8 @@ func TestWalkingSkeletonFixture(t *testing.T) {
t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType)
}
assertStructuralJSONEqual(t, output.OutputFiles[0].Bytes, expectedBytes)
if llmClient.calls != 2 {
t.Fatalf("LLM calls = %d, want chunk count 2", llmClient.calls)
if llmClient.calls != 3 {
t.Fatalf("LLM calls = %d, want extractor calls plus normalizer call", llmClient.calls)
}
}
@@ -216,6 +216,10 @@ func (chunker walkingSkeletonChunker) Key() string {
return "fake/chunk"
}
func (chunker walkingSkeletonChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker walkingSkeletonChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
if len(req.Source.Units) < 3 {
return contracts.ChunkResult{}, fmt.Errorf("fixture source must contain at least three units")
@@ -336,7 +340,20 @@ func (normalizer walkingSkeletonNormalizer) Key() string {
return DefaultNormalizeModule
}
func (normalizer walkingSkeletonNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer walkingSkeletonNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
var response struct {
Call int `json:"call"`
}
if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: "fake/normalize",
ResponseSchemaName: "fake_normalize",
}, &response); err != nil {
return contracts.NormalizeResult{}, err
}
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
}

View File

@@ -97,6 +97,57 @@ func TestRenderUserSystemWithReferencesRendersDeclaredSlots(t *testing.T) {
}
}
func TestRenderUserSystemWithReferencesSupportsChunkRequestData(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "scene_guide"}},
`Chunk system has guide={{ hasreference "scene_guide" }}`,
`Source={{ .SourceID }} Guide={{ reference "scene_guide" }}`,
)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"scene_guide": {
Slot: contracts.ReferenceSlot{Name: "scene_guide"},
Items: []contracts.ReferenceItem{
{SlotName: "scene_guide", Content: []byte("Keep combat scenes separate.")},
},
},
}}
system, user, _, err := bundle.RenderUserSystemWithReferences(map[string]any{"SourceID": "session-alpha"}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences: %v", err)
}
if !strings.Contains(system, "has guide=true") {
t.Fatalf("system = %q, want chunk reference presence", system)
}
if !strings.Contains(user, "Source=session-alpha") || !strings.Contains(user, "Keep combat scenes separate.") {
t.Fatalf("user = %q, want chunk request data and reference content", user)
}
}
func TestRenderUserSystemWithReferencesSupportsNormalizeRequestData(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "normalization_notes"}},
`Normalize system`,
`Lane={{ .LaneID }} Notes={{ reference "normalization_notes" }}`,
)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"normalization_notes": {
Slot: contracts.ReferenceSlot{Name: "normalization_notes"},
Items: []contracts.ReferenceItem{
{SlotName: "normalization_notes", Content: []byte("Prefer canonical item names.")},
},
},
}}
_, user, _, err := bundle.RenderUserSystemWithReferences(map[string]any{"LaneID": "spells"}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences: %v", err)
}
if !strings.Contains(user, "Lane=spells") || !strings.Contains(user, "Prefer canonical item names.") {
t.Fatalf("user = %q, want normalize request data and reference content", user)
}
}
func TestRenderUserSystemReferenceHasReferenceRequiresContent(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "roster"}},

View File

@@ -34,6 +34,10 @@ func (c *Chunker) Key() string {
return Key
}
func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (c *Chunker) ManifestMetadata() map[string]any {
promptMetadata := scenesPromptBundle.Metadata()
metadata := map[string]any{

View File

@@ -57,6 +57,9 @@ func TestNewModuleSpecAndRegister(t *testing.T) {
if built.Key() != Key {
t.Fatalf("built Key() = %q, want %q", built.Key(), Key)
}
if slots := built.ReferenceSlots(); len(slots) != 0 {
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
}
}
func TestRegisterNilRegistryReturnsError(t *testing.T) {

View File

@@ -31,6 +31,10 @@ func (c *Chunker) Key() string {
return Key
}
func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
if c == nil {
return contracts.ChunkResult{}, chunkerErrorf("chunker must not be nil")

View File

@@ -41,6 +41,9 @@ func TestModuleSpecAndRegister(t *testing.T) {
if chunker.Key() != Key {
t.Fatalf("Key() = %q, want %q", chunker.Key(), Key)
}
if slots := chunker.ReferenceSlots(); len(slots) != 0 {
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
}
}
func TestChunkUsesDefaultsForSingleChunk(t *testing.T) {

View File

@@ -236,6 +236,10 @@ func (dndSpellsChunker) Key() string {
return "fake/chunk"
}
func (dndSpellsChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (dndSpellsChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{

View File

@@ -114,7 +114,7 @@ func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T)
raw := readDNDSpellsFixture(t)
expectedDoc := parseDNDSpellsFixture(t, raw)
resolved := resolveDNDSpellsPipeline(t)
resolved.ResolvedPipeline.ArtifactLanes[0].ReferenceSet = dndSpellsReferenceSet(
resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet(
"Aria: party cleric\nBorin: fighter",
"Fire Bolt: evocation cantrip",
)
@@ -168,7 +168,7 @@ func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T)
func TestRunnerDoesNotExtractSpellMentionedOnlyInRoster(t *testing.T) {
raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t)
resolved.ResolvedPipeline.ArtifactLanes[0].ReferenceSet = dndSpellsReferenceSet(
resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet(
"Mira: wizard who can cast Lightning Bolt",
"",
)

View File

@@ -209,6 +209,8 @@ type fakeChunker struct{}
func (fakeChunker) Key() string { return "fake/chunk" }
func (fakeChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (fakeChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{}, nil
}

View File

@@ -157,6 +157,10 @@ func (runnerSeriatimChunker) Key() string {
return "fake/chunk"
}
func (runnerSeriatimChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (runnerSeriatimChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{

View File

@@ -25,6 +25,10 @@ func (n *Normalizer) Key() string {
return Key
}
func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (n *Normalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
if n == nil {
return contracts.NormalizeResult{}, normalizerErrorf("normalizer must not be nil")

View File

@@ -34,6 +34,13 @@ func TestModuleSpecAndRegister(t *testing.T) {
if !reflect.DeepEqual(spec, want) {
t.Fatalf("registered spec = %#v, want %#v", spec, want)
}
normalizer, err := registry.Build(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if slots := normalizer.ReferenceSlots(); len(slots) != 0 {
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
}
}
func TestNormalizePassesThroughOrderAndValues(t *testing.T) {

View File

@@ -148,6 +148,7 @@ func TestEncodeIncludesManifestReferences(t *testing.T) {
RunID: "run-1",
References: []artifacts.ReferenceProvenance{
{
Stage: "extract",
LaneID: "events",
SlotName: "roster",
OriginType: "file",