Compare commits
9 Commits
84c4c06712
...
3aa88ab9d3
| Author | SHA1 | Date | |
|---|---|---|---|
| 3aa88ab9d3 | |||
| c1ba94192d | |||
| 22032dfd6d | |||
| 4cafde2502 | |||
| 8c623b7ad8 | |||
| 43dc954440 | |||
| 51053d390d | |||
| 9278797aa9 | |||
| 39e49d7f77 |
61
docs/cli.md
61
docs/cli.md
@@ -20,7 +20,7 @@ a bearer token.
|
|||||||
|
|
||||||
```text
|
```text
|
||||||
notarius help
|
notarius help
|
||||||
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--reference slot=path] [--without-reference slot]
|
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--reference selector=path] [--without-reference selector]
|
||||||
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
|
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||||
notarius pipelines list --config path/to/config.yml [--json]
|
notarius pipelines list --config path/to/config.yml [--json]
|
||||||
```
|
```
|
||||||
@@ -46,34 +46,44 @@ Flags:
|
|||||||
invocation.
|
invocation.
|
||||||
- `--llm-profile id`: override every effective module binding to use one LLM
|
- `--llm-profile id`: override every effective module binding to use one LLM
|
||||||
profile.
|
profile.
|
||||||
- `--reference slot=path`: bind a reference path to an extractor reference
|
- `--reference selector=path`: bind a reference path to a chunk, extractor, or
|
||||||
slot. Repeatable. Use `lane.slot=path` when multiple selected lanes declare
|
normalizer reference slot. Repeatable.
|
||||||
the same slot.
|
- `--without-reference selector`: remove a configured optional reference binding.
|
||||||
- `--without-reference slot`: remove a configured optional reference binding.
|
Repeatable. It accepts the same selector forms as `--reference`, without
|
||||||
Repeatable. Use `lane.slot` when multiple selected lanes declare the same
|
`=path`.
|
||||||
slot.
|
|
||||||
|
|
||||||
On success, the command prints the completed pipeline ID, approved and rejected
|
On success, the command prints the completed pipeline ID, approved and rejected
|
||||||
artifact counts, and the output directory. If the run completes with warnings,
|
artifact counts, and the output directory. If the run completes with warnings,
|
||||||
the warning count is printed to stderr.
|
the warning count is printed to stderr.
|
||||||
|
|
||||||
Reference flags are resolved against selected artifact lanes before the run
|
Reference flags are resolved against selected chunk, extractor, and normalizer
|
||||||
starts. Flat slot names are accepted only when exactly one selected lane
|
targets before the run starts. Flat slot names are accepted only when exactly
|
||||||
declares that slot. Bound reference files are read before extraction, validated
|
one selected target declares that slot. Bound reference files are read before
|
||||||
as UTF-8 text, and passed only to the lane extractor that declares the slot.
|
pipeline work starts, validated as UTF-8 text, and recorded as provenance for
|
||||||
Notarius infers reference media types from file extensions for provenance and
|
the target that declares the slot. Runtime reference content is passed to the
|
||||||
for optional slot checks. Reference content is not written to diagnostics, logs,
|
chunker, extractor, or normalizer target that declares the slot. Notarius infers
|
||||||
errors, or manifests.
|
reference media types from file extensions for provenance and for optional slot
|
||||||
|
checks. Reference content is not written to diagnostics, logs, errors, or
|
||||||
|
manifests.
|
||||||
|
|
||||||
Reference binding precedence is:
|
Reference binding precedence is:
|
||||||
|
|
||||||
1. pipeline-level config `references`;
|
1. pipeline-level config `references`;
|
||||||
2. lane-level config `references`;
|
2. target-local config references, including legacy lane-level extractor
|
||||||
|
`references`;
|
||||||
3. `--reference` run flags;
|
3. `--reference` run flags;
|
||||||
4. `--without-reference` run flags.
|
4. `--without-reference` run flags.
|
||||||
|
|
||||||
`--reference` binds or replaces one slot for one selected lane. Use
|
`--reference` binds or replaces one slot for one selected target. Selectors are:
|
||||||
`slot=path` when the selected lanes declare the slot unambiguously:
|
|
||||||
|
- `slot=path`: valid when exactly one selected target declares `slot`;
|
||||||
|
- `chunk.slot=path`: target the chunker;
|
||||||
|
- `lane.slot=path`: valid when exactly one selected extractor or normalizer in
|
||||||
|
that lane declares `slot`;
|
||||||
|
- `lane.extract.slot=path`: target a lane extractor;
|
||||||
|
- `lane.normalize.slot=path`: target a lane normalizer.
|
||||||
|
|
||||||
|
Use `slot=path` when the selected targets declare the slot unambiguously:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go run ./cmd/notarius run dnd-session \
|
go run ./cmd/notarius run dnd-session \
|
||||||
@@ -82,14 +92,25 @@ go run ./cmd/notarius run dnd-session \
|
|||||||
--reference roster=./campaign-roster.txt
|
--reference roster=./campaign-roster.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
Use `lane.slot=path` when multiple selected lanes declare the same slot or when
|
Use an explicit selector when multiple selected targets declare the same slot or
|
||||||
you want to target a specific lane:
|
when you want to target a specific target:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go run ./cmd/notarius run dnd-session \
|
go run ./cmd/notarius run dnd-session \
|
||||||
--config examples/dnd-spells.config.yml \
|
--config examples/dnd-spells.config.yml \
|
||||||
--input examples/seriatim-minimal-transcript.json \
|
--input examples/seriatim-minimal-transcript.json \
|
||||||
--reference spells.glossary=./campaign-glossary.txt
|
--reference spells.extract.glossary=./campaign-glossary.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
The same grammar can target chunk and normalize slots when the configured
|
||||||
|
modules declare them:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/notarius run dnd-session \
|
||||||
|
--config path/to/config.yml \
|
||||||
|
--input examples/seriatim-minimal-transcript.json \
|
||||||
|
--reference chunk.scene_guide=./campaign-scenes.txt \
|
||||||
|
--reference spells.normalize.normalization_notes=./normalization-notes.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
Use `--without-reference` to remove a configured optional binding for a run:
|
Use `--without-reference` to remove a configured optional binding for a run:
|
||||||
|
|||||||
@@ -124,9 +124,9 @@ Pipeline fields:
|
|||||||
- `artifacts`: required for pipeline resolution. It maps artifact lane IDs to
|
- `artifacts`: required for pipeline resolution. It maps artifact lane IDs to
|
||||||
lane definitions.
|
lane definitions.
|
||||||
- `output`: optional module binding. Default module is `json`.
|
- `output`: optional module binding. Default module is `json`.
|
||||||
- `references`: optional map of extractor reference slot names to reference
|
- `references`: optional map of reference slot names to reference paths. These
|
||||||
paths. These bindings are defaults for artifact lanes whose extractor declares
|
bindings are defaults for eligible pipeline targets that declare the matching
|
||||||
the matching slot.
|
slot.
|
||||||
|
|
||||||
Artifact lane fields:
|
Artifact lane fields:
|
||||||
|
|
||||||
@@ -135,28 +135,32 @@ Artifact lane fields:
|
|||||||
- `normalize`: optional module binding. Default module is `noop`.
|
- `normalize`: optional module binding. Default module is `noop`.
|
||||||
- `validators`: optional list of module bindings. The production CLI currently
|
- `validators`: optional list of module bindings. The production CLI currently
|
||||||
does not register validator modules.
|
does not register validator modules.
|
||||||
- `references`: optional map of extractor reference slot names to reference
|
- `references`: optional compatibility alias for extractor reference bindings.
|
||||||
paths. Lane bindings override pipeline-level bindings for the same slot.
|
Lane bindings override pipeline-level bindings for the same slot.
|
||||||
|
|
||||||
`notarius run` and `notarius config validate --pipeline` resolve the pipeline
|
`notarius run` and `notarius config validate --pipeline` resolve the pipeline
|
||||||
against the production module catalog and fail fast for unknown or incompatible
|
against the production module catalog and fail fast for unknown or incompatible
|
||||||
module keys.
|
module keys.
|
||||||
|
|
||||||
Reference bindings are validated against extractor-declared slots during
|
Reference bindings are validated against reference slots declared by eligible
|
||||||
pipeline resolution. Required slots must be bound after config defaults,
|
chunk, extract, and normalize targets during pipeline resolution. Required slots
|
||||||
lane-level bindings, and run-time `--reference` or `--without-reference`
|
must be bound after config defaults, target-local references, lane-level
|
||||||
overrides are applied. Config-relative paths are resolved relative to the
|
compatibility bindings, and run-time `--reference` or `--without-reference`
|
||||||
config file; CLI reference paths are resolved relative to the current working
|
overrides are applied. Config-relative paths are resolved relative to the config
|
||||||
directory. Bound files must be UTF-8 text and are passed only to lane
|
file; CLI reference paths are resolved relative to the current working
|
||||||
extractors that declare the slot. Reference media types are inferred from file
|
directory. Materialized bound files must be UTF-8 text. Materialized reference
|
||||||
extensions, recorded as canonical base media types, and checked only when a
|
provenance is recorded for chunk, extractor, and normalizer targets, and runtime
|
||||||
module declares `AcceptedMediaTypes`; unknown extensions are recorded as
|
reference content is passed to the target that declares the slot. Reference
|
||||||
|
media types are inferred from file extensions, recorded as canonical base media
|
||||||
|
types, and checked only when a module declares
|
||||||
|
`AcceptedMediaTypes`; unknown extensions are recorded as
|
||||||
`application/octet-stream`. Reference content is not written to diagnostics,
|
`application/octet-stream`. Reference content is not written to diagnostics,
|
||||||
logs, errors, or manifests.
|
logs, errors, or manifests.
|
||||||
|
|
||||||
Pipeline-level `references` are defaults. They are valid when at least one
|
Pipeline-level `references` are defaults. They are valid when at least one
|
||||||
declared lane in the pipeline has an extractor that declares the slot. During a
|
eligible target in the full configured pipeline declares the slot, including
|
||||||
run, they apply only to selected lanes whose extractor declares the slot:
|
chunk, extractor, and normalizer targets. During a run, they apply only to the
|
||||||
|
selected targets that declare the slot:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
pipelines:
|
pipelines:
|
||||||
@@ -170,7 +174,9 @@ pipelines:
|
|||||||
extract: dnd/spells
|
extract: dnd/spells
|
||||||
```
|
```
|
||||||
|
|
||||||
Lane-level `references` override or add bindings for one lane:
|
Extractor binding `references` are the canonical lane-local location. The
|
||||||
|
legacy lane-level `references` field remains supported as an alias; when both
|
||||||
|
bind the same slot, `extract.references` wins:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
pipelines:
|
pipelines:
|
||||||
@@ -180,11 +186,27 @@ pipelines:
|
|||||||
glossary: ./campaign/glossary.txt
|
glossary: ./campaign/glossary.txt
|
||||||
artifacts:
|
artifacts:
|
||||||
spells:
|
spells:
|
||||||
extract: dnd/spells
|
|
||||||
references:
|
references:
|
||||||
roster: ./campaign/session-roster.txt
|
roster: ./campaign/legacy-roster.txt
|
||||||
|
extract:
|
||||||
|
module: dnd/spells
|
||||||
|
references:
|
||||||
|
roster: ./campaign/session-roster.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`chunk.references` and `normalize.references` are accepted in object-form
|
||||||
|
bindings. They override pipeline-level defaults for slots declared by the chunk
|
||||||
|
or normalizer module. Extractor-local references apply only to the extractor,
|
||||||
|
and normalizer-local references apply only to the normalizer.
|
||||||
|
|
||||||
|
Stage-local reference fields use the same map shape at:
|
||||||
|
|
||||||
|
- `pipelines.<id>.chunk.references`
|
||||||
|
- `pipelines.<id>.artifacts.<lane>.extract.references`
|
||||||
|
- `pipelines.<id>.artifacts.<lane>.normalize.references`
|
||||||
|
|
||||||
|
Each binding is valid only when that target module declares the slot.
|
||||||
|
|
||||||
## Module Bindings
|
## Module Bindings
|
||||||
|
|
||||||
Every module binding may use shorthand:
|
Every module binding may use shorthand:
|
||||||
@@ -208,6 +230,9 @@ Binding fields:
|
|||||||
- `module`: module key.
|
- `module`: module key.
|
||||||
- `llm_profile`: optional LLM profile ID. Empty means `default`.
|
- `llm_profile`: optional LLM profile ID. Empty means `default`.
|
||||||
- `options`: optional module-specific settings.
|
- `options`: optional module-specific settings.
|
||||||
|
- `references`: optional reference bindings. Supported only for `chunk`,
|
||||||
|
`extract`, and `normalize` bindings. `input`, `merge`, validator, and
|
||||||
|
`output` bindings reject this field during validation.
|
||||||
|
|
||||||
The `--llm-profile` run flag overrides every effective module binding to use
|
The `--llm-profile` run flag overrides every effective module binding to use
|
||||||
one configured profile.
|
one configured profile.
|
||||||
|
|||||||
@@ -100,8 +100,12 @@ Fields with empty values may be omitted by JSON encoding.
|
|||||||
|
|
||||||
`source_digests` contains source document digests only. Bound references are
|
`source_digests` contains source document digests only. Bound references are
|
||||||
recorded separately under `references`, which contains provenance only:
|
recorded separately under `references`, which contains provenance only:
|
||||||
lane ID, slot name, origin type and URI, digest, media type, byte size, and
|
target stage, lane ID when present, slot name, origin type and URI, digest,
|
||||||
binding source. Reference content is not written to durable output.
|
media type, byte size, and binding source. Reference content is not written to
|
||||||
|
durable output.
|
||||||
|
|
||||||
|
Reference `stage` is `chunk`, `extract`, or `normalize`. `lane_id` is omitted
|
||||||
|
for chunk references and present for extract and normalize references.
|
||||||
|
|
||||||
When references are bound, the manifest section has this shape:
|
When references are bound, the manifest section has this shape:
|
||||||
|
|
||||||
@@ -109,6 +113,7 @@ When references are bound, the manifest section has this shape:
|
|||||||
{
|
{
|
||||||
"references": [
|
"references": [
|
||||||
{
|
{
|
||||||
|
"stage": "extract",
|
||||||
"lane_id": "spells",
|
"lane_id": "spells",
|
||||||
"slot_name": "roster",
|
"slot_name": "roster",
|
||||||
"origin_type": "file",
|
"origin_type": "file",
|
||||||
|
|||||||
@@ -17,8 +17,8 @@ The request contains messages, optional model override, response schema name,
|
|||||||
and response schema JSON. The caller supplies a pointer target for decoded
|
and response schema JSON. The caller supplies a pointer target for decoded
|
||||||
structured output.
|
structured output.
|
||||||
|
|
||||||
Extractors own prompts and schemas. Provider adapters should not contain
|
Modules that call the LLM own their prompts and schemas. Provider adapters
|
||||||
domain-specific prompt logic.
|
should not contain domain-specific prompt logic.
|
||||||
|
|
||||||
## Production Client Construction
|
## Production Client Construction
|
||||||
|
|
||||||
|
|||||||
@@ -20,21 +20,24 @@ A production module package should provide:
|
|||||||
Module specs should describe capabilities accurately. Resolution uses specs to
|
Module specs should describe capabilities accurately. Resolution uses specs to
|
||||||
reject incompatible pipelines before execution.
|
reject incompatible pipelines before execution.
|
||||||
|
|
||||||
Extractor modules that accept auxiliary reference material must declare slots
|
Chunk, extract, and normalize modules that accept auxiliary reference material
|
||||||
through both `ReferenceSlots()` and `ModuleSpec().ReferenceSlots`. The runtime
|
must declare slots through both `ReferenceSlots()` and
|
||||||
slot list and registry metadata should match so config validation can inspect
|
`ModuleSpec().ReferenceSlots`. The runtime slot list and registry metadata
|
||||||
slots without constructing extractor instances. A slot declaration names the
|
should match so config validation can inspect slots without constructing module
|
||||||
slot, whether it is required, accepted media types, whether multiple items are
|
instances. A slot declaration names the slot, whether it is required, accepted
|
||||||
allowed, and any byte limit. Empty `AcceptedMediaTypes` means any inferred
|
media types, whether multiple items are allowed, and any byte limit. Empty
|
||||||
media type is accepted, though the file must still be UTF-8 text. When a slot
|
`AcceptedMediaTypes` means any inferred media type is accepted, though the file
|
||||||
declares accepted media types, Notarius compares the canonical base media type
|
must still be UTF-8 text. When a slot declares accepted media types, Notarius
|
||||||
inferred from the file extension, case-insensitively and without parameters.
|
compares the canonical base media type inferred from the file extension,
|
||||||
|
case-insensitively and without parameters.
|
||||||
|
|
||||||
Reference content is delivered only to the lane extractor through
|
The resolver materializes reference content for chunk, extractor, and
|
||||||
`contracts.ExtractionRequest.References`. It is not source evidence and must not
|
normalizer targets. Runtime delivery uses `contracts.ChunkRequest.References`,
|
||||||
be converted into `SourceRef` values. If a module prompt uses references, load
|
`contracts.ExtractionRequest.References`, and
|
||||||
the prompt bundle with the same declared slots and render with
|
`contracts.NormalizeRequest.References`. Reference material is not source
|
||||||
`RenderUserSystemWithReferences`. Prompt templates may use the `reference`
|
evidence and must not be converted into `SourceRef` values. If a module prompt
|
||||||
|
uses references, load the prompt bundle with the same declared slots and render
|
||||||
|
with `RenderUserSystemWithReferences`. Prompt templates may use the `reference`
|
||||||
function for content and the `hasreference` function for conditional sections.
|
function for content and the `hasreference` function for conditional sections.
|
||||||
Prompt metadata hashes remain based on template source, not rendered reference
|
Prompt metadata hashes remain based on template source, not rendered reference
|
||||||
bytes.
|
bytes.
|
||||||
@@ -44,6 +47,9 @@ when they need model-backed chunking. The pipeline runner validates generic
|
|||||||
chunk result invariants before extraction; module-owned policies may be stricter
|
chunk result invariants before extraction; module-owned policies may be stricter
|
||||||
but must stay within the module package.
|
but must stay within the module package.
|
||||||
|
|
||||||
|
Normalize modules receive the structured LLM client through
|
||||||
|
`contracts.NormalizeRequest` when they need model-backed reconciliation.
|
||||||
|
|
||||||
## `seriatim` Input
|
## `seriatim` Input
|
||||||
|
|
||||||
Package: `internal/modules/input/seriatim`
|
Package: `internal/modules/input/seriatim`
|
||||||
|
|||||||
@@ -31,32 +31,34 @@ before execution:
|
|||||||
The CLI writes the resolved pipeline and digest to diagnostics.
|
The CLI writes the resolved pipeline and digest to diagnostics.
|
||||||
|
|
||||||
Pipeline profiles and artifact lanes may include reference binding maps keyed by
|
Pipeline profiles and artifact lanes may include reference binding maps keyed by
|
||||||
extractor reference slot name. During resolution, pipeline-level bindings act as
|
reference slot name. During resolution, pipeline-level bindings act as defaults
|
||||||
defaults for selected lanes whose extractor declares the slot, lane-level
|
for selected chunk, extractor, and normalizer targets that declare the slot;
|
||||||
bindings override or add lane bindings, runtime `--reference` requests override
|
target-local bindings override or add bindings for that target. Runtime
|
||||||
config bindings, and runtime unbinds remove optional bindings. Flat runtime slot
|
`--reference` requests override target config bindings, and runtime unbinds
|
||||||
names are resolved only when exactly one selected lane declares the slot;
|
remove optional target bindings. Flat runtime slot names are resolved only when
|
||||||
otherwise the CLI requires `lane.slot`. Resolution validates bindings against
|
exactly one selected target declares the slot; otherwise the CLI requires a more
|
||||||
extractor specs and records lane-scoped binding metadata. It does not read
|
specific selector such as `chunk.slot`, `lane.extract.slot`, or
|
||||||
reference files or include reference bytes in source digests.
|
`lane.normalize.slot`. Resolution validates bindings against the declaring
|
||||||
|
target specs and stores the bindings in target-aware resolved reference holders.
|
||||||
|
It does not read reference files or include reference bytes in source digests.
|
||||||
|
|
||||||
During run preparation, resolved file references are materialized before any
|
During run preparation, resolved file references for chunk, extractor, and
|
||||||
LLM-backed pipeline work. Config bindings resolve relative to the config file,
|
normalizer targets are materialized before any LLM-backed pipeline work. Config
|
||||||
CLI bindings resolve relative to the current working directory, and materialized
|
bindings resolve relative to the config file, and CLI bindings resolve relative
|
||||||
reference content is passed only to the matching lane extractor through
|
to the current working directory. Materialization accepts UTF-8 text files,
|
||||||
`ExtractionRequest`. Materialization accepts UTF-8 text files, computes
|
computes `sha256:` content digests, records file origins, infers canonical base
|
||||||
`sha256:` content digests, records file origins, infers canonical base media
|
media types from file extensions, enforces declared byte limits, and warns for
|
||||||
types from file extensions, enforces declared byte limits, and warns for empty
|
empty bound files. Media-type acceptance is checked only when a slot declares
|
||||||
bound files. Media-type acceptance is checked only when a slot declares
|
|
||||||
`AcceptedMediaTypes`; unknown extensions are recorded as
|
`AcceptedMediaTypes`; unknown extensions are recorded as
|
||||||
`application/octet-stream`. Reference content is omitted from diagnostics and
|
`application/octet-stream`. Reference content is omitted from diagnostics and
|
||||||
manifests. The CLI writes provenance-only resolved reference diagnostics, and
|
manifests. The CLI writes provenance-only resolved reference diagnostics, and
|
||||||
the run manifest records lane-scoped reference provenance separately from source
|
the run manifest records target-stage reference provenance separately from
|
||||||
digests.
|
source digests. Runtime reference content is passed to the matching chunker,
|
||||||
|
extractor, or normalizer request.
|
||||||
|
|
||||||
Prompt bundles can declare reference slots and use `reference` and
|
Prompt bundles can declare reference slots and use `reference` and
|
||||||
`hasreference` template functions. Bundle loading validates string-literal slot
|
`hasreference` template functions. Bundle loading validates string-literal slot
|
||||||
names against the declaration. Rendering receives a lane reference set from the
|
names against the declaration. Rendering receives a target reference set from the
|
||||||
caller; unbound optional slots render as empty strings, and `hasreference`
|
caller; unbound optional slots render as empty strings, and `hasreference`
|
||||||
returns true only when at least one bound item has content.
|
returns true only when at least one bound item has content.
|
||||||
|
|
||||||
@@ -74,9 +76,10 @@ Every production module registers a `ModuleSpec` with:
|
|||||||
- `Provides`: capabilities added after that module runs;
|
- `Provides`: capabilities added after that module runs;
|
||||||
- `Requires`: capabilities that must already be available.
|
- `Requires`: capabilities that must already be available.
|
||||||
|
|
||||||
Extractor specs may also declare reference slots. Slot declarations are
|
Chunk, extract, and normalize specs may also declare reference slots. Slot
|
||||||
available from registry metadata without constructing extractor instances.
|
declarations are available from registry metadata without constructing module
|
||||||
Non-extractor module specs must not declare reference slots.
|
instances. Input, merge, validate, and output specs must not declare reference
|
||||||
|
slots.
|
||||||
|
|
||||||
Capability checks prevent incompatible pipeline composition before a run starts.
|
Capability checks prevent incompatible pipeline composition before a run starts.
|
||||||
|
|
||||||
@@ -114,10 +117,10 @@ The runner:
|
|||||||
## Chunk Results
|
## Chunk Results
|
||||||
|
|
||||||
Chunkers implement `contracts.Chunker` and receive a `contracts.ChunkRequest`
|
Chunkers implement `contracts.Chunker` and receive a `contracts.ChunkRequest`
|
||||||
with the validated source document, the structured LLM client, the configured
|
with the validated source document, reference set, structured LLM client, the
|
||||||
LLM profile, module options, and run metadata. Deterministic and LLM-backed
|
configured LLM profile, module options, and run metadata. Deterministic and
|
||||||
chunkers use the same contract; provider construction stays outside chunk
|
LLM-backed chunkers use the same contract; provider construction stays outside
|
||||||
modules.
|
chunk modules.
|
||||||
|
|
||||||
After `Chunk` returns, the runner appends chunker warnings before returning any
|
After `Chunk` returns, the runner appends chunker warnings before returning any
|
||||||
chunker error. When chunking succeeds, the runner validates generic chunk
|
chunker error. When chunking succeeds, the runner validates generic chunk
|
||||||
|
|||||||
@@ -63,9 +63,9 @@ Implemented diagnostics artifacts:
|
|||||||
path, selected lanes, run ID, and pipeline digest when available.
|
path, selected lanes, run ID, and pipeline digest when available.
|
||||||
- `effective-config.json`: resolved config with API keys redacted.
|
- `effective-config.json`: resolved config with API keys redacted.
|
||||||
- `resolved-pipeline.json`: resolved module bindings and pipeline digest.
|
- `resolved-pipeline.json`: resolved module bindings and pipeline digest.
|
||||||
- `resolved-references.json`: lane-scoped resolved reference provenance,
|
- `resolved-references.json`: resolved reference provenance, including target
|
||||||
including origin, digest, media type, byte size, and binding source, without
|
stage, lane ID when present, origin, digest, media type, byte size, and
|
||||||
reference content.
|
binding source, without reference content.
|
||||||
- `run-manifest.json`: the same run manifest written to durable output when it
|
- `run-manifest.json`: the same run manifest written to durable output when it
|
||||||
is available, including top-level module metadata when present.
|
is available, including top-level module metadata when present.
|
||||||
- `warnings.json`: warning list.
|
- `warnings.json`: warning list.
|
||||||
|
|||||||
@@ -8,11 +8,6 @@ future work only.
|
|||||||
|
|
||||||
- Additional input adapters, such as Markdown or note-export formats.
|
- Additional input adapters, such as Markdown or note-export formats.
|
||||||
- Additional D&D extractors beyond spell casts.
|
- Additional D&D extractors beyond spell casts.
|
||||||
- First-class LLM access for all LLM-eligible pipeline stages: `chunk`,
|
|
||||||
`extract`, and `normalize`.
|
|
||||||
- Expand reference support beyond extractors to `chunk` and `normalize`
|
|
||||||
modules, using the same declared-slot model and preserving stage-specific
|
|
||||||
ownership boundaries.
|
|
||||||
- Add non-file reference producers, such as prior-run artifacts, derived
|
- Add non-file reference producers, such as prior-run artifacts, derived
|
||||||
summaries, or entity registries, without changing module-facing reference
|
summaries, or entity registries, without changing module-facing reference
|
||||||
item contracts.
|
item contracts.
|
||||||
|
|||||||
@@ -1,463 +1,21 @@
|
|||||||
# LLM And Stage Reference Expansion Implementation Plan
|
# LLM And Reference Expansion Completed
|
||||||
|
|
||||||
## Purpose
|
This roadmap item is implemented.
|
||||||
|
|
||||||
Implement these future roadmap items:
|
Notarius now supports first-class LLM access for chunk, extract, and normalize
|
||||||
|
modules, plus target-aware reference declaration, binding, CLI override,
|
||||||
|
materialization, provenance, and runtime delivery for chunker, extractor, and
|
||||||
|
normalizer targets.
|
||||||
|
|
||||||
- First-class LLM access for all LLM-eligible pipeline stages: `chunk`,
|
Current behavior is documented in:
|
||||||
`extract`, and `normalize`.
|
|
||||||
- Expand reference support beyond extractors to `chunk` and `normalize`
|
|
||||||
modules, using the same declared-slot model and preserving stage-specific
|
|
||||||
ownership boundaries.
|
|
||||||
|
|
||||||
This is a decision-complete implementation plan for an LLM coding agent. Follow
|
- [CLI Reference](../cli.md)
|
||||||
the stages in order. Keep current-behavior docs unchanged until the
|
- [Configuration](../config.md)
|
||||||
corresponding behavior exists. Do not implement token budgeting, non-file
|
- [Pipeline Internals](../internal/pipeline.md)
|
||||||
reference producers, structured references, retrieval, or multiple effective LLM
|
- [Module Internals](../internal/modules.md)
|
||||||
profiles in this pass.
|
- [JSON Output](../integrations/json-output.md)
|
||||||
|
- [Troubleshooting](../troubleshooting.md)
|
||||||
|
|
||||||
## Target Behavior
|
Deferred reference work remains tracked in [Future Work](future.md), including
|
||||||
|
non-file reference producers, token budgeting, structured references, and
|
||||||
LLM access:
|
reference preprocessing or retrieval.
|
||||||
|
|
||||||
- `contracts.ChunkRequest`, `contracts.ExtractionRequest`, and
|
|
||||||
`contracts.NormalizeRequest` all carry `LLMClient`, `LLMProfile`, `Options`,
|
|
||||||
and `Metadata`.
|
|
||||||
- The runner passes the configured structured LLM client to chunkers,
|
|
||||||
extractors, and normalizers.
|
|
||||||
- The existing single-effective-LLM-profile CLI restriction remains in place.
|
|
||||||
|
|
||||||
References:
|
|
||||||
|
|
||||||
- `chunk`, `extract`, and `normalize` modules can declare reference slots.
|
|
||||||
- The framework treats references as opaque UTF-8 text plus provenance. Stage
|
|
||||||
modules own slot names and semantics.
|
|
||||||
- Reference slots are declared in both runtime module contracts and
|
|
||||||
`ModuleSpec.ReferenceSlots`, so config and pipeline validation can inspect
|
|
||||||
slots without constructing modules.
|
|
||||||
- `ModuleSpec.ReferenceSlots` is valid only for `chunk`, `extract`, and
|
|
||||||
`normalize` modules. `input`, `merge`, `validate`, and `output` modules must
|
|
||||||
not declare reference slots.
|
|
||||||
- Pipeline-level `references` remain defaults. A pipeline-level binding is
|
|
||||||
valid when any declared eligible module in the pipeline declares the slot. In
|
|
||||||
a run, it is applied only to selected targets that declare the slot.
|
|
||||||
- Stage-local reference bindings override pipeline-level defaults for that
|
|
||||||
target.
|
|
||||||
- CLI `--reference` and `--without-reference` can target chunk, extract, and
|
|
||||||
normalize references explicitly, while preserving existing unambiguous flat
|
|
||||||
forms.
|
|
||||||
- Reference materialization still accepts file producers only, validates UTF-8,
|
|
||||||
infers media types from extensions, enforces optional `AcceptedMediaTypes` and
|
|
||||||
`MaxBytes`, warns for empty files, and never writes content to diagnostics,
|
|
||||||
logs, errors, or manifests.
|
|
||||||
- Manifest and diagnostics provenance identifies the reference target stage and,
|
|
||||||
for lane-owned stages, the lane ID.
|
|
||||||
|
|
||||||
## Target Config Shape
|
|
||||||
|
|
||||||
Keep existing config compatibility and add stage-specific binding locations:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
pipelines:
|
|
||||||
dnd-session:
|
|
||||||
input: seriatim
|
|
||||||
references:
|
|
||||||
campaign_context: ./campaign/context.md
|
|
||||||
chunk:
|
|
||||||
module: dnd/scenes
|
|
||||||
references:
|
|
||||||
scene_guide: ./campaign/scenes.md
|
|
||||||
artifacts:
|
|
||||||
spells:
|
|
||||||
extract: dnd/spells
|
|
||||||
references:
|
|
||||||
glossary: ./campaign/glossary.md
|
|
||||||
normalize:
|
|
||||||
module: noop
|
|
||||||
references:
|
|
||||||
normalization_notes: ./campaign/normalization.md
|
|
||||||
```
|
|
||||||
|
|
||||||
Compatibility rule:
|
|
||||||
|
|
||||||
- Existing `artifacts.<lane>.references` remains supported as an alias for
|
|
||||||
extractor references.
|
|
||||||
- New `artifacts.<lane>.extract.references` is the canonical extract-stage
|
|
||||||
location.
|
|
||||||
- If both the legacy lane-level `references` map and
|
|
||||||
`extract.references` bind the same slot, `extract.references` wins. This keeps
|
|
||||||
migration deterministic and avoids breaking existing configs.
|
|
||||||
|
|
||||||
`chunk.references` and `normalize.references` are supported only when their
|
|
||||||
module binding uses object form. Scalar bindings continue to work when no
|
|
||||||
stage-local references are needed.
|
|
||||||
|
|
||||||
## Target CLI Reference Selectors
|
|
||||||
|
|
||||||
Preserve existing valid forms and add explicit stage selectors:
|
|
||||||
|
|
||||||
- `slot=path`: valid only when exactly one selected reference target across
|
|
||||||
chunk, selected extractors, and selected normalizers declares `slot`.
|
|
||||||
- `chunk.slot=path`: targets the singleton chunk module.
|
|
||||||
- `lane.slot=path`: valid only when exactly one selected module in that lane,
|
|
||||||
extract or normalize, declares `slot`.
|
|
||||||
- `lane.extract.slot=path`: targets a lane extractor.
|
|
||||||
- `lane.normalize.slot=path`: targets a lane normalizer.
|
|
||||||
|
|
||||||
`--without-reference` accepts the same selector forms without `=path`.
|
|
||||||
|
|
||||||
Ambiguous selectors must fail before any LLM call and should name the ambiguous
|
|
||||||
targets and the more specific selector forms.
|
|
||||||
|
|
||||||
## Stage 1: Contract And Registry Expansion
|
|
||||||
|
|
||||||
Add reference declaration support to chunkers and normalizers and LLM delivery
|
|
||||||
to normalizers. This stage should not change config shape or runtime reference
|
|
||||||
resolution yet.
|
|
||||||
|
|
||||||
Implementation steps:
|
|
||||||
|
|
||||||
- Add `References contracts.ReferenceSet` to `contracts.ChunkRequest`.
|
|
||||||
- Add `LLMClient contracts.StructuredLLMClient` and
|
|
||||||
`References contracts.ReferenceSet` to `contracts.NormalizeRequest`.
|
|
||||||
- Add `ReferenceSlots() []ReferenceSlot` to `contracts.Chunker` and
|
|
||||||
`contracts.Normalizer`.
|
|
||||||
- Update every concrete chunker and normalizer plus all test fakes to implement
|
|
||||||
`ReferenceSlots()`. Existing modules without references should return `nil`.
|
|
||||||
- Change `validateModuleSpec` so `ModuleSpec.ReferenceSlots` is allowed for
|
|
||||||
`StageChunk`, `StageExtract`, and `StageNormalize`, and rejected for all other
|
|
||||||
stages.
|
|
||||||
- Add registry/module tests proving:
|
|
||||||
- chunker specs may declare valid reference slots;
|
|
||||||
- normalizer specs may declare valid reference slots;
|
|
||||||
- extractor specs still may declare valid reference slots;
|
|
||||||
- input, merge, validate, and output specs reject reference slots;
|
|
||||||
- invalid slot names, duplicate slots, and negative `MaxBytes` are rejected
|
|
||||||
for all eligible stages.
|
|
||||||
- Add production registration tests comparing runtime `ReferenceSlots()` with
|
|
||||||
registered `ModuleSpec.ReferenceSlots` for chunkers, extractors, and
|
|
||||||
normalizers that declare slots. Modules with no slots should be covered by
|
|
||||||
simple nil/empty assertions.
|
|
||||||
|
|
||||||
Verification:
|
|
||||||
|
|
||||||
- `go test ./internal/framework/contracts`
|
|
||||||
- `go test ./internal/framework/pipeline`
|
|
||||||
- `go test ./internal/modules/chunk/...`
|
|
||||||
- `go test ./internal/modules/normalize/...`
|
|
||||||
- `go test ./...`
|
|
||||||
|
|
||||||
## Stage 2: Resolved Reference Target Model
|
|
||||||
|
|
||||||
Replace extractor-only resolved reference storage with a target-aware model.
|
|
||||||
Keep external config and CLI behavior unchanged in this stage where possible.
|
|
||||||
|
|
||||||
Implementation steps:
|
|
||||||
|
|
||||||
- Introduce a target model in `internal/framework/pipeline`, for example:
|
|
||||||
- target stage: `chunk`, `extract`, or `normalize`;
|
|
||||||
- optional lane ID for lane-owned targets;
|
|
||||||
- module key;
|
|
||||||
- reference bindings;
|
|
||||||
- materialized `ReferenceSet`.
|
|
||||||
- Add a resolved chunk reference holder to `ResolvedPipeline`.
|
|
||||||
- Split lane reference storage into extract and normalize holders on
|
|
||||||
`ResolvedArtifactLane`.
|
|
||||||
- Keep JSON field names deterministic and redaction-friendly. Do not expose
|
|
||||||
reference content through resolved pipeline JSON.
|
|
||||||
- Update `CloneReferenceSet`, `ReferenceProvenance`, redacted effective config,
|
|
||||||
resolved-pipeline diagnostics, and any affected tests for the target-aware
|
|
||||||
model.
|
|
||||||
- Preserve existing extractor behavior by mapping old lane extractor references
|
|
||||||
into the new extract target.
|
|
||||||
- Keep the pipeline digest deterministic and based on resolved reference
|
|
||||||
bindings, not materialized reference bytes. Materialized reference digests
|
|
||||||
remain manifest provenance.
|
|
||||||
|
|
||||||
Verification:
|
|
||||||
|
|
||||||
- `go test ./internal/framework/pipeline`
|
|
||||||
- `go test ./internal/core/config`
|
|
||||||
- `go test ./internal/core/diagnostics`
|
|
||||||
- `go test ./...`
|
|
||||||
|
|
||||||
## Stage 3: Config Parsing And Validation
|
|
||||||
|
|
||||||
Add stage-local reference fields while preserving existing configs.
|
|
||||||
|
|
||||||
Implementation steps:
|
|
||||||
|
|
||||||
- Extend the file config module binding object to accept `references`.
|
|
||||||
- Add `References map[string]string` to `pipeline.ModuleBinding`, or use a
|
|
||||||
narrower equivalent if the implementation can do so without duplicating
|
|
||||||
binding parsing. If added to `ModuleBinding`, validation must reject
|
|
||||||
references on non-eligible stages.
|
|
||||||
- Apply file config references from:
|
|
||||||
- `pipelines.<id>.chunk.references`;
|
|
||||||
- `pipelines.<id>.artifacts.<lane>.extract.references`;
|
|
||||||
- `pipelines.<id>.artifacts.<lane>.normalize.references`;
|
|
||||||
- existing `pipelines.<id>.artifacts.<lane>.references` as an extract-stage
|
|
||||||
compatibility alias.
|
|
||||||
- Validate reference maps for empty names, duplicate-after-trim names, and empty
|
|
||||||
sources at each new location.
|
|
||||||
- Reject references on `input`, `merge`, validators, and `output` bindings with
|
|
||||||
clear config errors.
|
|
||||||
- Preserve existing `llm_profile` validation for chunk, extract, and normalize
|
|
||||||
bindings. No new LLM profile semantics are introduced in this pass.
|
|
||||||
- Add config tests for:
|
|
||||||
- parsing `chunk.references`;
|
|
||||||
- parsing `extract.references`;
|
|
||||||
- parsing `normalize.references`;
|
|
||||||
- legacy lane `references` compatibility;
|
|
||||||
- duplicate conflicts where `extract.references` overrides legacy lane
|
|
||||||
references for the same slot;
|
|
||||||
- rejection of references on unsupported module binding locations;
|
|
||||||
- redaction/effective config cloning for all new reference maps.
|
|
||||||
|
|
||||||
Verification:
|
|
||||||
|
|
||||||
- `go test ./internal/core/config`
|
|
||||||
- `go test ./internal/framework/pipeline`
|
|
||||||
- `go test ./...`
|
|
||||||
|
|
||||||
## Stage 4: Pipeline Resolution
|
|
||||||
|
|
||||||
Resolve reference bindings against all eligible stage targets and selected lanes
|
|
||||||
without constructing modules.
|
|
||||||
|
|
||||||
Implementation steps:
|
|
||||||
|
|
||||||
- During `ResolvePipeline`, collect declared reference slots from:
|
|
||||||
- the configured chunker spec;
|
|
||||||
- every declared artifact lane extractor spec;
|
|
||||||
- every declared artifact lane normalizer spec.
|
|
||||||
- Keep pipeline-level reference validation broad: a pipeline-level binding is
|
|
||||||
valid if any declared eligible target in the full pipeline declares that slot,
|
|
||||||
even when `--only` selects lanes that do not consume it.
|
|
||||||
- During a selected run, apply pipeline-level defaults only to:
|
|
||||||
- the singleton chunk target if its chunker declares the slot;
|
|
||||||
- selected extractor targets that declare the slot;
|
|
||||||
- selected normalizer targets that declare the slot.
|
|
||||||
- Apply stage-local references after pipeline defaults:
|
|
||||||
- chunk-local references override chunk pipeline defaults;
|
|
||||||
- extract-local references override extract pipeline defaults;
|
|
||||||
- normalize-local references override normalize pipeline defaults.
|
|
||||||
- Enforce required slots independently per selected target after defaults,
|
|
||||||
local references, CLI overrides, and CLI unbinds.
|
|
||||||
- Keep lane-local strictness: extract-local references must be declared by that
|
|
||||||
lane's extractor, and normalize-local references must be declared by that
|
|
||||||
lane's normalizer.
|
|
||||||
- Return clear errors that name pipeline ID, stage, lane ID when present, slot,
|
|
||||||
and module key.
|
|
||||||
- Add pipeline tests for:
|
|
||||||
- pipeline default consumed by chunk only;
|
|
||||||
- pipeline default consumed by extract only;
|
|
||||||
- pipeline default consumed by normalize only;
|
|
||||||
- one pipeline default consumed by several eligible targets;
|
|
||||||
- valid pipeline default declared only by an unselected lane target;
|
|
||||||
- unknown pipeline-level slot rejected when no declared target has it;
|
|
||||||
- extract-local reference rejected by normalizer-only slot;
|
|
||||||
- normalize-local reference rejected by extractor-only slot;
|
|
||||||
- required chunk reference missing;
|
|
||||||
- required normalize reference missing;
|
|
||||||
- local overrides over pipeline defaults for each eligible stage.
|
|
||||||
|
|
||||||
Verification:
|
|
||||||
|
|
||||||
- `go test ./internal/framework/pipeline`
|
|
||||||
- `go test ./internal/core/config`
|
|
||||||
- `go test ./...`
|
|
||||||
|
|
||||||
## Stage 5: CLI Reference Selectors
|
|
||||||
|
|
||||||
Extend CLI reference override and unbind resolution to target chunk, extract,
|
|
||||||
and normalize references.
|
|
||||||
|
|
||||||
Implementation steps:
|
|
||||||
|
|
||||||
- Replace extractor-only CLI reference target lookup with target-aware lookup
|
|
||||||
over the resolved eligible targets for the selected run.
|
|
||||||
- Support selector forms listed in "Target CLI Reference Selectors".
|
|
||||||
- Preserve existing behavior for unambiguous `slot` and `lane.slot` forms.
|
|
||||||
- Reject ambiguous `slot` and `lane.slot` forms with a message naming matching
|
|
||||||
targets and suggesting `chunk.slot`, `lane.extract.slot`, or
|
|
||||||
`lane.normalize.slot` as appropriate.
|
|
||||||
- Reject selectors that target an unselected lane during `--only` runs.
|
|
||||||
- Reject selectors for stages that do not declare the slot.
|
|
||||||
- Keep `--reference` empty-path rejection and `--without-reference` optional
|
|
||||||
unbinding semantics.
|
|
||||||
- Add CLI tests for:
|
|
||||||
- `chunk.slot=path`;
|
|
||||||
- `lane.extract.slot=path`;
|
|
||||||
- `lane.normalize.slot=path`;
|
|
||||||
- flat slot resolution across exactly one target;
|
|
||||||
- lane slot resolution across exactly one lane-local target;
|
|
||||||
- ambiguous flat slot across chunk and extract;
|
|
||||||
- ambiguous lane slot across extract and normalize;
|
|
||||||
- unbind forms for chunk, extract, and normalize;
|
|
||||||
- unbind leaving required chunk or normalize slot missing;
|
|
||||||
- `--only` with an unselected lane selector.
|
|
||||||
|
|
||||||
Verification:
|
|
||||||
|
|
||||||
- `go test ./internal/cli`
|
|
||||||
- `go test ./internal/framework/pipeline`
|
|
||||||
- `go test ./...`
|
|
||||||
|
|
||||||
## Stage 6: Materialization And Provenance
|
|
||||||
|
|
||||||
Materialize references for chunk, extract, and normalize targets with one shared
|
|
||||||
implementation.
|
|
||||||
|
|
||||||
Implementation steps:
|
|
||||||
|
|
||||||
- Refactor materialization so it accepts a resolved target plus that target's
|
|
||||||
declared slots, rather than assuming an extractor lane.
|
|
||||||
- Reuse the existing file behavior:
|
|
||||||
- config paths relative to the config file;
|
|
||||||
- CLI paths relative to the current working directory;
|
|
||||||
- UTF-8 validation for all files;
|
|
||||||
- extension-based media type inference;
|
|
||||||
- optional `AcceptedMediaTypes` enforcement;
|
|
||||||
- optional `MaxBytes` enforcement;
|
|
||||||
- empty-file warning;
|
|
||||||
- `sha256:` content digest.
|
|
||||||
- Store materialized reference sets on the resolved chunk target, extract
|
|
||||||
targets, and normalize targets.
|
|
||||||
- Update warnings to include target context in `Scope`, such as:
|
|
||||||
- `pipeline.<id>.chunk.reference.<slot>`;
|
|
||||||
- `pipeline.<id>.lane.<lane>.extract.reference.<slot>`;
|
|
||||||
- `pipeline.<id>.lane.<lane>.normalize.reference.<slot>`.
|
|
||||||
- Update manifest and diagnostics provenance to include:
|
|
||||||
- `stage`: `chunk`, `extract`, or `normalize`;
|
|
||||||
- `lane_id`: omitted for chunk and present for extract/normalize;
|
|
||||||
- `slot_name`;
|
|
||||||
- origin type and URI;
|
|
||||||
- digest;
|
|
||||||
- media type;
|
|
||||||
- size bytes;
|
|
||||||
- binding source.
|
|
||||||
- Keep reference content out of manifests, diagnostics, logs, and errors.
|
|
||||||
- Add tests for materialization and provenance for all three eligible stages,
|
|
||||||
including media-type rejection and UTF-8 rejection on non-extractor targets.
|
|
||||||
|
|
||||||
Verification:
|
|
||||||
|
|
||||||
- `go test ./internal/framework/pipeline`
|
|
||||||
- `go test ./internal/core/artifacts`
|
|
||||||
- `go test ./internal/core/diagnostics`
|
|
||||||
- `go test ./internal/modules/output/json`
|
|
||||||
- `go test ./internal/cli`
|
|
||||||
- `go test ./...`
|
|
||||||
|
|
||||||
## Stage 7: Runner Delivery
|
|
||||||
|
|
||||||
Deliver LLM clients and references to eligible stage modules at runtime.
|
|
||||||
|
|
||||||
Implementation steps:
|
|
||||||
|
|
||||||
- Pass `CloneReferenceSet` of the materialized chunk reference set into
|
|
||||||
`contracts.ChunkRequest.References`.
|
|
||||||
- Continue passing `LLMClient` into `contracts.ChunkRequest`.
|
|
||||||
- Continue passing extract references and `LLMClient` into
|
|
||||||
`contracts.ExtractionRequest`.
|
|
||||||
- Pass `LLMClient` and `CloneReferenceSet` of the materialized normalize
|
|
||||||
reference set into `contracts.NormalizeRequest`.
|
|
||||||
- Ensure normalizers receive their configured `LLMProfile`, options, and
|
|
||||||
metadata as they do today.
|
|
||||||
- Add runner tests proving:
|
|
||||||
- chunkers receive LLM client and chunk references;
|
|
||||||
- extractors still receive LLM client and extract references;
|
|
||||||
- normalizers receive LLM client and normalize references;
|
|
||||||
- reference sets are cloned before delivery so modules cannot mutate resolved
|
|
||||||
pipeline state;
|
|
||||||
- missing LLM client still fails only when a module actually uses it, matching
|
|
||||||
existing chunk/extract behavior.
|
|
||||||
- Update the walking skeleton to exercise LLM-backed normalize access if a small
|
|
||||||
fake normalizer can do so without making the fixture noisy.
|
|
||||||
|
|
||||||
Verification:
|
|
||||||
|
|
||||||
- `go test ./internal/framework/pipeline`
|
|
||||||
- `go test ./...`
|
|
||||||
|
|
||||||
## Stage 8: Prompt Helpers And Module Usage
|
|
||||||
|
|
||||||
Make prompt reference helpers usable by chunk and normalize modules, then update
|
|
||||||
production modules only where needed.
|
|
||||||
|
|
||||||
Implementation steps:
|
|
||||||
|
|
||||||
- Confirm `internal/framework/prompt` reference rendering is contract-based and
|
|
||||||
not extractor-specific. If any naming or error text says extractor-only,
|
|
||||||
generalize it to module/stage language.
|
|
||||||
- Add prompt tests rendering references for a fake chunker or normalizer prompt
|
|
||||||
bundle if existing tests cover only extractor usage.
|
|
||||||
- Update production chunkers and normalizers:
|
|
||||||
- `generic` chunker: no reference slots.
|
|
||||||
- `dnd/scenes` chunker: no reference slots in this pass unless a clear
|
|
||||||
module-owned slot already exists.
|
|
||||||
- `noop` normalizer: no reference slots.
|
|
||||||
- Do not invent a new production normalize module in this pass. The goal is
|
|
||||||
framework support, not a domain normalizer.
|
|
||||||
|
|
||||||
Verification:
|
|
||||||
|
|
||||||
- `go test ./internal/framework/prompt`
|
|
||||||
- `go test ./internal/modules/chunk/...`
|
|
||||||
- `go test ./internal/modules/normalize/...`
|
|
||||||
- `go test ./...`
|
|
||||||
|
|
||||||
## Stage 9: Documentation And Examples
|
|
||||||
|
|
||||||
After behavior is implemented and tested, update canonical docs and examples.
|
|
||||||
|
|
||||||
Implementation steps:
|
|
||||||
|
|
||||||
- Update `docs/cli.md` with the expanded `--reference` and
|
|
||||||
`--without-reference` selector grammar and examples.
|
|
||||||
- Update `docs/config.md` with `chunk.references`,
|
|
||||||
`artifacts.<lane>.extract.references`, `artifacts.<lane>.normalize.references`,
|
|
||||||
and the legacy `artifacts.<lane>.references` compatibility rule.
|
|
||||||
- Update `docs/internal/modules.md` with module-author guidance for
|
|
||||||
`ReferenceSlots()` on chunkers, extractors, and normalizers, plus LLM access
|
|
||||||
expectations for chunk, extract, and normalize.
|
|
||||||
- Update `docs/internal/pipeline.md` with target-aware reference resolution,
|
|
||||||
materialization, provenance, and runtime delivery.
|
|
||||||
- Update `docs/integrations/json-output.md` with the new reference provenance
|
|
||||||
shape including `stage` and optional `lane_id`.
|
|
||||||
- Update `docs/troubleshooting.md` with ambiguous selector examples for chunk,
|
|
||||||
extract, and normalize targets.
|
|
||||||
- Update maintained examples only if they can remain small and validated. Do
|
|
||||||
not add speculative reference files for chunk or normalize modules without a
|
|
||||||
production module that actually declares those slots.
|
|
||||||
- Update `docs/roadmap/future.md` by removing or revising the two completed
|
|
||||||
items while leaving still-deferred work such as token budgeting and non-file
|
|
||||||
producers.
|
|
||||||
- Replace this implementation plan with a concise completed-note document after
|
|
||||||
implementation is finished.
|
|
||||||
|
|
||||||
Verification:
|
|
||||||
|
|
||||||
- `rg -n "chunk.references|normalize.references|lane.extract.slot|lane.normalize.slot" docs/cli.md docs/config.md docs/internal docs/integrations docs/troubleshooting.md`
|
|
||||||
- `go test ./...`
|
|
||||||
- `go vet ./...`
|
|
||||||
- `go build ./cmd/notarius`
|
|
||||||
|
|
||||||
## Final Acceptance Criteria
|
|
||||||
|
|
||||||
The feature is complete when:
|
|
||||||
|
|
||||||
- chunkers, extractors, and normalizers can declare reference slots through
|
|
||||||
runtime contracts and registry specs;
|
|
||||||
- normalizers receive the structured LLM client at runtime;
|
|
||||||
- config can bind references at pipeline, chunk, extract, and normalize levels;
|
|
||||||
- CLI reference overrides and unbinds can target chunk, extract, and normalize
|
|
||||||
references without ambiguous selector behavior;
|
|
||||||
- pipeline resolution validates all reference bindings before any LLM call;
|
|
||||||
- materialization and provenance work uniformly for chunk, extract, and
|
|
||||||
normalize references;
|
|
||||||
- reference manifest entries identify stage and lane context;
|
|
||||||
- existing extractor reference behavior remains backward compatible;
|
|
||||||
- canonical docs describe only implemented behavior;
|
|
||||||
- `go test ./...`, `go vet ./...`, and `go build ./cmd/notarius` pass.
|
|
||||||
|
|||||||
@@ -107,10 +107,10 @@ go run ./cmd/notarius run dnd-session \
|
|||||||
Symptoms include:
|
Symptoms include:
|
||||||
|
|
||||||
- `reference slot "..." is not declared`
|
- `reference slot "..." is not declared`
|
||||||
- `reference slot "..." is declared by multiple selected lanes`
|
- `reference slot "..." is declared by multiple selected targets`
|
||||||
- `required reference slot "..." is not bound`
|
- `required reference slot "..." is not bound`
|
||||||
- `--reference must use slot=path or lane.slot=path`
|
- `--reference must use slot=path`
|
||||||
- `--without-reference must use slot or lane.slot`
|
- `--without-reference must use a reference selector without =path`
|
||||||
- `read "...": no such file`
|
- `read "...": no such file`
|
||||||
- `must be UTF-8 text`
|
- `must be UTF-8 text`
|
||||||
- `media type "..." is not accepted`
|
- `media type "..." is not accepted`
|
||||||
@@ -118,17 +118,24 @@ Symptoms include:
|
|||||||
|
|
||||||
Fix:
|
Fix:
|
||||||
|
|
||||||
- Confirm the selected extractor declares the slot. The implemented
|
- Confirm the selected chunker, extractor, or normalizer declares the slot. The
|
||||||
`dnd/spells` extractor declares optional `roster` and `glossary` slots.
|
implemented `dnd/spells` extractor declares optional `roster` and `glossary`
|
||||||
- Use `lane.slot=path` when more than one selected lane declares the same slot.
|
slots.
|
||||||
- Use `--without-reference slot` to remove optional config bindings; do not pass
|
- Use a specific selector when more than one selected target declares the same
|
||||||
an empty `--reference slot=`.
|
slot. Examples include `chunk.context=./context.txt`,
|
||||||
|
`spells.extract.context=./extract-context.txt`, and
|
||||||
|
`spells.normalize.context=./normalize-context.txt`.
|
||||||
|
- `lane.slot=path` is valid only when exactly one selected extractor or
|
||||||
|
normalizer in that lane declares the slot. If both do, use
|
||||||
|
`lane.extract.slot=path` or `lane.normalize.slot=path`.
|
||||||
|
- Use `--without-reference selector` to remove optional config bindings; do not
|
||||||
|
pass an empty `--reference selector=`.
|
||||||
- Check whether a path came from config or CLI. Config paths are relative to
|
- Check whether a path came from config or CLI. Config paths are relative to
|
||||||
the config file. CLI reference paths are relative to the current working
|
the config file. CLI reference paths are relative to the current working
|
||||||
directory.
|
directory.
|
||||||
- Ensure the file is readable UTF-8 text and within any byte limit declared by
|
- Ensure the file is readable UTF-8 text and within any byte limit declared by
|
||||||
the extractor.
|
the declaring module.
|
||||||
- If the extractor narrows accepted media types, use a file extension that
|
- If the declaring module narrows accepted media types, use a file extension that
|
||||||
infers an accepted type such as `text/markdown` or `application/json`.
|
infers an accepted type such as `text/markdown` or `application/json`.
|
||||||
Unknown extensions infer `application/octet-stream`.
|
Unknown extensions infer `application/octet-stream`.
|
||||||
- If diagnostics are retained, inspect `resolved-pipeline.json`,
|
- If diagnostics are retained, inspect `resolved-pipeline.json`,
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ const defaultOutputRoot = "./notarius-output"
|
|||||||
|
|
||||||
const usage = `Usage:
|
const usage = `Usage:
|
||||||
notarius help
|
notarius help
|
||||||
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--reference slot=path] [--without-reference slot]
|
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--reference selector=path] [--without-reference selector]
|
||||||
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
|
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||||
notarius pipelines list --config path/to/config.yml [--json]
|
notarius pipelines list --config path/to/config.yml [--json]
|
||||||
`
|
`
|
||||||
@@ -97,8 +97,8 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
llmProfile := fs.String("llm-profile", "", "LLM profile override")
|
llmProfile := fs.String("llm-profile", "", "LLM profile override")
|
||||||
referenceFlags := stringListFlag{}
|
referenceFlags := stringListFlag{}
|
||||||
withoutReferenceFlags := stringListFlag{}
|
withoutReferenceFlags := stringListFlag{}
|
||||||
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path or lane.slot=path")
|
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, lane.slot=path, lane.extract.slot=path, or lane.normalize.slot=path")
|
||||||
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, as slot or lane.slot")
|
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
|
||||||
if err := fs.Parse(reorderRunArgs(args)); err != nil {
|
if err := fs.Parse(reorderRunArgs(args)); err != nil {
|
||||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||||
return 2
|
return 2
|
||||||
@@ -712,13 +712,17 @@ func (flag *stringListFlag) Set(value string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type cliReferenceRequest struct {
|
type cliReferenceRequest struct {
|
||||||
LaneID string
|
Selector cliReferenceSelector
|
||||||
SlotName string
|
|
||||||
Source string
|
Source string
|
||||||
}
|
}
|
||||||
|
|
||||||
type cliReferenceUnbindRequest struct {
|
type cliReferenceUnbindRequest struct {
|
||||||
|
Selector cliReferenceSelector
|
||||||
|
}
|
||||||
|
|
||||||
|
type cliReferenceSelector struct {
|
||||||
LaneID string
|
LaneID string
|
||||||
|
Stage pipeline.ModuleStage
|
||||||
SlotName string
|
SlotName string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -735,13 +739,12 @@ func parseReferenceFlags(values []string) ([]cliReferenceRequest, error) {
|
|||||||
if strings.TrimSpace(source) == "" {
|
if strings.TrimSpace(source) == "" {
|
||||||
return nil, fmt.Errorf("--reference path must not be empty; use --without-reference to unbind")
|
return nil, fmt.Errorf("--reference path must not be empty; use --without-reference to unbind")
|
||||||
}
|
}
|
||||||
laneID, slotName, err := parseReferenceSelector(name, "--reference")
|
selector, err := parseReferenceSelector(name, "--reference")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
requests = append(requests, cliReferenceRequest{
|
requests = append(requests, cliReferenceRequest{
|
||||||
LaneID: laneID,
|
Selector: selector,
|
||||||
SlotName: slotName,
|
|
||||||
Source: strings.TrimSpace(source),
|
Source: strings.TrimSpace(source),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -755,39 +758,51 @@ func parseReferenceUnbindFlags(values []string) ([]cliReferenceUnbindRequest, er
|
|||||||
requests := make([]cliReferenceUnbindRequest, 0, len(values))
|
requests := make([]cliReferenceUnbindRequest, 0, len(values))
|
||||||
for _, raw := range values {
|
for _, raw := range values {
|
||||||
if strings.Contains(raw, "=") {
|
if strings.Contains(raw, "=") {
|
||||||
return nil, fmt.Errorf("--without-reference must use slot or lane.slot")
|
return nil, fmt.Errorf("--without-reference must use a reference selector without =path")
|
||||||
}
|
}
|
||||||
laneID, slotName, err := parseReferenceSelector(raw, "--without-reference")
|
selector, err := parseReferenceSelector(raw, "--without-reference")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
requests = append(requests, cliReferenceUnbindRequest{
|
requests = append(requests, cliReferenceUnbindRequest{
|
||||||
LaneID: laneID,
|
Selector: selector,
|
||||||
SlotName: slotName,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return requests, nil
|
return requests, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func parseReferenceSelector(raw string, flagName string) (string, string, error) {
|
func parseReferenceSelector(raw string, flagName string) (cliReferenceSelector, error) {
|
||||||
selector := strings.TrimSpace(raw)
|
selector := strings.TrimSpace(raw)
|
||||||
if selector == "" {
|
if selector == "" {
|
||||||
return "", "", fmt.Errorf("%s reference slot must not be empty", flagName)
|
return cliReferenceSelector{}, fmt.Errorf("%s reference slot must not be empty", flagName)
|
||||||
}
|
}
|
||||||
if strings.Count(selector, ".") > 1 {
|
parts := strings.Split(selector, ".")
|
||||||
return "", "", fmt.Errorf("%s must use slot or lane.slot", flagName)
|
for _, part := range parts {
|
||||||
}
|
if strings.TrimSpace(part) == "" {
|
||||||
laneID := ""
|
return cliReferenceSelector{}, fmt.Errorf("%s must use non-empty reference selector values", flagName)
|
||||||
slotName := selector
|
|
||||||
if strings.Contains(selector, ".") {
|
|
||||||
before, after, _ := strings.Cut(selector, ".")
|
|
||||||
laneID = strings.TrimSpace(before)
|
|
||||||
slotName = strings.TrimSpace(after)
|
|
||||||
if laneID == "" || slotName == "" {
|
|
||||||
return "", "", fmt.Errorf("%s must use non-empty lane.slot values", flagName)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return laneID, slotName, nil
|
switch len(parts) {
|
||||||
|
case 1:
|
||||||
|
return cliReferenceSelector{SlotName: strings.TrimSpace(parts[0])}, nil
|
||||||
|
case 2:
|
||||||
|
first := strings.TrimSpace(parts[0])
|
||||||
|
slotName := strings.TrimSpace(parts[1])
|
||||||
|
if first == string(pipeline.StageChunk) {
|
||||||
|
return cliReferenceSelector{Stage: pipeline.StageChunk, SlotName: slotName}, nil
|
||||||
|
}
|
||||||
|
return cliReferenceSelector{LaneID: first, SlotName: slotName}, nil
|
||||||
|
case 3:
|
||||||
|
laneID := strings.TrimSpace(parts[0])
|
||||||
|
stage := pipeline.ModuleStage(strings.TrimSpace(parts[1]))
|
||||||
|
slotName := strings.TrimSpace(parts[2])
|
||||||
|
if stage != pipeline.StageExtract && stage != pipeline.StageNormalize {
|
||||||
|
return cliReferenceSelector{}, fmt.Errorf("%s lane-qualified selector must use lane.extract.slot or lane.normalize.slot", flagName)
|
||||||
|
}
|
||||||
|
return cliReferenceSelector{LaneID: laneID, Stage: stage, SlotName: slotName}, nil
|
||||||
|
default:
|
||||||
|
return cliReferenceSelector{}, fmt.Errorf("%s must use slot, chunk.slot, lane.slot, lane.extract.slot, or lane.normalize.slot", flagName)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveCLIReferenceRequests(
|
func resolveCLIReferenceRequests(
|
||||||
@@ -802,20 +817,21 @@ func resolveCLIReferenceRequests(
|
|||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
selected, err := selectedReferenceLanes(cfg, pipelineID, only, catalog)
|
targets, err := selectedReferenceTargets(cfg, pipelineID, only, catalog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
overrides := make([]pipeline.ReferenceBinding, 0, len(referenceRequests))
|
overrides := make([]pipeline.ReferenceBinding, 0, len(referenceRequests))
|
||||||
for _, request := range referenceRequests {
|
for _, request := range referenceRequests {
|
||||||
laneID, err := resolveCLIReferenceLane(selected, request.LaneID, request.SlotName)
|
target, err := resolveCLIReferenceTarget(targets, request.Selector)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
overrides = append(overrides, pipeline.ReferenceBinding{
|
overrides = append(overrides, pipeline.ReferenceBinding{
|
||||||
LaneID: laneID,
|
Stage: target.stage,
|
||||||
SlotName: request.SlotName,
|
LaneID: target.laneID,
|
||||||
|
SlotName: request.Selector.SlotName,
|
||||||
Source: request.Source,
|
Source: request.Source,
|
||||||
BindingSource: contracts.ReferenceBindingSourceCLI,
|
BindingSource: contracts.ReferenceBindingSourceCLI,
|
||||||
})
|
})
|
||||||
@@ -823,25 +839,28 @@ func resolveCLIReferenceRequests(
|
|||||||
|
|
||||||
unbinds := make([]pipeline.ReferenceUnbind, 0, len(unbindRequests))
|
unbinds := make([]pipeline.ReferenceUnbind, 0, len(unbindRequests))
|
||||||
for _, request := range unbindRequests {
|
for _, request := range unbindRequests {
|
||||||
laneID, err := resolveCLIReferenceLane(selected, request.LaneID, request.SlotName)
|
target, err := resolveCLIReferenceTarget(targets, request.Selector)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
unbinds = append(unbinds, pipeline.ReferenceUnbind{
|
unbinds = append(unbinds, pipeline.ReferenceUnbind{
|
||||||
LaneID: laneID,
|
Stage: target.stage,
|
||||||
SlotName: request.SlotName,
|
LaneID: target.laneID,
|
||||||
|
SlotName: request.Selector.SlotName,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
return overrides, unbinds, nil
|
return overrides, unbinds, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type selectedReferenceLane struct {
|
type selectedReferenceTarget struct {
|
||||||
id string
|
laneID string
|
||||||
slots map[string]struct{}
|
stage pipeline.ModuleStage
|
||||||
|
module string
|
||||||
|
slots map[string]struct{}
|
||||||
}
|
}
|
||||||
|
|
||||||
func selectedReferenceLanes(cfg config.Config, pipelineID string, only []string, catalog pipeline.ModuleCatalog) ([]selectedReferenceLane, error) {
|
func selectedReferenceTargets(cfg config.Config, pipelineID string, only []string, catalog pipeline.ModuleCatalog) ([]selectedReferenceTarget, error) {
|
||||||
profile, ok := lookupCLIReferencePipeline(cfg.Pipelines, pipelineID)
|
profile, ok := lookupCLIReferencePipeline(cfg.Pipelines, pipelineID)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("pipeline %q is not configured", strings.TrimSpace(pipelineID))
|
return nil, fmt.Errorf("pipeline %q is not configured", strings.TrimSpace(pipelineID))
|
||||||
@@ -882,27 +901,55 @@ func selectedReferenceLanes(cfg config.Config, pipelineID string, only []string,
|
|||||||
}
|
}
|
||||||
sort.Strings(selectedIDs)
|
sort.Strings(selectedIDs)
|
||||||
|
|
||||||
selected := make([]selectedReferenceLane, 0, len(selectedIDs))
|
targets := make([]selectedReferenceTarget, 0, 1+len(selectedIDs)*2)
|
||||||
|
chunk := pipeline.Binding(profile.Chunk.Module)
|
||||||
|
chunk.Module = strings.TrimSpace(profile.Chunk.Module)
|
||||||
|
if chunk.Module == "" {
|
||||||
|
chunk.Module = pipeline.DefaultChunkModule
|
||||||
|
}
|
||||||
|
chunkSpec, err := cliReferenceChunkerSpec(catalog, chunk.Module)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("pipeline %q chunk module %q: %w", strings.TrimSpace(pipelineID), chunk.Module, err)
|
||||||
|
}
|
||||||
|
targets = append(targets, selectedReferenceTarget{
|
||||||
|
stage: pipeline.StageChunk,
|
||||||
|
module: chunk.Module,
|
||||||
|
slots: referenceSlotSet(chunkSpec.ReferenceSlots),
|
||||||
|
})
|
||||||
|
|
||||||
for _, laneID := range selectedIDs {
|
for _, laneID := range selectedIDs {
|
||||||
lane := lanesByID[laneID]
|
lane := lanesByID[laneID]
|
||||||
extractModule := strings.TrimSpace(lane.Extract.Module)
|
extractModule := strings.TrimSpace(lane.Extract.Module)
|
||||||
if extractModule == "" {
|
if extractModule == "" {
|
||||||
return nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", strings.TrimSpace(pipelineID), laneID)
|
return nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", strings.TrimSpace(pipelineID), laneID)
|
||||||
}
|
}
|
||||||
if catalog.Extractors == nil {
|
extractSpec, err := cliReferenceExtractorSpec(catalog, extractModule)
|
||||||
return nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", strings.TrimSpace(pipelineID), laneID, extractModule, extractModule)
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("pipeline %q lane %q extract module %q: %w", strings.TrimSpace(pipelineID), laneID, extractModule, err)
|
||||||
}
|
}
|
||||||
spec, ok := catalog.Extractors.Spec(extractModule)
|
targets = append(targets, selectedReferenceTarget{
|
||||||
if !ok {
|
laneID: laneID,
|
||||||
return nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", strings.TrimSpace(pipelineID), laneID, extractModule, extractModule)
|
stage: pipeline.StageExtract,
|
||||||
|
module: extractModule,
|
||||||
|
slots: referenceSlotSet(extractSpec.ReferenceSlots),
|
||||||
|
})
|
||||||
|
|
||||||
|
normalizeModule := strings.TrimSpace(lane.Normalize.Module)
|
||||||
|
if normalizeModule == "" {
|
||||||
|
normalizeModule = pipeline.DefaultNormalizeModule
|
||||||
}
|
}
|
||||||
slotSet := make(map[string]struct{}, len(spec.ReferenceSlots))
|
normalizeSpec, err := cliReferenceNormalizerSpec(catalog, normalizeModule)
|
||||||
for _, slot := range spec.ReferenceSlots {
|
if err != nil {
|
||||||
slotSet[slot.Name] = struct{}{}
|
return nil, fmt.Errorf("pipeline %q lane %q normalize module %q: %w", strings.TrimSpace(pipelineID), laneID, normalizeModule, err)
|
||||||
}
|
}
|
||||||
selected = append(selected, selectedReferenceLane{id: laneID, slots: slotSet})
|
targets = append(targets, selectedReferenceTarget{
|
||||||
|
laneID: laneID,
|
||||||
|
stage: pipeline.StageNormalize,
|
||||||
|
module: normalizeModule,
|
||||||
|
slots: referenceSlotSet(normalizeSpec.ReferenceSlots),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
return selected, nil
|
return targets, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func lookupCLIReferencePipeline(profiles map[string]pipeline.PipelineProfile, pipelineID string) (pipeline.PipelineProfile, bool) {
|
func lookupCLIReferencePipeline(profiles map[string]pipeline.PipelineProfile, pipelineID string) (pipeline.PipelineProfile, bool) {
|
||||||
@@ -915,37 +962,152 @@ func lookupCLIReferencePipeline(profiles map[string]pipeline.PipelineProfile, pi
|
|||||||
return pipeline.PipelineProfile{}, false
|
return pipeline.PipelineProfile{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveCLIReferenceLane(selected []selectedReferenceLane, requestedLaneID string, slotName string) (string, error) {
|
func cliReferenceChunkerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
|
||||||
slotName = strings.TrimSpace(slotName)
|
if catalog.Chunkers == nil {
|
||||||
requestedLaneID = strings.TrimSpace(requestedLaneID)
|
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
||||||
if requestedLaneID != "" {
|
}
|
||||||
for _, lane := range selected {
|
spec, ok := catalog.Chunkers.Spec(module)
|
||||||
if lane.id == requestedLaneID {
|
if !ok {
|
||||||
if _, ok := lane.slots[slotName]; !ok {
|
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
||||||
return "", fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, requestedLaneID)
|
}
|
||||||
|
return spec, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cliReferenceExtractorSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
|
||||||
|
if catalog.Extractors == nil {
|
||||||
|
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
||||||
|
}
|
||||||
|
spec, ok := catalog.Extractors.Spec(module)
|
||||||
|
if !ok {
|
||||||
|
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
||||||
|
}
|
||||||
|
return spec, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cliReferenceNormalizerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
|
||||||
|
if catalog.Normalizers == nil {
|
||||||
|
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
||||||
|
}
|
||||||
|
spec, ok := catalog.Normalizers.Spec(module)
|
||||||
|
if !ok {
|
||||||
|
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
||||||
|
}
|
||||||
|
return spec, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func referenceSlotSet(slots []contracts.ReferenceSlot) map[string]struct{} {
|
||||||
|
slotSet := make(map[string]struct{}, len(slots))
|
||||||
|
for _, slot := range slots {
|
||||||
|
slotSet[slot.Name] = struct{}{}
|
||||||
|
}
|
||||||
|
return slotSet
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveCLIReferenceTarget(targets []selectedReferenceTarget, selector cliReferenceSelector) (selectedReferenceTarget, error) {
|
||||||
|
slotName := strings.TrimSpace(selector.SlotName)
|
||||||
|
if slotName == "" {
|
||||||
|
return selectedReferenceTarget{}, fmt.Errorf("reference slot must not be empty")
|
||||||
|
}
|
||||||
|
if selector.Stage == pipeline.StageChunk {
|
||||||
|
for _, target := range targets {
|
||||||
|
if target.stage != pipeline.StageChunk {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := target.slots[slotName]; !ok {
|
||||||
|
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by chunk module %q", slotName, target.module)
|
||||||
|
}
|
||||||
|
return target, nil
|
||||||
|
}
|
||||||
|
return selectedReferenceTarget{}, fmt.Errorf("reference chunk target is not selected")
|
||||||
|
}
|
||||||
|
if selector.Stage == pipeline.StageExtract || selector.Stage == pipeline.StageNormalize {
|
||||||
|
for _, target := range targets {
|
||||||
|
if target.laneID == selector.LaneID && target.stage == selector.Stage {
|
||||||
|
if _, ok := target.slots[slotName]; !ok {
|
||||||
|
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected %s target %q", slotName, selector.Stage, targetLabel(target))
|
||||||
}
|
}
|
||||||
return requestedLaneID, nil
|
return target, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return "", fmt.Errorf("reference lane %q is not selected", requestedLaneID)
|
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", selector.LaneID)
|
||||||
}
|
}
|
||||||
|
if strings.TrimSpace(selector.LaneID) != "" {
|
||||||
|
return resolveCLIReferenceLaneTarget(targets, strings.TrimSpace(selector.LaneID), slotName)
|
||||||
|
}
|
||||||
|
return resolveCLIReferenceFlatTarget(targets, slotName)
|
||||||
|
}
|
||||||
|
|
||||||
matches := make([]string, 0, 1)
|
func resolveCLIReferenceLaneTarget(targets []selectedReferenceTarget, laneID string, slotName string) (selectedReferenceTarget, error) {
|
||||||
for _, lane := range selected {
|
laneSelected := false
|
||||||
if _, ok := lane.slots[slotName]; ok {
|
matches := make([]selectedReferenceTarget, 0, 2)
|
||||||
matches = append(matches, lane.id)
|
for _, target := range targets {
|
||||||
|
if target.laneID != laneID {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
laneSelected = true
|
||||||
|
if _, ok := target.slots[slotName]; ok {
|
||||||
|
matches = append(matches, target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !laneSelected {
|
||||||
|
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", laneID)
|
||||||
|
}
|
||||||
|
switch len(matches) {
|
||||||
|
case 0:
|
||||||
|
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, laneID)
|
||||||
|
case 1:
|
||||||
|
return matches[0], nil
|
||||||
|
default:
|
||||||
|
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets in lane %q (%s); use %s.extract.%s or %s.normalize.%s", slotName, laneID, targetList(matches), laneID, slotName, laneID, slotName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveCLIReferenceFlatTarget(targets []selectedReferenceTarget, slotName string) (selectedReferenceTarget, error) {
|
||||||
|
matches := make([]selectedReferenceTarget, 0, 2)
|
||||||
|
for _, target := range targets {
|
||||||
|
if _, ok := target.slots[slotName]; ok {
|
||||||
|
matches = append(matches, target)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
switch len(matches) {
|
switch len(matches) {
|
||||||
case 0:
|
case 0:
|
||||||
return "", fmt.Errorf("reference slot %q is not declared by any selected lane", slotName)
|
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected reference target", slotName)
|
||||||
case 1:
|
case 1:
|
||||||
return matches[0], nil
|
return matches[0], nil
|
||||||
default:
|
default:
|
||||||
return "", fmt.Errorf("reference slot %q is declared by multiple selected lanes (%s); use lane.slot", slotName, strings.Join(matches, ", "))
|
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets (%s); use a more specific selector such as %s", slotName, targetList(matches), selectorSuggestions(matches, slotName))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func targetList(targets []selectedReferenceTarget) string {
|
||||||
|
labels := make([]string, 0, len(targets))
|
||||||
|
for _, target := range targets {
|
||||||
|
labels = append(labels, targetLabel(target))
|
||||||
|
}
|
||||||
|
sort.Strings(labels)
|
||||||
|
return strings.Join(labels, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func targetLabel(target selectedReferenceTarget) string {
|
||||||
|
if target.stage == pipeline.StageChunk {
|
||||||
|
return "chunk"
|
||||||
|
}
|
||||||
|
return target.laneID + "." + string(target.stage)
|
||||||
|
}
|
||||||
|
|
||||||
|
func selectorSuggestions(targets []selectedReferenceTarget, slotName string) string {
|
||||||
|
suggestions := make([]string, 0, len(targets))
|
||||||
|
for _, target := range targets {
|
||||||
|
if target.stage == pipeline.StageChunk {
|
||||||
|
suggestions = append(suggestions, "chunk."+slotName)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
suggestions = append(suggestions, target.laneID+"."+string(target.stage)+"."+slotName)
|
||||||
|
}
|
||||||
|
sort.Strings(suggestions)
|
||||||
|
return strings.Join(suggestions, " or ")
|
||||||
|
}
|
||||||
|
|
||||||
func sortedPipelineIDs(cfg config.Config) []string {
|
func sortedPipelineIDs(cfg config.Config) []string {
|
||||||
ids := make([]string, 0, len(cfg.Pipelines))
|
ids := make([]string, 0, len(cfg.Pipelines))
|
||||||
for id := range cfg.Pipelines {
|
for id := range cfg.Pipelines {
|
||||||
|
|||||||
@@ -769,7 +769,7 @@ func TestRunPipelineReferenceFlagBindsUnambiguousSlot(t *testing.T) {
|
|||||||
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
|
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
|
||||||
}
|
}
|
||||||
resolved := readResolvedPipeline(t, diagnosticsDir)
|
resolved := readResolvedPipeline(t, diagnosticsDir)
|
||||||
refs := resolved.ArtifactLanes[0].References
|
refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings
|
||||||
want := []pipeline.ReferenceBinding{
|
want := []pipeline.ReferenceBinding{
|
||||||
{LaneID: "events", SlotName: "roster", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI},
|
{LaneID: "events", SlotName: "roster", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI},
|
||||||
}
|
}
|
||||||
@@ -810,12 +810,199 @@ func TestRunPipelineReferenceFlagBindsLaneQualifiedSlot(t *testing.T) {
|
|||||||
}
|
}
|
||||||
resolved := readResolvedPipeline(t, diagnosticsDir)
|
resolved := readResolvedPipeline(t, diagnosticsDir)
|
||||||
events := resolvedArtifactLane(t, resolved, "events")
|
events := resolvedArtifactLane(t, resolved, "events")
|
||||||
if len(events.References) != 0 {
|
if len(events.ExtractReferences.Bindings) != 0 {
|
||||||
t.Fatalf("events references = %#v, want none", events.References)
|
t.Fatalf("events references = %#v, want none", events.ExtractReferences.Bindings)
|
||||||
}
|
}
|
||||||
notes := resolvedArtifactLane(t, resolved, "notes")
|
notes := resolvedArtifactLane(t, resolved, "notes")
|
||||||
if len(notes.References) != 1 || notes.References[0].Source != referencePath {
|
if len(notes.ExtractReferences.Bindings) != 1 || notes.ExtractReferences.Bindings[0].Source != referencePath {
|
||||||
t.Fatalf("notes references = %#v, want lane-qualified binding", notes.References)
|
t.Fatalf("notes references = %#v, want lane-qualified binding", notes.ExtractReferences.Bindings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPipelineReferenceFlagBindsChunkQualifiedSlot(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, testConfigYAML("example", "events"))
|
||||||
|
inputPath := filepath.Join(t.TempDir(), "missing.json")
|
||||||
|
referencePath := writeFile(t, "scenes.md", "Scenes\n")
|
||||||
|
diagnosticsDir := t.TempDir()
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{
|
||||||
|
"run", "example",
|
||||||
|
"--config", configPath,
|
||||||
|
"--input", inputPath,
|
||||||
|
"--diagnostics-dir", diagnosticsDir,
|
||||||
|
"--reference", "chunk.scene_guide=" + referencePath,
|
||||||
|
}, &stdout, &stderr, Options{
|
||||||
|
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
|
||||||
|
Key: "generic",
|
||||||
|
Stage: pipeline.StageChunk,
|
||||||
|
Requires: []string{"source"},
|
||||||
|
Provides: []string{"chunks"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "scene_guide"},
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
|
||||||
|
if code != 1 || !strings.Contains(stderr.String(), "read input") {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
|
||||||
|
}
|
||||||
|
resolved := readResolvedPipeline(t, diagnosticsDir)
|
||||||
|
refs := resolved.ChunkReferences.Bindings
|
||||||
|
want := []pipeline.ReferenceBinding{
|
||||||
|
{SlotName: "scene_guide", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(refs, want) {
|
||||||
|
t.Fatalf("chunk references = %#v, want %#v", refs, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPipelineReferenceFlagBindsExplicitExtractSlot(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, testConfigYAML("example", "events"))
|
||||||
|
inputPath := filepath.Join(t.TempDir(), "missing.json")
|
||||||
|
referencePath := writeFile(t, "roster.yml", "Aria\n")
|
||||||
|
diagnosticsDir := t.TempDir()
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{
|
||||||
|
"run", "example",
|
||||||
|
"--config", configPath,
|
||||||
|
"--input", inputPath,
|
||||||
|
"--diagnostics-dir", diagnosticsDir,
|
||||||
|
"--reference", "events.extract.roster=" + referencePath,
|
||||||
|
}, &stdout, &stderr, Options{
|
||||||
|
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
|
||||||
|
Key: "fake/extract",
|
||||||
|
Stage: pipeline.StageExtract,
|
||||||
|
Requires: []string{"chunks"},
|
||||||
|
Provides: []string{"artifact"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "roster"},
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
|
||||||
|
if code != 1 || !strings.Contains(stderr.String(), "read input") {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
|
||||||
|
}
|
||||||
|
resolved := readResolvedPipeline(t, diagnosticsDir)
|
||||||
|
refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings
|
||||||
|
want := []pipeline.ReferenceBinding{
|
||||||
|
{LaneID: "events", SlotName: "roster", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(refs, want) {
|
||||||
|
t.Fatalf("extract references = %#v, want %#v", refs, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPipelineReferenceFlagBindsExplicitNormalizeSlot(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, testConfigYAML("example", "events"))
|
||||||
|
inputPath := filepath.Join(t.TempDir(), "missing.json")
|
||||||
|
referencePath := writeFile(t, "normalize.md", "Normalize\n")
|
||||||
|
diagnosticsDir := t.TempDir()
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{
|
||||||
|
"run", "example",
|
||||||
|
"--config", configPath,
|
||||||
|
"--input", inputPath,
|
||||||
|
"--diagnostics-dir", diagnosticsDir,
|
||||||
|
"--reference", "events.normalize.notes=" + referencePath,
|
||||||
|
}, &stdout, &stderr, Options{
|
||||||
|
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
|
||||||
|
Key: "noop",
|
||||||
|
Stage: pipeline.StageNormalize,
|
||||||
|
Requires: []string{"merged"},
|
||||||
|
Provides: []string{"normalized"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "notes"},
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
|
||||||
|
if code != 1 || !strings.Contains(stderr.String(), "read input") {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
|
||||||
|
}
|
||||||
|
resolved := readResolvedPipeline(t, diagnosticsDir)
|
||||||
|
refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings
|
||||||
|
want := []pipeline.ReferenceBinding{
|
||||||
|
{LaneID: "events", SlotName: "notes", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(refs, want) {
|
||||||
|
t.Fatalf("normalize references = %#v, want %#v", refs, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPipelineReferenceFlagBindsFlatSlotAcrossOneTarget(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, testConfigYAML("example", "events"))
|
||||||
|
inputPath := filepath.Join(t.TempDir(), "missing.json")
|
||||||
|
referencePath := writeFile(t, "normalize.md", "Normalize\n")
|
||||||
|
diagnosticsDir := t.TempDir()
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{
|
||||||
|
"run", "example",
|
||||||
|
"--config", configPath,
|
||||||
|
"--input", inputPath,
|
||||||
|
"--diagnostics-dir", diagnosticsDir,
|
||||||
|
"--reference", "notes=" + referencePath,
|
||||||
|
}, &stdout, &stderr, Options{
|
||||||
|
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
|
||||||
|
Key: "noop",
|
||||||
|
Stage: pipeline.StageNormalize,
|
||||||
|
Requires: []string{"merged"},
|
||||||
|
Provides: []string{"normalized"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "notes"},
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
|
||||||
|
if code != 1 || !strings.Contains(stderr.String(), "read input") {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
|
||||||
|
}
|
||||||
|
resolved := readResolvedPipeline(t, diagnosticsDir)
|
||||||
|
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 1 || refs[0].Source != referencePath {
|
||||||
|
t.Fatalf("normalize references = %#v, want flat binding", refs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPipelineReferenceFlagBindsLaneSlotAcrossOneTarget(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, testConfigYAML("example", "events"))
|
||||||
|
inputPath := filepath.Join(t.TempDir(), "missing.json")
|
||||||
|
referencePath := writeFile(t, "normalize.md", "Normalize\n")
|
||||||
|
diagnosticsDir := t.TempDir()
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{
|
||||||
|
"run", "example",
|
||||||
|
"--config", configPath,
|
||||||
|
"--input", inputPath,
|
||||||
|
"--diagnostics-dir", diagnosticsDir,
|
||||||
|
"--reference", "events.notes=" + referencePath,
|
||||||
|
}, &stdout, &stderr, Options{
|
||||||
|
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
|
||||||
|
Key: "noop",
|
||||||
|
Stage: pipeline.StageNormalize,
|
||||||
|
Requires: []string{"merged"},
|
||||||
|
Provides: []string{"normalized"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "notes"},
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
|
||||||
|
if code != 1 || !strings.Contains(stderr.String(), "read input") {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
|
||||||
|
}
|
||||||
|
resolved := readResolvedPipeline(t, diagnosticsDir)
|
||||||
|
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 1 || refs[0].Source != referencePath {
|
||||||
|
t.Fatalf("normalize references = %#v, want lane-qualified binding", refs)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -847,11 +1034,99 @@ func TestRunPipelineReferenceFlagRejectsAmbiguousFlatSlot(t *testing.T) {
|
|||||||
if code != 1 {
|
if code != 1 {
|
||||||
t.Fatalf("RunWithOptions() code = %d, want 1", code)
|
t.Fatalf("RunWithOptions() code = %d, want 1", code)
|
||||||
}
|
}
|
||||||
if !strings.Contains(stderr.String(), "multiple selected lanes") || !strings.Contains(stderr.String(), "lane.slot") {
|
if !strings.Contains(stderr.String(), "multiple selected targets") || !strings.Contains(stderr.String(), "events.extract.roster") || !strings.Contains(stderr.String(), "notes.extract.roster") {
|
||||||
t.Fatalf("stderr = %q, want ambiguous reference error", stderr.String())
|
t.Fatalf("stderr = %q, want ambiguous reference error", stderr.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunPipelineReferenceFlagRejectsAmbiguousFlatSlotAcrossChunkAndExtract(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, testConfigYAML("example", "events"))
|
||||||
|
inputPath := writeSeriatimInput(t)
|
||||||
|
diagnosticsDir := t.TempDir()
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{
|
||||||
|
"run", "example",
|
||||||
|
"--config", configPath,
|
||||||
|
"--input", inputPath,
|
||||||
|
"--diagnostics-dir", diagnosticsDir,
|
||||||
|
"--reference", "context=./context.md",
|
||||||
|
}, &stdout, &stderr, Options{
|
||||||
|
Catalog: fakeCatalog(t,
|
||||||
|
pipeline.ModuleSpec{
|
||||||
|
Key: "generic",
|
||||||
|
Stage: pipeline.StageChunk,
|
||||||
|
Requires: []string{"source"},
|
||||||
|
Provides: []string{"chunks"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "context"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
pipeline.ModuleSpec{
|
||||||
|
Key: "fake/extract",
|
||||||
|
Stage: pipeline.StageExtract,
|
||||||
|
Requires: []string{"chunks"},
|
||||||
|
Provides: []string{"artifact"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "context"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
),
|
||||||
|
})
|
||||||
|
|
||||||
|
if code != 1 {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d, want 1", code)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "multiple selected targets") || !strings.Contains(stderr.String(), "chunk.context") || !strings.Contains(stderr.String(), "events.extract.context") {
|
||||||
|
t.Fatalf("stderr = %q, want ambiguous chunk/extract reference error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPipelineReferenceFlagRejectsAmbiguousLaneSlotAcrossExtractAndNormalize(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, testConfigYAML("example", "events"))
|
||||||
|
inputPath := writeSeriatimInput(t)
|
||||||
|
diagnosticsDir := t.TempDir()
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{
|
||||||
|
"run", "example",
|
||||||
|
"--config", configPath,
|
||||||
|
"--input", inputPath,
|
||||||
|
"--diagnostics-dir", diagnosticsDir,
|
||||||
|
"--reference", "events.context=./context.md",
|
||||||
|
}, &stdout, &stderr, Options{
|
||||||
|
Catalog: fakeCatalog(t,
|
||||||
|
pipeline.ModuleSpec{
|
||||||
|
Key: "fake/extract",
|
||||||
|
Stage: pipeline.StageExtract,
|
||||||
|
Requires: []string{"chunks"},
|
||||||
|
Provides: []string{"artifact"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "context"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
pipeline.ModuleSpec{
|
||||||
|
Key: "noop",
|
||||||
|
Stage: pipeline.StageNormalize,
|
||||||
|
Requires: []string{"merged"},
|
||||||
|
Provides: []string{"normalized"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "context"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
),
|
||||||
|
})
|
||||||
|
|
||||||
|
if code != 1 {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d, want 1", code)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "multiple selected targets in lane") || !strings.Contains(stderr.String(), "events.extract.context") || !strings.Contains(stderr.String(), "events.normalize.context") {
|
||||||
|
t.Fatalf("stderr = %q, want ambiguous lane reference error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunPipelineReferenceFlagsRejectMalformedValues(t *testing.T) {
|
func TestRunPipelineReferenceFlagsRejectMalformedValues(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -861,8 +1136,9 @@ func TestRunPipelineReferenceFlagsRejectMalformedValues(t *testing.T) {
|
|||||||
{name: "missing equals", args: []string{"--reference", "roster"}, want: "slot=path"},
|
{name: "missing equals", args: []string{"--reference", "roster"}, want: "slot=path"},
|
||||||
{name: "empty path", args: []string{"--reference", "roster="}, want: "path must not be empty"},
|
{name: "empty path", args: []string{"--reference", "roster="}, want: "path must not be empty"},
|
||||||
{name: "empty slot", args: []string{"--reference", "=./roster.yml"}, want: "slot must not be empty"},
|
{name: "empty slot", args: []string{"--reference", "=./roster.yml"}, want: "slot must not be empty"},
|
||||||
{name: "too many selector parts", args: []string{"--reference", "a.b.c=./roster.yml"}, want: "lane.slot"},
|
{name: "unsupported explicit stage", args: []string{"--reference", "a.b.c=./roster.yml"}, want: "lane.extract.slot or lane.normalize.slot"},
|
||||||
{name: "unbind with equals", args: []string{"--without-reference", "roster=./roster.yml"}, want: "slot or lane.slot"},
|
{name: "too many selector parts", args: []string{"--reference", "a.b.c.d=./roster.yml"}, want: "slot, chunk.slot, lane.slot, lane.extract.slot, or lane.normalize.slot"},
|
||||||
|
{name: "unbind with equals", args: []string{"--without-reference", "roster=./roster.yml"}, want: "without =path"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
@@ -885,6 +1161,72 @@ func TestRunPipelineReferenceFlagsRejectMalformedValues(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunPipelineWithoutReferenceRemovesConfigBindingsForEligibleTargets(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, testConfigYAMLWithPipelineReferences("example", "events", map[string]string{
|
||||||
|
"context": "./config-context.md",
|
||||||
|
"notes": "./config-notes.md",
|
||||||
|
"roster": "./config-roster.yml",
|
||||||
|
}))
|
||||||
|
inputPath := filepath.Join(t.TempDir(), "missing.json")
|
||||||
|
diagnosticsDir := t.TempDir()
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{
|
||||||
|
"run", "example",
|
||||||
|
"--config", configPath,
|
||||||
|
"--input", inputPath,
|
||||||
|
"--diagnostics-dir", diagnosticsDir,
|
||||||
|
"--without-reference", "chunk.context",
|
||||||
|
"--without-reference", "events.extract.roster",
|
||||||
|
"--without-reference", "events.normalize.notes",
|
||||||
|
}, &stdout, &stderr, Options{
|
||||||
|
Catalog: fakeCatalog(t,
|
||||||
|
pipeline.ModuleSpec{
|
||||||
|
Key: "generic",
|
||||||
|
Stage: pipeline.StageChunk,
|
||||||
|
Requires: []string{"source"},
|
||||||
|
Provides: []string{"chunks"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "context"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
pipeline.ModuleSpec{
|
||||||
|
Key: "fake/extract",
|
||||||
|
Stage: pipeline.StageExtract,
|
||||||
|
Requires: []string{"chunks"},
|
||||||
|
Provides: []string{"artifact"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "roster"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
pipeline.ModuleSpec{
|
||||||
|
Key: "noop",
|
||||||
|
Stage: pipeline.StageNormalize,
|
||||||
|
Requires: []string{"merged"},
|
||||||
|
Provides: []string{"normalized"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "notes"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
),
|
||||||
|
})
|
||||||
|
|
||||||
|
if code != 1 || !strings.Contains(stderr.String(), "read input") {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
|
||||||
|
}
|
||||||
|
resolved := readResolvedPipeline(t, diagnosticsDir)
|
||||||
|
if refs := resolved.ChunkReferences.Bindings; len(refs) != 0 {
|
||||||
|
t.Fatalf("chunk references = %#v, want none", refs)
|
||||||
|
}
|
||||||
|
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
|
||||||
|
t.Fatalf("extract references = %#v, want none", refs)
|
||||||
|
}
|
||||||
|
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
|
||||||
|
t.Fatalf("normalize references = %#v, want none", refs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunPipelineWithoutReferenceRemovesOptionalConfigBinding(t *testing.T) {
|
func TestRunPipelineWithoutReferenceRemovesOptionalConfigBinding(t *testing.T) {
|
||||||
configPath := writeTestConfig(t, testConfigYAMLWithReferences("example", "events", map[string]string{"roster": "./config-roster.yml"}))
|
configPath := writeTestConfig(t, testConfigYAMLWithReferences("example", "events", map[string]string{"roster": "./config-roster.yml"}))
|
||||||
inputPath := filepath.Join(t.TempDir(), "missing.json")
|
inputPath := filepath.Join(t.TempDir(), "missing.json")
|
||||||
@@ -914,11 +1256,112 @@ func TestRunPipelineWithoutReferenceRemovesOptionalConfigBinding(t *testing.T) {
|
|||||||
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
|
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
|
||||||
}
|
}
|
||||||
resolved := readResolvedPipeline(t, diagnosticsDir)
|
resolved := readResolvedPipeline(t, diagnosticsDir)
|
||||||
if refs := resolved.ArtifactLanes[0].References; len(refs) != 0 {
|
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
|
||||||
t.Fatalf("references = %#v, want unbound optional slot", refs)
|
t.Fatalf("references = %#v, want unbound optional slot", refs)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunPipelineWithoutReferenceFailsWhenRequiredChunkSlotWouldBeMissing(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, testConfigYAMLWithPipelineReferences("example", "events", map[string]string{"context": "./config-context.md"}))
|
||||||
|
inputPath := writeSeriatimInput(t)
|
||||||
|
diagnosticsDir := t.TempDir()
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{
|
||||||
|
"run", "example",
|
||||||
|
"--config", configPath,
|
||||||
|
"--input", inputPath,
|
||||||
|
"--diagnostics-dir", diagnosticsDir,
|
||||||
|
"--without-reference", "chunk.context",
|
||||||
|
}, &stdout, &stderr, Options{
|
||||||
|
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
|
||||||
|
Key: "generic",
|
||||||
|
Stage: pipeline.StageChunk,
|
||||||
|
Requires: []string{"source"},
|
||||||
|
Provides: []string{"chunks"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "context", Required: true},
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
|
||||||
|
if code != 1 {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d, want 1", code)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "required reference slot") || !strings.Contains(stderr.String(), "context") {
|
||||||
|
t.Fatalf("stderr = %q, want required chunk reference error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPipelineWithoutReferenceFailsWhenRequiredNormalizeSlotWouldBeMissing(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, testConfigYAMLWithPipelineReferences("example", "events", map[string]string{"notes": "./config-notes.md"}))
|
||||||
|
inputPath := writeSeriatimInput(t)
|
||||||
|
diagnosticsDir := t.TempDir()
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{
|
||||||
|
"run", "example",
|
||||||
|
"--config", configPath,
|
||||||
|
"--input", inputPath,
|
||||||
|
"--diagnostics-dir", diagnosticsDir,
|
||||||
|
"--without-reference", "events.normalize.notes",
|
||||||
|
}, &stdout, &stderr, Options{
|
||||||
|
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
|
||||||
|
Key: "noop",
|
||||||
|
Stage: pipeline.StageNormalize,
|
||||||
|
Requires: []string{"merged"},
|
||||||
|
Provides: []string{"normalized"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "notes", Required: true},
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
|
||||||
|
if code != 1 {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d, want 1", code)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "required reference slot") || !strings.Contains(stderr.String(), "notes") {
|
||||||
|
t.Fatalf("stderr = %q, want required normalize reference error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPipelineReferenceFlagRejectsUnselectedLaneSelector(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, testConfigYAML("example", "events", "notes"))
|
||||||
|
inputPath := writeSeriatimInput(t)
|
||||||
|
referencePath := writeFile(t, "notes.yml", "Notes\n")
|
||||||
|
diagnosticsDir := t.TempDir()
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{
|
||||||
|
"run", "example",
|
||||||
|
"--config", configPath,
|
||||||
|
"--input", inputPath,
|
||||||
|
"--only", "events",
|
||||||
|
"--diagnostics-dir", diagnosticsDir,
|
||||||
|
"--reference", "notes.extract.roster=" + referencePath,
|
||||||
|
}, &stdout, &stderr, Options{
|
||||||
|
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
|
||||||
|
Key: "fake/extract",
|
||||||
|
Stage: pipeline.StageExtract,
|
||||||
|
Requires: []string{"chunks"},
|
||||||
|
Provides: []string{"artifact"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "roster"},
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
|
||||||
|
if code != 1 {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d, want 1", code)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), `reference lane "notes" is not selected`) {
|
||||||
|
t.Fatalf("stderr = %q, want unselected lane error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunPipelineWithoutReferenceFailsWhenRequiredSlotWouldBeMissing(t *testing.T) {
|
func TestRunPipelineWithoutReferenceFailsWhenRequiredSlotWouldBeMissing(t *testing.T) {
|
||||||
configPath := writeTestConfig(t, testConfigYAMLWithReferences("example", "events", map[string]string{"roster": "./config-roster.yml"}))
|
configPath := writeTestConfig(t, testConfigYAMLWithReferences("example", "events", map[string]string{"roster": "./config-roster.yml"}))
|
||||||
inputPath := writeSeriatimInput(t)
|
inputPath := writeSeriatimInput(t)
|
||||||
@@ -1609,6 +2052,27 @@ func testConfigYAMLWithReferences(pipelineID string, laneID string, references m
|
|||||||
return b.String()
|
return b.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func testConfigYAMLWithPipelineReferences(pipelineID string, laneID string, references map[string]string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString("version: 1\n")
|
||||||
|
b.WriteString("pipelines:\n")
|
||||||
|
b.WriteString(" " + pipelineID + ":\n")
|
||||||
|
b.WriteString(" input: fake/input\n")
|
||||||
|
b.WriteString(" references:\n")
|
||||||
|
keys := make([]string, 0, len(references))
|
||||||
|
for key := range references {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
for _, key := range keys {
|
||||||
|
b.WriteString(" " + key + ": " + references[key] + "\n")
|
||||||
|
}
|
||||||
|
b.WriteString(" artifacts:\n")
|
||||||
|
b.WriteString(" " + laneID + ":\n")
|
||||||
|
b.WriteString(" extract: fake/extract\n")
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
func testConfigYAMLWithReferencesAndDiagnostics(pipelineID string, laneID string, diagnosticsDir string, references map[string]string) string {
|
func testConfigYAMLWithReferencesAndDiagnostics(pipelineID string, laneID string, diagnosticsDir string, references map[string]string) string {
|
||||||
var b strings.Builder
|
var b strings.Builder
|
||||||
b.WriteString("version: 1\n")
|
b.WriteString("version: 1\n")
|
||||||
@@ -1961,6 +2425,10 @@ func (fakeRunChunker) Key() string {
|
|||||||
return "generic"
|
return "generic"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (fakeRunChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (fakeRunChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (fakeRunChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||||
return contracts.ChunkResult{
|
return contracts.ChunkResult{
|
||||||
Chunks: []contracts.SourceChunk{
|
Chunks: []contracts.SourceChunk{
|
||||||
@@ -2024,6 +2492,10 @@ func (fakeRunNormalizer) Key() string {
|
|||||||
return "noop"
|
return "noop"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (fakeRunNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (fakeRunNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
func (fakeRunNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
||||||
return contracts.NormalizeResult{Candidates: append([]artifacts.ArtifactCandidate(nil), req.Candidates...)}, nil
|
return contracts.NormalizeResult{Candidates: append([]artifacts.ArtifactCandidate(nil), req.Candidates...)}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,7 +49,8 @@ type LLMProfileManifest struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ReferenceProvenance struct {
|
type ReferenceProvenance struct {
|
||||||
LaneID string `json:"lane_id"`
|
Stage string `json:"stage,omitempty"`
|
||||||
|
LaneID string `json:"lane_id,omitempty"`
|
||||||
SlotName string `json:"slot_name"`
|
SlotName string `json:"slot_name"`
|
||||||
OriginType string `json:"origin_type"`
|
OriginType string `json:"origin_type"`
|
||||||
OriginURI string `json:"origin_uri,omitempty"`
|
OriginURI string `json:"origin_uri,omitempty"`
|
||||||
|
|||||||
@@ -187,6 +187,7 @@ func TestRunManifestIncludesReferenceProvenance(t *testing.T) {
|
|||||||
manifest := RunManifest{
|
manifest := RunManifest{
|
||||||
References: []ReferenceProvenance{
|
References: []ReferenceProvenance{
|
||||||
{
|
{
|
||||||
|
Stage: "extract",
|
||||||
LaneID: "events",
|
LaneID: "events",
|
||||||
SlotName: "roster",
|
SlotName: "roster",
|
||||||
OriginType: "file",
|
OriginType: "file",
|
||||||
@@ -212,7 +213,7 @@ func TestRunManifestIncludesReferenceProvenance(t *testing.T) {
|
|||||||
t.Fatalf("len(References) = %d, want 1", len(got.References))
|
t.Fatalf("len(References) = %d, want 1", len(got.References))
|
||||||
}
|
}
|
||||||
reference := got.References[0]
|
reference := got.References[0]
|
||||||
if reference.LaneID != "events" || reference.SlotName != "roster" || reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" {
|
if reference.Stage != "extract" || reference.LaneID != "events" || reference.SlotName != "roster" || reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" {
|
||||||
t.Fatalf("reference provenance = %#v, want lane-scoped origin details", reference)
|
t.Fatalf("reference provenance = %#v, want lane-scoped origin details", reference)
|
||||||
}
|
}
|
||||||
if reference.Digest != "sha256:reference" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != 12 || reference.BindingSource != "config" {
|
if reference.Digest != "sha256:reference" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != 12 || reference.BindingSource != "config" {
|
||||||
|
|||||||
@@ -114,6 +114,7 @@ func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
|
|||||||
if len(in.Options) > 0 {
|
if len(in.Options) > 0 {
|
||||||
out.Options = cloneOptions(in.Options)
|
out.Options = cloneOptions(in.Options)
|
||||||
}
|
}
|
||||||
|
out.References = cloneStringMap(in.References)
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -99,6 +99,7 @@ type fileModuleBinding struct {
|
|||||||
Module string
|
Module string
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
Options map[string]any
|
Options map[string]any
|
||||||
|
References map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
||||||
@@ -133,6 +134,12 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
b.Options = normalizeOptions(options)
|
b.Options = normalizeOptions(options)
|
||||||
|
case "references":
|
||||||
|
var references map[string]string
|
||||||
|
if err := valueNode.Decode(&references); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
b.References = references
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("field %s not found in module binding", keyNode.Value)
|
return fmt.Errorf("field %s not found in module binding", keyNode.Value)
|
||||||
}
|
}
|
||||||
@@ -148,6 +155,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
|
|||||||
Module: strings.TrimSpace(b.Module),
|
Module: strings.TrimSpace(b.Module),
|
||||||
LLMProfile: strings.TrimSpace(b.LLMProfile),
|
LLMProfile: strings.TrimSpace(b.LLMProfile),
|
||||||
Options: cloneOptions(b.Options),
|
Options: cloneOptions(b.Options),
|
||||||
|
References: normalizedStringMap(b.References),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -217,6 +225,19 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
|||||||
if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
|
if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if filePipeline.Chunk != nil {
|
||||||
|
if _, _, err := normalizedMapKeys(filePipeline.Chunk.References, fmt.Sprintf("pipeline %q chunk reference slot", pipelineID)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, _, err := normalizedMapKeys(filePipeline.Input.References, fmt.Sprintf("pipeline %q input reference slot", pipelineID)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if filePipeline.Output != nil {
|
||||||
|
if _, _, err := normalizedMapKeys(filePipeline.Output.References, fmt.Sprintf("pipeline %q output reference slot", pipelineID)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
for rawLaneID, fileLane := range filePipeline.Artifacts {
|
for rawLaneID, fileLane := range filePipeline.Artifacts {
|
||||||
laneID := strings.TrimSpace(rawLaneID)
|
laneID := strings.TrimSpace(rawLaneID)
|
||||||
if laneID == "" {
|
if laneID == "" {
|
||||||
@@ -225,6 +246,24 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
|||||||
if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil {
|
if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if _, _, err := normalizedMapKeys(fileLane.Extract.References, fmt.Sprintf("pipeline %q lane %q extract reference slot", pipelineID, laneID)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if fileLane.Merge != nil {
|
||||||
|
if _, _, err := normalizedMapKeys(fileLane.Merge.References, fmt.Sprintf("pipeline %q lane %q merge reference slot", pipelineID, laneID)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if fileLane.Normalize != nil {
|
||||||
|
if _, _, err := normalizedMapKeys(fileLane.Normalize.References, fmt.Sprintf("pipeline %q lane %q normalize reference slot", pipelineID, laneID)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i, validator := range fileLane.Validators {
|
||||||
|
if _, _, err := normalizedMapKeys(validator.References, fmt.Sprintf("pipeline %q lane %q validator[%d] reference slot", pipelineID, laneID, i)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -280,8 +319,10 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
|||||||
}
|
}
|
||||||
for _, laneID := range laneIDs {
|
for _, laneID := range laneIDs {
|
||||||
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
|
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
|
||||||
|
extract := fileLane.Extract.toPipelineBinding()
|
||||||
|
extract.References = mergeStringMaps(normalizedStringMap(fileLane.References), extract.References)
|
||||||
lane := pipeline.ArtifactLaneProfile{
|
lane := pipeline.ArtifactLaneProfile{
|
||||||
Extract: fileLane.Extract.toPipelineBinding(),
|
Extract: extract,
|
||||||
References: normalizedStringMap(fileLane.References),
|
References: normalizedStringMap(fileLane.References),
|
||||||
}
|
}
|
||||||
if fileLane.Merge != nil {
|
if fileLane.Merge != nil {
|
||||||
@@ -353,6 +394,20 @@ func normalizedStringMap(values map[string]string) map[string]string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func mergeStringMaps(base map[string]string, override map[string]string) map[string]string {
|
||||||
|
if len(base) == 0 && len(override) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]string, len(base)+len(override))
|
||||||
|
for key, value := range base {
|
||||||
|
out[key] = value
|
||||||
|
}
|
||||||
|
for key, value := range override {
|
||||||
|
out[key] = value
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
|
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
|
||||||
name := strings.TrimSpace(envName)
|
name := strings.TrimSpace(envName)
|
||||||
if name == "" {
|
if name == "" {
|
||||||
|
|||||||
@@ -169,6 +169,53 @@ pipelines:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseFileConfigStageLocalReferenceMaps(t *testing.T) {
|
||||||
|
cfg := parseAndApplyConfig(t, `
|
||||||
|
version: 1
|
||||||
|
pipelines:
|
||||||
|
example:
|
||||||
|
input: fake/input
|
||||||
|
chunk:
|
||||||
|
module: generic
|
||||||
|
references:
|
||||||
|
" scene_guide ": " ./scenes.md "
|
||||||
|
artifacts:
|
||||||
|
events:
|
||||||
|
extract:
|
||||||
|
module: fake/extract
|
||||||
|
references:
|
||||||
|
" glossary ": " ./glossary.md "
|
||||||
|
" roster ": " ./extract-roster.yml "
|
||||||
|
references:
|
||||||
|
roster: ./legacy-roster.yml
|
||||||
|
lore: ./lore.md
|
||||||
|
normalize:
|
||||||
|
module: noop
|
||||||
|
references:
|
||||||
|
" normalization_notes ": " ./normalization.md "
|
||||||
|
`)
|
||||||
|
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
if !reflect.DeepEqual(profile.Chunk.References, map[string]string{"scene_guide": "./scenes.md"}) {
|
||||||
|
t.Fatalf("chunk references = %#v, want trimmed map", profile.Chunk.References)
|
||||||
|
}
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
if !reflect.DeepEqual(lane.References, map[string]string{"lore": "./lore.md", "roster": "./legacy-roster.yml"}) {
|
||||||
|
t.Fatalf("legacy lane references = %#v, want trimmed map", lane.References)
|
||||||
|
}
|
||||||
|
wantExtract := map[string]string{
|
||||||
|
"glossary": "./glossary.md",
|
||||||
|
"lore": "./lore.md",
|
||||||
|
"roster": "./extract-roster.yml",
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(lane.Extract.References, wantExtract) {
|
||||||
|
t.Fatalf("extract references = %#v, want legacy merged with extract override %#v", lane.Extract.References, wantExtract)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(lane.Normalize.References, map[string]string{"normalization_notes": "./normalization.md"}) {
|
||||||
|
t.Fatalf("normalize references = %#v, want trimmed map", lane.Normalize.References)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) {
|
func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) {
|
||||||
cfg := parseAndApplyConfig(t, `
|
cfg := parseAndApplyConfig(t, `
|
||||||
version: 1
|
version: 1
|
||||||
@@ -357,6 +404,56 @@ pipelines:
|
|||||||
`,
|
`,
|
||||||
want: `pipeline "example" lane "events" reference slot`,
|
want: `pipeline "example" lane "events" reference slot`,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "chunk",
|
||||||
|
raw: `
|
||||||
|
version: 1
|
||||||
|
pipelines:
|
||||||
|
example:
|
||||||
|
input: fake/input
|
||||||
|
chunk:
|
||||||
|
module: generic
|
||||||
|
references:
|
||||||
|
roster: ./first.yml
|
||||||
|
" roster ": ./second.yml
|
||||||
|
`,
|
||||||
|
want: `pipeline "example" chunk reference slot`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "extract",
|
||||||
|
raw: `
|
||||||
|
version: 1
|
||||||
|
pipelines:
|
||||||
|
example:
|
||||||
|
input: fake/input
|
||||||
|
artifacts:
|
||||||
|
events:
|
||||||
|
extract:
|
||||||
|
module: fake/extract
|
||||||
|
references:
|
||||||
|
roster: ./first.yml
|
||||||
|
" roster ": ./second.yml
|
||||||
|
`,
|
||||||
|
want: `pipeline "example" lane "events" extract reference slot`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "normalize",
|
||||||
|
raw: `
|
||||||
|
version: 1
|
||||||
|
pipelines:
|
||||||
|
example:
|
||||||
|
input: fake/input
|
||||||
|
artifacts:
|
||||||
|
events:
|
||||||
|
extract: fake/extract
|
||||||
|
normalize:
|
||||||
|
module: noop
|
||||||
|
references:
|
||||||
|
roster: ./first.yml
|
||||||
|
" roster ": ./second.yml
|
||||||
|
`,
|
||||||
|
want: `pipeline "example" lane "events" normalize reference slot`,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli
|
|||||||
out := in
|
out := in
|
||||||
out.Input = cloneModuleBinding(in.Input)
|
out.Input = cloneModuleBinding(in.Input)
|
||||||
out.Chunk = cloneModuleBinding(in.Chunk)
|
out.Chunk = cloneModuleBinding(in.Chunk)
|
||||||
|
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)
|
||||||
out.Output = cloneModuleBinding(in.Output)
|
out.Output = cloneModuleBinding(in.Output)
|
||||||
if len(in.ArtifactLanes) > 0 {
|
if len(in.ArtifactLanes) > 0 {
|
||||||
out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes))
|
out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes))
|
||||||
@@ -49,8 +50,8 @@ func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.Resolv
|
|||||||
out.Extract = cloneModuleBinding(in.Extract)
|
out.Extract = cloneModuleBinding(in.Extract)
|
||||||
out.Merge = cloneModuleBinding(in.Merge)
|
out.Merge = cloneModuleBinding(in.Merge)
|
||||||
out.Normalize = cloneModuleBinding(in.Normalize)
|
out.Normalize = cloneModuleBinding(in.Normalize)
|
||||||
out.References = append([]pipeline.ReferenceBinding(nil), in.References...)
|
out.ExtractReferences = pipeline.CloneReferenceTarget(in.ExtractReferences)
|
||||||
out.ReferenceSet = pipeline.CloneReferenceSet(in.ReferenceSet)
|
out.NormalizeReferences = pipeline.CloneReferenceTarget(in.NormalizeReferences)
|
||||||
if len(in.Validators) > 0 {
|
if len(in.Validators) > 0 {
|
||||||
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
|
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
|
||||||
for i, binding := range in.Validators {
|
for i, binding := range in.Validators {
|
||||||
|
|||||||
@@ -67,20 +67,46 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
|
|||||||
lane := cfg.Pipelines["example"].Artifacts["events"]
|
lane := cfg.Pipelines["example"].Artifacts["events"]
|
||||||
lane.Extract.Options = map[string]any{"temperature": 0.2}
|
lane.Extract.Options = map[string]any{"temperature": 0.2}
|
||||||
lane.References = map[string]string{"roster": "./roster.yml"}
|
lane.References = map[string]string{"roster": "./roster.yml"}
|
||||||
|
lane.Extract.References = map[string]string{"glossary": "./glossary.md"}
|
||||||
|
lane.Normalize.References = map[string]string{"notes": "./normalize.md"}
|
||||||
cfg.Pipelines["example"].Artifacts["events"] = lane
|
cfg.Pipelines["example"].Artifacts["events"] = lane
|
||||||
|
pipelineProfile := cfg.Pipelines["example"]
|
||||||
|
pipelineProfile.Chunk.References = map[string]string{"scene_guide": "./scene.md"}
|
||||||
|
cfg.Pipelines["example"] = pipelineProfile
|
||||||
|
|
||||||
effective, err := cfg.Resolve(ResolveInput{
|
effective, err := cfg.Resolve(ResolveInput{
|
||||||
PipelineID: "example",
|
PipelineID: "example",
|
||||||
Only: []string{"events"},
|
Only: []string{"events"},
|
||||||
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
|
Catalog: fakeCatalog(t,
|
||||||
Key: "fake/extract",
|
pipeline.ModuleSpec{
|
||||||
Stage: pipeline.StageExtract,
|
Key: "generic",
|
||||||
Requires: []string{"chunks"},
|
Stage: pipeline.StageChunk,
|
||||||
Provides: []string{"artifact"},
|
Requires: []string{"source"},
|
||||||
ReferenceSlots: []contracts.ReferenceSlot{
|
Provides: []string{"chunks"},
|
||||||
{Name: "roster"},
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "scene_guide"},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}),
|
pipeline.ModuleSpec{
|
||||||
|
Key: "fake/extract",
|
||||||
|
Stage: pipeline.StageExtract,
|
||||||
|
Requires: []string{"chunks"},
|
||||||
|
Provides: []string{"artifact"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "glossary"},
|
||||||
|
{Name: "roster"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
pipeline.ModuleSpec{
|
||||||
|
Key: "noop",
|
||||||
|
Stage: pipeline.StageNormalize,
|
||||||
|
Requires: []string{"merged"},
|
||||||
|
Provides: []string{"normalized"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "notes"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Resolve: %v", err)
|
t.Fatalf("Resolve: %v", err)
|
||||||
@@ -108,8 +134,52 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
|
|||||||
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.Options["temperature"] != 0.2 {
|
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.Options["temperature"] != 0.2 {
|
||||||
t.Fatalf("expected resolved pipeline options to be copied")
|
t.Fatalf("expected resolved pipeline options to be copied")
|
||||||
}
|
}
|
||||||
payload.ResolvedPipeline.ArtifactLanes[0].References[0].Source = "./changed.yml"
|
payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source = "./changed.yml"
|
||||||
if effective.ResolvedPipeline.ArtifactLanes[0].References[0].Source != "./roster.yml" {
|
if referenceBindingSource(effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings, "roster") != "./roster.yml" {
|
||||||
t.Fatalf("expected resolved pipeline references to be copied")
|
t.Fatalf("expected resolved pipeline references to be copied")
|
||||||
}
|
}
|
||||||
|
payload.ResolvedPipeline.Chunk.References["scene_guide"] = "./changed-scene.md"
|
||||||
|
if effective.ResolvedPipeline.Chunk.References["scene_guide"] != "./scene.md" {
|
||||||
|
t.Fatalf("expected chunk references to be copied")
|
||||||
|
}
|
||||||
|
payload.ResolvedPipeline.ArtifactLanes[0].Extract.References["glossary"] = "./changed-glossary.md"
|
||||||
|
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.References["glossary"] != "./glossary.md" {
|
||||||
|
t.Fatalf("expected extract references to be copied")
|
||||||
|
}
|
||||||
|
payload.ResolvedPipeline.ArtifactLanes[0].Normalize.References["notes"] = "./changed-normalize.md"
|
||||||
|
if effective.ResolvedPipeline.ArtifactLanes[0].Normalize.References["notes"] != "./normalize.md" {
|
||||||
|
t.Fatalf("expected normalize references to be copied")
|
||||||
|
}
|
||||||
|
|
||||||
|
effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
|
||||||
|
Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||||
|
"roster": {
|
||||||
|
Slot: contracts.ReferenceSlot{Name: "roster"},
|
||||||
|
Items: []contracts.ReferenceItem{
|
||||||
|
{
|
||||||
|
SlotName: "roster",
|
||||||
|
Content: []byte("reference content"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
payload, ok = effective.RedactedDiagnosticsPayload().(EffectiveConfig)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected EffectiveConfig payload, got %T", effective.RedactedDiagnosticsPayload())
|
||||||
|
}
|
||||||
|
payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content[0] = 'X'
|
||||||
|
got := effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content
|
||||||
|
if string(got) != "reference content" {
|
||||||
|
t.Fatalf("expected materialized reference content to be copied, got %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func referenceBindingSource(bindings []pipeline.ReferenceBinding, slotName string) string {
|
||||||
|
for _, binding := range bindings {
|
||||||
|
if binding.SlotName == slotName {
|
||||||
|
return binding.Source
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -89,13 +89,13 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
|
|||||||
if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
|
if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
|
||||||
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
|
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
|
||||||
}
|
}
|
||||||
if err := validateBindingLLMProfile(id, "", "input", profile.Input, llmProfiles); err != nil {
|
if err := validateBinding(id, "", "input", profile.Input, llmProfiles, false); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validateBindingLLMProfile(id, "", "chunk", profile.Chunk, llmProfiles); err != nil {
|
if err := validateBinding(id, "", "chunk", profile.Chunk, llmProfiles, true); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validateBindingLLMProfile(id, "", "output", profile.Output, llmProfiles); err != nil {
|
if err := validateBinding(id, "", "output", profile.Output, llmProfiles, false); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validateReferenceMap(id, "", profile.References); err != nil {
|
if err := validateReferenceMap(id, "", profile.References); err != nil {
|
||||||
@@ -109,17 +109,17 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
|
|||||||
if err := validateReferenceMap(id, laneID, lane.References); err != nil {
|
if err := validateReferenceMap(id, laneID, lane.References); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validateBindingLLMProfile(id, laneID, "extract", lane.Extract, llmProfiles); err != nil {
|
if err := validateBinding(id, laneID, "extract", lane.Extract, llmProfiles, true); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validateBindingLLMProfile(id, laneID, "merge", lane.Merge, llmProfiles); err != nil {
|
if err := validateBinding(id, laneID, "merge", lane.Merge, llmProfiles, false); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validateBindingLLMProfile(id, laneID, "normalize", lane.Normalize, llmProfiles); err != nil {
|
if err := validateBinding(id, laneID, "normalize", lane.Normalize, llmProfiles, true); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
for i, validator := range lane.Validators {
|
for i, validator := range lane.Validators {
|
||||||
if err := validateBindingLLMProfile(id, laneID, fmt.Sprintf("validator[%d]", i), validator, llmProfiles); err != nil {
|
if err := validateBinding(id, laneID, fmt.Sprintf("validator[%d]", i), validator, llmProfiles, false); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -128,33 +128,64 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateBinding(
|
||||||
|
pipelineID string,
|
||||||
|
laneID string,
|
||||||
|
slot string,
|
||||||
|
binding pipeline.ModuleBinding,
|
||||||
|
profiles map[string]LLMProfile,
|
||||||
|
referencesAllowed bool,
|
||||||
|
) error {
|
||||||
|
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding, profiles); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(binding.References) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !referencesAllowed {
|
||||||
|
if laneID != "" {
|
||||||
|
return fmt.Errorf("pipeline %q lane %q %s references are not supported", pipelineID, laneID, slot)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("pipeline %q %s references are not supported", pipelineID, slot)
|
||||||
|
}
|
||||||
|
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References)
|
||||||
|
}
|
||||||
|
|
||||||
func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error {
|
func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error {
|
||||||
|
return validateReferenceMapForContext(pipelineID, laneID, "", references)
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateReferenceMapForContext(pipelineID string, laneID string, slot string, references map[string]string) error {
|
||||||
seen := make(map[string]struct{}, len(references))
|
seen := make(map[string]struct{}, len(references))
|
||||||
for rawSlotName, rawSource := range references {
|
for rawSlotName, rawSource := range references {
|
||||||
slotName := strings.TrimSpace(rawSlotName)
|
slotName := strings.TrimSpace(rawSlotName)
|
||||||
if slotName == "" {
|
if slotName == "" {
|
||||||
if laneID != "" {
|
return fmt.Errorf("%s reference slot name must not be empty", referenceContext(pipelineID, laneID, slot))
|
||||||
return fmt.Errorf("pipeline %q lane %q reference slot name must not be empty", pipelineID, laneID)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("pipeline %q reference slot name must not be empty", pipelineID)
|
|
||||||
}
|
}
|
||||||
if _, ok := seen[slotName]; ok {
|
if _, ok := seen[slotName]; ok {
|
||||||
if laneID != "" {
|
return fmt.Errorf("%s reference slot %q is duplicated after trimming", referenceContext(pipelineID, laneID, slot), slotName)
|
||||||
return fmt.Errorf("pipeline %q lane %q reference slot %q is duplicated after trimming", pipelineID, laneID, slotName)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("pipeline %q reference slot %q is duplicated after trimming", pipelineID, slotName)
|
|
||||||
}
|
}
|
||||||
seen[slotName] = struct{}{}
|
seen[slotName] = struct{}{}
|
||||||
if strings.TrimSpace(rawSource) == "" {
|
if strings.TrimSpace(rawSource) == "" {
|
||||||
if laneID != "" {
|
return fmt.Errorf("%s reference slot %q source must not be empty", referenceContext(pipelineID, laneID, slot), slotName)
|
||||||
return fmt.Errorf("pipeline %q lane %q reference slot %q source must not be empty", pipelineID, laneID, slotName)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("pipeline %q reference slot %q source must not be empty", pipelineID, slotName)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func referenceContext(pipelineID string, laneID string, slot string) string {
|
||||||
|
if laneID != "" && slot != "" {
|
||||||
|
return fmt.Sprintf("pipeline %q lane %q %s", pipelineID, laneID, slot)
|
||||||
|
}
|
||||||
|
if laneID != "" {
|
||||||
|
return fmt.Sprintf("pipeline %q lane %q", pipelineID, laneID)
|
||||||
|
}
|
||||||
|
if slot != "" {
|
||||||
|
return fmt.Sprintf("pipeline %q %s", pipelineID, slot)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("pipeline %q", pipelineID)
|
||||||
|
}
|
||||||
|
|
||||||
func validateBindingLLMProfile(
|
func validateBindingLLMProfile(
|
||||||
pipelineID string,
|
pipelineID string,
|
||||||
laneID string,
|
laneID string,
|
||||||
|
|||||||
@@ -122,6 +122,74 @@ func TestValidateRejectsInvalidReferenceMaps(t *testing.T) {
|
|||||||
mutate func(Config) Config
|
mutate func(Config) Config
|
||||||
want []string
|
want []string
|
||||||
}{
|
}{
|
||||||
|
{
|
||||||
|
name: "empty chunk slot",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
profile.Chunk.References = map[string]string{" ": "./roster.yml"}
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "chunk", "reference slot", "empty"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty chunk source",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
profile.Chunk.References = map[string]string{"roster": " "}
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "chunk", "roster", "source", "empty"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty extract slot",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Extract.References = map[string]string{" ": "./roster.yml"}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "events", "extract", "reference slot", "empty"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty extract source",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Extract.References = map[string]string{"roster": " "}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "events", "extract", "roster", "source", "empty"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty normalize slot",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Normalize.References = map[string]string{" ": "./roster.yml"}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "events", "normalize", "reference slot", "empty"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty normalize source",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Normalize.References = map[string]string{"roster": " "}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "events", "normalize", "roster", "source", "empty"},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "empty pipeline slot",
|
name: "empty pipeline slot",
|
||||||
mutate: func(cfg Config) Config {
|
mutate: func(cfg Config) Config {
|
||||||
@@ -183,6 +251,76 @@ func TestValidateRejectsInvalidReferenceMaps(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidateRejectsReferencesOnUnsupportedBindings(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mutate func(Config) Config
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "input",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
profile.Input.References = map[string]string{"roster": "./roster.yml"}
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "input", "references", "not supported"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "merge",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Merge.References = map[string]string{"roster": "./roster.yml"}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "events", "merge", "references", "not supported"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validator",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Validators = []pipeline.ModuleBinding{{
|
||||||
|
Module: "fake/validator",
|
||||||
|
References: map[string]string{"roster": "./roster.yml"},
|
||||||
|
}}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "events", "validator[0]", "references", "not supported"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "output",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
profile.Output.References = map[string]string{"roster": "./roster.yml"}
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "output", "references", "not supported"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
err := tc.mutate(validConfig()).Validate()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Validate() error = nil, want error")
|
||||||
|
}
|
||||||
|
for _, want := range tc.want {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestValidateRejectsEmptyIDs(t *testing.T) {
|
func TestValidateRejectsEmptyIDs(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
@@ -162,6 +162,10 @@ func (chunker compositionChunker) Key() string {
|
|||||||
return "generic-chunker"
|
return "generic-chunker"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (chunker compositionChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (chunker compositionChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (chunker compositionChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||||
if req.Source == nil {
|
if req.Source == nil {
|
||||||
return contracts.ChunkResult{}, errors.New("source document is required")
|
return contracts.ChunkResult{}, errors.New("source document is required")
|
||||||
@@ -264,6 +268,10 @@ func (normalizer compositionNormalizer) Key() string {
|
|||||||
return "generic-normalizer"
|
return "generic-normalizer"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (normalizer compositionNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (normalizer compositionNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
func (normalizer compositionNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
||||||
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
|
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -58,6 +58,7 @@ type SourceChunk struct {
|
|||||||
|
|
||||||
type ChunkRequest struct {
|
type ChunkRequest struct {
|
||||||
Source *source.SourceDocument `json:"-"`
|
Source *source.SourceDocument `json:"-"`
|
||||||
|
References ReferenceSet `json:"references,omitempty"`
|
||||||
LLMClient StructuredLLMClient `json:"-"`
|
LLMClient StructuredLLMClient `json:"-"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
Options map[string]any `json:"options,omitempty"`
|
Options map[string]any `json:"options,omitempty"`
|
||||||
@@ -71,6 +72,7 @@ type ChunkResult struct {
|
|||||||
|
|
||||||
type Chunker interface {
|
type Chunker interface {
|
||||||
Key() string
|
Key() string
|
||||||
|
ReferenceSlots() []ReferenceSlot
|
||||||
Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error)
|
Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -165,6 +167,8 @@ type NormalizeRequest struct {
|
|||||||
Source *source.SourceDocument `json:"-"`
|
Source *source.SourceDocument `json:"-"`
|
||||||
LaneID string `json:"lane_id"`
|
LaneID string `json:"lane_id"`
|
||||||
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
|
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
|
||||||
|
References ReferenceSet `json:"references,omitempty"`
|
||||||
|
LLMClient StructuredLLMClient `json:"-"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
Options map[string]any `json:"options,omitempty"`
|
Options map[string]any `json:"options,omitempty"`
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
@@ -177,6 +181,7 @@ type NormalizeResult struct {
|
|||||||
|
|
||||||
type Normalizer interface {
|
type Normalizer interface {
|
||||||
Key() string
|
Key() string
|
||||||
|
ReferenceSlots() []ReferenceSlot
|
||||||
Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)
|
Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -378,6 +378,10 @@ func (chunker fakeChunker) Key() string {
|
|||||||
return chunker.key
|
return chunker.key
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (chunker fakeChunker) ReferenceSlots() []ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (chunker fakeChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
|
func (chunker fakeChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
|
||||||
return ChunkResult{
|
return ChunkResult{
|
||||||
Chunks: []SourceChunk{
|
Chunks: []SourceChunk{
|
||||||
@@ -400,6 +404,10 @@ func (chunker *recordingChunker) Key() string {
|
|||||||
return chunker.key
|
return chunker.key
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (chunker *recordingChunker) ReferenceSlots() []ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (chunker *recordingChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
|
func (chunker *recordingChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
|
||||||
chunker.request = req
|
chunker.request = req
|
||||||
return fakeChunker{key: chunker.key}.Chunk(ctx, req)
|
return fakeChunker{key: chunker.key}.Chunk(ctx, req)
|
||||||
@@ -487,6 +495,10 @@ func (normalizer fakeNormalizer) Key() string {
|
|||||||
return normalizer.key
|
return normalizer.key
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (normalizer fakeNormalizer) ReferenceSlots() []ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (normalizer fakeNormalizer) Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error) {
|
func (normalizer fakeNormalizer) Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error) {
|
||||||
return NormalizeResult{Candidates: req.Candidates}, nil
|
return NormalizeResult{Candidates: req.Candidates}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -330,6 +330,10 @@ func (chunker registryChunker) Key() string {
|
|||||||
return chunker.key
|
return chunker.key
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (chunker registryChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (chunker registryChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (chunker registryChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||||
return contracts.ChunkResult{}, nil
|
return contracts.ChunkResult{}, nil
|
||||||
}
|
}
|
||||||
@@ -354,6 +358,10 @@ func (normalizer registryNormalizer) Key() string {
|
|||||||
return normalizer.key
|
return normalizer.key
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (normalizer registryNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (normalizer registryNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
func (normalizer registryNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
||||||
return contracts.NormalizeResult{}, nil
|
return contracts.NormalizeResult{}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -87,7 +87,7 @@ func validateModuleSpec(kind string, expectedStage ModuleStage, spec ModuleSpec)
|
|||||||
if spec.Stage != expectedStage {
|
if spec.Stage != expectedStage {
|
||||||
return fmt.Errorf("%s %q must use %q stage, got %q", kind, spec.Key, expectedStage, spec.Stage)
|
return fmt.Errorf("%s %q must use %q stage, got %q", kind, spec.Key, expectedStage, spec.Stage)
|
||||||
}
|
}
|
||||||
if spec.Stage != StageExtract && len(spec.ReferenceSlots) > 0 {
|
if !referenceSlotStage(spec.Stage) && len(spec.ReferenceSlots) > 0 {
|
||||||
return fmt.Errorf("%s %q must not declare reference slots", kind, spec.Key)
|
return fmt.Errorf("%s %q must not declare reference slots", kind, spec.Key)
|
||||||
}
|
}
|
||||||
if err := validateReferenceSlots(spec.ReferenceSlots); err != nil {
|
if err := validateReferenceSlots(spec.ReferenceSlots); err != nil {
|
||||||
@@ -96,6 +96,10 @@ func validateModuleSpec(kind string, expectedStage ModuleStage, spec ModuleSpec)
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func referenceSlotStage(stage ModuleStage) bool {
|
||||||
|
return stage == StageChunk || stage == StageExtract || stage == StageNormalize
|
||||||
|
}
|
||||||
|
|
||||||
func sortedRegistryKeys[C any](constructors map[string]C) []string {
|
func sortedRegistryKeys[C any](constructors map[string]C) []string {
|
||||||
if len(constructors) == 0 {
|
if len(constructors) == 0 {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -7,19 +7,117 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestValidateModuleSpecRejectsReferenceSlotsForNonExtractors(t *testing.T) {
|
func TestValidateModuleSpecAllowsReferenceSlotsForEligibleStages(t *testing.T) {
|
||||||
err := validateModuleSpec("chunker", StageChunk, ModuleSpec{
|
tests := []struct {
|
||||||
Key: "generic",
|
name string
|
||||||
Stage: StageChunk,
|
kind string
|
||||||
ReferenceSlots: []contracts.ReferenceSlot{
|
stage ModuleStage
|
||||||
{Name: "roster"},
|
}{
|
||||||
},
|
{name: "chunker", kind: "chunker", stage: StageChunk},
|
||||||
})
|
{name: "extractor", kind: "extractor", stage: StageExtract},
|
||||||
|
{name: "normalizer", kind: "normalizer", stage: StageNormalize},
|
||||||
if err == nil {
|
|
||||||
t.Fatal("validateModuleSpec() error = nil, want error")
|
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "reference slots") {
|
|
||||||
t.Fatalf("validateModuleSpec() error = %q, want reference slots context", err.Error())
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
spec := normalizeModuleSpec(ModuleSpec{
|
||||||
|
Key: "module",
|
||||||
|
Stage: test.stage,
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "roster", Description: "Character roster", MaxBytes: 1024},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
err := validateModuleSpec(test.kind, test.stage, spec)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validateModuleSpec() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateModuleSpecRejectsReferenceSlotsForIneligibleStages(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
kind string
|
||||||
|
stage ModuleStage
|
||||||
|
}{
|
||||||
|
{name: "input", kind: "input adapter", stage: StageInput},
|
||||||
|
{name: "merge", kind: "merger", stage: StageMerge},
|
||||||
|
{name: "validate", kind: "validator", stage: StageValidate},
|
||||||
|
{name: "output", kind: "output encoder", stage: StageOutput},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
spec := normalizeModuleSpec(ModuleSpec{
|
||||||
|
Key: "module",
|
||||||
|
Stage: test.stage,
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "roster"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
err := validateModuleSpec(test.kind, test.stage, spec)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("validateModuleSpec() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "reference slots") {
|
||||||
|
t.Fatalf("validateModuleSpec() error = %q, want reference slots context", err.Error())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateModuleSpecRejectsInvalidReferenceSlotsForEligibleStages(t *testing.T) {
|
||||||
|
invalidSlots := []struct {
|
||||||
|
name string
|
||||||
|
slots []contracts.ReferenceSlot
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "empty name",
|
||||||
|
slots: []contracts.ReferenceSlot{{Name: " "}},
|
||||||
|
want: "name",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "duplicate name after trim",
|
||||||
|
slots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "roster"},
|
||||||
|
{Name: " roster "},
|
||||||
|
},
|
||||||
|
want: "duplicated",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "negative max bytes",
|
||||||
|
slots: []contracts.ReferenceSlot{{Name: "roster", MaxBytes: -1}},
|
||||||
|
want: "max_bytes",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
eligibleStages := []struct {
|
||||||
|
name string
|
||||||
|
kind string
|
||||||
|
stage ModuleStage
|
||||||
|
}{
|
||||||
|
{name: "chunk", kind: "chunker", stage: StageChunk},
|
||||||
|
{name: "extract", kind: "extractor", stage: StageExtract},
|
||||||
|
{name: "normalize", kind: "normalizer", stage: StageNormalize},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, stage := range eligibleStages {
|
||||||
|
for _, invalid := range invalidSlots {
|
||||||
|
t.Run(stage.name+"/"+invalid.name, func(t *testing.T) {
|
||||||
|
spec := normalizeModuleSpec(ModuleSpec{
|
||||||
|
Key: "module",
|
||||||
|
Stage: stage.stage,
|
||||||
|
ReferenceSlots: invalid.slots,
|
||||||
|
})
|
||||||
|
err := validateModuleSpec(stage.kind, stage.stage, spec)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("validateModuleSpec() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), invalid.want) {
|
||||||
|
t.Fatalf("validateModuleSpec() error = %q, want %q", err.Error(), invalid.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,9 +20,10 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type ModuleBinding struct {
|
type ModuleBinding struct {
|
||||||
Module string `json:"module"`
|
Module string `json:"module"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
Options map[string]any `json:"options,omitempty"`
|
Options map[string]any `json:"options,omitempty"`
|
||||||
|
References map[string]string `json:"references,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ArtifactLaneProfile struct {
|
type ArtifactLaneProfile struct {
|
||||||
@@ -49,34 +50,45 @@ type ResolveOptions struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ReferenceBinding struct {
|
type ReferenceBinding struct {
|
||||||
LaneID string `json:"lane_id,omitempty"`
|
Stage ModuleStage `json:"stage,omitempty"`
|
||||||
SlotName string `json:"slot_name"`
|
LaneID string `json:"lane_id,omitempty"`
|
||||||
Source string `json:"source"`
|
SlotName string `json:"slot_name"`
|
||||||
BindingSource string `json:"binding_source,omitempty"`
|
Source string `json:"source"`
|
||||||
|
BindingSource string `json:"binding_source,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ReferenceUnbind struct {
|
type ReferenceUnbind struct {
|
||||||
LaneID string `json:"lane_id"`
|
Stage ModuleStage `json:"stage,omitempty"`
|
||||||
SlotName string `json:"slot_name"`
|
LaneID string `json:"lane_id,omitempty"`
|
||||||
|
SlotName string `json:"slot_name"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ResolvedArtifactLane struct {
|
type ResolvedReferenceTarget struct {
|
||||||
ID string
|
Stage ModuleStage `json:"stage"`
|
||||||
Extract ModuleBinding
|
LaneID string `json:"lane_id,omitempty"`
|
||||||
Merge ModuleBinding
|
Module string `json:"module"`
|
||||||
Normalize ModuleBinding
|
Bindings []ReferenceBinding `json:"bindings,omitempty"`
|
||||||
Validators []ModuleBinding
|
|
||||||
References []ReferenceBinding `json:"references,omitempty"`
|
|
||||||
ReferenceSet contracts.ReferenceSet `json:"-"`
|
ReferenceSet contracts.ReferenceSet `json:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ResolvedArtifactLane struct {
|
||||||
|
ID string
|
||||||
|
Extract ModuleBinding
|
||||||
|
Merge ModuleBinding
|
||||||
|
Normalize ModuleBinding
|
||||||
|
Validators []ModuleBinding
|
||||||
|
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
|
||||||
|
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
|
||||||
|
}
|
||||||
|
|
||||||
type ResolvedPipeline struct {
|
type ResolvedPipeline struct {
|
||||||
ID string
|
ID string
|
||||||
Digest string
|
Digest string
|
||||||
Input ModuleBinding
|
Input ModuleBinding
|
||||||
Chunk ModuleBinding
|
Chunk ModuleBinding
|
||||||
ArtifactLanes []ResolvedArtifactLane
|
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
|
||||||
Output ModuleBinding
|
ArtifactLanes []ResolvedArtifactLane
|
||||||
|
Output ModuleBinding
|
||||||
}
|
}
|
||||||
|
|
||||||
type ModuleCatalog struct {
|
type ModuleCatalog struct {
|
||||||
@@ -135,15 +147,28 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
|
|||||||
if len(selectedLaneIDs) == 0 {
|
if len(selectedLaneIDs) == 0 {
|
||||||
return ResolvedPipeline{}, fmt.Errorf("pipeline %q must select at least one artifact lane", pipelineID)
|
return ResolvedPipeline{}, fmt.Errorf("pipeline %q must select at least one artifact lane", pipelineID)
|
||||||
}
|
}
|
||||||
if err := validatePipelineReferenceDefaults(pipelineID, profile.References, lanesByID, catalog); err != nil {
|
if err := validatePipelineReferenceDefaults(pipelineID, profile.References, chunkSpec, lanesByID, catalog); err != nil {
|
||||||
return ResolvedPipeline{}, err
|
return ResolvedPipeline{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
chunkReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
||||||
|
PipelineID: pipelineID,
|
||||||
|
Stage: StageChunk,
|
||||||
|
Module: chunk.Module,
|
||||||
|
Slots: chunkSpec.ReferenceSlots,
|
||||||
|
PipelineReferences: profile.References,
|
||||||
|
LocalReferences: chunk.References,
|
||||||
|
Options: options,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return ResolvedPipeline{}, err
|
||||||
|
}
|
||||||
resolved := ResolvedPipeline{
|
resolved := ResolvedPipeline{
|
||||||
ID: pipelineID,
|
ID: pipelineID,
|
||||||
Input: input,
|
Input: input,
|
||||||
Chunk: chunk,
|
Chunk: chunk,
|
||||||
Output: resolveBinding(profile.Output, DefaultOutputModule),
|
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
|
||||||
|
Output: resolveBinding(profile.Output, DefaultOutputModule),
|
||||||
}
|
}
|
||||||
outputCapabilities := capabilities.clone()
|
outputCapabilities := capabilities.clone()
|
||||||
|
|
||||||
@@ -202,11 +227,21 @@ func resolveArtifactLane(
|
|||||||
if missing, ok := capabilities.missing(extractSpec.Requires); ok {
|
if missing, ok := capabilities.missing(extractSpec.Requires); ok {
|
||||||
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
|
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
|
||||||
}
|
}
|
||||||
references, err := resolveReferenceBindings(pipelineID, laneID, lane.Extract.Module, extractSpec.ReferenceSlots, pipelineReferences, profile.References, options)
|
extractReferences := mergeReferenceMaps(profile.References, lane.Extract.References)
|
||||||
|
references, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
||||||
|
PipelineID: pipelineID,
|
||||||
|
LaneID: laneID,
|
||||||
|
Stage: StageExtract,
|
||||||
|
Module: lane.Extract.Module,
|
||||||
|
Slots: extractSpec.ReferenceSlots,
|
||||||
|
PipelineReferences: pipelineReferences,
|
||||||
|
LocalReferences: extractReferences,
|
||||||
|
Options: options,
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedArtifactLane{}, nil, err
|
return ResolvedArtifactLane{}, nil, err
|
||||||
}
|
}
|
||||||
lane.References = references
|
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
|
||||||
capabilities.add(extractSpec.Provides...)
|
capabilities.add(extractSpec.Provides...)
|
||||||
|
|
||||||
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
|
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
|
||||||
@@ -225,6 +260,20 @@ func resolveArtifactLane(
|
|||||||
if missing, ok := capabilities.missing(normalizeSpec.Requires); ok {
|
if missing, ok := capabilities.missing(normalizeSpec.Requires); ok {
|
||||||
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing)
|
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing)
|
||||||
}
|
}
|
||||||
|
normalizeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
||||||
|
PipelineID: pipelineID,
|
||||||
|
LaneID: laneID,
|
||||||
|
Stage: StageNormalize,
|
||||||
|
Module: lane.Normalize.Module,
|
||||||
|
Slots: normalizeSpec.ReferenceSlots,
|
||||||
|
PipelineReferences: pipelineReferences,
|
||||||
|
LocalReferences: lane.Normalize.References,
|
||||||
|
Options: options,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return ResolvedArtifactLane{}, nil, err
|
||||||
|
}
|
||||||
|
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
|
||||||
capabilities.add(normalizeSpec.Provides...)
|
capabilities.add(normalizeSpec.Provides...)
|
||||||
|
|
||||||
for _, validator := range lane.Validators {
|
for _, validator := range lane.Validators {
|
||||||
@@ -241,9 +290,33 @@ func resolveArtifactLane(
|
|||||||
return lane, capabilities, nil
|
return lane, capabilities, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func referenceTarget(stage ModuleStage, laneID string, module string, bindings []ReferenceBinding) ResolvedReferenceTarget {
|
||||||
|
return ResolvedReferenceTarget{
|
||||||
|
Stage: stage,
|
||||||
|
LaneID: strings.TrimSpace(laneID),
|
||||||
|
Module: strings.TrimSpace(module),
|
||||||
|
Bindings: append([]ReferenceBinding(nil), bindings...),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeReferenceMaps(base map[string]string, override map[string]string) map[string]string {
|
||||||
|
if len(base) == 0 && len(override) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]string, len(base)+len(override))
|
||||||
|
for key, value := range base {
|
||||||
|
out[key] = value
|
||||||
|
}
|
||||||
|
for key, value := range override {
|
||||||
|
out[key] = value
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func validatePipelineReferenceDefaults(
|
func validatePipelineReferenceDefaults(
|
||||||
pipelineID string,
|
pipelineID string,
|
||||||
pipelineReferences map[string]string,
|
pipelineReferences map[string]string,
|
||||||
|
chunkSpec ModuleSpec,
|
||||||
lanesByID map[string]ArtifactLaneProfile,
|
lanesByID map[string]ArtifactLaneProfile,
|
||||||
catalog ModuleCatalog,
|
catalog ModuleCatalog,
|
||||||
) error {
|
) error {
|
||||||
@@ -255,7 +328,10 @@ func validatePipelineReferenceDefaults(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
declaredByAnyLane := make(map[string]struct{}, len(normalizedPipelineReferences))
|
declaredByAnyTarget := make(map[string]struct{}, len(normalizedPipelineReferences))
|
||||||
|
for _, slot := range chunkSpec.ReferenceSlots {
|
||||||
|
declaredByAnyTarget[slot.Name] = struct{}{}
|
||||||
|
}
|
||||||
for _, laneID := range sortedArtifactLaneProfileKeys(lanesByID) {
|
for _, laneID := range sortedArtifactLaneProfileKeys(lanesByID) {
|
||||||
laneProfile := lanesByID[laneID]
|
laneProfile := lanesByID[laneID]
|
||||||
extract := resolveBinding(laneProfile.Extract, "")
|
extract := resolveBinding(laneProfile.Extract, "")
|
||||||
@@ -267,29 +343,41 @@ func validatePipelineReferenceDefaults(
|
|||||||
return moduleLookupError(pipelineID, laneID, StageExtract, extract.Module, err)
|
return moduleLookupError(pipelineID, laneID, StageExtract, extract.Module, err)
|
||||||
}
|
}
|
||||||
for _, slot := range extractSpec.ReferenceSlots {
|
for _, slot := range extractSpec.ReferenceSlots {
|
||||||
declaredByAnyLane[slot.Name] = struct{}{}
|
declaredByAnyTarget[slot.Name] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
normalize := resolveBinding(laneProfile.Normalize, DefaultNormalizeModule)
|
||||||
|
normalizeSpec, err := normalizerSpec(catalog, normalize.Module)
|
||||||
|
if err != nil {
|
||||||
|
return moduleLookupError(pipelineID, laneID, StageNormalize, normalize.Module, err)
|
||||||
|
}
|
||||||
|
for _, slot := range normalizeSpec.ReferenceSlots {
|
||||||
|
declaredByAnyTarget[slot.Name] = struct{}{}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, slotName := range sortedStringMapKeys(normalizedPipelineReferences) {
|
for _, slotName := range sortedStringMapKeys(normalizedPipelineReferences) {
|
||||||
if _, ok := declaredByAnyLane[slotName]; !ok {
|
if _, ok := declaredByAnyTarget[slotName]; !ok {
|
||||||
return fmt.Errorf("pipeline %q reference slot %q is not declared by any artifact lane", pipelineID, slotName)
|
return fmt.Errorf("pipeline %q reference slot %q is not declared by any eligible reference target", pipelineID, slotName)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveReferenceBindings(
|
type referenceResolutionTarget struct {
|
||||||
pipelineID string,
|
PipelineID string
|
||||||
laneID string,
|
LaneID string
|
||||||
extractorModule string,
|
Stage ModuleStage
|
||||||
slots []contracts.ReferenceSlot,
|
Module string
|
||||||
pipelineReferences map[string]string,
|
Slots []contracts.ReferenceSlot
|
||||||
laneReferences map[string]string,
|
PipelineReferences map[string]string
|
||||||
options ResolveOptions,
|
LocalReferences map[string]string
|
||||||
) ([]ReferenceBinding, error) {
|
Options ResolveOptions
|
||||||
slotByName := make(map[string]contracts.ReferenceSlot, len(slots))
|
}
|
||||||
for _, slot := range slots {
|
|
||||||
|
func resolveReferenceTargetBindings(target referenceResolutionTarget) ([]ReferenceBinding, error) {
|
||||||
|
slotByName := make(map[string]contracts.ReferenceSlot, len(target.Slots))
|
||||||
|
for _, slot := range target.Slots {
|
||||||
slotByName[slot.Name] = slot
|
slotByName[slot.Name] = slot
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -298,16 +386,16 @@ func resolveReferenceBindings(
|
|||||||
slotName = strings.TrimSpace(slotName)
|
slotName = strings.TrimSpace(slotName)
|
||||||
source = strings.TrimSpace(source)
|
source = strings.TrimSpace(source)
|
||||||
if slotName == "" {
|
if slotName == "" {
|
||||||
return fmt.Errorf("pipeline %q lane %q reference slot name must not be empty", pipelineID, laneID)
|
return fmt.Errorf("%s reference slot name must not be empty", referenceTargetErrorContext(target))
|
||||||
}
|
}
|
||||||
if source == "" {
|
if source == "" {
|
||||||
return fmt.Errorf("pipeline %q lane %q reference slot %q source must not be empty", pipelineID, laneID, slotName)
|
return fmt.Errorf("%s reference slot %q source must not be empty", referenceTargetErrorContext(target), slotName)
|
||||||
}
|
}
|
||||||
if _, ok := slotByName[slotName]; !ok {
|
if _, ok := slotByName[slotName]; !ok {
|
||||||
return fmt.Errorf("pipeline %q lane %q reference slot %q is not declared by extractor %q", pipelineID, laneID, slotName, extractorModule)
|
return fmt.Errorf("%s reference slot %q is not declared by %s module %q", referenceTargetErrorContext(target), slotName, target.Stage, target.Module)
|
||||||
}
|
}
|
||||||
bindings[slotName] = ReferenceBinding{
|
bindings[slotName] = ReferenceBinding{
|
||||||
LaneID: laneID,
|
LaneID: target.LaneID,
|
||||||
SlotName: slotName,
|
SlotName: slotName,
|
||||||
Source: source,
|
Source: source,
|
||||||
BindingSource: bindingSource,
|
BindingSource: bindingSource,
|
||||||
@@ -315,7 +403,7 @@ func resolveReferenceBindings(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
normalizedPipelineReferences, err := normalizedReferenceMap(pipelineReferences, fmt.Sprintf("pipeline %q reference slot", pipelineID))
|
normalizedPipelineReferences, err := normalizedReferenceMap(target.PipelineReferences, fmt.Sprintf("pipeline %q reference slot", target.PipelineID))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -328,22 +416,22 @@ func resolveReferenceBindings(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
normalizedLaneReferences, err := normalizedReferenceMap(laneReferences, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID))
|
normalizedLocalReferences, err := normalizedReferenceMap(target.LocalReferences, referenceTargetSlotLabel(target))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
for _, slotName := range sortedStringMapKeys(normalizedLaneReferences) {
|
for _, slotName := range sortedStringMapKeys(normalizedLocalReferences) {
|
||||||
if err := addBinding(slotName, normalizedLaneReferences[slotName], contracts.ReferenceBindingSourceConfig); err != nil {
|
if err := addBinding(slotName, normalizedLocalReferences[slotName], contracts.ReferenceBindingSourceConfig); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, override := range options.ReferenceOverrides {
|
for _, override := range target.Options.ReferenceOverrides {
|
||||||
optionLaneID := strings.TrimSpace(override.LaneID)
|
match, err := referenceOverrideMatchesTarget(target, override)
|
||||||
if optionLaneID == "" {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("pipeline %q reference override lane id must not be empty", pipelineID)
|
return nil, err
|
||||||
}
|
}
|
||||||
if optionLaneID != laneID {
|
if !match {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
source := override.BindingSource
|
source := override.BindingSource
|
||||||
@@ -355,38 +443,94 @@ func resolveReferenceBindings(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, unbind := range options.ReferenceUnbinds {
|
for _, unbind := range target.Options.ReferenceUnbinds {
|
||||||
optionLaneID := strings.TrimSpace(unbind.LaneID)
|
match, err := referenceUnbindMatchesTarget(target, unbind)
|
||||||
if optionLaneID == "" {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("pipeline %q reference unbind lane id must not be empty", pipelineID)
|
return nil, err
|
||||||
}
|
}
|
||||||
if optionLaneID != laneID {
|
if !match {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
slotName := strings.TrimSpace(unbind.SlotName)
|
slotName := strings.TrimSpace(unbind.SlotName)
|
||||||
if slotName == "" {
|
if slotName == "" {
|
||||||
return nil, fmt.Errorf("pipeline %q lane %q reference unbind slot name must not be empty", pipelineID, laneID)
|
return nil, fmt.Errorf("%s reference unbind slot name must not be empty", referenceTargetErrorContext(target))
|
||||||
}
|
}
|
||||||
if _, ok := slotByName[slotName]; !ok {
|
if _, ok := slotByName[slotName]; !ok {
|
||||||
return nil, fmt.Errorf("pipeline %q lane %q reference slot %q is not declared", pipelineID, laneID, slotName)
|
return nil, fmt.Errorf("%s reference slot %q is not declared", referenceTargetErrorContext(target), slotName)
|
||||||
}
|
}
|
||||||
delete(bindings, slotName)
|
delete(bindings, slotName)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, slot := range slots {
|
for _, slot := range target.Slots {
|
||||||
if slot.Required {
|
if slot.Required {
|
||||||
if _, ok := bindings[slot.Name]; !ok {
|
if _, ok := bindings[slot.Name]; !ok {
|
||||||
return nil, fmt.Errorf("pipeline %q lane %q required reference slot %q is not bound", pipelineID, laneID, slot.Name)
|
return nil, fmt.Errorf("%s required reference slot %q is not bound", referenceTargetErrorContext(target), slot.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return sortedReferenceBindings(bindings), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func referenceOverrideMatchesTarget(target referenceResolutionTarget, override ReferenceBinding) (bool, error) {
|
||||||
|
stage, laneID, err := normalizeReferenceOptionTarget(target.PipelineID, "override", override.Stage, override.LaneID)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return stage == target.Stage && laneID == target.LaneID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func referenceUnbindMatchesTarget(target referenceResolutionTarget, unbind ReferenceUnbind) (bool, error) {
|
||||||
|
stage, laneID, err := normalizeReferenceOptionTarget(target.PipelineID, "unbind", unbind.Stage, unbind.LaneID)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return stage == target.Stage && laneID == target.LaneID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeReferenceOptionTarget(pipelineID string, operation string, stage ModuleStage, laneID string) (ModuleStage, string, error) {
|
||||||
|
stage = ModuleStage(strings.TrimSpace(string(stage)))
|
||||||
|
if stage == "" {
|
||||||
|
stage = StageExtract
|
||||||
|
}
|
||||||
|
laneID = strings.TrimSpace(laneID)
|
||||||
|
|
||||||
|
switch stage {
|
||||||
|
case StageChunk:
|
||||||
|
if laneID != "" {
|
||||||
|
return "", "", fmt.Errorf("pipeline %q reference %s for chunk must not include a lane id", pipelineID, operation)
|
||||||
|
}
|
||||||
|
case StageExtract, StageNormalize:
|
||||||
|
if laneID == "" {
|
||||||
|
return "", "", fmt.Errorf("pipeline %q reference %s lane id must not be empty", pipelineID, operation)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return "", "", fmt.Errorf("pipeline %q reference %s stage %q is not supported", pipelineID, operation, stage)
|
||||||
|
}
|
||||||
|
return stage, laneID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedReferenceBindings(bindings map[string]ReferenceBinding) []ReferenceBinding {
|
||||||
keys := sortedReferenceBindingKeys(bindings)
|
keys := sortedReferenceBindingKeys(bindings)
|
||||||
resolved := make([]ReferenceBinding, 0, len(keys))
|
resolved := make([]ReferenceBinding, 0, len(keys))
|
||||||
for _, slotName := range keys {
|
for _, slotName := range keys {
|
||||||
resolved = append(resolved, bindings[slotName])
|
resolved = append(resolved, bindings[slotName])
|
||||||
}
|
}
|
||||||
return resolved, nil
|
return resolved
|
||||||
|
}
|
||||||
|
|
||||||
|
func referenceTargetErrorContext(target referenceResolutionTarget) string {
|
||||||
|
if target.LaneID != "" {
|
||||||
|
return fmt.Sprintf("pipeline %q lane %q %s module %q", target.PipelineID, target.LaneID, target.Stage, target.Module)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("pipeline %q %s module %q", target.PipelineID, target.Stage, target.Module)
|
||||||
|
}
|
||||||
|
|
||||||
|
func referenceTargetSlotLabel(target referenceResolutionTarget) string {
|
||||||
|
if target.LaneID != "" {
|
||||||
|
return fmt.Sprintf("pipeline %q lane %q %s reference slot", target.PipelineID, target.LaneID, target.Stage)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("pipeline %q %s reference slot", target.PipelineID, target.Stage)
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizedReferenceMap(values map[string]string, keyName string) (map[string]string, error) {
|
func normalizedReferenceMap(values map[string]string, keyName string) (map[string]string, error) {
|
||||||
@@ -460,6 +604,7 @@ func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
|
|||||||
Module: module,
|
Module: module,
|
||||||
LLMProfile: llmProfile,
|
LLMProfile: llmProfile,
|
||||||
Options: cloneOptions(binding.Options),
|
Options: cloneOptions(binding.Options),
|
||||||
|
References: normalizeReferenceMap(binding.References),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -488,6 +633,25 @@ func cloneOptions(options map[string]any) map[string]any {
|
|||||||
return copied
|
return copied
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func normalizeReferenceMap(values map[string]string) map[string]string {
|
||||||
|
if len(values) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]string, len(values))
|
||||||
|
keys := make([]string, 0, len(values))
|
||||||
|
rawByNormalized := make(map[string]string, len(values))
|
||||||
|
for rawKey := range values {
|
||||||
|
key := strings.TrimSpace(rawKey)
|
||||||
|
rawByNormalized[key] = rawKey
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
for _, key := range keys {
|
||||||
|
out[key] = strings.TrimSpace(values[rawByNormalized[key]])
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneProfile, options ResolveOptions) (map[string]ArtifactLaneProfile, []string, error) {
|
func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneProfile, options ResolveOptions) (map[string]ArtifactLaneProfile, []string, error) {
|
||||||
lanesByID := make(map[string]ArtifactLaneProfile, len(artifacts))
|
lanesByID := make(map[string]ArtifactLaneProfile, len(artifacts))
|
||||||
for rawLaneID, lane := range artifacts {
|
for rawLaneID, lane := range artifacts {
|
||||||
@@ -532,17 +696,19 @@ func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneP
|
|||||||
|
|
||||||
func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
|
func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
|
||||||
withoutDigest := struct {
|
withoutDigest := struct {
|
||||||
ID string
|
ID string
|
||||||
Input ModuleBinding
|
Input ModuleBinding
|
||||||
Chunk ModuleBinding
|
Chunk ModuleBinding
|
||||||
ArtifactLanes []ResolvedArtifactLane
|
ChunkReferences ResolvedReferenceTarget
|
||||||
Output ModuleBinding
|
ArtifactLanes []ResolvedArtifactLane
|
||||||
|
Output ModuleBinding
|
||||||
}{
|
}{
|
||||||
ID: resolved.ID,
|
ID: resolved.ID,
|
||||||
Input: resolved.Input,
|
Input: resolved.Input,
|
||||||
Chunk: resolved.Chunk,
|
Chunk: resolved.Chunk,
|
||||||
ArtifactLanes: resolved.ArtifactLanes,
|
ChunkReferences: resolved.ChunkReferences,
|
||||||
Output: resolved.Output,
|
ArtifactLanes: resolved.ArtifactLanes,
|
||||||
|
Output: resolved.Output,
|
||||||
}
|
}
|
||||||
encoded, err := json.Marshal(withoutDigest)
|
encoded, err := json.Marshal(withoutDigest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -154,19 +154,131 @@ func TestResolvePipelineAppliesReferenceBindings(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
events := resolvedLane(t, resolved.ArtifactLanes, "events")
|
events := resolvedLane(t, resolved.ArtifactLanes, "events")
|
||||||
|
if events.ExtractReferences.Stage != StageExtract || events.ExtractReferences.LaneID != "events" || events.ExtractReferences.Module != "event-extractor" {
|
||||||
|
t.Fatalf("extract reference target = %#v, want event extractor target", events.ExtractReferences)
|
||||||
|
}
|
||||||
|
if events.NormalizeReferences.Stage != StageNormalize || events.NormalizeReferences.LaneID != "events" || events.NormalizeReferences.Module != DefaultNormalizeModule {
|
||||||
|
t.Fatalf("normalize reference target = %#v, want event normalizer target", events.NormalizeReferences)
|
||||||
|
}
|
||||||
|
if resolved.ChunkReferences.Stage != StageChunk || resolved.ChunkReferences.Module != DefaultChunkModule {
|
||||||
|
t.Fatalf("chunk reference target = %#v, want chunk target", resolved.ChunkReferences)
|
||||||
|
}
|
||||||
want := []ReferenceBinding{
|
want := []ReferenceBinding{
|
||||||
{LaneID: "events", SlotName: "lore", Source: "./lore.md", BindingSource: contracts.ReferenceBindingSourceConfig},
|
{LaneID: "events", SlotName: "lore", Source: "./lore.md", BindingSource: contracts.ReferenceBindingSourceConfig},
|
||||||
{LaneID: "events", SlotName: "roster", Source: "./lane-roster.yml", BindingSource: contracts.ReferenceBindingSourceConfig},
|
{LaneID: "events", SlotName: "roster", Source: "./lane-roster.yml", BindingSource: contracts.ReferenceBindingSourceConfig},
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(events.References, want) {
|
if !reflect.DeepEqual(events.ExtractReferences.Bindings, want) {
|
||||||
t.Fatalf("events references = %#v, want %#v", events.References, want)
|
t.Fatalf("events references = %#v, want %#v", events.ExtractReferences.Bindings, want)
|
||||||
}
|
}
|
||||||
summaries := resolvedLane(t, resolved.ArtifactLanes, "summaries")
|
summaries := resolvedLane(t, resolved.ArtifactLanes, "summaries")
|
||||||
if len(summaries.References) != 0 {
|
if len(summaries.ExtractReferences.Bindings) != 0 {
|
||||||
t.Fatalf("summaries references = %#v, want none", summaries.References)
|
t.Fatalf("summaries references = %#v, want none", summaries.ExtractReferences.Bindings)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineAppliesPipelineReferenceDefaultToChunkTarget(t *testing.T) {
|
||||||
|
profile := baselineProfile()
|
||||||
|
profile.References = map[string]string{"scene_guide": "./scenes.md"}
|
||||||
|
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
|
||||||
|
Key: "generic",
|
||||||
|
Stage: StageChunk,
|
||||||
|
Requires: []string{"source"},
|
||||||
|
Provides: []string{"chunk"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{{Name: "scene_guide"}},
|
||||||
|
})
|
||||||
|
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []ReferenceBinding{{SlotName: "scene_guide", Source: "./scenes.md", BindingSource: contracts.ReferenceBindingSourceConfig}}
|
||||||
|
if !reflect.DeepEqual(resolved.ChunkReferences.Bindings, want) {
|
||||||
|
t.Fatalf("chunk references = %#v, want %#v", resolved.ChunkReferences.Bindings, want)
|
||||||
|
}
|
||||||
|
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
|
||||||
|
t.Fatalf("extract references = %#v, want none", refs)
|
||||||
|
}
|
||||||
|
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
|
||||||
|
t.Fatalf("normalize references = %#v, want none", refs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineAppliesPipelineReferenceDefaultToExtractorTarget(t *testing.T) {
|
||||||
|
profile := baselineProfile()
|
||||||
|
profile.References = map[string]string{"roster": "./roster.yml"}
|
||||||
|
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
|
||||||
|
Key: "event-extractor",
|
||||||
|
Stage: StageExtract,
|
||||||
|
Requires: []string{"chunk"},
|
||||||
|
Provides: []string{"candidate"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{{Name: "roster"}},
|
||||||
|
})
|
||||||
|
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []ReferenceBinding{{LaneID: "events", SlotName: "roster", Source: "./roster.yml", BindingSource: contracts.ReferenceBindingSourceConfig}}
|
||||||
|
if !reflect.DeepEqual(resolved.ArtifactLanes[0].ExtractReferences.Bindings, want) {
|
||||||
|
t.Fatalf("extract references = %#v, want %#v", resolved.ArtifactLanes[0].ExtractReferences.Bindings, want)
|
||||||
|
}
|
||||||
|
if refs := resolved.ChunkReferences.Bindings; len(refs) != 0 {
|
||||||
|
t.Fatalf("chunk references = %#v, want none", refs)
|
||||||
|
}
|
||||||
|
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
|
||||||
|
t.Fatalf("normalize references = %#v, want none", refs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineAppliesPipelineReferenceDefaultToNormalizerTarget(t *testing.T) {
|
||||||
|
profile := baselineProfile()
|
||||||
|
profile.References = map[string]string{"normalization_notes": "./normalize.md"}
|
||||||
|
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
|
||||||
|
Key: "noop",
|
||||||
|
Stage: StageNormalize,
|
||||||
|
Requires: []string{"merged"},
|
||||||
|
Provides: []string{"normalized"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes"}},
|
||||||
|
})
|
||||||
|
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []ReferenceBinding{{LaneID: "events", SlotName: "normalization_notes", Source: "./normalize.md", BindingSource: contracts.ReferenceBindingSourceConfig}}
|
||||||
|
if !reflect.DeepEqual(resolved.ArtifactLanes[0].NormalizeReferences.Bindings, want) {
|
||||||
|
t.Fatalf("normalize references = %#v, want %#v", resolved.ArtifactLanes[0].NormalizeReferences.Bindings, want)
|
||||||
|
}
|
||||||
|
if refs := resolved.ChunkReferences.Bindings; len(refs) != 0 {
|
||||||
|
t.Fatalf("chunk references = %#v, want none", refs)
|
||||||
|
}
|
||||||
|
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
|
||||||
|
t.Fatalf("extract references = %#v, want none", refs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineAppliesOnePipelineReferenceDefaultToMultipleTargets(t *testing.T) {
|
||||||
|
profile := baselineProfile()
|
||||||
|
profile.References = map[string]string{"context": "./context.md"}
|
||||||
|
catalog := newProfileCatalogWithOverrides(t,
|
||||||
|
ModuleSpec{Key: "generic", Stage: StageChunk, Requires: []string{"source"}, Provides: []string{"chunk"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
|
||||||
|
ModuleSpec{Key: "event-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
|
||||||
|
ModuleSpec{Key: "noop", Stage: StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
|
||||||
|
)
|
||||||
|
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertBindingSource(t, resolved.ChunkReferences.Bindings, "context", "./context.md")
|
||||||
|
assertBindingSource(t, resolved.ArtifactLanes[0].ExtractReferences.Bindings, "context", "./context.md")
|
||||||
|
assertBindingSource(t, resolved.ArtifactLanes[0].NormalizeReferences.Bindings, "context", "./context.md")
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolvePipelineAllowsPipelineReferenceDeclaredOnlyByUnselectedLane(t *testing.T) {
|
func TestResolvePipelineAllowsPipelineReferenceDeclaredOnlyByUnselectedLane(t *testing.T) {
|
||||||
profile := multiLaneProfile()
|
profile := multiLaneProfile()
|
||||||
profile.References = map[string]string{"notes_context": "./notes.md"}
|
profile.References = map[string]string{"notes_context": "./notes.md"}
|
||||||
@@ -184,11 +296,39 @@ func TestResolvePipelineAllowsPipelineReferenceDeclaredOnlyByUnselectedLane(t *t
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
if refs := resolved.ArtifactLanes[0].References; len(refs) != 0 {
|
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
|
||||||
t.Fatalf("selected lane references = %#v, want none", refs)
|
t.Fatalf("selected lane references = %#v, want none", refs)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineAllowsPipelineReferenceDeclaredOnlyByUnselectedNormalizer(t *testing.T) {
|
||||||
|
profile := multiLaneProfile()
|
||||||
|
profile.References = map[string]string{"notes_context": "./notes.md"}
|
||||||
|
lane := profile.Artifacts["notes"]
|
||||||
|
lane.Normalize = Binding("note-normalizer")
|
||||||
|
profile.Artifacts["notes"] = lane
|
||||||
|
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
|
||||||
|
Key: "note-normalizer",
|
||||||
|
Stage: StageNormalize,
|
||||||
|
Requires: []string{"merged"},
|
||||||
|
Provides: []string{"normalized"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "notes_context"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{"events"}}, catalog)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
|
||||||
|
t.Fatalf("selected extract references = %#v, want none", refs)
|
||||||
|
}
|
||||||
|
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
|
||||||
|
t.Fatalf("selected normalize references = %#v, want none", refs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolvePipelineRejectsPipelineReferenceNotDeclaredByAnyLane(t *testing.T) {
|
func TestResolvePipelineRejectsPipelineReferenceNotDeclaredByAnyLane(t *testing.T) {
|
||||||
profile := multiLaneProfile()
|
profile := multiLaneProfile()
|
||||||
profile.References = map[string]string{"missing": "./missing.md"}
|
profile.References = map[string]string{"missing": "./missing.md"}
|
||||||
@@ -213,6 +353,106 @@ func TestResolvePipelineRejectsUndeclaredReferenceSlot(t *testing.T) {
|
|||||||
assertErrorContains(t, err, "events", "missing", "not declared")
|
assertErrorContains(t, err, "events", "missing", "not declared")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineRejectsExtractLocalReferenceDeclaredOnlyByNormalizer(t *testing.T) {
|
||||||
|
profile := baselineProfile()
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Extract.References = map[string]string{"normalization_notes": "./normalize.md"}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
|
||||||
|
Key: "noop",
|
||||||
|
Stage: StageNormalize,
|
||||||
|
Requires: []string{"merged"},
|
||||||
|
Provides: []string{"normalized"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes"}},
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("ResolvePipeline() error = nil, want error")
|
||||||
|
}
|
||||||
|
assertErrorContains(t, err, "baseline", "events", "extract", "event-extractor", "normalization_notes", "not declared")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineRejectsNormalizeLocalReferenceDeclaredOnlyByExtractor(t *testing.T) {
|
||||||
|
profile := baselineProfile()
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Normalize.References = map[string]string{"roster": "./roster.yml"}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
|
||||||
|
Key: "event-extractor",
|
||||||
|
Stage: StageExtract,
|
||||||
|
Requires: []string{"chunk"},
|
||||||
|
Provides: []string{"candidate"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{{Name: "roster"}},
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("ResolvePipeline() error = nil, want error")
|
||||||
|
}
|
||||||
|
assertErrorContains(t, err, "baseline", "events", "normalize", "noop", "roster", "not declared")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineRequiresBoundChunkReference(t *testing.T) {
|
||||||
|
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
|
||||||
|
Key: "generic",
|
||||||
|
Stage: StageChunk,
|
||||||
|
Requires: []string{"source"},
|
||||||
|
Provides: []string{"chunk"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{{Name: "scene_guide", Required: true}},
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := ResolvePipeline(baselineProfile(), ResolveOptions{}, catalog)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("ResolvePipeline() error = nil, want error")
|
||||||
|
}
|
||||||
|
assertErrorContains(t, err, "baseline", "chunk", "generic", "required", "scene_guide", "not bound")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineRequiresBoundNormalizeReference(t *testing.T) {
|
||||||
|
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
|
||||||
|
Key: "noop",
|
||||||
|
Stage: StageNormalize,
|
||||||
|
Requires: []string{"merged"},
|
||||||
|
Provides: []string{"normalized"},
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes", Required: true}},
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := ResolvePipeline(baselineProfile(), ResolveOptions{}, catalog)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("ResolvePipeline() error = nil, want error")
|
||||||
|
}
|
||||||
|
assertErrorContains(t, err, "baseline", "events", "normalize", "noop", "required", "normalization_notes", "not bound")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineLocalReferencesOverridePipelineDefaultsForEligibleTargets(t *testing.T) {
|
||||||
|
profile := baselineProfile()
|
||||||
|
profile.References = map[string]string{
|
||||||
|
"context": "./shared-context.md",
|
||||||
|
"roster": "./shared-roster.yml",
|
||||||
|
"normalization_notes": "./shared-normalize.md",
|
||||||
|
}
|
||||||
|
profile.Chunk.References = map[string]string{"context": "./chunk-context.md"}
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Extract.References = map[string]string{"roster": "./extract-roster.yml"}
|
||||||
|
lane.Normalize.References = map[string]string{"normalization_notes": "./local-normalize.md"}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
catalog := newProfileCatalogWithOverrides(t,
|
||||||
|
ModuleSpec{Key: "generic", Stage: StageChunk, Requires: []string{"source"}, Provides: []string{"chunk"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
|
||||||
|
ModuleSpec{Key: "event-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "roster"}}},
|
||||||
|
ModuleSpec{Key: "noop", Stage: StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes"}}},
|
||||||
|
)
|
||||||
|
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertBindingSource(t, resolved.ChunkReferences.Bindings, "context", "./chunk-context.md")
|
||||||
|
assertBindingSource(t, resolved.ArtifactLanes[0].ExtractReferences.Bindings, "roster", "./extract-roster.yml")
|
||||||
|
assertBindingSource(t, resolved.ArtifactLanes[0].NormalizeReferences.Bindings, "normalization_notes", "./local-normalize.md")
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolvePipelineRequiresBoundReferenceSlotsForSelectedLanes(t *testing.T) {
|
func TestResolvePipelineRequiresBoundReferenceSlotsForSelectedLanes(t *testing.T) {
|
||||||
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
|
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
|
||||||
Key: "event-extractor",
|
Key: "event-extractor",
|
||||||
@@ -284,7 +524,7 @@ func TestResolvePipelineUsesReferenceSlotsFromSpecWithoutConstructingExtractor(t
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
if got := resolved.ArtifactLanes[0].References[0].Source; got != "./roster.yml" {
|
if got := resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].Source; got != "./roster.yml" {
|
||||||
t.Fatalf("reference source = %q, want ./roster.yml", got)
|
t.Fatalf("reference source = %q, want ./roster.yml", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -649,6 +889,21 @@ func assertErrorContains(t *testing.T, err error, values ...string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func assertBindingSource(t *testing.T, bindings []ReferenceBinding, slotName string, source string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
for _, binding := range bindings {
|
||||||
|
if binding.SlotName != slotName {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if binding.Source != source {
|
||||||
|
t.Fatalf("binding %q source = %q, want %q in %#v", slotName, binding.Source, source, bindings)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
t.Fatalf("binding %q not found in %#v", slotName, bindings)
|
||||||
|
}
|
||||||
|
|
||||||
func newProfileCatalog(t *testing.T) ModuleCatalog {
|
func newProfileCatalog(t *testing.T) ModuleCatalog {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -660,19 +915,29 @@ func newProfileCatalog(t *testing.T) ModuleCatalog {
|
|||||||
func newProfileCatalogWithOverride(t *testing.T, override ModuleSpec) ModuleCatalog {
|
func newProfileCatalogWithOverride(t *testing.T, override ModuleSpec) ModuleCatalog {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
return newProfileCatalogWithOverrides(t, override)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newProfileCatalogWithOverrides(t *testing.T, overrides ...ModuleSpec) ModuleCatalog {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
specs := defaultProfileSpecs()
|
specs := defaultProfileSpecs()
|
||||||
for index, spec := range specs {
|
for _, override := range overrides {
|
||||||
if spec.Stage == override.Stage && spec.Key == override.Key {
|
replaced := false
|
||||||
specs[index] = override
|
for index, spec := range specs {
|
||||||
catalog := emptyProfileCatalog()
|
if spec.Stage == override.Stage && spec.Key == override.Key {
|
||||||
registerProfileSpecs(t, catalog, specs...)
|
specs[index] = override
|
||||||
return catalog
|
replaced = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !replaced {
|
||||||
|
specs = append(specs, override)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
catalog := emptyProfileCatalog()
|
catalog := emptyProfileCatalog()
|
||||||
registerProfileSpecs(t, catalog, specs...)
|
registerProfileSpecs(t, catalog, specs...)
|
||||||
registerProfileSpecs(t, catalog, override)
|
|
||||||
return catalog
|
return catalog
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -29,40 +29,52 @@ type ReferenceMaterializationOptions struct {
|
|||||||
|
|
||||||
func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, options ReferenceMaterializationOptions) (ResolvedPipeline, []contracts.Warning, error) {
|
func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, options ReferenceMaterializationOptions) (ResolvedPipeline, []contracts.Warning, error) {
|
||||||
out := resolved
|
out := resolved
|
||||||
|
out.ChunkReferences = CloneReferenceTarget(resolved.ChunkReferences)
|
||||||
|
chunkReferenceSet, chunkWarnings, err := materializeReferenceTarget(resolved.ID, resolved.ChunkReferences, catalog, options)
|
||||||
|
if err != nil {
|
||||||
|
return ResolvedPipeline{}, nil, err
|
||||||
|
}
|
||||||
|
out.ChunkReferences.ReferenceSet = chunkReferenceSet
|
||||||
|
warnings := append([]contracts.Warning(nil), chunkWarnings...)
|
||||||
if len(resolved.ArtifactLanes) == 0 {
|
if len(resolved.ArtifactLanes) == 0 {
|
||||||
return out, nil, nil
|
return out, warnings, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
warnings := []contracts.Warning(nil)
|
|
||||||
out.ArtifactLanes = make([]ResolvedArtifactLane, len(resolved.ArtifactLanes))
|
out.ArtifactLanes = make([]ResolvedArtifactLane, len(resolved.ArtifactLanes))
|
||||||
for i, lane := range resolved.ArtifactLanes {
|
for i, lane := range resolved.ArtifactLanes {
|
||||||
materializedLane := lane
|
materializedLane := lane
|
||||||
referenceSet, laneWarnings, err := materializeLaneReferences(resolved.ID, lane, catalog, options)
|
materializedLane.ExtractReferences = CloneReferenceTarget(lane.ExtractReferences)
|
||||||
|
materializedLane.NormalizeReferences = CloneReferenceTarget(lane.NormalizeReferences)
|
||||||
|
extractReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.ExtractReferences, catalog, options)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedPipeline{}, nil, err
|
return ResolvedPipeline{}, nil, err
|
||||||
}
|
}
|
||||||
materializedLane.ReferenceSet = referenceSet
|
materializedLane.ExtractReferences.ReferenceSet = extractReferenceSet
|
||||||
out.ArtifactLanes[i] = materializedLane
|
|
||||||
warnings = append(warnings, laneWarnings...)
|
warnings = append(warnings, laneWarnings...)
|
||||||
|
|
||||||
|
normalizeReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.NormalizeReferences, catalog, options)
|
||||||
|
if err != nil {
|
||||||
|
return ResolvedPipeline{}, nil, err
|
||||||
|
}
|
||||||
|
materializedLane.NormalizeReferences.ReferenceSet = normalizeReferenceSet
|
||||||
|
warnings = append(warnings, laneWarnings...)
|
||||||
|
out.ArtifactLanes[i] = materializedLane
|
||||||
}
|
}
|
||||||
return out, warnings, nil
|
return out, warnings, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func materializeLaneReferences(
|
func materializeReferenceTarget(
|
||||||
pipelineID string,
|
pipelineID string,
|
||||||
lane ResolvedArtifactLane,
|
target ResolvedReferenceTarget,
|
||||||
catalog ModuleCatalog,
|
catalog ModuleCatalog,
|
||||||
options ReferenceMaterializationOptions,
|
options ReferenceMaterializationOptions,
|
||||||
) (contracts.ReferenceSet, []contracts.Warning, error) {
|
) (contracts.ReferenceSet, []contracts.Warning, error) {
|
||||||
if len(lane.References) == 0 {
|
if len(target.Bindings) == 0 {
|
||||||
return contracts.ReferenceSet{}, nil, nil
|
return contracts.ReferenceSet{}, nil, nil
|
||||||
}
|
}
|
||||||
if catalog.Extractors == nil {
|
spec, err := referenceTargetSpec(target, catalog)
|
||||||
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", pipelineID, lane.ID, lane.Extract.Module, lane.Extract.Module)
|
if err != nil {
|
||||||
}
|
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s: %w", referenceTargetContext(pipelineID, target), err)
|
||||||
spec, ok := catalog.Extractors.Spec(lane.Extract.Module)
|
|
||||||
if !ok {
|
|
||||||
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", pipelineID, lane.ID, lane.Extract.Module, lane.Extract.Module)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
slotByName := make(map[string]contracts.ReferenceSlot, len(spec.ReferenceSlots))
|
slotByName := make(map[string]contracts.ReferenceSlot, len(spec.ReferenceSlots))
|
||||||
@@ -70,38 +82,38 @@ func materializeLaneReferences(
|
|||||||
slotByName[slot.Name] = slot
|
slotByName[slot.Name] = slot
|
||||||
}
|
}
|
||||||
|
|
||||||
set := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(lane.References))}
|
set := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(target.Bindings))}
|
||||||
var warnings []contracts.Warning
|
var warnings []contracts.Warning
|
||||||
for _, binding := range lane.References {
|
for _, binding := range target.Bindings {
|
||||||
slotName := strings.TrimSpace(binding.SlotName)
|
slotName := strings.TrimSpace(binding.SlotName)
|
||||||
slot, ok := slotByName[slotName]
|
slot, ok := slotByName[slotName]
|
||||||
if !ok {
|
if !ok {
|
||||||
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q is not declared by extractor %q", pipelineID, lane.ID, slotName, lane.Extract.Module)
|
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q is not declared by %s module %q", referenceTargetContext(pipelineID, target), slotName, target.Stage, target.Module)
|
||||||
}
|
}
|
||||||
|
|
||||||
path, err := referencePath(binding, options)
|
path, err := referencePath(binding, options)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q: %w", pipelineID, lane.ID, slotName, binding.Source, err)
|
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q: %w", referenceTargetContext(pipelineID, target), slotName, binding.Source, err)
|
||||||
}
|
}
|
||||||
content, err := os.ReadFile(path)
|
content, err := os.ReadFile(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q read %q: %w", pipelineID, lane.ID, slotName, path, err)
|
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q read %q: %w", referenceTargetContext(pipelineID, target), slotName, path, err)
|
||||||
}
|
}
|
||||||
if !utf8.Valid(content) {
|
if !utf8.Valid(content) {
|
||||||
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q must be UTF-8 text", pipelineID, lane.ID, slotName, path)
|
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q must be UTF-8 text", referenceTargetContext(pipelineID, target), slotName, path)
|
||||||
}
|
}
|
||||||
mediaType := referenceMediaTypeForPath(path)
|
mediaType := referenceMediaTypeForPath(path)
|
||||||
if !referenceMediaTypeAccepted(mediaType, slot.AcceptedMediaTypes) {
|
if !referenceMediaTypeAccepted(mediaType, slot.AcceptedMediaTypes) {
|
||||||
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q media type %q is not accepted", pipelineID, lane.ID, slotName, path, mediaType)
|
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q media type %q is not accepted", referenceTargetContext(pipelineID, target), slotName, path, mediaType)
|
||||||
}
|
}
|
||||||
if slot.MaxBytes > 0 && int64(len(content)) > slot.MaxBytes {
|
if slot.MaxBytes > 0 && int64(len(content)) > slot.MaxBytes {
|
||||||
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q is %d bytes, limit %d", pipelineID, lane.ID, slotName, path, len(content), slot.MaxBytes)
|
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q is %d bytes, limit %d", referenceTargetContext(pipelineID, target), slotName, path, len(content), slot.MaxBytes)
|
||||||
}
|
}
|
||||||
if len(content) == 0 {
|
if len(content) == 0 {
|
||||||
warnings = append(warnings, contracts.Warning{
|
warnings = append(warnings, contracts.Warning{
|
||||||
Scope: fmt.Sprintf("pipeline.%s.lane.%s.reference.%s", pipelineID, lane.ID, slotName),
|
Scope: referenceWarningScope(pipelineID, target, slotName),
|
||||||
ReasonCode: "empty_reference",
|
ReasonCode: "empty_reference",
|
||||||
Message: fmt.Sprintf("reference slot %q for lane %q is bound to an empty file", slotName, lane.ID),
|
Message: fmt.Sprintf("reference slot %q for %s is bound to an empty file", slotName, referenceTargetLabel(target)),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -122,6 +134,40 @@ func materializeLaneReferences(
|
|||||||
return set, warnings, nil
|
return set, warnings, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func referenceTargetSpec(target ResolvedReferenceTarget, catalog ModuleCatalog) (ModuleSpec, error) {
|
||||||
|
switch target.Stage {
|
||||||
|
case StageChunk:
|
||||||
|
return registrySpec(catalog.Chunkers, target.Module)
|
||||||
|
case StageExtract:
|
||||||
|
return registrySpec(catalog.Extractors, target.Module)
|
||||||
|
case StageNormalize:
|
||||||
|
return registrySpec(catalog.Normalizers, target.Module)
|
||||||
|
default:
|
||||||
|
return ModuleSpec{}, fmt.Errorf("reference target stage %q is not supported", target.Stage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func referenceTargetContext(pipelineID string, target ResolvedReferenceTarget) string {
|
||||||
|
if target.LaneID != "" {
|
||||||
|
return fmt.Sprintf("pipeline %q lane %q %s module %q", pipelineID, target.LaneID, target.Stage, target.Module)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("pipeline %q %s module %q", pipelineID, target.Stage, target.Module)
|
||||||
|
}
|
||||||
|
|
||||||
|
func referenceTargetLabel(target ResolvedReferenceTarget) string {
|
||||||
|
if target.LaneID != "" {
|
||||||
|
return fmt.Sprintf("lane %q %s target", target.LaneID, target.Stage)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s target", target.Stage)
|
||||||
|
}
|
||||||
|
|
||||||
|
func referenceWarningScope(pipelineID string, target ResolvedReferenceTarget, slotName string) string {
|
||||||
|
if target.LaneID != "" {
|
||||||
|
return fmt.Sprintf("pipeline.%s.lane.%s.%s.reference.%s", pipelineID, target.LaneID, target.Stage, slotName)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("pipeline.%s.%s.reference.%s", pipelineID, target.Stage, slotName)
|
||||||
|
}
|
||||||
|
|
||||||
func referenceMediaTypeForPath(path string) string {
|
func referenceMediaTypeForPath(path string) string {
|
||||||
extension := strings.ToLower(filepath.Ext(path))
|
extension := strings.ToLower(filepath.Ext(path))
|
||||||
mediaType := mime.TypeByExtension(extension)
|
mediaType := mime.TypeByExtension(extension)
|
||||||
@@ -230,31 +276,47 @@ func CloneReferenceSet(in contracts.ReferenceSet) contracts.ReferenceSet {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func CloneReferenceTarget(in ResolvedReferenceTarget) ResolvedReferenceTarget {
|
||||||
|
out := in
|
||||||
|
out.Bindings = append([]ReferenceBinding(nil), in.Bindings...)
|
||||||
|
out.ReferenceSet = CloneReferenceSet(in.ReferenceSet)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func ReferenceProvenance(resolved ResolvedPipeline) []artifacts.ReferenceProvenance {
|
func ReferenceProvenance(resolved ResolvedPipeline) []artifacts.ReferenceProvenance {
|
||||||
provenance := []artifacts.ReferenceProvenance{}
|
provenance := []artifacts.ReferenceProvenance{}
|
||||||
|
provenance = append(provenance, referenceTargetProvenance(resolved.ChunkReferences)...)
|
||||||
for _, lane := range resolved.ArtifactLanes {
|
for _, lane := range resolved.ArtifactLanes {
|
||||||
if len(lane.ReferenceSet.Slots) == 0 {
|
provenance = append(provenance, referenceTargetProvenance(lane.ExtractReferences)...)
|
||||||
continue
|
provenance = append(provenance, referenceTargetProvenance(lane.NormalizeReferences)...)
|
||||||
}
|
}
|
||||||
slotNames := make([]string, 0, len(lane.ReferenceSet.Slots))
|
return provenance
|
||||||
for slotName := range lane.ReferenceSet.Slots {
|
}
|
||||||
slotNames = append(slotNames, slotName)
|
|
||||||
}
|
func referenceTargetProvenance(target ResolvedReferenceTarget) []artifacts.ReferenceProvenance {
|
||||||
sort.Strings(slotNames)
|
if len(target.ReferenceSet.Slots) == 0 {
|
||||||
for _, slotName := range slotNames {
|
return nil
|
||||||
slot := lane.ReferenceSet.Slots[slotName]
|
}
|
||||||
for _, item := range slot.Items {
|
provenance := []artifacts.ReferenceProvenance{}
|
||||||
provenance = append(provenance, artifacts.ReferenceProvenance{
|
slotNames := make([]string, 0, len(target.ReferenceSet.Slots))
|
||||||
LaneID: lane.ID,
|
for slotName := range target.ReferenceSet.Slots {
|
||||||
SlotName: item.SlotName,
|
slotNames = append(slotNames, slotName)
|
||||||
OriginType: item.Origin.Type,
|
}
|
||||||
OriginURI: item.Origin.URI,
|
sort.Strings(slotNames)
|
||||||
Digest: item.Digest,
|
for _, slotName := range slotNames {
|
||||||
MediaType: item.MediaType,
|
slot := target.ReferenceSet.Slots[slotName]
|
||||||
SizeBytes: item.SizeBytes,
|
for _, item := range slot.Items {
|
||||||
BindingSource: item.BindingSource,
|
provenance = append(provenance, artifacts.ReferenceProvenance{
|
||||||
})
|
Stage: string(target.Stage),
|
||||||
}
|
LaneID: target.LaneID,
|
||||||
|
SlotName: item.SlotName,
|
||||||
|
OriginType: item.Origin.Type,
|
||||||
|
OriginURI: item.Origin.URI,
|
||||||
|
Digest: item.Digest,
|
||||||
|
MediaType: item.MediaType,
|
||||||
|
SizeBytes: item.SizeBytes,
|
||||||
|
BindingSource: item.BindingSource,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return provenance
|
return provenance
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -54,12 +56,12 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
|
|||||||
t.Fatalf("MaterializeReferences(second) error = %v, want nil", err)
|
t.Fatalf("MaterializeReferences(second) error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
referenceSet := first.ArtifactLanes[0].ReferenceSet
|
referenceSet := first.ArtifactLanes[0].ExtractReferences.ReferenceSet
|
||||||
roster := referenceSet.Slots["roster"].Items[0]
|
roster := referenceSet.Slots["roster"].Items[0]
|
||||||
if string(roster.Content) != "config text" {
|
if string(roster.Content) != "config text" {
|
||||||
t.Fatalf("roster content = %q, want config text", roster.Content)
|
t.Fatalf("roster content = %q, want config text", roster.Content)
|
||||||
}
|
}
|
||||||
if roster.Digest != referenceDigest([]byte("config text")) || roster.Digest != second.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0].Digest {
|
if roster.Digest != referenceDigest([]byte("config text")) || roster.Digest != second.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Digest {
|
||||||
t.Fatalf("roster digest = %q, want stable digest", roster.Digest)
|
t.Fatalf("roster digest = %q, want stable digest", roster.Digest)
|
||||||
}
|
}
|
||||||
if roster.BindingSource != contracts.ReferenceBindingSourceConfig {
|
if roster.BindingSource != contracts.ReferenceBindingSourceConfig {
|
||||||
@@ -87,10 +89,10 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
|
|||||||
if len(provenance) != 2 {
|
if len(provenance) != 2 {
|
||||||
t.Fatalf("ReferenceProvenance() = %#v, want two entries", provenance)
|
t.Fatalf("ReferenceProvenance() = %#v, want two entries", provenance)
|
||||||
}
|
}
|
||||||
if provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest {
|
if provenance[0].Stage != string(StageExtract) || provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest {
|
||||||
t.Fatalf("ReferenceProvenance()[0] = %#v, want sorted glossary provenance", provenance[0])
|
t.Fatalf("ReferenceProvenance()[0] = %#v, want sorted glossary provenance", provenance[0])
|
||||||
}
|
}
|
||||||
if provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest {
|
if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest {
|
||||||
t.Fatalf("ReferenceProvenance()[1] = %#v, want roster provenance", provenance[1])
|
t.Fatalf("ReferenceProvenance()[1] = %#v, want roster provenance", provenance[1])
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -103,6 +105,66 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMaterializeReferencesStoresSetsAndProvenanceForAllTargets(t *testing.T) {
|
||||||
|
configDir := t.TempDir()
|
||||||
|
writeReferenceFile(t, filepath.Join(configDir, "chunk.txt"), []byte("chunk text"))
|
||||||
|
writeReferenceFile(t, filepath.Join(configDir, "extract.txt"), []byte("extract text"))
|
||||||
|
writeReferenceFile(t, filepath.Join(configDir, "normalize.txt"), []byte("normalize text"))
|
||||||
|
|
||||||
|
profile := baselineProfile()
|
||||||
|
profile.References = map[string]string{
|
||||||
|
"scene_guide": "chunk.txt",
|
||||||
|
"roster": "extract.txt",
|
||||||
|
"normalization_notes": "normalize.txt",
|
||||||
|
}
|
||||||
|
catalog := referenceCatalogForTargets(t,
|
||||||
|
[]contracts.ReferenceSlot{{Name: "scene_guide"}},
|
||||||
|
[]contracts.ReferenceSlot{{Name: "roster"}},
|
||||||
|
[]contracts.ReferenceSlot{{Name: "normalization_notes"}},
|
||||||
|
)
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
materialized, warnings, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
|
||||||
|
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if len(warnings) != 0 {
|
||||||
|
t.Fatalf("warnings = %#v, want none", warnings)
|
||||||
|
}
|
||||||
|
|
||||||
|
chunkItem := materialized.ChunkReferences.ReferenceSet.Slots["scene_guide"].Items[0]
|
||||||
|
if string(chunkItem.Content) != "chunk text" {
|
||||||
|
t.Fatalf("chunk content = %q, want chunk text", chunkItem.Content)
|
||||||
|
}
|
||||||
|
extractItem := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
|
||||||
|
if string(extractItem.Content) != "extract text" {
|
||||||
|
t.Fatalf("extract content = %q, want extract text", extractItem.Content)
|
||||||
|
}
|
||||||
|
normalizeItem := materialized.ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["normalization_notes"].Items[0]
|
||||||
|
if string(normalizeItem.Content) != "normalize text" {
|
||||||
|
t.Fatalf("normalize content = %q, want normalize text", normalizeItem.Content)
|
||||||
|
}
|
||||||
|
|
||||||
|
provenance := ReferenceProvenance(materialized)
|
||||||
|
if len(provenance) != 3 {
|
||||||
|
t.Fatalf("ReferenceProvenance() = %#v, want three entries", provenance)
|
||||||
|
}
|
||||||
|
if provenance[0].Stage != string(StageChunk) || provenance[0].LaneID != "" || provenance[0].SlotName != "scene_guide" || provenance[0].Digest != chunkItem.Digest {
|
||||||
|
t.Fatalf("ReferenceProvenance()[0] = %#v, want chunk scene guide provenance", provenance[0])
|
||||||
|
}
|
||||||
|
if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != extractItem.Digest {
|
||||||
|
t.Fatalf("ReferenceProvenance()[1] = %#v, want extract roster provenance", provenance[1])
|
||||||
|
}
|
||||||
|
if provenance[2].Stage != string(StageNormalize) || provenance[2].LaneID != "events" || provenance[2].SlotName != "normalization_notes" || provenance[2].Digest != normalizeItem.Digest {
|
||||||
|
t.Fatalf("ReferenceProvenance()[2] = %#v, want normalize notes provenance", provenance[2])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestMaterializeReferencesRejectsNonUTF8Content(t *testing.T) {
|
func TestMaterializeReferencesRejectsNonUTF8Content(t *testing.T) {
|
||||||
configDir := t.TempDir()
|
configDir := t.TempDir()
|
||||||
path := filepath.Join(configDir, "bad.txt")
|
path := filepath.Join(configDir, "bad.txt")
|
||||||
@@ -117,6 +179,20 @@ func TestMaterializeReferencesRejectsNonUTF8Content(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMaterializeReferencesRejectsNonUTF8ContentForChunkTarget(t *testing.T) {
|
||||||
|
configDir := t.TempDir()
|
||||||
|
path := filepath.Join(configDir, "bad.txt")
|
||||||
|
writeReferenceFile(t, path, []byte{0xff, 0xfe})
|
||||||
|
|
||||||
|
resolved := resolvedPipelineWithTargetReference(t, StageChunk, "", "scene_guide", "bad.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "scene_guide"})
|
||||||
|
_, _, err := MaterializeReferences(resolved, referenceCatalogForTargets(t, []contracts.ReferenceSlot{{Name: "scene_guide"}}, nil, nil), ReferenceMaterializationOptions{
|
||||||
|
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "chunk") || !strings.Contains(err.Error(), "UTF-8") || !strings.Contains(err.Error(), "scene_guide") || !strings.Contains(err.Error(), path) {
|
||||||
|
t.Fatalf("error = %v, want chunk UTF-8 path error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestMaterializeReferencesAllowsAnyMediaTypeWhenSlotDoesNotRestrictIt(t *testing.T) {
|
func TestMaterializeReferencesAllowsAnyMediaTypeWhenSlotDoesNotRestrictIt(t *testing.T) {
|
||||||
configDir := t.TempDir()
|
configDir := t.TempDir()
|
||||||
path := filepath.Join(configDir, "roster.reference")
|
path := filepath.Join(configDir, "roster.reference")
|
||||||
@@ -129,7 +205,7 @@ func TestMaterializeReferencesAllowsAnyMediaTypeWhenSlotDoesNotRestrictIt(t *tes
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
|
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
|
||||||
if item.MediaType != unknownMediaType {
|
if item.MediaType != unknownMediaType {
|
||||||
t.Fatalf("MediaType = %q, want %q", item.MediaType, unknownMediaType)
|
t.Fatalf("MediaType = %q, want %q", item.MediaType, unknownMediaType)
|
||||||
}
|
}
|
||||||
@@ -148,7 +224,7 @@ func TestMaterializeReferencesAcceptsDeclaredMarkdownMediaType(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["glossary"].Items[0]
|
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["glossary"].Items[0]
|
||||||
if item.MediaType != "text/markdown" {
|
if item.MediaType != "text/markdown" {
|
||||||
t.Fatalf("MediaType = %q, want text/markdown", item.MediaType)
|
t.Fatalf("MediaType = %q, want text/markdown", item.MediaType)
|
||||||
}
|
}
|
||||||
@@ -167,7 +243,7 @@ func TestMaterializeReferencesAcceptsDeclaredJSONMediaType(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
|
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
|
||||||
if item.MediaType != "application/json" {
|
if item.MediaType != "application/json" {
|
||||||
t.Fatalf("MediaType = %q, want application/json", item.MediaType)
|
t.Fatalf("MediaType = %q, want application/json", item.MediaType)
|
||||||
}
|
}
|
||||||
@@ -201,12 +277,27 @@ func TestMaterializeReferencesMatchesAcceptedMediaTypesIgnoringParameters(t *tes
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
|
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
|
||||||
if item.MediaType != referenceMediaType {
|
if item.MediaType != referenceMediaType {
|
||||||
t.Fatalf("MediaType = %q, want %q", item.MediaType, referenceMediaType)
|
t.Fatalf("MediaType = %q, want %q", item.MediaType, referenceMediaType)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMaterializeReferencesRejectsUnacceptedMediaTypeForNormalizeTarget(t *testing.T) {
|
||||||
|
configDir := t.TempDir()
|
||||||
|
path := filepath.Join(configDir, "notes.json")
|
||||||
|
writeReferenceFile(t, path, []byte(`{"notes":true}`))
|
||||||
|
|
||||||
|
slot := contracts.ReferenceSlot{Name: "normalization_notes", AcceptedMediaTypes: []string{"text/markdown"}}
|
||||||
|
resolved := resolvedPipelineWithTargetReference(t, StageNormalize, "events", "normalization_notes", "notes.json", contracts.ReferenceBindingSourceConfig, slot)
|
||||||
|
_, _, err := MaterializeReferences(resolved, referenceCatalogForTargets(t, nil, nil, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
|
||||||
|
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "normalize") || !strings.Contains(err.Error(), "media type") || !strings.Contains(err.Error(), "application/json") || !strings.Contains(err.Error(), "normalization_notes") {
|
||||||
|
t.Fatalf("error = %v, want normalize media type rejection", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestMaterializeReferencesWarnsForEmptyFiles(t *testing.T) {
|
func TestMaterializeReferencesWarnsForEmptyFiles(t *testing.T) {
|
||||||
configDir := t.TempDir()
|
configDir := t.TempDir()
|
||||||
path := filepath.Join(configDir, "empty.txt")
|
path := filepath.Join(configDir, "empty.txt")
|
||||||
@@ -222,12 +313,51 @@ func TestMaterializeReferencesWarnsForEmptyFiles(t *testing.T) {
|
|||||||
if len(warnings) != 1 || warnings[0].ReasonCode != "empty_reference" {
|
if len(warnings) != 1 || warnings[0].ReasonCode != "empty_reference" {
|
||||||
t.Fatalf("warnings = %#v, want empty reference warning", warnings)
|
t.Fatalf("warnings = %#v, want empty reference warning", warnings)
|
||||||
}
|
}
|
||||||
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
|
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
|
||||||
if item.SizeBytes != 0 || item.Digest != referenceDigest(nil) {
|
if item.SizeBytes != 0 || item.Digest != referenceDigest(nil) {
|
||||||
t.Fatalf("empty item = %#v, want zero size and empty digest", item)
|
t.Fatalf("empty item = %#v, want zero size and empty digest", item)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMaterializeReferencesWarningScopesIncludeTargetContext(t *testing.T) {
|
||||||
|
configDir := t.TempDir()
|
||||||
|
writeReferenceFile(t, filepath.Join(configDir, "chunk.txt"), nil)
|
||||||
|
writeReferenceFile(t, filepath.Join(configDir, "extract.txt"), nil)
|
||||||
|
writeReferenceFile(t, filepath.Join(configDir, "normalize.txt"), nil)
|
||||||
|
|
||||||
|
profile := baselineProfile()
|
||||||
|
profile.References = map[string]string{
|
||||||
|
"scene_guide": "chunk.txt",
|
||||||
|
"roster": "extract.txt",
|
||||||
|
"normalization_notes": "normalize.txt",
|
||||||
|
}
|
||||||
|
catalog := referenceCatalogForTargets(t,
|
||||||
|
[]contracts.ReferenceSlot{{Name: "scene_guide"}},
|
||||||
|
[]contracts.ReferenceSlot{{Name: "roster"}},
|
||||||
|
[]contracts.ReferenceSlot{{Name: "normalization_notes"}},
|
||||||
|
)
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, warnings, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
|
||||||
|
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
got := warningScopes(warnings)
|
||||||
|
want := []string{
|
||||||
|
"pipeline.baseline.chunk.reference.scene_guide",
|
||||||
|
"pipeline.baseline.lane.events.extract.reference.roster",
|
||||||
|
"pipeline.baseline.lane.events.normalize.reference.normalization_notes",
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("warning scopes = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestMaterializeReferencesEnforcesMaxBytes(t *testing.T) {
|
func TestMaterializeReferencesEnforcesMaxBytes(t *testing.T) {
|
||||||
configDir := t.TempDir()
|
configDir := t.TempDir()
|
||||||
path := filepath.Join(configDir, "large.txt")
|
path := filepath.Join(configDir, "large.txt")
|
||||||
@@ -244,31 +374,99 @@ func TestMaterializeReferencesEnforcesMaxBytes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func resolvedPipelineWithReference(t *testing.T, slotName, source, bindingSource string, slot contracts.ReferenceSlot) ResolvedPipeline {
|
func resolvedPipelineWithReference(t *testing.T, slotName, source, bindingSource string, slot contracts.ReferenceSlot) ResolvedPipeline {
|
||||||
|
t.Helper()
|
||||||
|
return resolvedPipelineWithTargetReference(t, StageExtract, "events", slotName, source, bindingSource, slot)
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolvedPipelineWithTargetReference(t *testing.T, stage ModuleStage, laneID string, slotName, source, bindingSource string, slot contracts.ReferenceSlot) ResolvedPipeline {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
profile := baselineProfile()
|
profile := baselineProfile()
|
||||||
lane := profile.Artifacts["events"]
|
switch stage {
|
||||||
lane.References = map[string]string{slotName: source}
|
case StageChunk:
|
||||||
profile.Artifacts["events"] = lane
|
profile.Chunk.References = map[string]string{slotName: source}
|
||||||
catalog := referenceCatalog(t, []contracts.ReferenceSlot{slot})
|
case StageExtract:
|
||||||
|
lane := profile.Artifacts[laneID]
|
||||||
|
lane.References = map[string]string{slotName: source}
|
||||||
|
profile.Artifacts[laneID] = lane
|
||||||
|
case StageNormalize:
|
||||||
|
lane := profile.Artifacts[laneID]
|
||||||
|
lane.Normalize.References = map[string]string{slotName: source}
|
||||||
|
profile.Artifacts[laneID] = lane
|
||||||
|
default:
|
||||||
|
t.Fatalf("unsupported reference target stage %q", stage)
|
||||||
|
}
|
||||||
|
catalog := referenceCatalogForStage(t, stage, []contracts.ReferenceSlot{slot})
|
||||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
if bindingSource != contracts.ReferenceBindingSourceConfig {
|
if bindingSource != contracts.ReferenceBindingSourceConfig {
|
||||||
resolved.ArtifactLanes[0].References[0].BindingSource = bindingSource
|
switch stage {
|
||||||
|
case StageChunk:
|
||||||
|
resolved.ChunkReferences.Bindings[0].BindingSource = bindingSource
|
||||||
|
case StageExtract:
|
||||||
|
resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].BindingSource = bindingSource
|
||||||
|
case StageNormalize:
|
||||||
|
resolved.ArtifactLanes[0].NormalizeReferences.Bindings[0].BindingSource = bindingSource
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return resolved
|
return resolved
|
||||||
}
|
}
|
||||||
|
|
||||||
func referenceCatalog(t *testing.T, slots []contracts.ReferenceSlot) ModuleCatalog {
|
func referenceCatalog(t *testing.T, slots []contracts.ReferenceSlot) ModuleCatalog {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
return newProfileCatalogWithOverride(t, ModuleSpec{
|
return referenceCatalogForStage(t, StageExtract, slots)
|
||||||
Key: "event-extractor",
|
}
|
||||||
Stage: StageExtract,
|
|
||||||
Requires: []string{"chunk"},
|
func referenceCatalogForStage(t *testing.T, stage ModuleStage, slots []contracts.ReferenceSlot) ModuleCatalog {
|
||||||
Provides: []string{"candidate"},
|
t.Helper()
|
||||||
ReferenceSlots: slots,
|
switch stage {
|
||||||
})
|
case StageChunk:
|
||||||
|
return referenceCatalogForTargets(t, slots, nil, nil)
|
||||||
|
case StageExtract:
|
||||||
|
return referenceCatalogForTargets(t, nil, slots, nil)
|
||||||
|
case StageNormalize:
|
||||||
|
return referenceCatalogForTargets(t, nil, nil, slots)
|
||||||
|
default:
|
||||||
|
t.Fatalf("unsupported reference target stage %q", stage)
|
||||||
|
return ModuleCatalog{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func referenceCatalogForTargets(t *testing.T, chunkSlots, extractSlots, normalizeSlots []contracts.ReferenceSlot) ModuleCatalog {
|
||||||
|
t.Helper()
|
||||||
|
return newProfileCatalogWithOverrides(t,
|
||||||
|
ModuleSpec{
|
||||||
|
Key: "generic",
|
||||||
|
Stage: StageChunk,
|
||||||
|
Requires: []string{"source"},
|
||||||
|
Provides: []string{"chunk"},
|
||||||
|
ReferenceSlots: chunkSlots,
|
||||||
|
},
|
||||||
|
ModuleSpec{
|
||||||
|
Key: "event-extractor",
|
||||||
|
Stage: StageExtract,
|
||||||
|
Requires: []string{"chunk"},
|
||||||
|
Provides: []string{"candidate"},
|
||||||
|
ReferenceSlots: extractSlots,
|
||||||
|
},
|
||||||
|
ModuleSpec{
|
||||||
|
Key: "noop",
|
||||||
|
Stage: StageNormalize,
|
||||||
|
Requires: []string{"merged"},
|
||||||
|
Provides: []string{"normalized"},
|
||||||
|
ReferenceSlots: normalizeSlots,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func warningScopes(warnings []contracts.Warning) []string {
|
||||||
|
scopes := make([]string, 0, len(warnings))
|
||||||
|
for _, warning := range warnings {
|
||||||
|
scopes = append(scopes, warning.Scope)
|
||||||
|
}
|
||||||
|
sort.Strings(scopes)
|
||||||
|
return scopes
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeReferenceFile(t *testing.T, path string, content []byte) {
|
func writeReferenceFile(t *testing.T, path string, content []byte) {
|
||||||
|
|||||||
@@ -118,6 +118,10 @@ func (chunker integrationChunker) Key() string {
|
|||||||
return "chunk"
|
return "chunk"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (chunker integrationChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (chunker integrationChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (chunker integrationChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||||
return contracts.ChunkResult{
|
return contracts.ChunkResult{
|
||||||
Chunks: []contracts.SourceChunk{
|
Chunks: []contracts.SourceChunk{
|
||||||
@@ -186,6 +190,10 @@ func (normalizer integrationNormalizer) Key() string {
|
|||||||
return "normalize"
|
return "normalize"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (normalizer integrationNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (normalizer integrationNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
func (normalizer integrationNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
||||||
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
|
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -95,6 +95,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
attachModuleManifestMetadata(&output, "chunker", chunker)
|
attachModuleManifestMetadata(&output, "chunker", chunker)
|
||||||
chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
|
chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
|
||||||
Source: doc,
|
Source: doc,
|
||||||
|
References: CloneReferenceSet(input.Pipeline.ChunkReferences.ReferenceSet),
|
||||||
LLMClient: input.LLMClient,
|
LLMClient: input.LLMClient,
|
||||||
LLMProfile: input.Pipeline.Chunk.LLMProfile,
|
LLMProfile: input.Pipeline.Chunk.LLMProfile,
|
||||||
Options: cloneOptions(input.Pipeline.Chunk.Options),
|
Options: cloneOptions(input.Pipeline.Chunk.Options),
|
||||||
@@ -186,7 +187,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
|
|||||||
result, err := extractor.Extract(ctx, contracts.ExtractionRequest{
|
result, err := extractor.Extract(ctx, contracts.ExtractionRequest{
|
||||||
Source: doc,
|
Source: doc,
|
||||||
Chunk: &chunk,
|
Chunk: &chunk,
|
||||||
References: CloneReferenceSet(lane.ReferenceSet),
|
References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet),
|
||||||
LLMClient: input.LLMClient,
|
LLMClient: input.LLMClient,
|
||||||
LLMProfile: lane.Extract.LLMProfile,
|
LLMProfile: lane.Extract.LLMProfile,
|
||||||
Options: cloneOptions(lane.Extract.Options),
|
Options: cloneOptions(lane.Extract.Options),
|
||||||
@@ -224,6 +225,8 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
|
|||||||
Source: doc,
|
Source: doc,
|
||||||
LaneID: lane.ID,
|
LaneID: lane.ID,
|
||||||
Candidates: mergeResult.Candidates,
|
Candidates: mergeResult.Candidates,
|
||||||
|
References: CloneReferenceSet(lane.NormalizeReferences.ReferenceSet),
|
||||||
|
LLMClient: input.LLMClient,
|
||||||
LLMProfile: lane.Normalize.LLMProfile,
|
LLMProfile: lane.Normalize.LLMProfile,
|
||||||
Options: cloneOptions(lane.Normalize.Options),
|
Options: cloneOptions(lane.Normalize.Options),
|
||||||
Metadata: input.Metadata,
|
Metadata: input.Metadata,
|
||||||
|
|||||||
@@ -464,6 +464,10 @@ func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) {
|
|||||||
if len(extractor.seenLLMClients) != 2 || extractor.seenLLMClients[0] == nil || extractor.seenLLMClients[1] == nil {
|
if len(extractor.seenLLMClients) != 2 || extractor.seenLLMClients[0] == nil || extractor.seenLLMClients[1] == nil {
|
||||||
t.Fatalf("seen LLM clients = %#v, want client for each chunk", extractor.seenLLMClients)
|
t.Fatalf("seen LLM clients = %#v, want client for each chunk", extractor.seenLLMClients)
|
||||||
}
|
}
|
||||||
|
normalizer := modules.normalizers["normalize"]
|
||||||
|
if len(normalizer.requests) != 1 || normalizer.requests[0].LLMClient == nil {
|
||||||
|
t.Fatalf("normalizer LLM client = %#v, want client on normalize request", normalizer.requests)
|
||||||
|
}
|
||||||
if extractor.seenMetadata[0]["request"] != "test" {
|
if extractor.seenMetadata[0]["request"] != "test" {
|
||||||
t.Fatalf("seen metadata = %#v, want request metadata", extractor.seenMetadata)
|
t.Fatalf("seen metadata = %#v, want request metadata", extractor.seenMetadata)
|
||||||
}
|
}
|
||||||
@@ -562,24 +566,7 @@ func TestRunPassesModuleBindingConfigToStageRequests(t *testing.T) {
|
|||||||
func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) {
|
func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
pipeline := resolvedPipeline()
|
pipeline := resolvedPipeline()
|
||||||
pipeline.ArtifactLanes[0].ReferenceSet = contracts.ReferenceSet{
|
pipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = testReferenceSet("roster", "reference text")
|
||||||
Slots: map[string]contracts.ResolvedReferenceSlot{
|
|
||||||
"roster": {
|
|
||||||
Slot: contracts.ReferenceSlot{Name: "roster"},
|
|
||||||
Items: []contracts.ReferenceItem{
|
|
||||||
{
|
|
||||||
SlotName: "roster",
|
|
||||||
MediaType: "text/plain; charset=utf-8",
|
|
||||||
Content: []byte("reference text"),
|
|
||||||
Digest: "sha256:test",
|
|
||||||
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/reference.txt"},
|
|
||||||
SizeBytes: int64(len("reference text")),
|
|
||||||
BindingSource: contracts.ReferenceBindingSourceConfig,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
|
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -592,11 +579,60 @@ func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) {
|
|||||||
t.Fatalf("reference content = %q, want reference text", item.Content)
|
t.Fatalf("reference content = %q, want reference text", item.Content)
|
||||||
}
|
}
|
||||||
item.Content[0] = 'R'
|
item.Content[0] = 'R'
|
||||||
if got := string(pipeline.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0].Content); got != "reference text" {
|
if got := string(pipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content); got != "reference text" {
|
||||||
t.Fatalf("runner mutated reference set content = %q", got)
|
t.Fatalf("runner mutated reference set content = %q", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunPassesChunkReferencesToChunkerRequest(t *testing.T) {
|
||||||
|
modules := defaultRunnerModules()
|
||||||
|
pipeline := resolvedPipeline()
|
||||||
|
pipeline.ChunkReferences.ReferenceSet = testReferenceSet("scene_guide", "chunk reference text")
|
||||||
|
|
||||||
|
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req := modules.chunker.requests[0]
|
||||||
|
item := req.References.Slots["scene_guide"].Items[0]
|
||||||
|
if string(item.Content) != "chunk reference text" {
|
||||||
|
t.Fatalf("chunk reference content = %q, want chunk reference text", item.Content)
|
||||||
|
}
|
||||||
|
item.Content[0] = 'C'
|
||||||
|
if got := string(pipeline.ChunkReferences.ReferenceSet.Slots["scene_guide"].Items[0].Content); got != "chunk reference text" {
|
||||||
|
t.Fatalf("runner mutated chunk reference set content = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPassesNormalizeReferencesToNormalizerRequest(t *testing.T) {
|
||||||
|
modules := defaultRunnerModules()
|
||||||
|
pipeline := resolvedPipeline()
|
||||||
|
pipeline.ArtifactLanes[0].NormalizeReferences.ReferenceSet = testReferenceSet("normalization_notes", "normalize reference text")
|
||||||
|
|
||||||
|
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req := modules.normalizers["normalize"].requests[0]
|
||||||
|
item := req.References.Slots["normalization_notes"].Items[0]
|
||||||
|
if string(item.Content) != "normalize reference text" {
|
||||||
|
t.Fatalf("normalize reference content = %q, want normalize reference text", item.Content)
|
||||||
|
}
|
||||||
|
item.Content[0] = 'N'
|
||||||
|
if got := string(pipeline.ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["normalization_notes"].Items[0].Content); got != "normalize reference text" {
|
||||||
|
t.Fatalf("runner mutated normalize reference set content = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunAllowsNilLLMClientWhenModulesDoNotUseIt(t *testing.T) {
|
||||||
|
_, err := New(newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil with nil LLM client when modules do not use it", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunIncludesInputWarnings(t *testing.T) {
|
func TestRunIncludesInputWarnings(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
warning := contracts.Warning{Scope: "reference", ReasonCode: "empty_reference", Message: "empty reference"}
|
warning := contracts.Warning{Scope: "reference", ReasonCode: "empty_reference", Message: "empty reference"}
|
||||||
@@ -961,7 +997,25 @@ func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
|
func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
|
||||||
resolved := resolvedPipelineWithValidators("configured")
|
resolved := resolvedPipelineWithValidators("configured")
|
||||||
resolved.ArtifactLanes[0].ReferenceSet = contracts.ReferenceSet{
|
resolved.ChunkReferences.ReferenceSet = contracts.ReferenceSet{
|
||||||
|
Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||||
|
"scene_guide": {
|
||||||
|
Slot: contracts.ReferenceSlot{Name: "scene_guide"},
|
||||||
|
Items: []contracts.ReferenceItem{
|
||||||
|
{
|
||||||
|
SlotName: "scene_guide",
|
||||||
|
MediaType: "text/plain; charset=utf-8",
|
||||||
|
Content: []byte("chunk reference content"),
|
||||||
|
Digest: "sha256:chunk-reference",
|
||||||
|
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/scene-guide.txt"},
|
||||||
|
SizeBytes: int64(len("chunk reference content")),
|
||||||
|
BindingSource: contracts.ReferenceBindingSourceCLI,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
resolved.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
|
||||||
Slots: map[string]contracts.ResolvedReferenceSlot{
|
Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||||
"roster": {
|
"roster": {
|
||||||
Slot: contracts.ReferenceSlot{Name: "roster"},
|
Slot: contracts.ReferenceSlot{Name: "roster"},
|
||||||
@@ -979,6 +1033,24 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
resolved.ArtifactLanes[0].NormalizeReferences.ReferenceSet = contracts.ReferenceSet{
|
||||||
|
Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||||
|
"normalization_notes": {
|
||||||
|
Slot: contracts.ReferenceSlot{Name: "normalization_notes"},
|
||||||
|
Items: []contracts.ReferenceItem{
|
||||||
|
{
|
||||||
|
SlotName: "normalization_notes",
|
||||||
|
MediaType: "text/plain; charset=utf-8",
|
||||||
|
Content: []byte("normalize reference content"),
|
||||||
|
Digest: "sha256:normalize-reference",
|
||||||
|
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/normalize.txt"},
|
||||||
|
SizeBytes: int64(len("normalize reference content")),
|
||||||
|
BindingSource: contracts.ReferenceBindingSourceConfig,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: resolved})
|
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: resolved})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -995,15 +1067,26 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
|
|||||||
if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:source"}) {
|
if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:source"}) {
|
||||||
t.Fatalf("SourceDigests = %#v, want source digest", manifest.SourceDigests)
|
t.Fatalf("SourceDigests = %#v, want source digest", manifest.SourceDigests)
|
||||||
}
|
}
|
||||||
if len(manifest.References) != 1 {
|
if len(manifest.References) != 3 {
|
||||||
t.Fatalf("References = %#v, want one reference provenance entry", manifest.References)
|
t.Fatalf("References = %#v, want three reference provenance entries", manifest.References)
|
||||||
}
|
}
|
||||||
reference := manifest.References[0]
|
chunkReference := manifest.References[0]
|
||||||
if reference.LaneID != "alpha" || reference.SlotName != "roster" || reference.Digest != "sha256:reference" {
|
if chunkReference.Stage != string(StageChunk) || chunkReference.LaneID != "" || chunkReference.SlotName != "scene_guide" || chunkReference.Digest != "sha256:chunk-reference" {
|
||||||
t.Fatalf("reference provenance = %#v, want lane slot digest", reference)
|
t.Fatalf("chunk reference provenance = %#v, want chunk slot digest", chunkReference)
|
||||||
}
|
}
|
||||||
if reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != int64(len("reference content")) || reference.BindingSource != contracts.ReferenceBindingSourceConfig {
|
if chunkReference.OriginType != "file" || chunkReference.OriginURI != "file:///tmp/scene-guide.txt" || chunkReference.MediaType != "text/plain; charset=utf-8" || chunkReference.SizeBytes != int64(len("chunk reference content")) || chunkReference.BindingSource != contracts.ReferenceBindingSourceCLI {
|
||||||
t.Fatalf("reference provenance = %#v, want origin/media/size/source", reference)
|
t.Fatalf("chunk reference provenance = %#v, want origin/media/size/source", chunkReference)
|
||||||
|
}
|
||||||
|
extractReference := manifest.References[1]
|
||||||
|
if extractReference.Stage != string(StageExtract) || extractReference.LaneID != "alpha" || extractReference.SlotName != "roster" || extractReference.Digest != "sha256:reference" {
|
||||||
|
t.Fatalf("extract reference provenance = %#v, want lane slot digest", extractReference)
|
||||||
|
}
|
||||||
|
if extractReference.OriginType != "file" || extractReference.OriginURI != "file:///tmp/roster.txt" || extractReference.MediaType != "text/plain; charset=utf-8" || extractReference.SizeBytes != int64(len("reference content")) || extractReference.BindingSource != contracts.ReferenceBindingSourceConfig {
|
||||||
|
t.Fatalf("extract reference provenance = %#v, want origin/media/size/source", extractReference)
|
||||||
|
}
|
||||||
|
normalizeReference := manifest.References[2]
|
||||||
|
if normalizeReference.Stage != string(StageNormalize) || normalizeReference.LaneID != "alpha" || normalizeReference.SlotName != "normalization_notes" || normalizeReference.Digest != "sha256:normalize-reference" {
|
||||||
|
t.Fatalf("normalize reference provenance = %#v, want lane slot digest", normalizeReference)
|
||||||
}
|
}
|
||||||
if manifest.ValidationStatus != "approved" {
|
if manifest.ValidationStatus != "approved" {
|
||||||
t.Fatalf("ValidationStatus = %q, want approved", manifest.ValidationStatus)
|
t.Fatalf("ValidationStatus = %q, want approved", manifest.ValidationStatus)
|
||||||
@@ -1157,16 +1240,19 @@ func TestRunRejectsNilDefaultValidator(t *testing.T) {
|
|||||||
|
|
||||||
func resolvedPipeline() ResolvedPipeline {
|
func resolvedPipeline() ResolvedPipeline {
|
||||||
return ResolvedPipeline{
|
return ResolvedPipeline{
|
||||||
ID: "pipeline-1",
|
ID: "pipeline-1",
|
||||||
Digest: "sha256:pipeline",
|
Digest: "sha256:pipeline",
|
||||||
Input: Binding("input"),
|
Input: Binding("input"),
|
||||||
Chunk: Binding("chunk"),
|
Chunk: Binding("chunk"),
|
||||||
|
ChunkReferences: referenceTarget(StageChunk, "", "chunk", nil),
|
||||||
ArtifactLanes: []ResolvedArtifactLane{
|
ArtifactLanes: []ResolvedArtifactLane{
|
||||||
{
|
{
|
||||||
ID: "alpha",
|
ID: "alpha",
|
||||||
Extract: Binding("extract-alpha"),
|
Extract: Binding("extract-alpha"),
|
||||||
Merge: Binding("merge"),
|
Merge: Binding("merge"),
|
||||||
Normalize: Binding("normalize"),
|
Normalize: Binding("normalize"),
|
||||||
|
ExtractReferences: referenceTarget(StageExtract, "alpha", "extract-alpha", nil),
|
||||||
|
NormalizeReferences: referenceTarget(StageNormalize, "alpha", "normalize", nil),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
Output: Binding("output"),
|
Output: Binding("output"),
|
||||||
@@ -1181,6 +1267,27 @@ func resolvedPipelineWithValidators(validators ...string) ResolvedPipeline {
|
|||||||
return pipeline
|
return pipeline
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func testReferenceSet(slotName string, content string) contracts.ReferenceSet {
|
||||||
|
return contracts.ReferenceSet{
|
||||||
|
Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||||
|
slotName: {
|
||||||
|
Slot: contracts.ReferenceSlot{Name: slotName},
|
||||||
|
Items: []contracts.ReferenceItem{
|
||||||
|
{
|
||||||
|
SlotName: slotName,
|
||||||
|
MediaType: "text/plain; charset=utf-8",
|
||||||
|
Content: []byte(content),
|
||||||
|
Digest: "sha256:test",
|
||||||
|
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/reference.txt"},
|
||||||
|
SizeBytes: int64(len(content)),
|
||||||
|
BindingSource: contracts.ReferenceBindingSourceConfig,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type runnerModules struct {
|
type runnerModules struct {
|
||||||
input *runnerInputAdapter
|
input *runnerInputAdapter
|
||||||
chunker *runnerChunker
|
chunker *runnerChunker
|
||||||
@@ -1314,6 +1421,10 @@ func (chunker *runnerChunker) Key() string {
|
|||||||
return chunker.key
|
return chunker.key
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (chunker *runnerChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||||
chunker.requests = append(chunker.requests, req)
|
chunker.requests = append(chunker.requests, req)
|
||||||
return contracts.ChunkResult{
|
return contracts.ChunkResult{
|
||||||
@@ -1421,6 +1532,10 @@ func (normalizer *runnerNormalizer) Key() string {
|
|||||||
return normalizer.key
|
return normalizer.key
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (normalizer *runnerNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (normalizer *runnerNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
func (normalizer *runnerNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
||||||
normalizer.requests = append(normalizer.requests, req)
|
normalizer.requests = append(normalizer.requests, req)
|
||||||
candidates := append([]artifacts.ArtifactCandidate(nil), normalizer.result...)
|
candidates := append([]artifacts.ArtifactCandidate(nil), normalizer.result...)
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"manifest": {
|
"manifest": {
|
||||||
"pipeline_id": "walking-skeleton",
|
"pipeline_id": "walking-skeleton",
|
||||||
"pipeline_digest": "sha256:5df1e501a2307ef75bbfeb59d315b3710571d52e5466a9c7f8320248740e6fca",
|
"pipeline_digest": "sha256:25084e39a0cadace375c896551d1752413755c1a6772f6b24cfe91c029ea2631",
|
||||||
"validation_status": "approved",
|
"validation_status": "approved",
|
||||||
"artifact_lanes": [
|
"artifact_lanes": [
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -43,8 +43,8 @@ func TestWalkingSkeletonFixture(t *testing.T) {
|
|||||||
t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType)
|
t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType)
|
||||||
}
|
}
|
||||||
assertStructuralJSONEqual(t, output.OutputFiles[0].Bytes, expectedBytes)
|
assertStructuralJSONEqual(t, output.OutputFiles[0].Bytes, expectedBytes)
|
||||||
if llmClient.calls != 2 {
|
if llmClient.calls != 3 {
|
||||||
t.Fatalf("LLM calls = %d, want chunk count 2", llmClient.calls)
|
t.Fatalf("LLM calls = %d, want extractor calls plus normalizer call", llmClient.calls)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -216,6 +216,10 @@ func (chunker walkingSkeletonChunker) Key() string {
|
|||||||
return "fake/chunk"
|
return "fake/chunk"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (chunker walkingSkeletonChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (chunker walkingSkeletonChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (chunker walkingSkeletonChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||||
if len(req.Source.Units) < 3 {
|
if len(req.Source.Units) < 3 {
|
||||||
return contracts.ChunkResult{}, fmt.Errorf("fixture source must contain at least three units")
|
return contracts.ChunkResult{}, fmt.Errorf("fixture source must contain at least three units")
|
||||||
@@ -336,7 +340,20 @@ func (normalizer walkingSkeletonNormalizer) Key() string {
|
|||||||
return DefaultNormalizeModule
|
return DefaultNormalizeModule
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (normalizer walkingSkeletonNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (normalizer walkingSkeletonNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
func (normalizer walkingSkeletonNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
||||||
|
var response struct {
|
||||||
|
Call int `json:"call"`
|
||||||
|
}
|
||||||
|
if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||||
|
StageName: "fake/normalize",
|
||||||
|
ResponseSchemaName: "fake_normalize",
|
||||||
|
}, &response); err != nil {
|
||||||
|
return contracts.NormalizeResult{}, err
|
||||||
|
}
|
||||||
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
|
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -97,6 +97,57 @@ func TestRenderUserSystemWithReferencesRendersDeclaredSlots(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRenderUserSystemWithReferencesSupportsChunkRequestData(t *testing.T) {
|
||||||
|
bundle := loadReferenceBundle(t,
|
||||||
|
[]contracts.ReferenceSlot{{Name: "scene_guide"}},
|
||||||
|
`Chunk system has guide={{ hasreference "scene_guide" }}`,
|
||||||
|
`Source={{ .SourceID }} Guide={{ reference "scene_guide" }}`,
|
||||||
|
)
|
||||||
|
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||||
|
"scene_guide": {
|
||||||
|
Slot: contracts.ReferenceSlot{Name: "scene_guide"},
|
||||||
|
Items: []contracts.ReferenceItem{
|
||||||
|
{SlotName: "scene_guide", Content: []byte("Keep combat scenes separate.")},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
|
||||||
|
system, user, _, err := bundle.RenderUserSystemWithReferences(map[string]any{"SourceID": "session-alpha"}, references)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("RenderUserSystemWithReferences: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(system, "has guide=true") {
|
||||||
|
t.Fatalf("system = %q, want chunk reference presence", system)
|
||||||
|
}
|
||||||
|
if !strings.Contains(user, "Source=session-alpha") || !strings.Contains(user, "Keep combat scenes separate.") {
|
||||||
|
t.Fatalf("user = %q, want chunk request data and reference content", user)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderUserSystemWithReferencesSupportsNormalizeRequestData(t *testing.T) {
|
||||||
|
bundle := loadReferenceBundle(t,
|
||||||
|
[]contracts.ReferenceSlot{{Name: "normalization_notes"}},
|
||||||
|
`Normalize system`,
|
||||||
|
`Lane={{ .LaneID }} Notes={{ reference "normalization_notes" }}`,
|
||||||
|
)
|
||||||
|
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||||
|
"normalization_notes": {
|
||||||
|
Slot: contracts.ReferenceSlot{Name: "normalization_notes"},
|
||||||
|
Items: []contracts.ReferenceItem{
|
||||||
|
{SlotName: "normalization_notes", Content: []byte("Prefer canonical item names.")},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
|
||||||
|
_, user, _, err := bundle.RenderUserSystemWithReferences(map[string]any{"LaneID": "spells"}, references)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("RenderUserSystemWithReferences: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(user, "Lane=spells") || !strings.Contains(user, "Prefer canonical item names.") {
|
||||||
|
t.Fatalf("user = %q, want normalize request data and reference content", user)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRenderUserSystemReferenceHasReferenceRequiresContent(t *testing.T) {
|
func TestRenderUserSystemReferenceHasReferenceRequiresContent(t *testing.T) {
|
||||||
bundle := loadReferenceBundle(t,
|
bundle := loadReferenceBundle(t,
|
||||||
[]contracts.ReferenceSlot{{Name: "roster"}},
|
[]contracts.ReferenceSlot{{Name: "roster"}},
|
||||||
|
|||||||
@@ -34,6 +34,10 @@ func (c *Chunker) Key() string {
|
|||||||
return Key
|
return Key
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Chunker) ManifestMetadata() map[string]any {
|
func (c *Chunker) ManifestMetadata() map[string]any {
|
||||||
promptMetadata := scenesPromptBundle.Metadata()
|
promptMetadata := scenesPromptBundle.Metadata()
|
||||||
metadata := map[string]any{
|
metadata := map[string]any{
|
||||||
|
|||||||
@@ -57,6 +57,9 @@ func TestNewModuleSpecAndRegister(t *testing.T) {
|
|||||||
if built.Key() != Key {
|
if built.Key() != Key {
|
||||||
t.Fatalf("built Key() = %q, want %q", built.Key(), Key)
|
t.Fatalf("built Key() = %q, want %q", built.Key(), Key)
|
||||||
}
|
}
|
||||||
|
if slots := built.ReferenceSlots(); len(slots) != 0 {
|
||||||
|
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRegisterNilRegistryReturnsError(t *testing.T) {
|
func TestRegisterNilRegistryReturnsError(t *testing.T) {
|
||||||
|
|||||||
@@ -31,6 +31,10 @@ func (c *Chunker) Key() string {
|
|||||||
return Key
|
return Key
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||||
if c == nil {
|
if c == nil {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("chunker must not be nil")
|
return contracts.ChunkResult{}, chunkerErrorf("chunker must not be nil")
|
||||||
|
|||||||
@@ -41,6 +41,9 @@ func TestModuleSpecAndRegister(t *testing.T) {
|
|||||||
if chunker.Key() != Key {
|
if chunker.Key() != Key {
|
||||||
t.Fatalf("Key() = %q, want %q", chunker.Key(), Key)
|
t.Fatalf("Key() = %q, want %q", chunker.Key(), Key)
|
||||||
}
|
}
|
||||||
|
if slots := chunker.ReferenceSlots(); len(slots) != 0 {
|
||||||
|
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestChunkUsesDefaultsForSingleChunk(t *testing.T) {
|
func TestChunkUsesDefaultsForSingleChunk(t *testing.T) {
|
||||||
|
|||||||
@@ -236,6 +236,10 @@ func (dndSpellsChunker) Key() string {
|
|||||||
return "fake/chunk"
|
return "fake/chunk"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (dndSpellsChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (dndSpellsChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (dndSpellsChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||||
return contracts.ChunkResult{
|
return contracts.ChunkResult{
|
||||||
Chunks: []contracts.SourceChunk{
|
Chunks: []contracts.SourceChunk{
|
||||||
|
|||||||
@@ -114,7 +114,7 @@ func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T)
|
|||||||
raw := readDNDSpellsFixture(t)
|
raw := readDNDSpellsFixture(t)
|
||||||
expectedDoc := parseDNDSpellsFixture(t, raw)
|
expectedDoc := parseDNDSpellsFixture(t, raw)
|
||||||
resolved := resolveDNDSpellsPipeline(t)
|
resolved := resolveDNDSpellsPipeline(t)
|
||||||
resolved.ResolvedPipeline.ArtifactLanes[0].ReferenceSet = dndSpellsReferenceSet(
|
resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet(
|
||||||
"Aria: party cleric\nBorin: fighter",
|
"Aria: party cleric\nBorin: fighter",
|
||||||
"Fire Bolt: evocation cantrip",
|
"Fire Bolt: evocation cantrip",
|
||||||
)
|
)
|
||||||
@@ -168,7 +168,7 @@ func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T)
|
|||||||
func TestRunnerDoesNotExtractSpellMentionedOnlyInRoster(t *testing.T) {
|
func TestRunnerDoesNotExtractSpellMentionedOnlyInRoster(t *testing.T) {
|
||||||
raw := readDNDSpellsFixture(t)
|
raw := readDNDSpellsFixture(t)
|
||||||
resolved := resolveDNDSpellsPipeline(t)
|
resolved := resolveDNDSpellsPipeline(t)
|
||||||
resolved.ResolvedPipeline.ArtifactLanes[0].ReferenceSet = dndSpellsReferenceSet(
|
resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet(
|
||||||
"Mira: wizard who can cast Lightning Bolt",
|
"Mira: wizard who can cast Lightning Bolt",
|
||||||
"",
|
"",
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -209,6 +209,8 @@ type fakeChunker struct{}
|
|||||||
|
|
||||||
func (fakeChunker) Key() string { return "fake/chunk" }
|
func (fakeChunker) Key() string { return "fake/chunk" }
|
||||||
|
|
||||||
|
func (fakeChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||||
|
|
||||||
func (fakeChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (fakeChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||||
return contracts.ChunkResult{}, nil
|
return contracts.ChunkResult{}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -157,6 +157,10 @@ func (runnerSeriatimChunker) Key() string {
|
|||||||
return "fake/chunk"
|
return "fake/chunk"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (runnerSeriatimChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (runnerSeriatimChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (runnerSeriatimChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||||
return contracts.ChunkResult{
|
return contracts.ChunkResult{
|
||||||
Chunks: []contracts.SourceChunk{
|
Chunks: []contracts.SourceChunk{
|
||||||
|
|||||||
@@ -25,6 +25,10 @@ func (n *Normalizer) Key() string {
|
|||||||
return Key
|
return Key
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (n *Normalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
func (n *Normalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
||||||
if n == nil {
|
if n == nil {
|
||||||
return contracts.NormalizeResult{}, normalizerErrorf("normalizer must not be nil")
|
return contracts.NormalizeResult{}, normalizerErrorf("normalizer must not be nil")
|
||||||
|
|||||||
@@ -34,6 +34,13 @@ func TestModuleSpecAndRegister(t *testing.T) {
|
|||||||
if !reflect.DeepEqual(spec, want) {
|
if !reflect.DeepEqual(spec, want) {
|
||||||
t.Fatalf("registered spec = %#v, want %#v", spec, want)
|
t.Fatalf("registered spec = %#v, want %#v", spec, want)
|
||||||
}
|
}
|
||||||
|
normalizer, err := registry.Build(Key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
|
||||||
|
}
|
||||||
|
if slots := normalizer.ReferenceSlots(); len(slots) != 0 {
|
||||||
|
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNormalizePassesThroughOrderAndValues(t *testing.T) {
|
func TestNormalizePassesThroughOrderAndValues(t *testing.T) {
|
||||||
|
|||||||
@@ -148,6 +148,7 @@ func TestEncodeIncludesManifestReferences(t *testing.T) {
|
|||||||
RunID: "run-1",
|
RunID: "run-1",
|
||||||
References: []artifacts.ReferenceProvenance{
|
References: []artifacts.ReferenceProvenance{
|
||||||
{
|
{
|
||||||
|
Stage: "extract",
|
||||||
LaneID: "events",
|
LaneID: "events",
|
||||||
SlotName: "roster",
|
SlotName: "roster",
|
||||||
OriginType: "file",
|
OriginType: "file",
|
||||||
|
|||||||
Reference in New Issue
Block a user