Add stage-local reference config bindings
This commit is contained in:
@@ -135,8 +135,8 @@ Artifact lane fields:
|
|||||||
- `normalize`: optional module binding. Default module is `noop`.
|
- `normalize`: optional module binding. Default module is `noop`.
|
||||||
- `validators`: optional list of module bindings. The production CLI currently
|
- `validators`: optional list of module bindings. The production CLI currently
|
||||||
does not register validator modules.
|
does not register validator modules.
|
||||||
- `references`: optional map of extractor reference slot names to reference
|
- `references`: optional compatibility alias for extractor reference bindings.
|
||||||
paths. Lane bindings override pipeline-level bindings for the same slot.
|
Lane bindings override pipeline-level bindings for the same slot.
|
||||||
|
|
||||||
`notarius run` and `notarius config validate --pipeline` resolve the pipeline
|
`notarius run` and `notarius config validate --pipeline` resolve the pipeline
|
||||||
against the production module catalog and fail fast for unknown or incompatible
|
against the production module catalog and fail fast for unknown or incompatible
|
||||||
@@ -144,15 +144,15 @@ module keys.
|
|||||||
|
|
||||||
Reference bindings are validated against extractor-declared slots during
|
Reference bindings are validated against extractor-declared slots during
|
||||||
pipeline resolution. Required slots must be bound after config defaults,
|
pipeline resolution. Required slots must be bound after config defaults,
|
||||||
lane-level bindings, and run-time `--reference` or `--without-reference`
|
extractor binding references, lane-level compatibility bindings, and run-time
|
||||||
overrides are applied. Config-relative paths are resolved relative to the
|
`--reference` or `--without-reference` overrides are applied. Config-relative
|
||||||
config file; CLI reference paths are resolved relative to the current working
|
paths are resolved relative to the config file; CLI reference paths are resolved
|
||||||
directory. Bound files must be UTF-8 text and are passed only to lane
|
relative to the current working directory. Bound files must be UTF-8 text and
|
||||||
extractors that declare the slot. Reference media types are inferred from file
|
are passed only to lane extractors that declare the slot. Reference media types
|
||||||
extensions, recorded as canonical base media types, and checked only when a
|
are inferred from file extensions, recorded as canonical base media types, and
|
||||||
module declares `AcceptedMediaTypes`; unknown extensions are recorded as
|
checked only when a module declares `AcceptedMediaTypes`; unknown extensions are
|
||||||
`application/octet-stream`. Reference content is not written to diagnostics,
|
recorded as `application/octet-stream`. Reference content is not written to
|
||||||
logs, errors, or manifests.
|
diagnostics, logs, errors, or manifests.
|
||||||
|
|
||||||
Pipeline-level `references` are defaults. They are valid when at least one
|
Pipeline-level `references` are defaults. They are valid when at least one
|
||||||
declared lane in the pipeline has an extractor that declares the slot. During a
|
declared lane in the pipeline has an extractor that declares the slot. During a
|
||||||
@@ -170,7 +170,9 @@ pipelines:
|
|||||||
extract: dnd/spells
|
extract: dnd/spells
|
||||||
```
|
```
|
||||||
|
|
||||||
Lane-level `references` override or add bindings for one lane:
|
Extractor binding `references` are the canonical lane-local location. The
|
||||||
|
legacy lane-level `references` field remains supported as an alias; when both
|
||||||
|
bind the same slot, `extract.references` wins:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
pipelines:
|
pipelines:
|
||||||
@@ -180,11 +182,19 @@ pipelines:
|
|||||||
glossary: ./campaign/glossary.txt
|
glossary: ./campaign/glossary.txt
|
||||||
artifacts:
|
artifacts:
|
||||||
spells:
|
spells:
|
||||||
extract: dnd/spells
|
references:
|
||||||
|
roster: ./campaign/legacy-roster.txt
|
||||||
|
extract:
|
||||||
|
module: dnd/spells
|
||||||
references:
|
references:
|
||||||
roster: ./campaign/session-roster.txt
|
roster: ./campaign/session-roster.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`chunk.references` and `normalize.references` are accepted in object-form
|
||||||
|
bindings and preserved in the effective configuration. They are validated as
|
||||||
|
reference maps, but current reference materialization still delivers content
|
||||||
|
only to extractor bindings.
|
||||||
|
|
||||||
## Module Bindings
|
## Module Bindings
|
||||||
|
|
||||||
Every module binding may use shorthand:
|
Every module binding may use shorthand:
|
||||||
@@ -208,6 +218,9 @@ Binding fields:
|
|||||||
- `module`: module key.
|
- `module`: module key.
|
||||||
- `llm_profile`: optional LLM profile ID. Empty means `default`.
|
- `llm_profile`: optional LLM profile ID. Empty means `default`.
|
||||||
- `options`: optional module-specific settings.
|
- `options`: optional module-specific settings.
|
||||||
|
- `references`: optional reference bindings. Supported only for `chunk`,
|
||||||
|
`extract`, and `normalize` bindings. `input`, `merge`, validator, and
|
||||||
|
`output` bindings reject this field during validation.
|
||||||
|
|
||||||
The `--llm-profile` run flag overrides every effective module binding to use
|
The `--llm-profile` run flag overrides every effective module binding to use
|
||||||
one configured profile.
|
one configured profile.
|
||||||
|
|||||||
@@ -114,6 +114,7 @@ func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
|
|||||||
if len(in.Options) > 0 {
|
if len(in.Options) > 0 {
|
||||||
out.Options = cloneOptions(in.Options)
|
out.Options = cloneOptions(in.Options)
|
||||||
}
|
}
|
||||||
|
out.References = cloneStringMap(in.References)
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -99,6 +99,7 @@ type fileModuleBinding struct {
|
|||||||
Module string
|
Module string
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
Options map[string]any
|
Options map[string]any
|
||||||
|
References map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
||||||
@@ -133,6 +134,12 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
b.Options = normalizeOptions(options)
|
b.Options = normalizeOptions(options)
|
||||||
|
case "references":
|
||||||
|
var references map[string]string
|
||||||
|
if err := valueNode.Decode(&references); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
b.References = references
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("field %s not found in module binding", keyNode.Value)
|
return fmt.Errorf("field %s not found in module binding", keyNode.Value)
|
||||||
}
|
}
|
||||||
@@ -148,6 +155,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
|
|||||||
Module: strings.TrimSpace(b.Module),
|
Module: strings.TrimSpace(b.Module),
|
||||||
LLMProfile: strings.TrimSpace(b.LLMProfile),
|
LLMProfile: strings.TrimSpace(b.LLMProfile),
|
||||||
Options: cloneOptions(b.Options),
|
Options: cloneOptions(b.Options),
|
||||||
|
References: normalizedStringMap(b.References),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -217,6 +225,19 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
|||||||
if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
|
if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if filePipeline.Chunk != nil {
|
||||||
|
if _, _, err := normalizedMapKeys(filePipeline.Chunk.References, fmt.Sprintf("pipeline %q chunk reference slot", pipelineID)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, _, err := normalizedMapKeys(filePipeline.Input.References, fmt.Sprintf("pipeline %q input reference slot", pipelineID)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if filePipeline.Output != nil {
|
||||||
|
if _, _, err := normalizedMapKeys(filePipeline.Output.References, fmt.Sprintf("pipeline %q output reference slot", pipelineID)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
for rawLaneID, fileLane := range filePipeline.Artifacts {
|
for rawLaneID, fileLane := range filePipeline.Artifacts {
|
||||||
laneID := strings.TrimSpace(rawLaneID)
|
laneID := strings.TrimSpace(rawLaneID)
|
||||||
if laneID == "" {
|
if laneID == "" {
|
||||||
@@ -225,6 +246,24 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
|||||||
if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil {
|
if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if _, _, err := normalizedMapKeys(fileLane.Extract.References, fmt.Sprintf("pipeline %q lane %q extract reference slot", pipelineID, laneID)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if fileLane.Merge != nil {
|
||||||
|
if _, _, err := normalizedMapKeys(fileLane.Merge.References, fmt.Sprintf("pipeline %q lane %q merge reference slot", pipelineID, laneID)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if fileLane.Normalize != nil {
|
||||||
|
if _, _, err := normalizedMapKeys(fileLane.Normalize.References, fmt.Sprintf("pipeline %q lane %q normalize reference slot", pipelineID, laneID)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i, validator := range fileLane.Validators {
|
||||||
|
if _, _, err := normalizedMapKeys(validator.References, fmt.Sprintf("pipeline %q lane %q validator[%d] reference slot", pipelineID, laneID, i)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -280,8 +319,10 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
|||||||
}
|
}
|
||||||
for _, laneID := range laneIDs {
|
for _, laneID := range laneIDs {
|
||||||
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
|
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
|
||||||
|
extract := fileLane.Extract.toPipelineBinding()
|
||||||
|
extract.References = mergeStringMaps(normalizedStringMap(fileLane.References), extract.References)
|
||||||
lane := pipeline.ArtifactLaneProfile{
|
lane := pipeline.ArtifactLaneProfile{
|
||||||
Extract: fileLane.Extract.toPipelineBinding(),
|
Extract: extract,
|
||||||
References: normalizedStringMap(fileLane.References),
|
References: normalizedStringMap(fileLane.References),
|
||||||
}
|
}
|
||||||
if fileLane.Merge != nil {
|
if fileLane.Merge != nil {
|
||||||
@@ -353,6 +394,20 @@ func normalizedStringMap(values map[string]string) map[string]string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func mergeStringMaps(base map[string]string, override map[string]string) map[string]string {
|
||||||
|
if len(base) == 0 && len(override) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]string, len(base)+len(override))
|
||||||
|
for key, value := range base {
|
||||||
|
out[key] = value
|
||||||
|
}
|
||||||
|
for key, value := range override {
|
||||||
|
out[key] = value
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
|
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
|
||||||
name := strings.TrimSpace(envName)
|
name := strings.TrimSpace(envName)
|
||||||
if name == "" {
|
if name == "" {
|
||||||
|
|||||||
@@ -169,6 +169,53 @@ pipelines:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseFileConfigStageLocalReferenceMaps(t *testing.T) {
|
||||||
|
cfg := parseAndApplyConfig(t, `
|
||||||
|
version: 1
|
||||||
|
pipelines:
|
||||||
|
example:
|
||||||
|
input: fake/input
|
||||||
|
chunk:
|
||||||
|
module: generic
|
||||||
|
references:
|
||||||
|
" scene_guide ": " ./scenes.md "
|
||||||
|
artifacts:
|
||||||
|
events:
|
||||||
|
extract:
|
||||||
|
module: fake/extract
|
||||||
|
references:
|
||||||
|
" glossary ": " ./glossary.md "
|
||||||
|
" roster ": " ./extract-roster.yml "
|
||||||
|
references:
|
||||||
|
roster: ./legacy-roster.yml
|
||||||
|
lore: ./lore.md
|
||||||
|
normalize:
|
||||||
|
module: noop
|
||||||
|
references:
|
||||||
|
" normalization_notes ": " ./normalization.md "
|
||||||
|
`)
|
||||||
|
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
if !reflect.DeepEqual(profile.Chunk.References, map[string]string{"scene_guide": "./scenes.md"}) {
|
||||||
|
t.Fatalf("chunk references = %#v, want trimmed map", profile.Chunk.References)
|
||||||
|
}
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
if !reflect.DeepEqual(lane.References, map[string]string{"lore": "./lore.md", "roster": "./legacy-roster.yml"}) {
|
||||||
|
t.Fatalf("legacy lane references = %#v, want trimmed map", lane.References)
|
||||||
|
}
|
||||||
|
wantExtract := map[string]string{
|
||||||
|
"glossary": "./glossary.md",
|
||||||
|
"lore": "./lore.md",
|
||||||
|
"roster": "./extract-roster.yml",
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(lane.Extract.References, wantExtract) {
|
||||||
|
t.Fatalf("extract references = %#v, want legacy merged with extract override %#v", lane.Extract.References, wantExtract)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(lane.Normalize.References, map[string]string{"normalization_notes": "./normalization.md"}) {
|
||||||
|
t.Fatalf("normalize references = %#v, want trimmed map", lane.Normalize.References)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) {
|
func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) {
|
||||||
cfg := parseAndApplyConfig(t, `
|
cfg := parseAndApplyConfig(t, `
|
||||||
version: 1
|
version: 1
|
||||||
@@ -357,6 +404,56 @@ pipelines:
|
|||||||
`,
|
`,
|
||||||
want: `pipeline "example" lane "events" reference slot`,
|
want: `pipeline "example" lane "events" reference slot`,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "chunk",
|
||||||
|
raw: `
|
||||||
|
version: 1
|
||||||
|
pipelines:
|
||||||
|
example:
|
||||||
|
input: fake/input
|
||||||
|
chunk:
|
||||||
|
module: generic
|
||||||
|
references:
|
||||||
|
roster: ./first.yml
|
||||||
|
" roster ": ./second.yml
|
||||||
|
`,
|
||||||
|
want: `pipeline "example" chunk reference slot`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "extract",
|
||||||
|
raw: `
|
||||||
|
version: 1
|
||||||
|
pipelines:
|
||||||
|
example:
|
||||||
|
input: fake/input
|
||||||
|
artifacts:
|
||||||
|
events:
|
||||||
|
extract:
|
||||||
|
module: fake/extract
|
||||||
|
references:
|
||||||
|
roster: ./first.yml
|
||||||
|
" roster ": ./second.yml
|
||||||
|
`,
|
||||||
|
want: `pipeline "example" lane "events" extract reference slot`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "normalize",
|
||||||
|
raw: `
|
||||||
|
version: 1
|
||||||
|
pipelines:
|
||||||
|
example:
|
||||||
|
input: fake/input
|
||||||
|
artifacts:
|
||||||
|
events:
|
||||||
|
extract: fake/extract
|
||||||
|
normalize:
|
||||||
|
module: noop
|
||||||
|
references:
|
||||||
|
roster: ./first.yml
|
||||||
|
" roster ": ./second.yml
|
||||||
|
`,
|
||||||
|
want: `pipeline "example" lane "events" normalize reference slot`,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
|
|||||||
@@ -67,7 +67,12 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
|
|||||||
lane := cfg.Pipelines["example"].Artifacts["events"]
|
lane := cfg.Pipelines["example"].Artifacts["events"]
|
||||||
lane.Extract.Options = map[string]any{"temperature": 0.2}
|
lane.Extract.Options = map[string]any{"temperature": 0.2}
|
||||||
lane.References = map[string]string{"roster": "./roster.yml"}
|
lane.References = map[string]string{"roster": "./roster.yml"}
|
||||||
|
lane.Extract.References = map[string]string{"glossary": "./glossary.md"}
|
||||||
|
lane.Normalize.References = map[string]string{"notes": "./normalize.md"}
|
||||||
cfg.Pipelines["example"].Artifacts["events"] = lane
|
cfg.Pipelines["example"].Artifacts["events"] = lane
|
||||||
|
pipelineProfile := cfg.Pipelines["example"]
|
||||||
|
pipelineProfile.Chunk.References = map[string]string{"scene_guide": "./scene.md"}
|
||||||
|
cfg.Pipelines["example"] = pipelineProfile
|
||||||
|
|
||||||
effective, err := cfg.Resolve(ResolveInput{
|
effective, err := cfg.Resolve(ResolveInput{
|
||||||
PipelineID: "example",
|
PipelineID: "example",
|
||||||
@@ -78,6 +83,7 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
|
|||||||
Requires: []string{"chunks"},
|
Requires: []string{"chunks"},
|
||||||
Provides: []string{"artifact"},
|
Provides: []string{"artifact"},
|
||||||
ReferenceSlots: []contracts.ReferenceSlot{
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "glossary"},
|
||||||
{Name: "roster"},
|
{Name: "roster"},
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
@@ -109,9 +115,21 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
|
|||||||
t.Fatalf("expected resolved pipeline options to be copied")
|
t.Fatalf("expected resolved pipeline options to be copied")
|
||||||
}
|
}
|
||||||
payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source = "./changed.yml"
|
payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source = "./changed.yml"
|
||||||
if effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source != "./roster.yml" {
|
if referenceBindingSource(effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings, "roster") != "./roster.yml" {
|
||||||
t.Fatalf("expected resolved pipeline references to be copied")
|
t.Fatalf("expected resolved pipeline references to be copied")
|
||||||
}
|
}
|
||||||
|
payload.ResolvedPipeline.Chunk.References["scene_guide"] = "./changed-scene.md"
|
||||||
|
if effective.ResolvedPipeline.Chunk.References["scene_guide"] != "./scene.md" {
|
||||||
|
t.Fatalf("expected chunk references to be copied")
|
||||||
|
}
|
||||||
|
payload.ResolvedPipeline.ArtifactLanes[0].Extract.References["glossary"] = "./changed-glossary.md"
|
||||||
|
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.References["glossary"] != "./glossary.md" {
|
||||||
|
t.Fatalf("expected extract references to be copied")
|
||||||
|
}
|
||||||
|
payload.ResolvedPipeline.ArtifactLanes[0].Normalize.References["notes"] = "./changed-normalize.md"
|
||||||
|
if effective.ResolvedPipeline.ArtifactLanes[0].Normalize.References["notes"] != "./normalize.md" {
|
||||||
|
t.Fatalf("expected normalize references to be copied")
|
||||||
|
}
|
||||||
|
|
||||||
effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
|
effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
|
||||||
Slots: map[string]contracts.ResolvedReferenceSlot{
|
Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||||
@@ -136,3 +154,12 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
|
|||||||
t.Fatalf("expected materialized reference content to be copied, got %q", got)
|
t.Fatalf("expected materialized reference content to be copied, got %q", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func referenceBindingSource(bindings []pipeline.ReferenceBinding, slotName string) string {
|
||||||
|
for _, binding := range bindings {
|
||||||
|
if binding.SlotName == slotName {
|
||||||
|
return binding.Source
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|||||||
@@ -89,13 +89,13 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
|
|||||||
if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
|
if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
|
||||||
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
|
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
|
||||||
}
|
}
|
||||||
if err := validateBindingLLMProfile(id, "", "input", profile.Input, llmProfiles); err != nil {
|
if err := validateBinding(id, "", "input", profile.Input, llmProfiles, false); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validateBindingLLMProfile(id, "", "chunk", profile.Chunk, llmProfiles); err != nil {
|
if err := validateBinding(id, "", "chunk", profile.Chunk, llmProfiles, true); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validateBindingLLMProfile(id, "", "output", profile.Output, llmProfiles); err != nil {
|
if err := validateBinding(id, "", "output", profile.Output, llmProfiles, false); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validateReferenceMap(id, "", profile.References); err != nil {
|
if err := validateReferenceMap(id, "", profile.References); err != nil {
|
||||||
@@ -109,17 +109,17 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
|
|||||||
if err := validateReferenceMap(id, laneID, lane.References); err != nil {
|
if err := validateReferenceMap(id, laneID, lane.References); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validateBindingLLMProfile(id, laneID, "extract", lane.Extract, llmProfiles); err != nil {
|
if err := validateBinding(id, laneID, "extract", lane.Extract, llmProfiles, true); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validateBindingLLMProfile(id, laneID, "merge", lane.Merge, llmProfiles); err != nil {
|
if err := validateBinding(id, laneID, "merge", lane.Merge, llmProfiles, false); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validateBindingLLMProfile(id, laneID, "normalize", lane.Normalize, llmProfiles); err != nil {
|
if err := validateBinding(id, laneID, "normalize", lane.Normalize, llmProfiles, true); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
for i, validator := range lane.Validators {
|
for i, validator := range lane.Validators {
|
||||||
if err := validateBindingLLMProfile(id, laneID, fmt.Sprintf("validator[%d]", i), validator, llmProfiles); err != nil {
|
if err := validateBinding(id, laneID, fmt.Sprintf("validator[%d]", i), validator, llmProfiles, false); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -128,33 +128,64 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateBinding(
|
||||||
|
pipelineID string,
|
||||||
|
laneID string,
|
||||||
|
slot string,
|
||||||
|
binding pipeline.ModuleBinding,
|
||||||
|
profiles map[string]LLMProfile,
|
||||||
|
referencesAllowed bool,
|
||||||
|
) error {
|
||||||
|
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding, profiles); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(binding.References) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !referencesAllowed {
|
||||||
|
if laneID != "" {
|
||||||
|
return fmt.Errorf("pipeline %q lane %q %s references are not supported", pipelineID, laneID, slot)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("pipeline %q %s references are not supported", pipelineID, slot)
|
||||||
|
}
|
||||||
|
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References)
|
||||||
|
}
|
||||||
|
|
||||||
func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error {
|
func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error {
|
||||||
|
return validateReferenceMapForContext(pipelineID, laneID, "", references)
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateReferenceMapForContext(pipelineID string, laneID string, slot string, references map[string]string) error {
|
||||||
seen := make(map[string]struct{}, len(references))
|
seen := make(map[string]struct{}, len(references))
|
||||||
for rawSlotName, rawSource := range references {
|
for rawSlotName, rawSource := range references {
|
||||||
slotName := strings.TrimSpace(rawSlotName)
|
slotName := strings.TrimSpace(rawSlotName)
|
||||||
if slotName == "" {
|
if slotName == "" {
|
||||||
if laneID != "" {
|
return fmt.Errorf("%s reference slot name must not be empty", referenceContext(pipelineID, laneID, slot))
|
||||||
return fmt.Errorf("pipeline %q lane %q reference slot name must not be empty", pipelineID, laneID)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("pipeline %q reference slot name must not be empty", pipelineID)
|
|
||||||
}
|
}
|
||||||
if _, ok := seen[slotName]; ok {
|
if _, ok := seen[slotName]; ok {
|
||||||
if laneID != "" {
|
return fmt.Errorf("%s reference slot %q is duplicated after trimming", referenceContext(pipelineID, laneID, slot), slotName)
|
||||||
return fmt.Errorf("pipeline %q lane %q reference slot %q is duplicated after trimming", pipelineID, laneID, slotName)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("pipeline %q reference slot %q is duplicated after trimming", pipelineID, slotName)
|
|
||||||
}
|
}
|
||||||
seen[slotName] = struct{}{}
|
seen[slotName] = struct{}{}
|
||||||
if strings.TrimSpace(rawSource) == "" {
|
if strings.TrimSpace(rawSource) == "" {
|
||||||
if laneID != "" {
|
return fmt.Errorf("%s reference slot %q source must not be empty", referenceContext(pipelineID, laneID, slot), slotName)
|
||||||
return fmt.Errorf("pipeline %q lane %q reference slot %q source must not be empty", pipelineID, laneID, slotName)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("pipeline %q reference slot %q source must not be empty", pipelineID, slotName)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func referenceContext(pipelineID string, laneID string, slot string) string {
|
||||||
|
if laneID != "" && slot != "" {
|
||||||
|
return fmt.Sprintf("pipeline %q lane %q %s", pipelineID, laneID, slot)
|
||||||
|
}
|
||||||
|
if laneID != "" {
|
||||||
|
return fmt.Sprintf("pipeline %q lane %q", pipelineID, laneID)
|
||||||
|
}
|
||||||
|
if slot != "" {
|
||||||
|
return fmt.Sprintf("pipeline %q %s", pipelineID, slot)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("pipeline %q", pipelineID)
|
||||||
|
}
|
||||||
|
|
||||||
func validateBindingLLMProfile(
|
func validateBindingLLMProfile(
|
||||||
pipelineID string,
|
pipelineID string,
|
||||||
laneID string,
|
laneID string,
|
||||||
|
|||||||
@@ -122,6 +122,74 @@ func TestValidateRejectsInvalidReferenceMaps(t *testing.T) {
|
|||||||
mutate func(Config) Config
|
mutate func(Config) Config
|
||||||
want []string
|
want []string
|
||||||
}{
|
}{
|
||||||
|
{
|
||||||
|
name: "empty chunk slot",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
profile.Chunk.References = map[string]string{" ": "./roster.yml"}
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "chunk", "reference slot", "empty"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty chunk source",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
profile.Chunk.References = map[string]string{"roster": " "}
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "chunk", "roster", "source", "empty"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty extract slot",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Extract.References = map[string]string{" ": "./roster.yml"}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "events", "extract", "reference slot", "empty"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty extract source",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Extract.References = map[string]string{"roster": " "}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "events", "extract", "roster", "source", "empty"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty normalize slot",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Normalize.References = map[string]string{" ": "./roster.yml"}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "events", "normalize", "reference slot", "empty"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty normalize source",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Normalize.References = map[string]string{"roster": " "}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "events", "normalize", "roster", "source", "empty"},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "empty pipeline slot",
|
name: "empty pipeline slot",
|
||||||
mutate: func(cfg Config) Config {
|
mutate: func(cfg Config) Config {
|
||||||
@@ -183,6 +251,76 @@ func TestValidateRejectsInvalidReferenceMaps(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidateRejectsReferencesOnUnsupportedBindings(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mutate func(Config) Config
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "input",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
profile.Input.References = map[string]string{"roster": "./roster.yml"}
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "input", "references", "not supported"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "merge",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Merge.References = map[string]string{"roster": "./roster.yml"}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "events", "merge", "references", "not supported"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validator",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Validators = []pipeline.ModuleBinding{{
|
||||||
|
Module: "fake/validator",
|
||||||
|
References: map[string]string{"roster": "./roster.yml"},
|
||||||
|
}}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "events", "validator[0]", "references", "not supported"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "output",
|
||||||
|
mutate: func(cfg Config) Config {
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
profile.Output.References = map[string]string{"roster": "./roster.yml"}
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
return cfg
|
||||||
|
},
|
||||||
|
want: []string{"example", "output", "references", "not supported"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
err := tc.mutate(validConfig()).Validate()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Validate() error = nil, want error")
|
||||||
|
}
|
||||||
|
for _, want := range tc.want {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestValidateRejectsEmptyIDs(t *testing.T) {
|
func TestValidateRejectsEmptyIDs(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ type ModuleBinding struct {
|
|||||||
Module string `json:"module"`
|
Module string `json:"module"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
Options map[string]any `json:"options,omitempty"`
|
Options map[string]any `json:"options,omitempty"`
|
||||||
|
References map[string]string `json:"references,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ArtifactLaneProfile struct {
|
type ArtifactLaneProfile struct {
|
||||||
@@ -212,7 +213,8 @@ func resolveArtifactLane(
|
|||||||
if missing, ok := capabilities.missing(extractSpec.Requires); ok {
|
if missing, ok := capabilities.missing(extractSpec.Requires); ok {
|
||||||
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
|
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
|
||||||
}
|
}
|
||||||
references, err := resolveReferenceBindings(pipelineID, laneID, lane.Extract.Module, extractSpec.ReferenceSlots, pipelineReferences, profile.References, options)
|
extractReferences := mergeReferenceMaps(profile.References, lane.Extract.References)
|
||||||
|
references, err := resolveReferenceBindings(pipelineID, laneID, lane.Extract.Module, extractSpec.ReferenceSlots, pipelineReferences, extractReferences, options)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedArtifactLane{}, nil, err
|
return ResolvedArtifactLane{}, nil, err
|
||||||
}
|
}
|
||||||
@@ -261,6 +263,20 @@ func referenceTarget(stage ModuleStage, laneID string, module string, bindings [
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func mergeReferenceMaps(base map[string]string, override map[string]string) map[string]string {
|
||||||
|
if len(base) == 0 && len(override) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]string, len(base)+len(override))
|
||||||
|
for key, value := range base {
|
||||||
|
out[key] = value
|
||||||
|
}
|
||||||
|
for key, value := range override {
|
||||||
|
out[key] = value
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func validatePipelineReferenceDefaults(
|
func validatePipelineReferenceDefaults(
|
||||||
pipelineID string,
|
pipelineID string,
|
||||||
pipelineReferences map[string]string,
|
pipelineReferences map[string]string,
|
||||||
@@ -480,6 +496,7 @@ func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
|
|||||||
Module: module,
|
Module: module,
|
||||||
LLMProfile: llmProfile,
|
LLMProfile: llmProfile,
|
||||||
Options: cloneOptions(binding.Options),
|
Options: cloneOptions(binding.Options),
|
||||||
|
References: normalizeReferenceMap(binding.References),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -508,6 +525,25 @@ func cloneOptions(options map[string]any) map[string]any {
|
|||||||
return copied
|
return copied
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func normalizeReferenceMap(values map[string]string) map[string]string {
|
||||||
|
if len(values) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]string, len(values))
|
||||||
|
keys := make([]string, 0, len(values))
|
||||||
|
rawByNormalized := make(map[string]string, len(values))
|
||||||
|
for rawKey := range values {
|
||||||
|
key := strings.TrimSpace(rawKey)
|
||||||
|
rawByNormalized[key] = rawKey
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
for _, key := range keys {
|
||||||
|
out[key] = strings.TrimSpace(values[rawByNormalized[key]])
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneProfile, options ResolveOptions) (map[string]ArtifactLaneProfile, []string, error) {
|
func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneProfile, options ResolveOptions) (map[string]ArtifactLaneProfile, []string, error) {
|
||||||
lanesByID := make(map[string]ArtifactLaneProfile, len(artifacts))
|
lanesByID := make(map[string]ArtifactLaneProfile, len(artifacts))
|
||||||
for rawLaneID, lane := range artifacts {
|
for rawLaneID, lane := range artifacts {
|
||||||
|
|||||||
Reference in New Issue
Block a user