Expand family publish policies into concrete rules
This commit is contained in:
@@ -439,7 +439,7 @@ For each `pipeline.scriptorium.artifact_families.<name>`:
|
|||||||
| `output_path_pattern` | string | Yes | safe path beneath `artifacts/` with exactly one `{character_id}` token and no other brace syntax |
|
| `output_path_pattern` | string | Yes | safe path beneath `artifacts/` with exactly one `{character_id}` token and no other brace syntax |
|
||||||
| `member_vars` | map | No | maps an ordinary Scriptorium variable name to a supported canonical character selector |
|
| `member_vars` | map | No | maps an ordinary Scriptorium variable name to a supported canonical character selector |
|
||||||
| `member_dependencies` | list | No | unique family keys; each generated member depends on the corresponding generated member of each listed family |
|
| `member_dependencies` | list | No | unique family keys; each generated member depends on the corresponding generated member of each listed family |
|
||||||
| `publish` | map | No | typed family publish metadata (`enabled`, `required`, `dest_pattern`); it does not yet create publish outputs |
|
| `publish` | map | No | typed family publish policy (`enabled`, `required`, `dest_pattern`) expanded into concrete publish outputs when enabled |
|
||||||
|
|
||||||
Generated keys are `<family>_<character_id>` and generated output paths must
|
Generated keys are `<family>_<character_id>` and generated output paths must
|
||||||
not collide with explicit artifacts or another generated artifact. Families
|
not collide with explicit artifacts or another generated artifact. Families
|
||||||
|
|||||||
@@ -64,6 +64,8 @@ Configuration may normalize a family selection into its concrete generated
|
|||||||
members before this resolver runs. The effective set retains optional family
|
members before this resolver runs. The effective set retains optional family
|
||||||
and character origin metadata, but its keys, catalog sources, and runtime
|
and character origin metadata, but its keys, catalog sources, and runtime
|
||||||
lookups remain concrete configured-artifact identities.
|
lookups remain concrete configured-artifact identities.
|
||||||
|
Family publish policies are likewise expanded into ordinary configured-source
|
||||||
|
publish rules during configuration resolution.
|
||||||
Configured outputs, including non-executable prerequisites, become available
|
Configured outputs, including non-executable prerequisites, become available
|
||||||
only when the versioned analyze state identifies a current result whose source,
|
only when the versioned analyze state identifies a current result whose source,
|
||||||
contract, canonical configured path, size, and checksum match a confined
|
contract, canonical configured path, size, and checksum match a confined
|
||||||
|
|||||||
@@ -134,4 +134,6 @@ the family-only `narratio.member_artifact.<family>` input form is rewritten to
|
|||||||
the matching ordinary configured-artifact source. The catalog records those
|
the matching ordinary configured-artifact source. The catalog records those
|
||||||
resolved dependency and input identities with their declaring family and party
|
resolved dependency and input identities with their declaring family and party
|
||||||
member. No member-artifact source is registered as a runtime policy source.
|
member. No member-artifact source is registered as a runtime policy source.
|
||||||
The typed publish declaration still does not create a publish rule here.
|
An enabled family publish declaration expands to ordinary configured-artifact
|
||||||
|
publish rules before the existing publish and lock validators run. Runtime
|
||||||
|
publication consequently receives no family wildcard or special matcher.
|
||||||
|
|||||||
@@ -911,7 +911,7 @@ concrete execution, artifact-granular fingerprints, and manifest authority.
|
|||||||
|
|
||||||
## Stage 16 — Family Publish Rule Expansion
|
## Stage 16 — Family Publish Rule Expansion
|
||||||
|
|
||||||
**Status: Pending**
|
**Status: Completed**
|
||||||
|
|
||||||
### Goal
|
### Goal
|
||||||
|
|
||||||
|
|||||||
@@ -121,6 +121,9 @@ func expandPipelineArtifactFamilies(cfg *PipelineConfig, party ResolvedParty) er
|
|||||||
catalog.Families[familyKey] = familyOrigin
|
catalog.Families[familyKey] = familyOrigin
|
||||||
}
|
}
|
||||||
cfg.Scriptorium.Artifacts = expanded
|
cfg.Scriptorium.Artifacts = expanded
|
||||||
|
if err := expandFamilyPublishRules(cfg, catalog); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
cfg.Scriptorium.ArtifactFamilies = nil
|
cfg.Scriptorium.ArtifactFamilies = nil
|
||||||
cfg.familyCatalog = catalog
|
cfg.familyCatalog = catalog
|
||||||
cfg.resolution.artifactFamiliesExpanded = true
|
cfg.resolution.artifactFamiliesExpanded = true
|
||||||
@@ -130,12 +133,71 @@ func expandPipelineArtifactFamilies(cfg *PipelineConfig, party ResolvedParty) er
|
|||||||
if err := validateScriptorium(cfg.Scriptorium, cfg.Notarius); err != nil {
|
if err := validateScriptorium(cfg.Scriptorium, cfg.Notarius); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if err := validatePublish(cfg.Publish, cfg.Scriptorium, cfg.Notarius); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if err := recomputePipelineEffectiveDigest(cfg); err != nil {
|
if err := recomputePipelineEffectiveDigest(cfg); err != nil {
|
||||||
return fmt.Errorf("refresh expanded pipeline digest: %w", err)
|
return fmt.Errorf("refresh expanded pipeline digest: %w", err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func expandFamilyPublishRules(cfg *PipelineConfig, catalog ArtifactFamilyCatalog) error {
|
||||||
|
if cfg == nil || cfg.Scriptorium == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if cfg.Publish == nil {
|
||||||
|
for familyKey, family := range catalog.Families {
|
||||||
|
if family.Publish != nil && family.Publish.Enabled {
|
||||||
|
return fmt.Errorf("pipeline.scriptorium.artifact_families.%s.publish requires pipeline.publish", familyKey)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
generated := make([]PublishOutputRule, 0)
|
||||||
|
explicitSources := make(map[string]struct{}, len(cfg.Publish.Outputs))
|
||||||
|
for _, rule := range cfg.Publish.Outputs {
|
||||||
|
explicitSources[strings.TrimSpace(rule.Source)] = struct{}{}
|
||||||
|
}
|
||||||
|
for familyKey, family := range catalog.Families {
|
||||||
|
if family.Publish == nil || !family.Publish.Enabled {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if cfg.resolution == nil || !cfg.resolution.publishDeclared {
|
||||||
|
return fmt.Errorf("pipeline.scriptorium.artifact_families.%s.publish requires pipeline.publish", familyKey)
|
||||||
|
}
|
||||||
|
pattern := strings.TrimSpace(family.Publish.DestPattern)
|
||||||
|
if pattern != "" {
|
||||||
|
if err := validateFamilyOutputPattern("pipeline.scriptorium.artifact_families."+familyKey+".publish.dest_pattern", pattern); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, key := range family.Members {
|
||||||
|
if _, conflict := explicitSources[artifactpolicy.ConfiguredSourceID(key)]; conflict {
|
||||||
|
return fmt.Errorf("pipeline.scriptorium.artifact_families.%s.publish conflicts with explicit publish source %q", familyKey, artifactpolicy.ConfiguredSourceID(key))
|
||||||
|
}
|
||||||
|
origin := catalog.Members[key]
|
||||||
|
dest := cfg.Scriptorium.Artifacts[key].OutputPath
|
||||||
|
if pattern != "" {
|
||||||
|
dest = strings.ReplaceAll(pattern, characterIDToken, origin.CharacterID)
|
||||||
|
}
|
||||||
|
required := family.Publish.Required
|
||||||
|
generated = append(generated, PublishOutputRule{Source: artifactpolicy.ConfiguredSourceID(key), Dest: dest, Required: &required})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(generated) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cfg.Publish.Outputs = append(cfg.Publish.Outputs, generated...)
|
||||||
|
sort.Slice(cfg.Publish.Outputs, func(i, j int) bool {
|
||||||
|
if cfg.Publish.Outputs[i].Dest == cfg.Publish.Outputs[j].Dest {
|
||||||
|
return cfg.Publish.Outputs[i].Source < cfg.Publish.Outputs[j].Source
|
||||||
|
}
|
||||||
|
return cfg.Publish.Outputs[i].Dest < cfg.Publish.Outputs[j].Dest
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func validateFamilyMemberDependencies(families map[string]ScriptoriumArtifactFamilyConfig, familyKeys []string) error {
|
func validateFamilyMemberDependencies(families map[string]ScriptoriumArtifactFamilyConfig, familyKeys []string) error {
|
||||||
for _, familyKey := range familyKeys {
|
for _, familyKey := range familyKeys {
|
||||||
family := families[familyKey]
|
family := families[familyKey]
|
||||||
|
|||||||
@@ -297,6 +297,35 @@ func TestArtifactFamiliesRejectInvalidMemberDependenciesAndSources(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFamilyPublishRulesExpandToConcreteArtifacts(t *testing.T) {
|
||||||
|
required := true
|
||||||
|
cfg := &PipelineConfig{
|
||||||
|
Scriptorium: &ScriptoriumConfig{Artifacts: map[string]ScriptoriumArtifactConfig{
|
||||||
|
"character_meta_alpha": {OutputPath: "artifacts/characters/alpha/meta.md"},
|
||||||
|
"character_meta_zeta": {OutputPath: "artifacts/characters/zeta/meta.md"},
|
||||||
|
}},
|
||||||
|
Publish: &PublishConfig{},
|
||||||
|
resolution: &pipelineResolutionMetadata{publishDeclared: true},
|
||||||
|
}
|
||||||
|
catalog := ArtifactFamilyCatalog{Families: map[string]ArtifactFamilyOrigin{
|
||||||
|
"character_meta": {Members: []string{"character_meta_alpha", "character_meta_zeta"}, Publish: &ScriptoriumArtifactFamilyPublishConfig{Enabled: true, Required: required, DestPattern: "published/{character_id}.md"}},
|
||||||
|
}, Members: map[string]ArtifactFamilyMemberOrigin{
|
||||||
|
"character_meta_alpha": {CharacterID: "alpha"}, "character_meta_zeta": {CharacterID: "zeta"},
|
||||||
|
}}
|
||||||
|
if err := expandFamilyPublishRules(cfg, catalog); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := cfg.Publish.Outputs; len(got) != 2 || got[0].Source != "narratio.artifact.character_meta_alpha" || got[0].Dest != "published/alpha.md" || got[0].Required == nil || !*got[0].Required {
|
||||||
|
t.Fatalf("generated publish rules = %#v", got)
|
||||||
|
}
|
||||||
|
cfg.Publish.Outputs = nil
|
||||||
|
catalog.Families["character_meta"] = ArtifactFamilyOrigin{Members: []string{"character_meta_alpha"}, Publish: &ScriptoriumArtifactFamilyPublishConfig{Enabled: true}}
|
||||||
|
cfg.resolution.publishDeclared = false
|
||||||
|
if err := expandFamilyPublishRules(cfg, catalog); err == nil || !strings.Contains(err.Error(), "requires pipeline.publish") {
|
||||||
|
t.Fatalf("missing publish error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const validFamilyYAML = `
|
const validFamilyYAML = `
|
||||||
character_meta:
|
character_meta:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ func LoadPipelineWithOptions(path string, opts PipelineLoadOptions) (*PipelineCo
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||||
}
|
}
|
||||||
|
cfg.resolution.publishDeclared = cfg.Publish != nil
|
||||||
applyPipelineDefaults(cfg)
|
applyPipelineDefaults(cfg)
|
||||||
if err := resolveNotariusPaths(cfg, path); err != nil {
|
if err := resolveNotariusPaths(cfg, path); err != nil {
|
||||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ type pipelineResolutionMetadata struct {
|
|||||||
ownership []pipelineFieldOwnership
|
ownership []pipelineFieldOwnership
|
||||||
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
|
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
|
||||||
artifactFamiliesExpanded bool
|
artifactFamiliesExpanded bool
|
||||||
|
publishDeclared bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type pipelineProfileSelection struct {
|
type pipelineProfileSelection struct {
|
||||||
|
|||||||
Reference in New Issue
Block a user