diff --git a/docs/config.md b/docs/config.md index d1dda97..1ede008 100644 --- a/docs/config.md +++ b/docs/config.md @@ -438,7 +438,7 @@ For each `pipeline.scriptorium.artifact_families.`: | `for_each` | string | Yes | exactly `party.characters` | | `output_path_pattern` | string | Yes | safe path beneath `artifacts/` with exactly one `{character_id}` token and no other brace syntax | | `member_vars` | map | No | maps an ordinary Scriptorium variable name to a supported canonical character selector | -| `member_dependencies` | list | No | retained as typed family metadata; it does not yet add ordinary dependencies | +| `member_dependencies` | list | No | unique family keys; each generated member depends on the corresponding generated member of each listed family | | `publish` | map | No | typed family publish metadata (`enabled`, `required`, `dest_pattern`); it does not yet create publish outputs | Generated keys are `_` and generated output paths must @@ -452,6 +452,13 @@ Their resolved values are strings. A member variable may not reuse a static `vars` name; `session_id` remains owned and overwritten by Narratio as for any other Scriptorium artifact. +Within a family only, an input source may use +`narratio.member_artifact.`. The referenced family must be named in +that family's `member_dependencies`; resolution rewrites the source to the +corresponding ordinary `narratio.artifact._` source. +This syntax is rejected in explicit artifacts and never reaches runtime stages +or Scriptorium. + ### Notifications Narratio currently supports only `notification.mode: noop`, which is also the diff --git a/docs/internal/artifacts.md b/docs/internal/artifacts.md index 830f89e..fceaf1c 100644 --- a/docs/internal/artifacts.md +++ b/docs/internal/artifacts.md @@ -102,6 +102,10 @@ Configured sources (`narratio.artifact.*`): rewriting manifest state. Catalog construction iterates current configuration, so removed or renamed records are not advertised. +`narratio.member_artifact.*` is not a runtime source family. Configuration +resolution accepts it only in an artifact-family declaration and rewrites it +to the corresponding configured source before this catalog is built. + Prepared stable sources (`narratio.input.*`): - resolve only from the current manifest's exact prepared-input record; diff --git a/docs/internal/configuration.md b/docs/internal/configuration.md index c2a880b..dd14a27 100644 --- a/docs/internal/configuration.md +++ b/docs/internal/configuration.md @@ -129,5 +129,9 @@ therefore receive only concrete artifact maps. The catalog retains sorted family member keys plus family/character/source origins and the typed dependency/publish declarations for their later owners. -It does not create a new runtime artifact type, dependency edge, or publish -rule during this resolution step. +`member_dependencies` add corresponding ordinary concrete dependencies, while +the family-only `narratio.member_artifact.` input form is rewritten to +the matching ordinary configured-artifact source. The catalog records those +resolved dependency and input identities with their declaring family and party +member. No member-artifact source is registered as a runtime policy source. +The typed publish declaration still does not create a publish rule here. diff --git a/docs/internal/stage-analyze.md b/docs/internal/stage-analyze.md index af8d7e1..de74a42 100644 --- a/docs/internal/stage-analyze.md +++ b/docs/internal/stage-analyze.md @@ -9,7 +9,8 @@ dependency order, and safely materialize validated outputs. - ordinary configured artifacts from `pipeline.scriptorium.artifacts`; canonical party artifact families have already expanded into this map during - configuration resolution + configuration resolution, including corresponding member dependencies and + rewritten member-artifact input sources - optional selected artifact keys supplied through the stage environment - built-in, configured, extraction, and previous-session source references in artifact inputs diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 05d4320..34eddfd 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -813,7 +813,7 @@ Scriptorium artifacts with member-specific identity, paths, and variables. ## Stage 14 — Same-Member Dependencies And Member Artifact Sources -**Status: Pending** +**Status: Completed** ### Goal diff --git a/internal/config/artifact_families.go b/internal/config/artifact_families.go index dad1829..f94faae 100644 --- a/internal/config/artifact_families.go +++ b/internal/config/artifact_families.go @@ -11,6 +11,7 @@ import ( const ( partyCharactersFamilySource = "party.characters" characterIDToken = "{character_id}" + memberArtifactSourcePrefix = "narratio.member_artifact." ) var memberVariableSelectors = map[string]func(PartyCharacter) string{ @@ -44,6 +45,9 @@ func expandPipelineArtifactFamilies(cfg *PipelineConfig, party ResolvedParty) er } expanded := cloneArtifactDefinitions(cfg.Scriptorium.Artifacts) + if err := rejectConcreteMemberArtifactSources(expanded); err != nil { + return err + } familyKeys := sortedFamilyKeys(families) for _, familyKey := range familyKeys { if !artifactpolicy.IsConfiguredKey(familyKey) { @@ -53,6 +57,9 @@ func expandPipelineArtifactFamilies(cfg *PipelineConfig, party ResolvedParty) er return fmt.Errorf("pipeline.scriptorium.artifact_families.%s collides with configured artifact key %q", familyKey, familyKey) } } + if err := validateFamilyMemberDependencies(families, familyKeys); err != nil { + return err + } characters := append([]PartyCharacter(nil), party.Canonical.Characters...) sort.Slice(characters, func(left, right int) bool { return characters[left].ID < characters[right].ID }) @@ -94,14 +101,22 @@ func expandPipelineArtifactFamilies(cfg *PipelineConfig, party ResolvedParty) er if err != nil { return err } + dependencies, err := expandedFamilyDependencies(prefix, family, character.ID) + if err != nil { + return err + } + inputs, err := expandFamilyInputs(prefix, family, character.ID) + if err != nil { + return err + } expanded[key] = ScriptoriumArtifactConfig{ - Enabled: family.Enabled, DependsOn: append([]string(nil), family.DependsOn...), RenderDebug: family.RenderDebug, + Enabled: family.Enabled, DependsOn: dependencies, RenderDebug: family.RenderDebug, PromptID: family.PromptID, ProfileID: family.ProfileID, OutputPath: outputPath, Timeout: family.Timeout, - Inputs: cloneArtifactInputs(family.Inputs), Vars: vars, + Inputs: inputs, Vars: vars, } outputOwners[outputPath] = key familyOrigin.Members = append(familyOrigin.Members, key) - catalog.Members[key] = ArtifactFamilyMemberOrigin{Family: familyKey, CharacterID: character.ID, Source: familyOrigin.Source} + catalog.Members[key] = ArtifactFamilyMemberOrigin{Family: familyKey, CharacterID: character.ID, Source: familyOrigin.Source, Dependencies: append([]string(nil), dependencies...), Inputs: inputSourceMap(inputs)} } catalog.Families[familyKey] = familyOrigin } @@ -109,6 +124,9 @@ func expandPipelineArtifactFamilies(cfg *PipelineConfig, party ResolvedParty) er cfg.Scriptorium.ArtifactFamilies = nil cfg.familyCatalog = catalog cfg.resolution.artifactFamiliesExpanded = true + if err := rejectConcreteMemberArtifactSources(expanded); err != nil { + return err + } if err := validateScriptorium(cfg.Scriptorium, cfg.Notarius); err != nil { return err } @@ -118,6 +136,127 @@ func expandPipelineArtifactFamilies(cfg *PipelineConfig, party ResolvedParty) er return nil } +func validateFamilyMemberDependencies(families map[string]ScriptoriumArtifactFamilyConfig, familyKeys []string) error { + for _, familyKey := range familyKeys { + family := families[familyKey] + seen := make(map[string]struct{}, len(family.MemberDependencies)) + for index, raw := range family.MemberDependencies { + dependency := strings.TrimSpace(raw) + prefix := fmt.Sprintf("pipeline.scriptorium.artifact_families.%s.member_dependencies[%d]", familyKey, index) + if !artifactpolicy.IsConfiguredKey(dependency) { + return fmt.Errorf("%s must be a valid family key", prefix) + } + if dependency == familyKey { + return fmt.Errorf("%s must not reference its own family", prefix) + } + if _, duplicate := seen[dependency]; duplicate { + return fmt.Errorf("%s duplicates family %q", prefix, dependency) + } + seen[dependency] = struct{}{} + dependencyFamily, exists := families[dependency] + if !exists { + return fmt.Errorf("%s references unknown family %q", prefix, dependency) + } + if strings.TrimSpace(dependencyFamily.ForEach) != partyCharactersFamilySource { + return fmt.Errorf("%s family %q must use %q", prefix, dependency, partyCharactersFamilySource) + } + } + } + return nil +} + +func expandedFamilyDependencies(prefix string, family ScriptoriumArtifactFamilyConfig, characterID string) ([]string, error) { + dependencies := append([]string(nil), family.DependsOn...) + memberDependencies := append([]string(nil), family.MemberDependencies...) + sort.Strings(memberDependencies) + for _, familyKey := range memberDependencies { + dependencies = append(dependencies, familyKey+"_"+characterID) + } + return normalizedFamilyDependencies(prefix, dependencies), nil +} + +func normalizedFamilyDependencies(_ string, dependencies []string) []string { + seen := make(map[string]struct{}, len(dependencies)) + normalized := make([]string, 0, len(dependencies)) + for _, dependency := range dependencies { + dependency = strings.TrimSpace(dependency) + if _, exists := seen[dependency]; exists { + continue + } + seen[dependency] = struct{}{} + normalized = append(normalized, dependency) + } + return normalized +} + +func expandFamilyInputs(prefix string, family ScriptoriumArtifactFamilyConfig, characterID string) (map[string]ScriptoriumInputConfig, error) { + inputs := cloneArtifactInputs(family.Inputs) + for name, input := range inputs { + familyKey, matched, err := parseMemberArtifactSource(input.Source) + if err != nil { + return nil, fmt.Errorf("%s.inputs.%s.source: %w", prefix, name, err) + } + if !matched { + continue + } + if !familyDependencyContains(family.MemberDependencies, familyKey) { + return nil, fmt.Errorf("%s.inputs.%s.source %q requires member_dependencies entry %q", prefix, name, input.Source, familyKey) + } + input.Source = artifactpolicy.ConfiguredSourceID(familyKey + "_" + characterID) + inputs[name] = input + } + return inputs, nil +} + +func parseMemberArtifactSource(source string) (string, bool, error) { + trimmed := strings.TrimSpace(source) + if !strings.HasPrefix(trimmed, "narratio.member_artifact") { + return "", false, nil + } + if !strings.HasPrefix(trimmed, memberArtifactSourcePrefix) { + return "", true, fmt.Errorf("malformed member artifact source %q", source) + } + family := strings.TrimPrefix(trimmed, memberArtifactSourcePrefix) + if !artifactpolicy.IsConfiguredKey(family) { + return "", true, fmt.Errorf("malformed member artifact source %q", source) + } + return family, true, nil +} + +func rejectConcreteMemberArtifactSources(artifacts map[string]ScriptoriumArtifactConfig) error { + for artifactKey, artifact := range artifacts { + for inputName, input := range artifact.Inputs { + if _, matched, err := parseMemberArtifactSource(input.Source); matched { + if err != nil { + return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs.%s.source: %w", artifactKey, inputName, err) + } + return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs.%s.source uses member artifact syntax outside an artifact family", artifactKey, inputName) + } + } + } + return nil +} + +func familyDependencyContains(values []string, want string) bool { + for _, value := range values { + if strings.TrimSpace(value) == want { + return true + } + } + return false +} + +func inputSourceMap(inputs map[string]ScriptoriumInputConfig) map[string]string { + if len(inputs) == 0 { + return nil + } + out := make(map[string]string, len(inputs)) + for name, input := range inputs { + out[name] = input.Source + } + return out +} + func validateFamilyOutputPattern(field, pattern string) error { if strings.Count(pattern, characterIDToken) != 1 { return fmt.Errorf("%s must contain exactly one %s token", field, characterIDToken) @@ -248,6 +387,14 @@ func cloneArtifactFamilyCatalog(in ArtifactFamilyCatalog) ArtifactFamilyCatalog out.Families[key] = family } for key, member := range in.Members { + member.Dependencies = append([]string(nil), member.Dependencies...) + if len(member.Inputs) > 0 { + inputs := make(map[string]string, len(member.Inputs)) + for name, source := range member.Inputs { + inputs[name] = source + } + member.Inputs = inputs + } out.Members[key] = member } return out diff --git a/internal/config/artifact_families_test.go b/internal/config/artifact_families_test.go index 0165fd1..1894fde 100644 --- a/internal/config/artifact_families_test.go +++ b/internal/config/artifact_families_test.go @@ -188,6 +188,115 @@ func TestArtifactFamiliesRejectGeneratedKeyAndOutputCollisions(t *testing.T) { } } +func TestArtifactFamiliesExpandSameMemberDependenciesAndInputs(t *testing.T) { + dir := t.TempDir() + writePartyResolutionFile(t, filepath.Join(dir, "party.yml"), `schema_version: narratio.party.v1 +characters: + arannis: + player: {name: Eric} + character: {name: Arannis, classes: [{name: wizard}]} + bryn: + player: {name: Bri} + character: {name: Bryn, classes: [{name: fighter}]} +`) + families := ` + character_meta: + enabled: true + for_each: party.characters + prompt_id: dnd.character_meta + output_path_pattern: artifacts/characters/{character_id}/meta.md + character_items: + enabled: true + for_each: party.characters + prompt_id: dnd.character_items + output_path_pattern: artifacts/characters/{character_id}/items.md + depends_on: [session_recap] + member_dependencies: [character_meta] + inputs: + transcript: {source: narratio.transcript.final_trimmed, required: true} + prior_meta: {source: narratio.member_artifact.character_meta, required: true} +` + pipelinePath, campaignPath, sessionPath := writeArtifactFamilyConfig(t, dir, strings.Replace(artifactFamilyPipelineYAML(families), " artifact_families:", " artifacts:\n session_recap:\n enabled: false\n output_path: artifacts/session_recap.md\n artifact_families:", 1)) + cfg, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{}) + if err != nil { + t.Fatal(err) + } + items := cfg.Pipeline.Scriptorium.Artifacts["character_items_arannis"] + if !reflect.DeepEqual(items.DependsOn, []string{"session_recap", "character_meta_arannis"}) { + t.Fatalf("member dependencies = %#v", items.DependsOn) + } + if got := items.Inputs["prior_meta"].Source; got != "narratio.artifact.character_meta_arannis" { + t.Fatalf("rewritten member source = %q", got) + } + catalog := ArtifactFamilies(cfg.Pipeline) + if got := catalog.Members["character_items_arannis"]; !reflect.DeepEqual(got.Dependencies, items.DependsOn) || got.Inputs["prior_meta"] != "narratio.artifact.character_meta_arannis" { + t.Fatalf("member provenance = %#v", got) + } + for key, artifact := range cfg.Pipeline.Scriptorium.Artifacts { + for name, input := range artifact.Inputs { + if strings.Contains(input.Source, "narratio.member_artifact.") { + t.Fatalf("unresolved source at %s.%s = %q", key, name, input.Source) + } + } + } +} + +func TestArtifactFamiliesRejectInvalidMemberDependenciesAndSources(t *testing.T) { + for _, test := range []struct{ name, families, want string }{ + {name: "missing family", families: strings.Replace(validFamilyYAML, "member_vars:", "member_dependencies: [missing]\n member_vars:", 1), want: "unknown family"}, + {name: "self dependency", families: strings.Replace(validFamilyYAML, "member_vars:", "member_dependencies: [character_meta]\n member_vars:", 1), want: "must not reference its own family"}, + {name: "duplicate dependency", families: strings.Replace(validFamilyYAML, "member_vars:", "member_dependencies: [other, other]\n member_vars:", 1) + ` + other: + for_each: party.characters + output_path_pattern: artifacts/characters/{character_id}/other.md +`, want: "duplicates family"}, + {name: "undeclared source dependency", families: strings.Replace(validFamilyYAML, "member_vars:", "inputs:\n prior: {source: narratio.member_artifact.other, required: true}\n member_vars:", 1) + ` + other: + for_each: party.characters + output_path_pattern: artifacts/characters/{character_id}/other.md +`, want: "requires member_dependencies"}, + {name: "transitive cycle", families: strings.Replace(validFamilyYAML, "member_vars:", "member_dependencies: [other]\n member_vars:", 1) + ` + other: + for_each: party.characters + output_path_pattern: artifacts/characters/{character_id}/other.md + member_dependencies: [character_meta] +`, want: "dependencies must not contain cycles"}, + } { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + writePartyResolutionFile(t, filepath.Join(dir, "party.yml"), canonicalPartyFixture) + pipelinePath, campaignPath, sessionPath := writeArtifactFamilyConfig(t, dir, artifactFamilyPipelineYAML(test.families)) + _, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{}) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("error = %v, want %q", err, test.want) + } + }) + } + + pipeline := &PipelineConfig{ + Scriptorium: &ScriptoriumConfig{Artifacts: map[string]ScriptoriumArtifactConfig{ + "explicit": {Inputs: map[string]ScriptoriumInputConfig{ + "bad": {Source: "narratio.member_artifact.character_meta"}, + }}, + }}, + resolution: &pipelineResolutionMetadata{artifactFamilies: map[string]ScriptoriumArtifactFamilyConfig{ + "character_meta": {ForEach: partyCharactersFamilySource, OutputPathPattern: "artifacts/{character_id}.md"}, + }}, + } + document, err := ParseParty([]byte(canonicalPartyFixture)) + if err != nil { + t.Fatal(err) + } + if err := expandPipelineArtifactFamilies(pipeline, ResolvedParty{Mode: PartyModeCanonical, Canonical: document.Canonical}); err == nil || !strings.Contains(err.Error(), "outside an artifact family") { + t.Fatalf("concrete member source error = %v", err) + } + pipeline.Scriptorium.Artifacts["explicit"] = ScriptoriumArtifactConfig{Inputs: map[string]ScriptoriumInputConfig{"bad": {Source: "narratio.member_artifact."}}} + pipeline.resolution.artifactFamiliesExpanded = false + if err := expandPipelineArtifactFamilies(pipeline, ResolvedParty{Mode: PartyModeCanonical, Canonical: document.Canonical}); err == nil || !strings.Contains(err.Error(), "malformed member artifact source") { + t.Fatalf("malformed concrete member source error = %v", err) + } +} + const validFamilyYAML = ` character_meta: enabled: true diff --git a/internal/config/config.go b/internal/config/config.go index 32e614c..fda3728 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -310,9 +310,11 @@ type ArtifactFamilyOrigin struct { // ArtifactFamilyMemberOrigin identifies the canonical party member that // produced one ordinary concrete artifact. type ArtifactFamilyMemberOrigin struct { - Family string - CharacterID string - Source string + Family string + CharacterID string + Source string + Dependencies []string + Inputs map[string]string } // ScriptoriumInputConfig configures one named prompt input source.