From fcb5f825e15f94087dbb6e3c0d4c30f5ce41d940 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 30 Aug 2026 15:17:46 +0000 Subject: [PATCH] Add split pipeline configuration example bundle --- docs/config.md | 4 + docs/roadmap/implementation.md | 2 +- examples/README.md | 20 +++- .../campaigns/sample-campaign/campaign.yml | 1 - examples/campaigns/sample-campaign/party.yml | 28 ++++- .../campaigns/sample-campaign/players.yml | 2 - .../production-testing/conf.d/artifacts.yml | 32 +++++ .../production-testing/conf.d/platform.yml | 12 ++ .../production-testing/conf.d/publish.yml | 7 ++ .../production-testing/conf.d/storage.yml | 2 + .../production-testing/conf.d/transcript.yml | 16 +++ examples/production-testing/pipeline.yml | 15 +++ .../profiles/production.yml | 10 ++ .../production-testing/profiles/testing.yml | 14 +++ internal/app/config_commands_test.go | 26 ++++ internal/config/load_validate_test.go | 113 ++++++++++++++++++ 16 files changed, 294 insertions(+), 10 deletions(-) delete mode 100644 examples/campaigns/sample-campaign/players.yml create mode 100644 examples/production-testing/conf.d/artifacts.yml create mode 100644 examples/production-testing/conf.d/platform.yml create mode 100644 examples/production-testing/conf.d/publish.yml create mode 100644 examples/production-testing/conf.d/storage.yml create mode 100644 examples/production-testing/conf.d/transcript.yml create mode 100644 examples/production-testing/pipeline.yml create mode 100644 examples/production-testing/profiles/production.yml create mode 100644 examples/production-testing/profiles/testing.yml diff --git a/docs/config.md b/docs/config.md index 4ada2ee..14cf1bb 100644 --- a/docs/config.md +++ b/docs/config.md @@ -135,6 +135,10 @@ campaigns: root: /usr/local/share/narratio/campaigns ``` +The maintained [production/testing bundle](../examples/production-testing/pipeline.yml) +is a complete copyable example of this structure, including canonical-party +artifact families. + Imports are resolved relative to the directory containing the root pipeline file and are loaded in declaration order. Narratio does not scan directories or infer fragments. Each import must be a confined regular `.yml` or `.yaml` file: diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index e75c1ff..cd1cef4 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1097,7 +1097,7 @@ layout or formatting. ## Stage 20 — Maintained Split Configuration Bundle -**Status: Pending** +**Status: Completed** ### Goal diff --git a/examples/README.md b/examples/README.md index f9938c6..507d5a4 100644 --- a/examples/README.md +++ b/examples/README.md @@ -11,6 +11,18 @@ in the [configuration reference](../docs/config.md). WhisperX URL. - [Production-shaped pipeline](pipeline.production.yml): S3 storage, publish, external tools, and configured Scriptorium artifacts. +- [Production/testing split bundle](production-testing/pipeline.yml): explicit + `conf.d` imports, a production default, and selectable production/testing + overlays. It also demonstrates canonical-party artifact families and a + testing-only disabled artifact. Validate it with: + + ```sh + narratio config validate --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml + narratio config diff production testing --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml + ``` + + `config show` and `config sources` accept the same selectors and remain + read-only. - [Full annotated pipeline](pipeline.full.annotated.yml): every implemented pipeline section with explanatory comments. - [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused @@ -37,11 +49,11 @@ with the sample campaign and a compatible local- or S3-audio session. - The sample campaign references its local [speakers](campaigns/sample-campaign/speakers.yml), [autocorrect](campaigns/sample-campaign/autocorrect.yml), - [glossary](campaigns/sample-campaign/glossary.yml), - [players](campaigns/sample-campaign/players.yml), and - [party](campaigns/sample-campaign/party.yml) fixtures, plus an optional + [glossary](campaigns/sample-campaign/glossary.yml), and canonical + [party](campaigns/sample-campaign/party.yml) fixture, plus an optional [spell-catalog overlay](campaigns/sample-campaign/spell_catalog.json) that - follows the Notarius v0.6 contract. + follows the Notarius v0.6 contract. Narratio derives the players projection + from this party source; the campaign deliberately has no `players_file`. - [Sample speaker audio](audio/sample-speaker.flac) is a text placeholder that reserves the expected filename and directory shape. Replace it with a real FLAC file before running transcription. diff --git a/examples/campaigns/sample-campaign/campaign.yml b/examples/campaigns/sample-campaign/campaign.yml index 6a0d32e..454bded 100644 --- a/examples/campaigns/sample-campaign/campaign.yml +++ b/examples/campaigns/sample-campaign/campaign.yml @@ -4,6 +4,5 @@ inputs: speakers_file: ./speakers.yml autocorrect_file: ./autocorrect.yml glossary_file: ./glossary.yml - players_file: ./players.yml party_file: ./party.yml spell_catalog_file: ./spell_catalog.json diff --git a/examples/campaigns/sample-campaign/party.yml b/examples/campaigns/sample-campaign/party.yml index 4ef5724..ad3a9d0 100644 --- a/examples/campaigns/sample-campaign/party.yml +++ b/examples/campaigns/sample-campaign/party.yml @@ -1,2 +1,26 @@ -- name: Example Hero - type: pc +schema_version: narratio.party.v1 + +characters: + arannis: + player: + name: Rowan Hale + character: + name: Arannis + alias: + - Ari + - The Grey Owl + classes: + - name: wizard + level: 8 + brenna: + player: + name: Rowan Hale + character: + name: Brenna + alias: + - Shield of Dawn + classes: + - name: paladin + level: 6 + - name: warlock + level: 2 diff --git a/examples/campaigns/sample-campaign/players.yml b/examples/campaigns/sample-campaign/players.yml deleted file mode 100644 index 1f1d75d..0000000 --- a/examples/campaigns/sample-campaign/players.yml +++ /dev/null @@ -1,2 +0,0 @@ -- name: Example Player - role: player diff --git a/examples/production-testing/conf.d/artifacts.yml b/examples/production-testing/conf.d/artifacts.yml new file mode 100644 index 0000000..d5984c9 --- /dev/null +++ b/examples/production-testing/conf.d/artifacts.yml @@ -0,0 +1,32 @@ +scriptorium: + binary: scriptorium + config_path: ./scriptorium/config.yml + artifact_families: + character_meta: + enabled: true + for_each: party.characters + prompt_id: dnd.character_meta + output_path_pattern: artifacts/characters/{character_id}/meta.md + member_vars: + character_id: character_id + character_name: character.name + player_name: player.name + class_summary: character.class_summary + character_items: + enabled: true + for_each: party.characters + prompt_id: dnd.character_items + output_path_pattern: artifacts/characters/{character_id}/items.md + member_dependencies: [character_meta] + inputs: + character_meta: + source: narratio.member_artifact.character_meta + required: true + member_vars: + character_id: character_id + character_name: character.name + aliases: character.alias_summary + publish: + enabled: true + required: false + dest_pattern: artifacts/characters/{character_id}/items.md diff --git a/examples/production-testing/conf.d/platform.yml b/examples/production-testing/conf.d/platform.yml new file mode 100644 index 0000000..e6c8311 --- /dev/null +++ b/examples/production-testing/conf.d/platform.yml @@ -0,0 +1,12 @@ +workspace: + root: ./workspace + +campaigns: + root: ../../campaigns + default_campaign_id: sample-campaign + +cache: + root: ./cache + +spool: + root: ./spool diff --git a/examples/production-testing/conf.d/publish.yml b/examples/production-testing/conf.d/publish.yml new file mode 100644 index 0000000..44ad202 --- /dev/null +++ b/examples/production-testing/conf.d/publish.yml @@ -0,0 +1,7 @@ +publish: + enabled: true + upload_run: false + outputs: + - source: narratio.transcript.final_markdown + dest: transcripts/final.md + required: true diff --git a/examples/production-testing/conf.d/storage.yml b/examples/production-testing/conf.d/storage.yml new file mode 100644 index 0000000..0b054d7 --- /dev/null +++ b/examples/production-testing/conf.d/storage.yml @@ -0,0 +1,2 @@ +storage: + backend: local diff --git a/examples/production-testing/conf.d/transcript.yml b/examples/production-testing/conf.d/transcript.yml new file mode 100644 index 0000000..52b8bf4 --- /dev/null +++ b/examples/production-testing/conf.d/transcript.yml @@ -0,0 +1,16 @@ +whisperx: + transcribe_url: https://transcription.example.com/transcribe + language: en + +seriatim: + binary: seriatim + output_schema: seriatim-intermediate + +audita: + binary: audita + modules: [glossary, grammar] + output_schema: audita-v1 + +normalize: + output_path: transcripts/final.json + output_schema: seriatim-intermediate diff --git a/examples/production-testing/pipeline.yml b/examples/production-testing/pipeline.yml new file mode 100644 index 0000000..64294fe --- /dev/null +++ b/examples/production-testing/pipeline.yml @@ -0,0 +1,15 @@ +# Copyable production/testing pipeline entry point. Every fragment is named +# explicitly; Narratio never scans conf.d automatically. +composition: + imports: + - conf.d/platform.yml + - conf.d/storage.yml + - conf.d/transcript.yml + - conf.d/artifacts.yml + - conf.d/publish.yml + default_profile: production + profiles: + production: + overlay: profiles/production.yml + testing: + overlay: profiles/testing.yml diff --git a/examples/production-testing/profiles/production.yml b/examples/production-testing/profiles/production.yml new file mode 100644 index 0000000..4219373 --- /dev/null +++ b/examples/production-testing/profiles/production.yml @@ -0,0 +1,10 @@ +audita: + model: narratio-production-model-placeholder + validation_model: narratio-production-validator-placeholder + +scriptorium: + artifact_families: + character_meta: + profile_id: production-placeholder + character_items: + profile_id: production-placeholder diff --git a/examples/production-testing/profiles/testing.yml b/examples/production-testing/profiles/testing.yml new file mode 100644 index 0000000..ff285fc --- /dev/null +++ b/examples/production-testing/profiles/testing.yml @@ -0,0 +1,14 @@ +audita: + model: narratio-testing-model-placeholder + validation_model: narratio-testing-validator-placeholder + +scriptorium: + artifacts: + testing_notes: + enabled: false + output_path: artifacts/testing-notes.md + artifact_families: + character_meta: + profile_id: testing-placeholder + character_items: + profile_id: testing-placeholder diff --git a/internal/app/config_commands_test.go b/internal/app/config_commands_test.go index c06e4d2..b7376bd 100644 --- a/internal/app/config_commands_test.go +++ b/internal/app/config_commands_test.go @@ -242,6 +242,32 @@ func TestConfigDiffReportsEqualityAndRejectsInvalidInput(t *testing.T) { } } +func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testing.T) { + examplesDir := filepath.Join("..", "..", "examples") + pipelinePath := filepath.Join(examplesDir, "production-testing", "pipeline.yml") + campaignPath := filepath.Join(examplesDir, "campaigns", "sample-campaign", "campaign.yml") + workspacePath := filepath.Join(examplesDir, "production-testing", "workspace") + if _, err := os.Stat(workspacePath); !os.IsNotExist(err) { + t.Fatalf("example workspace stat = %v, want absent", err) + } + + for _, command := range []func(context.Context, []string, io.Writer) error{ConfigValidate, ConfigShow, ConfigSources} { + if err := command(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "production"}, io.Discard); err != nil { + t.Fatalf("inspection command %T error = %v", command, err) + } + } + var diff bytes.Buffer + if err := ConfigDiff(context.Background(), []string{"production", "testing", "--config", pipelinePath, "--campaign-file", campaignPath}, &diff); err != nil { + t.Fatalf("ConfigDiff() error = %v", err) + } + if !strings.Contains(diff.String(), "audita.model") { + t.Fatalf("split bundle profile diff = %q, want model change", diff.String()) + } + if _, err := os.Stat(workspacePath); !os.IsNotExist(err) { + t.Fatalf("inspection created example workspace: %v", err) + } +} + func writeInspectionProfiles(t *testing.T, pipelinePath, production, testing string) { t.Helper() data, err := os.ReadFile(pipelinePath) diff --git a/internal/config/load_validate_test.go b/internal/config/load_validate_test.go index 40e6c50..2ad464b 100644 --- a/internal/config/load_validate_test.go +++ b/internal/config/load_validate_test.go @@ -1,9 +1,11 @@ package config import ( + "io/fs" "os" "path/filepath" "reflect" + "sort" "strings" "testing" ) @@ -101,6 +103,15 @@ func TestValidateMissingAudioSource(t *testing.T) { func TestExamplesLoadAndValidate(t *testing.T) { examplesDir := filepath.Join("..", "..", "examples") + if got, want := maintainedPipelineRoots(t, examplesDir), []string{ + "pipeline.extraction-subset.yml", + "pipeline.full.annotated.yml", + "pipeline.minimal.yml", + "pipeline.production.yml", + filepath.Join("production-testing", "pipeline.yml"), + }; !reflect.DeepEqual(got, want) { + t.Fatalf("maintained pipeline roots = %#v, want %#v", got, want) + } tests := []struct { name string pipelineFile string @@ -143,6 +154,108 @@ func TestExamplesLoadAndValidate(t *testing.T) { } }) } + + t.Run("split production and testing bundle", func(t *testing.T) { + pipelinePath := filepath.Join(examplesDir, "production-testing", "pipeline.yml") + campaignPath := filepath.Join(examplesDir, "campaigns", "sample-campaign", "campaign.yml") + sessionPath := filepath.Join(examplesDir, "session.local-audio.yml") + campaign, err := LoadCampaign(campaignPath) + if err != nil { + t.Fatalf("load split bundle campaign: %v", err) + } + for _, profile := range []struct { + name string + model string + }{ + {name: "production", model: "narratio-production-model-placeholder"}, + {name: "testing", model: "narratio-testing-model-placeholder"}, + } { + t.Run(profile.name, func(t *testing.T) { + selected := profile.name + pipeline, err := LoadPipelineWithOptions(pipelinePath, PipelineLoadOptions{Profile: &selected}) + if err != nil { + t.Fatalf("load split bundle pipeline: %v", err) + } + loaded, err := LoadPipelineCampaign(pipelinePath, pipeline, campaignPath, campaign) + if err != nil { + t.Fatalf("resolve split bundle campaign: %v", err) + } + cfg, err := LoadSessionWithPipelineCampaignOptions(loaded, sessionPath, SessionLoadOptions{}) + if err != nil { + t.Fatalf("load split bundle session: %v", err) + } + if err := Validate(cfg); err != nil { + t.Fatalf("validate split bundle: %v", err) + } + if cfg.Pipeline.Audita.Model != profile.model { + t.Fatalf("audita model = %q, want %q", cfg.Pipeline.Audita.Model, profile.model) + } + if cfg.Pipeline.Secrets != nil || cfg.Pipeline.Audita.LLMAPIKeyEnv != "" || cfg.Pipeline.Storage.Backend != StorageBackendLocal || cfg.Pipeline.Storage.S3 != nil { + t.Fatalf("split bundle must remain offline and secret-free: secrets=%#v audita=%#v storage=%#v", cfg.Pipeline.Secrets, cfg.Pipeline.Audita, cfg.Pipeline.Storage) + } + if selectedProfile, ok := SelectedPipelineProfile(cfg.Pipeline); !ok || selectedProfile.Name != profile.name { + t.Fatalf("selected profile = %#v, want %q", selectedProfile, profile.name) + } + for _, key := range []string{ + "character_meta_arannis", + "character_meta_brenna", + "character_items_arannis", + "character_items_brenna", + } { + if _, exists := cfg.Pipeline.Scriptorium.Artifacts[key]; !exists { + t.Fatalf("expanded artifact %q is absent", key) + } + } + if got := cfg.Pipeline.Scriptorium.Artifacts["character_items_arannis"].Inputs["character_meta"].Source; got != "narratio.artifact.character_meta_arannis" { + t.Fatalf("same-member input source = %q", got) + } + if cfg.Party.Mode != PartyModeCanonical || cfg.Party.Canonical == nil || len(cfg.Party.Canonical.Characters) != 2 { + t.Fatalf("canonical party = %#v", cfg.Party) + } + characters := cfg.Party.Canonical.Characters + if characters[0].Player.Name != characters[1].Player.Name || len(characters[0].Character.Aliases) != 2 || len(characters[1].Character.Classes) != 2 { + t.Fatalf("canonical party does not preserve repeated player, aliases, and multiclass data: %#v", characters) + } + if len(cfg.Pipeline.Publish.Outputs) != 3 { + t.Fatalf("publish outputs = %#v, want transcript plus two family outputs", cfg.Pipeline.Publish.Outputs) + } + if profile.name == "testing" { + artifact, exists := cfg.Pipeline.Scriptorium.Artifacts["testing_notes"] + if !exists || artifact.Enabled { + t.Fatalf("testing-only disabled artifact = %#v", artifact) + } + } + }) + } + }) +} + +func maintainedPipelineRoots(t *testing.T, examplesDir string) []string { + t.Helper() + var roots []string + err := filepath.WalkDir(examplesDir, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + return nil + } + name := entry.Name() + if name != "pipeline.yml" && !(strings.HasPrefix(name, "pipeline.") && (strings.HasSuffix(name, ".yml") || strings.HasSuffix(name, ".yaml"))) { + return nil + } + relative, err := filepath.Rel(examplesDir, path) + if err != nil { + return err + } + roots = append(roots, relative) + return nil + }) + if err != nil { + t.Fatalf("discover maintained pipeline roots: %v", err) + } + sort.Strings(roots) + return roots } func TestMaintainedExtractionExamplesPreservePublishedContracts(t *testing.T) {