Add split pipeline configuration example bundle
This commit is contained in:
@@ -135,6 +135,10 @@ campaigns:
|
|||||||
root: /usr/local/share/narratio/campaigns
|
root: /usr/local/share/narratio/campaigns
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The maintained [production/testing bundle](../examples/production-testing/pipeline.yml)
|
||||||
|
is a complete copyable example of this structure, including canonical-party
|
||||||
|
artifact families.
|
||||||
|
|
||||||
Imports are resolved relative to the directory containing the root pipeline
|
Imports are resolved relative to the directory containing the root pipeline
|
||||||
file and are loaded in declaration order. Narratio does not scan directories or
|
file and are loaded in declaration order. Narratio does not scan directories or
|
||||||
infer fragments. Each import must be a confined regular `.yml` or `.yaml` file:
|
infer fragments. Each import must be a confined regular `.yml` or `.yaml` file:
|
||||||
|
|||||||
@@ -1097,7 +1097,7 @@ layout or formatting.
|
|||||||
|
|
||||||
## Stage 20 — Maintained Split Configuration Bundle
|
## Stage 20 — Maintained Split Configuration Bundle
|
||||||
|
|
||||||
**Status: Pending**
|
**Status: Completed**
|
||||||
|
|
||||||
### Goal
|
### Goal
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,18 @@ in the [configuration reference](../docs/config.md).
|
|||||||
WhisperX URL.
|
WhisperX URL.
|
||||||
- [Production-shaped pipeline](pipeline.production.yml): S3 storage, publish,
|
- [Production-shaped pipeline](pipeline.production.yml): S3 storage, publish,
|
||||||
external tools, and configured Scriptorium artifacts.
|
external tools, and configured Scriptorium artifacts.
|
||||||
|
- [Production/testing split bundle](production-testing/pipeline.yml): explicit
|
||||||
|
`conf.d` imports, a production default, and selectable production/testing
|
||||||
|
overlays. It also demonstrates canonical-party artifact families and a
|
||||||
|
testing-only disabled artifact. Validate it with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
narratio config validate --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
|
||||||
|
narratio config diff production testing --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
`config show` and `config sources` accept the same selectors and remain
|
||||||
|
read-only.
|
||||||
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
|
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
|
||||||
pipeline section with explanatory comments.
|
pipeline section with explanatory comments.
|
||||||
- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused
|
- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused
|
||||||
@@ -37,11 +49,11 @@ with the sample campaign and a compatible local- or S3-audio session.
|
|||||||
- The sample campaign references its local
|
- The sample campaign references its local
|
||||||
[speakers](campaigns/sample-campaign/speakers.yml),
|
[speakers](campaigns/sample-campaign/speakers.yml),
|
||||||
[autocorrect](campaigns/sample-campaign/autocorrect.yml),
|
[autocorrect](campaigns/sample-campaign/autocorrect.yml),
|
||||||
[glossary](campaigns/sample-campaign/glossary.yml),
|
[glossary](campaigns/sample-campaign/glossary.yml), and canonical
|
||||||
[players](campaigns/sample-campaign/players.yml), and
|
[party](campaigns/sample-campaign/party.yml) fixture, plus an optional
|
||||||
[party](campaigns/sample-campaign/party.yml) fixtures, plus an optional
|
|
||||||
[spell-catalog overlay](campaigns/sample-campaign/spell_catalog.json) that
|
[spell-catalog overlay](campaigns/sample-campaign/spell_catalog.json) that
|
||||||
follows the Notarius v0.6 contract.
|
follows the Notarius v0.6 contract. Narratio derives the players projection
|
||||||
|
from this party source; the campaign deliberately has no `players_file`.
|
||||||
- [Sample speaker audio](audio/sample-speaker.flac) is a text placeholder that
|
- [Sample speaker audio](audio/sample-speaker.flac) is a text placeholder that
|
||||||
reserves the expected filename and directory shape. Replace it with a real
|
reserves the expected filename and directory shape. Replace it with a real
|
||||||
FLAC file before running transcription.
|
FLAC file before running transcription.
|
||||||
|
|||||||
@@ -4,6 +4,5 @@ inputs:
|
|||||||
speakers_file: ./speakers.yml
|
speakers_file: ./speakers.yml
|
||||||
autocorrect_file: ./autocorrect.yml
|
autocorrect_file: ./autocorrect.yml
|
||||||
glossary_file: ./glossary.yml
|
glossary_file: ./glossary.yml
|
||||||
players_file: ./players.yml
|
|
||||||
party_file: ./party.yml
|
party_file: ./party.yml
|
||||||
spell_catalog_file: ./spell_catalog.json
|
spell_catalog_file: ./spell_catalog.json
|
||||||
|
|||||||
@@ -1,2 +1,26 @@
|
|||||||
- name: Example Hero
|
schema_version: narratio.party.v1
|
||||||
type: pc
|
|
||||||
|
characters:
|
||||||
|
arannis:
|
||||||
|
player:
|
||||||
|
name: Rowan Hale
|
||||||
|
character:
|
||||||
|
name: Arannis
|
||||||
|
alias:
|
||||||
|
- Ari
|
||||||
|
- The Grey Owl
|
||||||
|
classes:
|
||||||
|
- name: wizard
|
||||||
|
level: 8
|
||||||
|
brenna:
|
||||||
|
player:
|
||||||
|
name: Rowan Hale
|
||||||
|
character:
|
||||||
|
name: Brenna
|
||||||
|
alias:
|
||||||
|
- Shield of Dawn
|
||||||
|
classes:
|
||||||
|
- name: paladin
|
||||||
|
level: 6
|
||||||
|
- name: warlock
|
||||||
|
level: 2
|
||||||
|
|||||||
@@ -1,2 +0,0 @@
|
|||||||
- name: Example Player
|
|
||||||
role: player
|
|
||||||
32
examples/production-testing/conf.d/artifacts.yml
Normal file
32
examples/production-testing/conf.d/artifacts.yml
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
scriptorium:
|
||||||
|
binary: scriptorium
|
||||||
|
config_path: ./scriptorium/config.yml
|
||||||
|
artifact_families:
|
||||||
|
character_meta:
|
||||||
|
enabled: true
|
||||||
|
for_each: party.characters
|
||||||
|
prompt_id: dnd.character_meta
|
||||||
|
output_path_pattern: artifacts/characters/{character_id}/meta.md
|
||||||
|
member_vars:
|
||||||
|
character_id: character_id
|
||||||
|
character_name: character.name
|
||||||
|
player_name: player.name
|
||||||
|
class_summary: character.class_summary
|
||||||
|
character_items:
|
||||||
|
enabled: true
|
||||||
|
for_each: party.characters
|
||||||
|
prompt_id: dnd.character_items
|
||||||
|
output_path_pattern: artifacts/characters/{character_id}/items.md
|
||||||
|
member_dependencies: [character_meta]
|
||||||
|
inputs:
|
||||||
|
character_meta:
|
||||||
|
source: narratio.member_artifact.character_meta
|
||||||
|
required: true
|
||||||
|
member_vars:
|
||||||
|
character_id: character_id
|
||||||
|
character_name: character.name
|
||||||
|
aliases: character.alias_summary
|
||||||
|
publish:
|
||||||
|
enabled: true
|
||||||
|
required: false
|
||||||
|
dest_pattern: artifacts/characters/{character_id}/items.md
|
||||||
12
examples/production-testing/conf.d/platform.yml
Normal file
12
examples/production-testing/conf.d/platform.yml
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
workspace:
|
||||||
|
root: ./workspace
|
||||||
|
|
||||||
|
campaigns:
|
||||||
|
root: ../../campaigns
|
||||||
|
default_campaign_id: sample-campaign
|
||||||
|
|
||||||
|
cache:
|
||||||
|
root: ./cache
|
||||||
|
|
||||||
|
spool:
|
||||||
|
root: ./spool
|
||||||
7
examples/production-testing/conf.d/publish.yml
Normal file
7
examples/production-testing/conf.d/publish.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
publish:
|
||||||
|
enabled: true
|
||||||
|
upload_run: false
|
||||||
|
outputs:
|
||||||
|
- source: narratio.transcript.final_markdown
|
||||||
|
dest: transcripts/final.md
|
||||||
|
required: true
|
||||||
2
examples/production-testing/conf.d/storage.yml
Normal file
2
examples/production-testing/conf.d/storage.yml
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
storage:
|
||||||
|
backend: local
|
||||||
16
examples/production-testing/conf.d/transcript.yml
Normal file
16
examples/production-testing/conf.d/transcript.yml
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
whisperx:
|
||||||
|
transcribe_url: https://transcription.example.com/transcribe
|
||||||
|
language: en
|
||||||
|
|
||||||
|
seriatim:
|
||||||
|
binary: seriatim
|
||||||
|
output_schema: seriatim-intermediate
|
||||||
|
|
||||||
|
audita:
|
||||||
|
binary: audita
|
||||||
|
modules: [glossary, grammar]
|
||||||
|
output_schema: audita-v1
|
||||||
|
|
||||||
|
normalize:
|
||||||
|
output_path: transcripts/final.json
|
||||||
|
output_schema: seriatim-intermediate
|
||||||
15
examples/production-testing/pipeline.yml
Normal file
15
examples/production-testing/pipeline.yml
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
# Copyable production/testing pipeline entry point. Every fragment is named
|
||||||
|
# explicitly; Narratio never scans conf.d automatically.
|
||||||
|
composition:
|
||||||
|
imports:
|
||||||
|
- conf.d/platform.yml
|
||||||
|
- conf.d/storage.yml
|
||||||
|
- conf.d/transcript.yml
|
||||||
|
- conf.d/artifacts.yml
|
||||||
|
- conf.d/publish.yml
|
||||||
|
default_profile: production
|
||||||
|
profiles:
|
||||||
|
production:
|
||||||
|
overlay: profiles/production.yml
|
||||||
|
testing:
|
||||||
|
overlay: profiles/testing.yml
|
||||||
10
examples/production-testing/profiles/production.yml
Normal file
10
examples/production-testing/profiles/production.yml
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
audita:
|
||||||
|
model: narratio-production-model-placeholder
|
||||||
|
validation_model: narratio-production-validator-placeholder
|
||||||
|
|
||||||
|
scriptorium:
|
||||||
|
artifact_families:
|
||||||
|
character_meta:
|
||||||
|
profile_id: production-placeholder
|
||||||
|
character_items:
|
||||||
|
profile_id: production-placeholder
|
||||||
14
examples/production-testing/profiles/testing.yml
Normal file
14
examples/production-testing/profiles/testing.yml
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
audita:
|
||||||
|
model: narratio-testing-model-placeholder
|
||||||
|
validation_model: narratio-testing-validator-placeholder
|
||||||
|
|
||||||
|
scriptorium:
|
||||||
|
artifacts:
|
||||||
|
testing_notes:
|
||||||
|
enabled: false
|
||||||
|
output_path: artifacts/testing-notes.md
|
||||||
|
artifact_families:
|
||||||
|
character_meta:
|
||||||
|
profile_id: testing-placeholder
|
||||||
|
character_items:
|
||||||
|
profile_id: testing-placeholder
|
||||||
@@ -242,6 +242,32 @@ func TestConfigDiffReportsEqualityAndRejectsInvalidInput(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testing.T) {
|
||||||
|
examplesDir := filepath.Join("..", "..", "examples")
|
||||||
|
pipelinePath := filepath.Join(examplesDir, "production-testing", "pipeline.yml")
|
||||||
|
campaignPath := filepath.Join(examplesDir, "campaigns", "sample-campaign", "campaign.yml")
|
||||||
|
workspacePath := filepath.Join(examplesDir, "production-testing", "workspace")
|
||||||
|
if _, err := os.Stat(workspacePath); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("example workspace stat = %v, want absent", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigValidate, ConfigShow, ConfigSources} {
|
||||||
|
if err := command(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "production"}, io.Discard); err != nil {
|
||||||
|
t.Fatalf("inspection command %T error = %v", command, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var diff bytes.Buffer
|
||||||
|
if err := ConfigDiff(context.Background(), []string{"production", "testing", "--config", pipelinePath, "--campaign-file", campaignPath}, &diff); err != nil {
|
||||||
|
t.Fatalf("ConfigDiff() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(diff.String(), "audita.model") {
|
||||||
|
t.Fatalf("split bundle profile diff = %q, want model change", diff.String())
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(workspacePath); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("inspection created example workspace: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func writeInspectionProfiles(t *testing.T, pipelinePath, production, testing string) {
|
func writeInspectionProfiles(t *testing.T, pipelinePath, production, testing string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
data, err := os.ReadFile(pipelinePath)
|
data, err := os.ReadFile(pipelinePath)
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
@@ -101,6 +103,15 @@ func TestValidateMissingAudioSource(t *testing.T) {
|
|||||||
|
|
||||||
func TestExamplesLoadAndValidate(t *testing.T) {
|
func TestExamplesLoadAndValidate(t *testing.T) {
|
||||||
examplesDir := filepath.Join("..", "..", "examples")
|
examplesDir := filepath.Join("..", "..", "examples")
|
||||||
|
if got, want := maintainedPipelineRoots(t, examplesDir), []string{
|
||||||
|
"pipeline.extraction-subset.yml",
|
||||||
|
"pipeline.full.annotated.yml",
|
||||||
|
"pipeline.minimal.yml",
|
||||||
|
"pipeline.production.yml",
|
||||||
|
filepath.Join("production-testing", "pipeline.yml"),
|
||||||
|
}; !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("maintained pipeline roots = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
pipelineFile string
|
pipelineFile string
|
||||||
@@ -143,6 +154,108 @@ func TestExamplesLoadAndValidate(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
t.Run("split production and testing bundle", func(t *testing.T) {
|
||||||
|
pipelinePath := filepath.Join(examplesDir, "production-testing", "pipeline.yml")
|
||||||
|
campaignPath := filepath.Join(examplesDir, "campaigns", "sample-campaign", "campaign.yml")
|
||||||
|
sessionPath := filepath.Join(examplesDir, "session.local-audio.yml")
|
||||||
|
campaign, err := LoadCampaign(campaignPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load split bundle campaign: %v", err)
|
||||||
|
}
|
||||||
|
for _, profile := range []struct {
|
||||||
|
name string
|
||||||
|
model string
|
||||||
|
}{
|
||||||
|
{name: "production", model: "narratio-production-model-placeholder"},
|
||||||
|
{name: "testing", model: "narratio-testing-model-placeholder"},
|
||||||
|
} {
|
||||||
|
t.Run(profile.name, func(t *testing.T) {
|
||||||
|
selected := profile.name
|
||||||
|
pipeline, err := LoadPipelineWithOptions(pipelinePath, PipelineLoadOptions{Profile: &selected})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load split bundle pipeline: %v", err)
|
||||||
|
}
|
||||||
|
loaded, err := LoadPipelineCampaign(pipelinePath, pipeline, campaignPath, campaign)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve split bundle campaign: %v", err)
|
||||||
|
}
|
||||||
|
cfg, err := LoadSessionWithPipelineCampaignOptions(loaded, sessionPath, SessionLoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load split bundle session: %v", err)
|
||||||
|
}
|
||||||
|
if err := Validate(cfg); err != nil {
|
||||||
|
t.Fatalf("validate split bundle: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Audita.Model != profile.model {
|
||||||
|
t.Fatalf("audita model = %q, want %q", cfg.Pipeline.Audita.Model, profile.model)
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Secrets != nil || cfg.Pipeline.Audita.LLMAPIKeyEnv != "" || cfg.Pipeline.Storage.Backend != StorageBackendLocal || cfg.Pipeline.Storage.S3 != nil {
|
||||||
|
t.Fatalf("split bundle must remain offline and secret-free: secrets=%#v audita=%#v storage=%#v", cfg.Pipeline.Secrets, cfg.Pipeline.Audita, cfg.Pipeline.Storage)
|
||||||
|
}
|
||||||
|
if selectedProfile, ok := SelectedPipelineProfile(cfg.Pipeline); !ok || selectedProfile.Name != profile.name {
|
||||||
|
t.Fatalf("selected profile = %#v, want %q", selectedProfile, profile.name)
|
||||||
|
}
|
||||||
|
for _, key := range []string{
|
||||||
|
"character_meta_arannis",
|
||||||
|
"character_meta_brenna",
|
||||||
|
"character_items_arannis",
|
||||||
|
"character_items_brenna",
|
||||||
|
} {
|
||||||
|
if _, exists := cfg.Pipeline.Scriptorium.Artifacts[key]; !exists {
|
||||||
|
t.Fatalf("expanded artifact %q is absent", key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got := cfg.Pipeline.Scriptorium.Artifacts["character_items_arannis"].Inputs["character_meta"].Source; got != "narratio.artifact.character_meta_arannis" {
|
||||||
|
t.Fatalf("same-member input source = %q", got)
|
||||||
|
}
|
||||||
|
if cfg.Party.Mode != PartyModeCanonical || cfg.Party.Canonical == nil || len(cfg.Party.Canonical.Characters) != 2 {
|
||||||
|
t.Fatalf("canonical party = %#v", cfg.Party)
|
||||||
|
}
|
||||||
|
characters := cfg.Party.Canonical.Characters
|
||||||
|
if characters[0].Player.Name != characters[1].Player.Name || len(characters[0].Character.Aliases) != 2 || len(characters[1].Character.Classes) != 2 {
|
||||||
|
t.Fatalf("canonical party does not preserve repeated player, aliases, and multiclass data: %#v", characters)
|
||||||
|
}
|
||||||
|
if len(cfg.Pipeline.Publish.Outputs) != 3 {
|
||||||
|
t.Fatalf("publish outputs = %#v, want transcript plus two family outputs", cfg.Pipeline.Publish.Outputs)
|
||||||
|
}
|
||||||
|
if profile.name == "testing" {
|
||||||
|
artifact, exists := cfg.Pipeline.Scriptorium.Artifacts["testing_notes"]
|
||||||
|
if !exists || artifact.Enabled {
|
||||||
|
t.Fatalf("testing-only disabled artifact = %#v", artifact)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func maintainedPipelineRoots(t *testing.T, examplesDir string) []string {
|
||||||
|
t.Helper()
|
||||||
|
var roots []string
|
||||||
|
err := filepath.WalkDir(examplesDir, func(path string, entry fs.DirEntry, walkErr error) error {
|
||||||
|
if walkErr != nil {
|
||||||
|
return walkErr
|
||||||
|
}
|
||||||
|
if entry.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
name := entry.Name()
|
||||||
|
if name != "pipeline.yml" && !(strings.HasPrefix(name, "pipeline.") && (strings.HasSuffix(name, ".yml") || strings.HasSuffix(name, ".yaml"))) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
relative, err := filepath.Rel(examplesDir, path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
roots = append(roots, relative)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("discover maintained pipeline roots: %v", err)
|
||||||
|
}
|
||||||
|
sort.Strings(roots)
|
||||||
|
return roots
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMaintainedExtractionExamplesPreservePublishedContracts(t *testing.T) {
|
func TestMaintainedExtractionExamplesPreservePublishedContracts(t *testing.T) {
|
||||||
|
|||||||
Reference in New Issue
Block a user