From 77f67adbc0f700e88bc15554e02052eaa22f1033 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 21 Sep 2026 03:37:02 +0000 Subject: [PATCH] Squashed 'roles/pipeline/' changes from c13f726..51566af 51566af Enable D&D storage and publishing git-subtree-dir: roles/pipeline git-subtree-split: 51566afcf69083cdbb5b5718f2544c86b079001c --- README.md | 59 ++++++++++++++++++++++++++++++++++++++--- defaults/main.yml | 29 +++++++++++++++++++- meta/argument_specs.yml | 52 ++++++++++++++++++++++++++++++++++++ meta/main.yml | 13 +++++++++ tests/integration | 55 +++++++++++++++++++++++++++++++++++++- 5 files changed, 203 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 53ed4c2..d164006 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ The compatible application versions are selected through these variables: | Variable | Default | | --- | --- | | `dnd_pipeline_audita_version` | `v0.9.1` | -| `dnd_pipeline_narratio_version` | `v2.1.0` | +| `dnd_pipeline_narratio_version` | `v2.1.1` | | `dnd_pipeline_notarius_version` | `v0.7.0` | | `dnd_pipeline_seriatim_version` | `v1.5.0` | | `dnd_pipeline_scriptorium_version` | `v0.13.0` | @@ -152,6 +152,40 @@ it is the shared cross-application boundary. Other Narratio-local controls, including reports, timeouts, debug rendering, presentation, and bounds prompt profile selection, remain Narratio defaults. +### Storage and publication policy + +The suite selects Narratio's S3 backend and the established Backblaze-compatible +archive. These are ordinary inventory variables, so deployments may replace +the selected policy through the role's public interface. + +| Variable | Default policy | +| --- | --- | +| `dnd_pipeline_narratio_workspace_cleanup_after_publish` | `true` | +| `dnd_pipeline_narratio_storage_backend` | `s3` | +| `dnd_pipeline_narratio_storage_s3_bucket` | `rakestraw-assets` | +| `dnd_pipeline_narratio_storage_s3_root_prefix` | `dnd` | +| `dnd_pipeline_narratio_storage_s3_region` | `us-east-1` | +| `dnd_pipeline_narratio_storage_s3_endpoint` | `https://s3.us-east-005.backblazeb2.com` | +| `dnd_pipeline_narratio_storage_s3_force_path_style` | `true` | +| `dnd_pipeline_narratio_storage_s3_access_key_id` | `OBJECT_STORAGE_KEY_ID` | +| `dnd_pipeline_narratio_storage_s3_secret_access_key` | `OBJECT_STORAGE_KEY` | +| `dnd_pipeline_narratio_cache_s3_audio` | `true` | +| `dnd_pipeline_narratio_publish_enabled` | `true` | +| `dnd_pipeline_narratio_publish_upload_run` | `true` | + +`dnd_pipeline_narratio_publish_outputs` is a complete native list, not a list +merged with the suite default. It publishes final and trimmed JSON, final and +trimmed Markdown, session recap, hostile rumors, and optional deranged rumors. +Every rule omits `dest`: Narratio derives transcript destinations from its +built-in sources and artifact destinations from the configured artifact output +paths. + +S3 session audio uses the maximum available native cache beneath +`/var/cache/narratio`. After successful run publication, Narratio cleans the +published workspace target while preserving that cache. The suite deliberately +does not set `narratio_spool_delete_audio_after_publish`; spool-audio retention +continues to use Narratio's application default. + ### Artifacts and profiles The role owns these complete native mapping variables. Replacing one mapping @@ -227,6 +261,9 @@ that input. The accepted sources are `narratio.extraction.item_registry`, `npc_registry`, `location_registry`, `scene_descriptions`, `item_occurrences`, `spells`, `combat_turns`, `npc_occurrences`, `location_occurrences`, and `enemy_events`. +They remain available as `narratio.extraction.` inputs for future +analyze artifacts whether or not a source has a stable publication rule; +accepted extraction and published output selection are independent interfaces. Narratio forwards the typed `OPENROUTER_API_KEY` reference to the selected Notarius profile and ordinary extraction requests checkpoint reuse with @@ -260,8 +297,15 @@ inspection shows enabled sources, inferred dependencies, and generated family members; the diff shows the overlay's profile selections without modifying managed files. -Storage and publishing, scheduling, and end-to-end external execution remain -deferred. +With valid object-storage credentials, the selected policy enables Narratio's +native [S3 session-audio](https://gitea.maximumdirect.net/eric/narratio/src/tag/v2.1.1/docs/config.md#session), +[publication](https://gitea.maximumdirect.net/eric/narratio/src/tag/v2.1.1/docs/operations.md#publish-workflow), +[restore](https://gitea.maximumdirect.net/eric/narratio/src/tag/v2.1.1/docs/operations.md#restore-workflow), +[publication locks](https://gitea.maximumdirect.net/eric/narratio/src/tag/v2.1.1/docs/operations.md#publish-locks), +[session status](https://gitea.maximumdirect.net/eric/narratio/src/tag/v2.1.1/docs/operations.md#standard-session-workflow), +and [cleanup](https://gitea.maximumdirect.net/eric/narratio/src/tag/v2.1.1/docs/operations.md#cleanup) +workflows. These remain application commands and lifecycle rules; scheduling +and real end-to-end execution remain deployment concerns. ### Shared execution filesystem @@ -311,8 +355,17 @@ consumer workflow, not role functionality: dnd_pipeline_secrets: AUDITA_LLM_API_KEY: "{{ vault_dnd_audita_llm_api_key }}" OPENROUTER_API_KEY: "{{ vault_dnd_openrouter_api_key }}" + OBJECT_STORAGE_KEY_ID: "{{ vault_dnd_object_storage_key_id }}" + OBJECT_STORAGE_KEY: "{{ vault_dnd_object_storage_key }}" ``` +Only the reference names enter ordinary Narratio configuration; credential +values remain in encrypted inventory and the restricted managed secrets +directory. A deployment using Narratio's ambient AWS SDK provider chain can +set both `dnd_pipeline_narratio_storage_s3_access_key_id` and +`dnd_pipeline_narratio_storage_s3_secret_access_key` to `null`. The pair must +be omitted together. + Scriptorium profiles continue to own their `api_key_env` selection. To supply that credential through the suite, add a `dnd_pipeline_secrets` entry whose name matches the selected profile's `api_key_env`. Narratio's secrets bootstrap diff --git a/defaults/main.yml b/defaults/main.yml index 8237cbe..e678ee2 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,6 +1,6 @@ --- dnd_pipeline_audita_version: v0.9.1 -dnd_pipeline_narratio_version: v2.1.0 +dnd_pipeline_narratio_version: v2.1.1 dnd_pipeline_notarius_version: v0.7.0 dnd_pipeline_seriatim_version: v1.5.0 dnd_pipeline_scriptorium_version: v0.13.0 @@ -207,6 +207,33 @@ dnd_pipeline_notarius_pipelines: dnd_pipeline_notarius_cache_root: /var/cache/notarius dnd_pipeline_notarius_cache_chunk_plans_mode: auto dnd_pipeline_notarius_cache_checkpoints_enabled: true +dnd_pipeline_narratio_workspace_cleanup_after_publish: true +dnd_pipeline_narratio_storage_backend: s3 +dnd_pipeline_narratio_storage_s3_bucket: rakestraw-assets +dnd_pipeline_narratio_storage_s3_root_prefix: dnd +dnd_pipeline_narratio_storage_s3_region: us-east-1 +dnd_pipeline_narratio_storage_s3_endpoint: https://s3.us-east-005.backblazeb2.com +dnd_pipeline_narratio_storage_s3_force_path_style: true +dnd_pipeline_narratio_storage_s3_access_key_id: OBJECT_STORAGE_KEY_ID +dnd_pipeline_narratio_storage_s3_secret_access_key: OBJECT_STORAGE_KEY +dnd_pipeline_narratio_cache_s3_audio: true +dnd_pipeline_narratio_publish_enabled: true +dnd_pipeline_narratio_publish_upload_run: true +dnd_pipeline_narratio_publish_outputs: + - source: narratio.transcript.final + required: true + - source: narratio.transcript.final_trimmed + required: true + - source: narratio.transcript.final_markdown + required: true + - source: narratio.transcript.final_trimmed_markdown + required: true + - source: narratio.artifact.session_recap + required: true + - source: narratio.artifact.hostile_rumors + required: true + - source: narratio.artifact.deranged_rumors + required: false dnd_pipeline_narratio_seriatim_output_schema: seriatim-intermediate dnd_pipeline_narratio_normalize_output_path: transcripts/final.json dnd_pipeline_narratio_normalize_output_schema: seriatim-intermediate diff --git a/meta/argument_specs.yml b/meta/argument_specs.yml index b8bc6fc..ea96720 100644 --- a/meta/argument_specs.yml +++ b/meta/argument_specs.yml @@ -221,6 +221,58 @@ argument_specs: type: bool required: true description: Whether Notarius records checkpoints. + dnd_pipeline_narratio_workspace_cleanup_after_publish: + type: bool + required: true + description: Whether Narratio cleans successfully published workspace state. + dnd_pipeline_narratio_storage_backend: + type: str + required: true + description: Narratio object-storage backend. + dnd_pipeline_narratio_storage_s3_bucket: + type: str + required: false + description: S3 bucket used by Narratio. + dnd_pipeline_narratio_storage_s3_root_prefix: + type: str + required: false + description: Root prefix for Narratio objects in the S3 bucket. + dnd_pipeline_narratio_storage_s3_region: + type: str + required: false + description: S3 region used by Narratio. + dnd_pipeline_narratio_storage_s3_endpoint: + type: str + required: false + description: S3-compatible endpoint used by Narratio. + dnd_pipeline_narratio_storage_s3_force_path_style: + type: bool + required: false + description: Whether Narratio uses path-style S3 requests. + dnd_pipeline_narratio_storage_s3_access_key_id: + type: str + required: false + description: Nullable access-key environment-variable reference. + dnd_pipeline_narratio_storage_s3_secret_access_key: + type: str + required: false + description: Nullable secret-key environment-variable reference. + dnd_pipeline_narratio_cache_s3_audio: + type: bool + required: true + description: Whether Narratio caches downloaded S3 audio. + dnd_pipeline_narratio_publish_enabled: + type: bool + required: true + description: Whether Narratio publication is enabled. + dnd_pipeline_narratio_publish_upload_run: + type: bool + required: true + description: Whether Narratio uploads immutable run content. + dnd_pipeline_narratio_publish_outputs: + type: list + required: true + description: Complete native list of Narratio output publication rules. dnd_pipeline_narratio_seriatim_output_schema: type: str required: true diff --git a/meta/main.yml b/meta/main.yml index d0b175d..e20aaab 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -39,6 +39,19 @@ dependencies: - path: artifacts/deranged-rumors.yml - path: artifacts/scene-map.yml - path: artifact-families/character-meta-analysis.yml + narratio_workspace_cleanup_after_publish: "{{ dnd_pipeline_narratio_workspace_cleanup_after_publish }}" + narratio_storage_backend: "{{ dnd_pipeline_narratio_storage_backend }}" + narratio_storage_s3_bucket: "{{ dnd_pipeline_narratio_storage_s3_bucket }}" + narratio_storage_s3_root_prefix: "{{ dnd_pipeline_narratio_storage_s3_root_prefix }}" + narratio_storage_s3_region: "{{ dnd_pipeline_narratio_storage_s3_region }}" + narratio_storage_s3_endpoint: "{{ dnd_pipeline_narratio_storage_s3_endpoint }}" + narratio_storage_s3_force_path_style: "{{ dnd_pipeline_narratio_storage_s3_force_path_style }}" + narratio_storage_s3_access_key_id: "{{ dnd_pipeline_narratio_storage_s3_access_key_id }}" + narratio_storage_s3_secret_access_key: "{{ dnd_pipeline_narratio_storage_s3_secret_access_key }}" + narratio_cache_s3_audio: "{{ dnd_pipeline_narratio_cache_s3_audio }}" + narratio_publish_enabled: "{{ dnd_pipeline_narratio_publish_enabled }}" + narratio_publish_upload_run: "{{ dnd_pipeline_narratio_publish_upload_run }}" + narratio_publish_outputs: "{{ dnd_pipeline_narratio_publish_outputs }}" narratio_seriatim_binary: /usr/local/bin/seriatim narratio_audita_binary: /usr/local/bin/audita narratio_audita_config_path: /usr/local/etc/audita/config.yml diff --git a/tests/integration b/tests/integration index cfd963e..c90905c 100755 --- a/tests/integration +++ b/tests/integration @@ -24,6 +24,8 @@ service_fixture_file="$shared_fixture_root/service.txt" operator_fixture_file="$shared_fixture_root/operator.txt" synthetic_audita_api_key='audita-pipeline-synthetic-value' synthetic_notarius_api_key='notarius-pipeline-synthetic-value' +synthetic_object_storage_key_id='object-storage-key-id-pipeline-synthetic-value' +synthetic_object_storage_key='object-storage-key-pipeline-synthetic-value' baseline_args=( -e 'dnd_pipeline_whisperx_transcribe_url=http://127.0.0.1:8000/transcribe' -e "dnd_pipeline_campaigns_repository_url=file://$campaigns_repository" @@ -32,7 +34,16 @@ baseline_args=( -e "dnd_pipeline_prompt_library_revision=$prompt_library_revision" -e "dnd_pipeline_prompt_library_root=$prompt_library_root" -e "{\"dnd_pipeline_operator_users\":[\"$operator_user\"]}" - -e "{\"dnd_pipeline_secrets\":{\"AUDITA_LLM_API_KEY\":\"$synthetic_audita_api_key\",\"OPENROUTER_API_KEY\":\"$synthetic_notarius_api_key\"}}" + -e "{\"dnd_pipeline_secrets\":{\"AUDITA_LLM_API_KEY\":\"$synthetic_audita_api_key\",\"OPENROUTER_API_KEY\":\"$synthetic_notarius_api_key\",\"OBJECT_STORAGE_KEY_ID\":\"$synthetic_object_storage_key_id\",\"OBJECT_STORAGE_KEY\":\"$synthetic_object_storage_key\"}}" + -e 'dnd_pipeline_narratio_storage_backend=s3' + -e 'dnd_pipeline_narratio_storage_s3_bucket=dnd-pipeline-fixture' + -e 'dnd_pipeline_narratio_storage_s3_root_prefix=fixture/dnd' + -e 'dnd_pipeline_narratio_storage_s3_region=fixture-1' + -e 'dnd_pipeline_narratio_storage_s3_endpoint=https://storage.fixture.invalid' + -e 'dnd_pipeline_narratio_storage_s3_access_key_id=OBJECT_STORAGE_KEY_ID' + -e 'dnd_pipeline_narratio_storage_s3_secret_access_key=OBJECT_STORAGE_KEY' + -e '{"dnd_pipeline_narratio_workspace_cleanup_after_publish":false,"dnd_pipeline_narratio_storage_s3_force_path_style":false,"dnd_pipeline_narratio_cache_s3_audio":false,"dnd_pipeline_narratio_publish_enabled":true,"dnd_pipeline_narratio_publish_upload_run":true}' + -e '{"dnd_pipeline_narratio_publish_outputs":[{"source":"narratio.transcript.final","required":true},{"source":"narratio.artifact.session_recap","required":true}]}' ) cleanup() { @@ -54,6 +65,7 @@ run_converge() { output_file=$1 shift env -u AUDITA_LLM_API_KEY -u OPENROUTER_API_KEY \ + -u OBJECT_STORAGE_KEY_ID -u OBJECT_STORAGE_KEY \ "$ansible_playbook" -i "$inventory" "$@" "$converge" | tee "$output_file" } @@ -104,8 +116,49 @@ grep -Fq 'resume: true' "$narratio_effective_config" ! grep -Fq 'chunk_cache:' "$narratio_effective_config" grep -Fq 'party: narratio.input.party' "$narratio_effective_config" grep -Fq 'lane_id: item-registry' "$narratio_effective_config" +"$workspace_venv/bin/python" - "$narratio_effective_config" <<'PY' +import sys + +import yaml + +with open(sys.argv[1], encoding="utf-8") as config_file: + config = yaml.safe_load(config_file) + +assert config["workspace"]["cleanup_after_publish"] is False +assert config["storage"] == { + "backend": "s3", + "s3": { + "bucket": "dnd-pipeline-fixture", + "root_prefix": "fixture/dnd", + "region": "fixture-1", + "endpoint": "https://storage.fixture.invalid", + "force_path_style": False, + "access_key_id": "OBJECT_STORAGE_KEY_ID", + "secret_access_key": "OBJECT_STORAGE_KEY", + }, +} +assert config["cache"]["s3_audio"] is False +assert config["publish"] == { + "enabled": True, + "upload_run": True, + "outputs": [ + { + "source": "narratio.transcript.final", + "dest": "transcripts/final.json", + "required": True, + }, + { + "source": "narratio.artifact.session_recap", + "dest": "artifacts/session_recap.md", + "required": True, + }, + ], +} +PY ! grep -Fq "$synthetic_audita_api_key" "$baseline_log" "$narratio_effective_config" "$notarius_effective_config" ! grep -Fq "$synthetic_notarius_api_key" "$baseline_log" "$narratio_effective_config" "$notarius_effective_config" +! grep -Fq "$synthetic_object_storage_key_id" "$baseline_log" "$narratio_effective_config" "$notarius_effective_config" +! grep -Fq "$synthetic_object_storage_key" "$baseline_log" "$narratio_effective_config" "$notarius_effective_config" /usr/bin/sudo -u narratio /usr/local/bin/narratio config artifacts \ --config /usr/local/etc/narratio/pipeline.yml --campaign centaurs | \ awk -F '\t' '$1 == "session_recap" && $4 == "true" { found = 1 } END { exit !found }'