16 Commits

Author SHA1 Message Date
083c01cfa0 Implemented narratio analyze as a shortcut to run the analyze stage only
Some checks failed
ci/woodpecker/tag/release Pipeline failed
2026-05-21 23:02:19 -05:00
2937696024 Add clean command 2026-05-21 22:48:18 -05:00
b817a5b772 Implemented shared S3 audio caching for prepare and restore --include-audio 2026-05-21 22:22:08 -05:00
3022f20beb Simplified the output of narratio artifacts list --remote and narratio status --session-id 2026-05-21 21:20:41 -05:00
ca1ded1821 Bugfix for commands that list artifacts in the S3 backend 2026-05-21 21:00:51 -05:00
3752f3ed28 Added remote artifact listing to narratio status 2026-05-21 20:49:28 -05:00
870c2d69d5 Consolidated addition, removal, and listing of locks under a single narratio locks command 2026-05-21 19:36:14 -05:00
135407ba7c Implemented a centralized secret-backed object-store helper 2026-05-21 19:13:10 -05:00
228c348e42 Implemented operations helper commands for validation, locking, and status 2026-05-21 11:50:20 -05:00
a813bd5a50 Fixed a redundant path bug for previous session artifacts 2026-05-21 10:58:49 -05:00
d8f58dce31 Normalize the default configuration discovery paths for all three config files, and update documentation and tests accordingly 2026-05-21 09:55:56 -05:00
7111edeca4 Add archive promotion locks 2026-05-20 21:40:09 -05:00
3aae4bbb12 Add remote session loading 2026-05-20 20:55:13 -05:00
b29d8eeb50 Add campaign configuration support 2026-05-20 20:41:28 -05:00
dffb432537 Removed completed roadmap for previous_session artifacts 2026-05-20 20:15:47 -05:00
2dd38c7913 Refine campaign and remote session roadmap 2026-05-20 20:15:05 -05:00
77 changed files with 6194 additions and 1080 deletions

View File

@@ -6,7 +6,9 @@
narratio run --session-id 2026-04-04 narratio run --session-id 2026-04-04
``` ```
This command uses default discovery for `pipeline.yml` and `session.yml`; both files must be discoverable unless you pass explicit `--config` and `--session` paths. This command uses default system discovery for `pipeline.yml`, `campaign.yml`, and local `session.yml`. If local session discovery misses and S3 storage is configured, `--session-id` can load remote `session.yml` from the canonical session prefix.
Default discovery checks system config locations only. Pass `--config`, `--campaign`, and `--session` to use files from the current working directory.
## Command Overview ## Command Overview
@@ -15,9 +17,15 @@ Implemented commands:
- `run`: execute pipeline stages and persist manifest state. - `run`: execute pipeline stages and persist manifest state.
- `plan`: validate config, prepare workspace layout, and print stage run/skip decisions. - `plan`: validate config, prepare workspace layout, and print stage run/skip decisions.
- `resume`: continue from first non-succeeded stage unless forced. - `resume`: continue from first non-succeeded stage unless forced.
- `status`: read and print stage statuses from an existing manifest. - `status`: read an existing manifest or inspect local/remote state for a session.
- `run-stage`: execute exactly one stage. - `run-stage`: execute exactly one stage.
- `analyze`: force-rerun the analyze stage.
- `restore`: restore durable local session state from the committed remote archive state. - `restore`: restore durable local session state from the committed remote archive state.
- `session validate`: run read-only preflight checks for a session.
- `session init`: create local or remote `session.yml`.
- `artifacts list`: list effective artifact source IDs.
- `locks`: list, add, and remove archive promotion locks.
- `clean`: remove local workspace/spool state for one session or all local sessions.
Unknown commands print usage and exit non-zero. Unknown commands print usage and exit non-zero.
@@ -28,6 +36,7 @@ For config semantics, see [docs/config.md](./config.md). For operator lifecycle
### `run` ### `run`
- `--config <path>`: optional explicit `pipeline.yml` path. - `--config <path>`: optional explicit `pipeline.yml` path.
- `--campaign <path>`: optional explicit `campaign.yml` path.
- `--session <path>`: optional explicit `session.yml` path. - `--session <path>`: optional explicit `session.yml` path.
- `--session-id <value>`: session template variable value. - `--session-id <value>`: session template variable value.
- `--previous-session-id <value>`: previous-session template variable value. - `--previous-session-id <value>`: previous-session template variable value.
@@ -37,6 +46,7 @@ For config semantics, see [docs/config.md](./config.md). For operator lifecycle
### `plan` ### `plan`
- `--config <path>` - `--config <path>`
- `--campaign <path>`
- `--session <path>` - `--session <path>`
- `--session-id <value>` - `--session-id <value>`
- `--previous-session-id <value>` - `--previous-session-id <value>`
@@ -45,6 +55,7 @@ For config semantics, see [docs/config.md](./config.md). For operator lifecycle
### `resume` ### `resume`
- `--config <path>` - `--config <path>`
- `--campaign <path>`
- `--session <path>` - `--session <path>`
- `--session-id <value>` - `--session-id <value>`
- `--previous-session-id <value>` - `--previous-session-id <value>`
@@ -54,6 +65,7 @@ For config semantics, see [docs/config.md](./config.md). For operator lifecycle
### `run-stage` ### `run-stage`
- `--config <path>` - `--config <path>`
- `--campaign <path>`
- `--session <path>` - `--session <path>`
- `--session-id <value>` - `--session-id <value>`
- `--previous-session-id <value>` - `--previous-session-id <value>`
@@ -61,6 +73,17 @@ For config semantics, see [docs/config.md](./config.md). For operator lifecycle
- `--artifacts <names>`: analyze artifact keys to execute (repeatable or comma-separated). - `--artifacts <names>`: analyze artifact keys to execute (repeatable or comma-separated).
- positional `<stage>`: required stage name. - positional `<stage>`: required stage name.
### `analyze`
- `--config <path>`
- `--campaign <path>`
- `--session <path>`
- `--session-id <value>`
- `--previous-session-id <value>`
- `--artifacts <names>`: analyze artifact keys to execute (repeatable or comma-separated).
`analyze` is force-by-design and does not accept `--force`.
Valid stage names: Valid stage names:
- `prepare` - `prepare`
@@ -76,6 +99,7 @@ Valid stage names:
### `restore` ### `restore`
- `--config <path>` - `--config <path>`
- `--campaign <path>`
- `--session <path>` - `--session <path>`
- `--session-id <value>` - `--session-id <value>`
- `--previous-session-id <value>` - `--previous-session-id <value>`
@@ -83,9 +107,68 @@ Valid stage names:
- `--force`: overwrite local conflicting files with remote archive files. - `--force`: overwrite local conflicting files with remote archive files.
- `--include-audio`: include durable archived `audio/**` files in restore scope. - `--include-audio`: include durable archived `audio/**` files in restore scope.
### `clean`
- `--session-id <value>`: required for session cleanup unless `--all` is set.
- `--config <path>`
- `--campaign <path>`
- `--session <path>`
- `--previous-session-id <value>`
- `--all`: clean all local session work/spool state using pipeline config only.
- `--dry-run`: print cleanup targets without deleting.
- `--clear-cache`: also remove matching S3 audio cache entries.
### `status` ### `status`
- `--manifest <path>`: required manifest path. - `--manifest <path>`: inspect one manifest file.
- `--config <path>`
- `--campaign <path>`
- `--session <path>`
- `--session-id <value>`
- `--previous-session-id <value>`
### `session validate`
- `--config <path>`
- `--campaign <path>`
- `--session <path>`
- `--session-id <value>`
- `--previous-session-id <value>`
### `session init`
- `--config <path>`: required.
- `--campaign <path>`: required.
- `--session-id <value>`: required.
- `--output <path>`: local `session.yml` target; mutually exclusive with `--remote`.
- `--remote`: write remote `session.yml` to the canonical session prefix; mutually exclusive with `--output`.
- `--previous-session-id <value>`
- `--date <value>`
- `--title <value>`
- `--audio-s3-prefix <prefix>`: defaults to `audio/` when neither audio flag is provided.
- `--audio-dir <path>`: local audio directory; mutually exclusive with `--audio-s3-prefix`.
- `--force`: overwrite existing local or remote target.
### `artifacts list`
- `--config <path>`
- `--campaign <path>`
- `--session <path>`
- `--session-id <value>`
- `--previous-session-id <value>`
- `--remote`: check remote availability for configured archive promotion destinations.
### `locks`
- `--session-id <value>`: required for list, add, and remove.
- `--config <path>`: optional explicit `pipeline.yml` path.
- `--campaign <path>`: optional explicit `campaign.yml` path.
- `--session <path>`: optional explicit `session.yml` path.
- `--previous-session-id <value>`: optional session template value.
- `add <source>`: add a remote lock for one artifact or transcript source.
- `add --reason <text>`: record an optional remote lock reason.
- `add --force`: update the reason for an existing remote lock.
- `remove <source>`: remove one remote lock.
## Command Reference ## Command Reference
@@ -97,14 +180,15 @@ Purpose:
Syntax: Syntax:
```bash ```bash
narratio run [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>] narratio run [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>]
``` ```
Success output: Success output:
- `narratio run: session <session_id>; executed=<n> skipped=<n>; manifest=<path>` - `narratio run: session <session_id>; executed=<n> skipped=<n>; manifest=<path>`
Common failure cases: Common failure cases:
- missing default config/session paths when flags omitted. - missing system default config/campaign/session paths when flags omitted.
- missing local session plus missing/unavailable remote `session.yml`.
- invalid template/rendered session mismatch. - invalid template/rendered session mismatch.
- unknown/invalid `--artifacts` value. - unknown/invalid `--artifacts` value.
- `--artifacts` with unknown configured artifact key. - `--artifacts` with unknown configured artifact key.
@@ -117,7 +201,7 @@ Purpose:
Syntax: Syntax:
```bash ```bash
narratio plan [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] narratio plan [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force]
``` ```
Success output includes: Success output includes:
@@ -126,7 +210,8 @@ Success output includes:
- `totals: run=<n> skip=<n>` - `totals: run=<n> skip=<n>`
Common failure cases: Common failure cases:
- same config/session discovery and validation failures as `run`. - same config/campaign/session discovery and validation failures as `run`.
- remote session fallback failures when local session discovery misses.
- secrets directory read failures when `pipeline.secrets.env_dir` is configured. - secrets directory read failures when `pipeline.secrets.env_dir` is configured.
### `resume` ### `resume`
@@ -137,7 +222,7 @@ Purpose:
Syntax: Syntax:
```bash ```bash
narratio resume [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>] narratio resume [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>]
``` ```
Success output: Success output:
@@ -152,22 +237,111 @@ Common failure cases:
### `status` ### `status`
Purpose: Purpose:
- Inspect one manifest file without executing stages. - Inspect one manifest file, or inspect configured local/remote state for a session.
Syntax: Syntax:
```bash ```bash
narratio status --manifest <manifest.json> narratio status --manifest <manifest.json>
narratio status [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>]
``` ```
Success output includes: Manifest output includes:
- `session_id: <id>` - `session_id: <id>`
- `updated_at: <timestamp>` - `updated_at: <timestamp>`
- `stages:` entries (`- <stage>: <status>`) - `stages:` entries (`- <stage>: <status>`)
Session output includes:
- session ID, campaign, workspace, session config source.
- local manifest state when present.
- remote current archive state when storage is configured.
- catalog-based remote output availability for expected transcript and artifact sources.
- effective archive locks and conservative next actions.
Common failure cases: Common failure cases:
- missing `--manifest`. - missing `--manifest` when no config/session flags are provided.
- unreadable or invalid manifest path. - unreadable or invalid manifest path.
- invalid config or remote session fallback failure in session mode.
### `session validate`
Purpose:
- Run read-only preflight checks for a session.
Syntax:
```bash
narratio session validate [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>]
```
Checks include:
- effective config and session source.
- stable input files.
- local or remote audio availability.
- previous-session requirements.
- archive promotions and effective locks.
Warnings do not fail the command. Any `ERROR` finding exits non-zero.
### `session init`
Purpose:
- Create a strict-decoded session skeleton locally or in object storage.
Syntax:
```bash
narratio session init --config <pipeline.yml> --campaign <campaign.yml> --session-id <id> --output ./session.yml
narratio session init --config <pipeline.yml> --campaign <campaign.yml> --session-id <id> --remote
```
Behavior:
- exactly one of `--output` or `--remote` is required.
- remote writes target `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`.
- existing local or remote targets fail unless `--force` is passed.
- remote writes use existence checks, not compare-and-swap.
### `artifacts list`
Purpose:
- List built-in, configured, previous-session, promoted, and locked artifact sources.
Syntax:
```bash
narratio artifacts list [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--remote]
```
`--remote` checks promoted top-level object availability through the storage adapter. Remote markers appear only in the `Promoted` section, which reports each configured archive promotion destination and includes `dest=<path>` when that destination differs from the source's canonical path.
### `locks`
Purpose:
- Inspect and mutate source-based archive promotion locks for one session.
Syntax:
```bash
narratio locks --session-id <id>
narratio locks add --session-id <id> [--reason <text>] [--force] <source>
narratio locks remove --session-id <id> <source>
```
Behavior:
- `--session-id` is required for list, add, and remove.
- optional `--config`, `--campaign`, and `--session` override default config discovery.
- list mode prints effective locks from static `pipeline.archive.locks` and remote `{session_prefix}/locks.yml`.
- `locks add` writes only the remote lock store and fails if the source is already locked by pipeline config.
- `locks remove` removes only remote locks and cannot remove static pipeline locks.
- `locks add --force` is required to update an existing remote lock reason.
Examples:
```bash
narratio locks --session-id 2026-04-04
narratio locks add --session-id 2026-04-04 --reason "manual transcript review" narratio.transcript.trimmed
narratio locks remove --session-id 2026-04-04 narratio.transcript.trimmed
```
### `run-stage` ### `run-stage`
@@ -177,7 +351,7 @@ Purpose:
Syntax: Syntax:
```bash ```bash
narratio run-stage [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>] <stage> narratio run-stage [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>] <stage>
``` ```
Success output: Success output:
@@ -193,6 +367,26 @@ Common failure cases:
- unknown stage name. - unknown stage name.
- using `--artifacts` with any non-`analyze` stage. - using `--artifacts` with any non-`analyze` stage.
### `analyze`
Purpose:
- Force-rerun the analyze stage.
- Provide a shorter equivalent for `narratio run-stage --force analyze`.
Syntax:
```bash
narratio analyze [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--artifacts <name[,name...]>]
```
Success output:
- `narratio analyze: executed=<n> skipped=<n> force=true; manifest=<path>`
Common failure cases:
- positional arguments.
- `--force`, because force is implicit.
- unknown configured artifact keys.
### `restore` ### `restore`
Purpose: Purpose:
@@ -201,7 +395,7 @@ Purpose:
Syntax: Syntax:
```bash ```bash
narratio restore [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--dry-run] [--force] [--include-audio] narratio restore [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--dry-run] [--force] [--include-audio]
``` ```
Success output (dry-run): Success output (dry-run):
@@ -226,6 +420,40 @@ Common failure cases:
- local conflicts without `--force`. - local conflicts without `--force`.
- session lock conflict. - session lock conflict.
When `--include-audio` is set, S3 audio files are restored through the shared audio cache. Cache hits avoid re-downloading large audio objects.
### `clean`
Purpose:
- Remove local Narratio work/spool state for testing, reruns, or recovery from corrupted local files.
- Preserve durable S3 audio cache state unless `--clear-cache` is passed.
Syntax:
```bash
narratio clean --session-id <id> [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--previous-session-id <id>] [--dry-run] [--clear-cache]
narratio clean --all [--config <pipeline.yml>] [--dry-run] [--clear-cache]
```
Session cleanup deletes:
- `{workspace.root}/work/{campaign}/{session_id}`
- `{spool.root}/{campaign}/{session_id}`
All-session cleanup deletes:
- `{workspace.root}/work`
- the contents of `{spool.root}`, while preserving the spool root directory itself.
Cache behavior:
- cache is preserved by default.
- `--clear-cache` in session mode removes cached S3 audio files for the resolved session.
- `--all --clear-cache` removes the configured Narratio S3 audio cache namespace for the configured bucket/root prefix.
- `--clear-cache` does not delete arbitrary files under `pipeline.cache.root`.
Common failure cases:
- missing `--session-id` when `--all` is not set.
- combining `--all` with `--campaign`, `--session`, `--session-id`, or `--previous-session-id`.
- unsafe cleanup target, such as a symlink, a non-directory session target, a configured root directory, or a path outside the configured root.
## Common Workflows ## Common Workflows
Default-discovery run: Default-discovery run:
@@ -246,10 +474,10 @@ Resume with selected analyze artifacts:
narratio resume --session-id 2026-04-04 --artifacts player_handout narratio resume --session-id 2026-04-04 --artifacts player_handout
``` ```
Run only analyze stage with selected artifacts: Force-rerun analyze with selected artifacts:
```bash ```bash
narratio run-stage --session-id 2026-04-04 --artifacts player_handout analyze narratio analyze --session-id 2026-04-04 --artifacts player_handout
``` ```
Preview restore actions without writes: Preview restore actions without writes:
@@ -262,7 +490,7 @@ Restore and then force analyze:
```bash ```bash
narratio restore --session-id 2026-04-04 narratio restore --session-id 2026-04-04
narratio run-stage --session-id 2026-04-04 --force analyze narratio analyze --session-id 2026-04-04
``` ```
Rehydrate canonical previous-session inputs after artifact-input changes: Rehydrate canonical previous-session inputs after artifact-input changes:
@@ -271,6 +499,20 @@ Rehydrate canonical previous-session inputs after artifact-input changes:
narratio run-stage --session-id 2026-04-04 --force prepare narratio run-stage --session-id 2026-04-04 --force prepare
``` ```
Reset local state before testing restore:
```bash
narratio clean --session-id 2026-04-04 --dry-run
narratio clean --session-id 2026-04-04
narratio restore --session-id 2026-04-04 --include-audio
```
Clean all local sessions while keeping cached S3 audio:
```bash
narratio clean --all
```
## Diagnostic / Recovery Commands ## Diagnostic / Recovery Commands
Inspect stage status: Inspect stage status:
@@ -280,7 +522,7 @@ narratio status --manifest <manifest.json>
``` ```
Get manifest path from previous output: Get manifest path from previous output:
- `run`, `resume`, and `run-stage` print `manifest=<path>` on success. - `run`, `resume`, `run-stage`, and `analyze` print `manifest=<path>` on success.
## `--artifacts` and `--force` ## `--artifacts` and `--force`

View File

@@ -2,12 +2,13 @@
## 1. Overview ## 1. Overview
Narratio loads two YAML files: Narratio loads three YAML files:
- `pipeline.yml`: pipeline-level runtime configuration. - `pipeline.yml`: pipeline-level runtime configuration.
- `session.yml`: per-session metadata and input selection. - `campaign.yml`: stable campaign identity and campaign-level input defaults.
- `session.yml`: per-session metadata and input selection, loaded locally or from the configured S3 backend.
These commands load and validate both files before running: These commands load and validate all three files before running:
- `narratio run` - `narratio run`
- `narratio plan` - `narratio plan`
@@ -19,12 +20,23 @@ Behavior:
- strict YAML decode is enabled (`KnownFields(true)`): unknown fields fail. - strict YAML decode is enabled (`KnownFields(true)`): unknown fields fail.
- session templates render before session YAML decode. - session templates render before session YAML decode.
- remote `session.yml` uses the same strict decode and template behavior as local `session.yml`.
- defaults are applied for optional pipeline fields. - defaults are applied for optional pipeline fields.
- campaign-level stable input paths fill missing session input paths.
- session-level stable input paths override campaign-level input paths.
- validation enforces required fields, value formats, and cross-field constraints. - validation enforces required fields, value formats, and cross-field constraints.
## 2. Config file discovery ## 2. Config file discovery
Pipeline config lookup for `run`, `plan`, `resume`, `run-stage`, and `restore`: These commands use the same config discovery behavior:
- `narratio run`
- `narratio plan`
- `narratio resume`
- `narratio run-stage`
- `narratio restore`
Pipeline config lookup:
- if `--config <path>` is provided, that path is used. - if `--config <path>` is provided, that path is used.
- if omitted, Narratio searches in order: - if omitted, Narratio searches in order:
@@ -32,18 +44,29 @@ Pipeline config lookup for `run`, `plan`, `resume`, `run-stage`, and `restore`:
2. `/etc/narratio/pipeline.yml` 2. `/etc/narratio/pipeline.yml`
- first existing file wins. - first existing file wins.
## 3. Session file discovery and templating Campaign config lookup:
Session config lookup for `run`, `plan`, `resume`, `run-stage`, and `restore`: - if `--campaign <path>` is provided, that path is used.
- if `--session <path>` is provided, that path is used.
- if omitted, Narratio searches in order: - if omitted, Narratio searches in order:
1. `./session.yml` 1. `/usr/local/etc/narratio/campaign.yml`
2. `/usr/local/etc/narratio/session.yml` 2. `/etc/narratio/campaign.yml`
3. `/etc/narratio/session.yml`
- first existing file wins. - first existing file wins.
Template behavior: Session config lookup:
- if `--session <path>` is provided, that path is used.
- if `--session` is omitted, Narratio searches locally in order:
1. `/usr/local/etc/narratio/session.yml`
2. `/etc/narratio/session.yml`
- first existing local file wins.
- if no local session file is found, `--session-id <value>` is present, storage is configured, and campaign identity is resolved, Narratio loads remote `session.yml` from:
- `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`
- local discovery always runs before remote fallback.
- local files in the current working directory are used only when passed explicitly, for example `--config ./pipeline.yml --campaign ./campaign.yml --session ./session.yml`.
## 3. Session templating
Template behavior for local and remote `session.yml`:
- supported placeholders: - supported placeholders:
- `{{session_id}}` - `{{session_id}}`
@@ -56,7 +79,9 @@ Template behavior:
- if rendered `session_id` mismatches `--session-id`, load fails. - if rendered `session_id` mismatches `--session-id`, load fails.
- if rendered `previous_session_id` mismatches `--previous-session-id`, load fails. - if rendered `previous_session_id` mismatches `--previous-session-id`, load fails.
## 4. Minimal pipeline config ## 4. Minimal config set
### `pipeline.yml`
```yaml ```yaml
whisperx: whisperx:
@@ -69,22 +94,40 @@ Why this is sufficient:
- `workspace.root` defaults to `/var/lib/narratio`. - `workspace.root` defaults to `/var/lib/narratio`.
- optional sections (`seriatim`, `audita`, `archive`, `scriptorium`, `trim`, `normalize`, etc.) receive defaults or stay inactive. - optional sections (`seriatim`, `audita`, `archive`, `scriptorium`, `trim`, `normalize`, etc.) receive defaults or stay inactive.
## 5. Minimal session template ### `campaign.yml`
```yaml
campaign: sample-campaign
inputs:
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
```
Why this is sufficient:
- `campaign` supplies the stable campaign identity.
- stable input files are required and resolve relative to `campaign.yml` when copied during `prepare`.
### `session.yml`
```yaml ```yaml
session_id: "{{ session_id }}" session_id: "{{ session_id }}"
campaign: sample-campaign
inputs: inputs:
audio_dir: ./audio audio_dir: ./audio
speakers_file: ./examples/speakers.yml
autocorrect_file: ./examples/autocorrect.yml
glossary_file: ./examples/glossary.yml
``` ```
Usage: Why this is sufficient:
- `session_id` is required and can be rendered from `--session-id`.
- `campaign` can be omitted because it is supplied by `campaign.yml`.
- stable input paths can be omitted because `campaign.yml` supplies defaults.
- local `audio_dir` resolves relative to `session.yml`.
Minimal local-file usage:
```bash ```bash
narratio run --config /path/to/pipeline.yml --session ./session.yml --session-id 2026-05-03 narratio run --config /path/to/pipeline.yml --campaign ./campaign.yml --session ./session.yml --session-id 2026-05-03
``` ```
Previous-session-enabled variant: Previous-session-enabled variant:
@@ -92,19 +135,17 @@ Previous-session-enabled variant:
```yaml ```yaml
session_id: "{{ session_id }}" session_id: "{{ session_id }}"
previous_session_id: "{{ previous_session_id }}" previous_session_id: "{{ previous_session_id }}"
campaign: sample-campaign
inputs: inputs:
audio_dir: ./audio audio_dir: ./audio
speakers_file: ./examples/speakers.yml
autocorrect_file: ./examples/autocorrect.yml
glossary_file: ./examples/glossary.yml
``` ```
```bash ```bash
narratio run --config /path/to/pipeline.yml --session ./session.yml --session-id 2026-05-03 --previous-session-id 2026-04-26 narratio run --config /path/to/pipeline.yml --campaign ./campaign.yml --session ./session.yml --session-id 2026-05-03 --previous-session-id 2026-04-26
``` ```
## 6. Production-oriented config ## 5. Production-oriented config set
### `pipeline.yml`
```yaml ```yaml
workspace: workspace:
@@ -124,6 +165,10 @@ spool:
root: /var/spool/narratio root: /var/spool/narratio
delete_audio_after_archive: true delete_audio_after_archive: true
cache:
root: /var/cache/narratio
s3_audio: true
archive: archive:
enabled: true enabled: true
upload_run: true upload_run: true
@@ -134,6 +179,9 @@ archive:
- source: narratio.artifact.session_recap - source: narratio.artifact.session_recap
dest: artifacts/session_recap.md dest: artifacts/session_recap.md
required: true required: true
locks:
- source: narratio.artifact.session_recap
reason: Final recap was manually edited.
whisperx: whisperx:
transcribe_url: "https://transcription.example.com/transcribe" transcribe_url: "https://transcription.example.com/transcribe"
@@ -153,14 +201,53 @@ scriptorium:
required: false required: false
``` ```
### `campaign.yml`
```yaml
campaign: forsaken
inputs:
speakers_file: /srv/narratio/campaigns/forsaken/speakers.yml
autocorrect_file: /srv/narratio/campaigns/forsaken/autocorrect.yml
glossary_file: /srv/narratio/campaigns/forsaken/glossary.yml
```
### Local `session.yml`
```yaml
session_id: "{{ session_id }}"
previous_session_id: "{{ previous_session_id }}"
date: 2026-05-03
title: The Black Cabin
inputs:
audio_s3:
prefix: audio/
```
### S3-first session config
For S3-first operation, upload the same `session.yml` content to:
```text
{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml
```
Then run with explicit or discovered pipeline/campaign config and no `--session`:
```bash
narratio run --config /usr/local/etc/narratio/pipeline.yml --campaign /usr/local/etc/narratio/campaign.yml --session-id 2026-05-03 --previous-session-id 2026-04-26
```
Operational notes: Operational notes:
- archive promotion is explicit and source-based via `archive.promote_artifacts`. - archive promotion is explicit and source-based via `archive.promote_artifacts`.
- `source` is required; `dest` is optional and derived when omitted. - `source` is required; `dest` is optional and derived when omitted.
- `archive.locks` skips top-level promotion overwrites for static locked sources while preserving run-local uploads.
- operator-created mutable locks are stored at `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/locks.yml` and are merged with static locks.
- Narratio does not auto-promote all generated analyze artifacts. - Narratio does not auto-promote all generated analyze artifacts.
- `restore` reads the same config/session inputs and restore scope is bounded by committed archive current state. - `restore` reads the same config/campaign/session inputs and restore scope is bounded by committed archive current state.
- `clean` removes workspace/spool state by default and preserves `pipeline.cache.root` unless `--clear-cache` is passed.
## 7. Full pipeline reference ## 6. Full pipeline reference
| Path | Type | Required | Default | | Path | Type | Required | Default |
| --- | --- | --- | --- | | --- | --- | --- | --- |
@@ -179,12 +266,17 @@ Operational notes:
| `pipeline.storage.s3.secret_access_key_env` | string | No | `OBJECT_STORAGE_KEY` | | `pipeline.storage.s3.secret_access_key_env` | string | No | `OBJECT_STORAGE_KEY` |
| `pipeline.spool.root` | string | No | `/var/spool/narratio` | | `pipeline.spool.root` | string | No | `/var/spool/narratio` |
| `pipeline.spool.delete_audio_after_archive` | bool | No | `false` | | `pipeline.spool.delete_audio_after_archive` | bool | No | `false` |
| `pipeline.cache.root` | string | No | `/var/cache/narratio` |
| `pipeline.cache.s3_audio` | bool | No | `true` |
| `pipeline.archive.enabled` | bool | No | `true` | | `pipeline.archive.enabled` | bool | No | `true` |
| `pipeline.archive.upload_run` | bool | No | `true` | | `pipeline.archive.upload_run` | bool | No | `true` |
| `pipeline.archive.promote_artifacts[]` | list | No | trimmed transcript rule | | `pipeline.archive.promote_artifacts[]` | list | No | trimmed transcript rule |
| `pipeline.archive.promote_artifacts[].source` | string | Yes (per rule) | none | | `pipeline.archive.promote_artifacts[].source` | string | Yes (per rule) | none |
| `pipeline.archive.promote_artifacts[].dest` | string | No | derived from source | | `pipeline.archive.promote_artifacts[].dest` | string | No | derived from source |
| `pipeline.archive.promote_artifacts[].required` | bool | No | `true` | | `pipeline.archive.promote_artifacts[].required` | bool | No | `true` |
| `pipeline.archive.locks[]` | list | No | empty |
| `pipeline.archive.locks[].source` | string | Yes (per lock) | none |
| `pipeline.archive.locks[].reason` | string | No | empty |
| `pipeline.whisperx.transcribe_url` | string | Yes | none | | `pipeline.whisperx.transcribe_url` | string | Yes | none |
| `pipeline.whisperx.language` | string | No | `en` | | `pipeline.whisperx.language` | string | No | `en` |
| `pipeline.whisperx.timeout` | duration string | No | `30m` | | `pipeline.whisperx.timeout` | duration string | No | `30m` |
@@ -285,6 +377,8 @@ Allowed `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source` values:
- `narratio.bounds.session` - `narratio.bounds.session`
- `narratio.artifact.<configured_artifact_key>` - `narratio.artifact.<configured_artifact_key>`
`pipeline.archive.locks[].source` accepts the same source values as `pipeline.archive.promote_artifacts[].source`.
Archive promotion destination rules: Archive promotion destination rules:
- `dest` must be a clean relative path (not absolute, no traversal). - `dest` must be a clean relative path (not absolute, no traversal).
@@ -294,10 +388,39 @@ Archive promotion destination rules:
- configured sources derive from `pipeline.scriptorium.artifacts.<name>.output_path`; - configured sources derive from `pipeline.scriptorium.artifacts.<name>.output_path`;
- derivation failure is a config validation error. - derivation failure is a config validation error.
Archive lock rules:
- locks are source-based and do not accept `dest`.
- duplicate lock sources are rejected.
- static `pipeline.archive.locks` win over remote mutable locks for the same source.
- locked promotions are recorded as intentional skips in archive metadata.
- locked required promotions do not fail archive by default.
- ordinary `--force` reruns do not override locks.
Remote mutable lock store:
- path: `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/locks.yml`.
- strict YAML shape: top-level `locks`, each with `source` and optional `reason`.
- `narratio locks add` and `narratio locks remove` mutate only the remote lock store.
- writes use existence checks plus `--force` for updates; they are not compare-and-swap atomic.
Restore-related implications: Restore-related implications:
- restore remote identity requires archive S3 identity to resolve (`pipeline.storage.s3.bucket` and session prefix derivation inputs). - restore remote identity requires archive S3 identity to resolve (`pipeline.storage.s3.bucket` and session prefix derivation inputs).
- restore scope considers committed current state and durable paths (`manifest.json`, `transcripts/**`, `artifacts/**`, `previous/**`, optional `audio/**`). - restore scope considers committed current state and durable paths (`manifest.json`, `transcripts/**`, `artifacts/**`, `previous/**`, optional `audio/**`).
- S3 audio downloads use `pipeline.spool.root` for active downloads and `pipeline.cache.root` for reusable cached audio when `pipeline.cache.s3_audio` is true.
- `pipeline.cache.root` is durable local cache state. It is not workspace state and is preserved by default by `narratio clean`.
## 7. Full campaign reference
| Path | Type | Required | Default |
| --- | --- | --- | --- |
| `campaign.campaign` | string | Yes | none |
| `campaign.inputs.speakers_file` | string | Yes | none |
| `campaign.inputs.autocorrect_file` | string | Yes | none |
| `campaign.inputs.glossary_file` | string | Yes | none |
Campaign input paths may be absolute or relative. Relative paths resolve from the directory containing `campaign.yml`.
## 8. Full session reference ## 8. Full session reference
@@ -305,15 +428,17 @@ Restore-related implications:
| --- | --- | --- | --- | | --- | --- | --- | --- |
| `session.session_id` | string | Yes | none | | `session.session_id` | string | Yes | none |
| `session.previous_session_id` | string | No | empty | | `session.previous_session_id` | string | No | empty |
| `session.campaign` | string | Yes | none | | `session.campaign` | string | No | `campaign.campaign` |
| `session.date` | string | No | empty | | `session.date` | string | No | empty |
| `session.title` | string | No | empty | | `session.title` | string | No | empty |
| `session.inputs.audio_dir` | string | Conditional | empty | | `session.inputs.audio_dir` | string | Conditional | empty |
| `session.inputs.audio_files[]` | list[string] | Conditional | empty | | `session.inputs.audio_files[]` | list[string] | Conditional | empty |
| `session.inputs.audio_s3.prefix` | string | Conditional | none | | `session.inputs.audio_s3.prefix` | string | Conditional | none |
| `session.inputs.speakers_file` | string | Yes | none | | `session.inputs.speakers_file` | string | No | `campaign.inputs.speakers_file` |
| `session.inputs.autocorrect_file` | string | Yes | none | | `session.inputs.autocorrect_file` | string | No | `campaign.inputs.autocorrect_file` |
| `session.inputs.glossary_file` | string | Yes | none | | `session.inputs.glossary_file` | string | No | `campaign.inputs.glossary_file` |
Session input paths may be absolute or relative. Relative audio paths and session-level stable input overrides resolve from the directory containing `session.yml`. If both `campaign.yml` and `session.yml` specify campaign identity, the values must match.
Audio-source rule: Audio-source rule:
@@ -354,6 +479,7 @@ Maintained examples:
- `examples/pipeline.minimal.yml` - `examples/pipeline.minimal.yml`
- `examples/pipeline.production.yml` - `examples/pipeline.production.yml`
- `examples/pipeline.full.annotated.yml` - `examples/pipeline.full.annotated.yml`
- `examples/campaign.yml`
- `examples/session.template.yml` - `examples/session.template.yml`
- `examples/session.local-audio.yml` - `examples/session.local-audio.yml`
- `examples/session.s3-audio.yml` - `examples/session.s3-audio.yml`

View File

@@ -69,6 +69,8 @@ For design principles and invariants, see [docs/architecture.md](./architecture.
2. Add or update command tests (`TestExecute` and command-specific tests). 2. Add or update command tests (`TestExecute` and command-specific tests).
3. Update [docs/cli.md](./cli.md) and, if operator workflow changes, [docs/operations.md](./operations.md). 3. Update [docs/cli.md](./cli.md) and, if operator workflow changes, [docs/operations.md](./operations.md).
Remote-storage commands must obtain object storage through the app-level command object-store helper. Do not call `storage.NewObjectStoreFromConfig` directly from command handlers; the helper loads configured filesystem secrets before constructing the storage adapter.
### Add or modify stages/adapters ### Add or modify stages/adapters
1. Implement stage behavior in `internal/stage` with clear input/output boundaries. 1. Implement stage behavior in `internal/stage` with clear input/output boundaries.

View File

@@ -47,6 +47,7 @@ Current execution usage:
Default construction in app runner: Default construction in app runner:
- Auto-constructed when not injected: WhisperX HTTP client, Seriatim subprocess runner, Audita subprocess runner, Scriptorium subprocess runner, object store (only when needed), and `notify.NoopSender`. - Auto-constructed when not injected: WhisperX HTTP client, Seriatim subprocess runner, Audita subprocess runner, Scriptorium subprocess runner, object store (only when needed), and `notify.NoopSender`.
- Object-store construction goes through app command orchestration so configured filesystem secrets are loaded before the storage adapter is initialized.
- Callers can inject test/fake implementations through `app.RunOptions.Env`. - Callers can inject test/fake implementations through `app.RunOptions.Env`.
## State and manifest behavior ## State and manifest behavior

View File

@@ -16,6 +16,7 @@ Outputs:
- resolved artifact path + provenance (`ResolvedSessionArtifact`); - resolved artifact path + provenance (`ResolvedSessionArtifact`);
- runtime catalog entries for built-ins and configured artifacts; - runtime catalog entries for built-ins and configured artifacts;
- requirement sets for canonical previous-session inputs. - requirement sets for canonical previous-session inputs.
- canonical S3 session, run, current, session config, session locks, audio, and promoted artifact keys.
## Boundaries ## Boundaries
Owns: Owns:
@@ -43,6 +44,15 @@ Does not own:
- configured artifact: `narratio.artifact.<artifact_key>` - configured artifact: `narratio.artifact.<artifact_key>`
- canonical previous-session artifact: `narratio.previous_session.artifact.<artifact_key>` - canonical previous-session artifact: `narratio.previous_session.artifact.<artifact_key>`
## S3 key helpers
- session prefix: `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
- session config: `{session_prefix}/session.yml`
- session lock store: `{session_prefix}/locks.yml`
- run prefix: `{session_prefix}/runs/{run_id}/`
- audio prefix: `{session_prefix}/{session.inputs.audio_s3.prefix}`
- current manifest: `{session_prefix}/current/manifest.json`
- current run pointer: `{session_prefix}/current/run_id.txt`
## Runtime catalog model ## Runtime catalog model
Catalog entries track: Catalog entries track:
- `planned`: source is registered for this run; - `planned`: source is registered for this run;
@@ -61,6 +71,7 @@ Previous-session canonical provenance values include:
- Built-ins resolve via manifest producer outputs first, then canonical fallback paths. - Built-ins resolve via manifest producer outputs first, then canonical fallback paths.
- Configured `narratio.artifact.<name>` sources resolve through catalog availability. - Configured `narratio.artifact.<name>` sources resolve through catalog availability.
- Canonical previous-session sources resolve to current-session `previous/` cache candidates derived from configured artifact canonical output paths. - Canonical previous-session sources resolve to current-session `previous/` cache candidates derived from configured artifact canonical output paths.
- Archive-relative configured artifact paths under `artifacts/` are cached without a redundant nested `artifacts/` segment.
- Previous-session canonical resolution prefers manifest-recorded input paths when present, then filesystem fallback under `previous/artifacts/**`. - Previous-session canonical resolution prefers manifest-recorded input paths when present, then filesystem fallback under `previous/artifacts/**`.
## Previous-session requirement scanning ## Previous-session requirement scanning

View File

@@ -34,6 +34,8 @@ Does not own:
- `pipeline.workspace.root` (local restore target root). - `pipeline.workspace.root` (local restore target root).
- `pipeline.storage.*` (remote backend + archive identity derivation). - `pipeline.storage.*` (remote backend + archive identity derivation).
- `pipeline.storage.s3.*` identity components used by archive prefix helpers. - `pipeline.storage.s3.*` identity components used by archive prefix helpers.
- `pipeline.spool.root` for active audio downloads.
- `pipeline.cache.root` and `pipeline.cache.s3_audio` for reusable S3 audio cache.
- `session.session_id` - `session.session_id`
- `session.campaign` - `session.campaign`
@@ -48,6 +50,7 @@ Does not own:
- `current/run_id.txt` must exist and be non-empty. - `current/run_id.txt` must exist and be non-empty.
- `current/manifest.json` must decode and match requested session/campaign. - `current/manifest.json` must decode and match requested session/campaign.
- Non-dry-run writes restore files to canonical session paths. - Non-dry-run writes restore files to canonical session paths.
- With `--include-audio`, restore uses the shared S3 audio cache for `audio/**` objects. Cache hits avoid object downloads; cache misses download through spool, install the work file, and populate cache.
- Manifest install behavior: - Manifest install behavior:
- validated before replacement. - validated before replacement.
- installed last among download actions. - installed last among download actions.
@@ -72,6 +75,7 @@ Restore path scope:
## Skip and resume behavior ## Skip and resume behavior
- Restore does not participate in stage skip/resume decisions. - Restore does not participate in stage skip/resume decisions.
- Restore provides durable local state so subsequent stage commands can resume or rerun based on restored manifest state. - Restore provides durable local state so subsequent stage commands can resume or rerun based on restored manifest state.
- Audio cache is outside the workspace and is reused across restore and prepare invocations.
- Dry-run is read-only and returns plan output only. - Dry-run is read-only and returns plan output only.
## Failure behavior ## Failure behavior

View File

@@ -8,6 +8,7 @@ Inputs:
- session manifest and prerequisite stage records - session manifest and prerequisite stage records
- run root contents under `runs/{run_id}/` - run root contents under `runs/{run_id}/`
- promotion rules with artifact `source` IDs and archive `dest` paths (`archive.promote_artifacts`) - promotion rules with artifact `source` IDs and archive `dest` paths (`archive.promote_artifacts`)
- effective source-based promotion locks from static config and remote session lock store
- session-level `previous/**` cache files when present - session-level `previous/**` cache files when present
Outputs: Outputs:
@@ -23,6 +24,7 @@ Owns:
- Prerequisite stage success enforcement - Prerequisite stage success enforcement
- Run file collection and upload (excluding `audio/`) - Run file collection and upload (excluding `audio/`)
- Promotion rule resolution and upload - Promotion rule resolution and upload
- Promotion lock enforcement
- Session previous-cache file collection/upload - Session previous-cache file collection/upload
- Commit pointer publish order - Commit pointer publish order
@@ -34,6 +36,8 @@ Does not own:
- `pipeline.archive.enabled` - `pipeline.archive.enabled`
- `pipeline.archive.upload_run` - `pipeline.archive.upload_run`
- `pipeline.archive.promote_artifacts` - `pipeline.archive.promote_artifacts`
- `pipeline.archive.locks`
- `{session_prefix}/locks.yml` loaded by app orchestration before archive execution
- `pipeline.storage.s3.bucket` - `pipeline.storage.s3.bucket`
- `pipeline.storage.s3.root_prefix` - `pipeline.storage.s3.root_prefix`
- `pipeline.workspace.root` - `pipeline.workspace.root`
@@ -47,9 +51,12 @@ Does not own:
- Requires `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, and `analyze` status `succeeded`. - Requires `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, and `analyze` status `succeeded`.
- Resolves bucket/prefix from manifest identity first, then config fallback. - Resolves bucket/prefix from manifest identity first, then config fallback.
- Uploads session `previous/**` files as durable session state when the local `previous/` directory exists. - Uploads session `previous/**` files as durable session state when the local `previous/` directory exists.
- Skips top-level promotion uploads for effective locked sources; run-local uploads still publish.
- Effective locks are the union of `pipeline.archive.locks` and remote `{session_prefix}/locks.yml`; static pipeline locks win on duplicate sources.
- Writes metadata including: - Writes metadata including:
- upload counts/paths - upload counts/paths
- `previous_files_uploaded` and `previous_uploaded_paths` - `previous_files_uploaded` and `previous_uploaded_paths`
- `locked_promotion_count` and `locked_promotions`
- `current_manifest_key` - `current_manifest_key`
- `current_run_id_key` - `current_run_id_key`
- `current_pointer_written` - `current_pointer_written`
@@ -60,7 +67,8 @@ Does not own:
- Runner-level skip also applies for previously succeeded stage unless forced. - Runner-level skip also applies for previously succeeded stage unless forced.
## Failure Behavior ## Failure Behavior
- Fails on missing prerequisite success, missing object store when required, missing run root, missing required promotion source, upload failures, or pointer write failures. - Fails on missing prerequisite success, missing object store when required, missing run root, missing unlocked required promotion source, upload failures, or pointer write failures.
- Locked required promotions are intentional skips and do not fail archive.
- Pointer semantics are fail-safe: `current/run_id.txt` is not written if prior required uploads fail. - Pointer semantics are fail-safe: `current/run_id.txt` is not written if prior required uploads fail.
## Tests to Inspect Before Changing ## Tests to Inspect Before Changing
@@ -70,5 +78,7 @@ Does not own:
## Architectural Invariants ## Architectural Invariants
- Run upload excludes `audio/` subtree. - Run upload excludes `audio/` subtree.
- Session `previous/**` is archiveable durable input/provenance state, not run-local output. - Session `previous/**` is archiveable durable input/provenance state, not run-local output.
- Ordinary `--force` does not override archive locks.
- Malformed or unreadable remote lock store fails archive-capable execution before promotion.
- `current/manifest.json` uploads before `current/run_id.txt`. - `current/manifest.json` uploads before `current/run_id.txt`.
- `current/run_id.txt` is the remote publish commit marker. - `current/run_id.txt` is the remote publish commit marker.

View File

@@ -10,8 +10,9 @@ Prepare owns:
## Inputs and outputs ## Inputs and outputs
Inputs: Inputs:
- resolved config/session (`pipeline.yml`, `session.yml`); - resolved config/campaign/session (`pipeline.yml`, `campaign.yml`, `session.yml`);
- session-local input files (`speakers`, `autocorrect`, `glossary`); - remote session provenance when `session.yml` was loaded from S3;
- campaign or session input files (`speakers`, `autocorrect`, `glossary`);
- audio source: - audio source:
- local: `session.inputs.audio_dir` or `session.inputs.audio_files`; - local: `session.inputs.audio_dir` or `session.inputs.audio_files`;
- S3: `session.inputs.audio_s3.prefix`; - S3: `session.inputs.audio_s3.prefix`;
@@ -19,6 +20,7 @@ Inputs:
- remote previous-session current archive state when previous hydration is required. - remote previous-session current archive state when previous hydration is required.
Outputs: Outputs:
- `inputs/campaign.yml`;
- `inputs/session.yml`; - `inputs/session.yml`;
- `inputs/pipeline.resolved.yml`; - `inputs/pipeline.resolved.yml`;
- `inputs/speakers.yml`; - `inputs/speakers.yml`;
@@ -53,11 +55,17 @@ Does not own:
- `session.inputs.audio_s3.prefix` - `session.inputs.audio_s3.prefix`
- `pipeline.workspace.root` - `pipeline.workspace.root`
- `pipeline.spool.root` - `pipeline.spool.root`
- `pipeline.cache.root`
- `pipeline.cache.s3_audio`
- `pipeline.storage.s3.bucket` - `pipeline.storage.s3.bucket`
- `pipeline.storage.s3.root_prefix` - `pipeline.storage.s3.root_prefix`
- `pipeline.scriptorium.artifacts.<name>.enabled` - `pipeline.scriptorium.artifacts.<name>.enabled`
- `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source` - `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source`
- `pipeline.scriptorium.artifacts.<name>.inputs.<key>.required` - `pipeline.scriptorium.artifacts.<name>.inputs.<key>.required`
- `campaign.campaign`
- `campaign.inputs.speakers_file`
- `campaign.inputs.autocorrect_file`
- `campaign.inputs.glossary_file`
## External adapters used ## External adapters used
- `storage.ObjectStore` for: - `storage.ObjectStore` for:
@@ -67,15 +75,21 @@ Does not own:
## State and manifest behavior ## State and manifest behavior
- Ensures workspace layout exists. - Ensures workspace layout exists.
- Materializes canonical input files and audio files. - Materializes canonical input files and audio files.
- For S3 audio, uses run-scoped spool for active downloads and durable cache for reusable audio files; cache hits copy directly to work audio without downloading the object again.
- Records `inputs/session.yml` provenance as local `session_config` or remote `session_config.s3`.
- Resolves campaign-provided stable input paths relative to `campaign.yml`.
- Resolves session-provided stable input overrides relative to `session.yml`.
- Scans enabled configured artifact inputs for canonical sources: - Scans enabled configured artifact inputs for canonical sources:
- `narratio.previous_session.artifact.<artifact_key>` - `narratio.previous_session.artifact.<artifact_key>`
- If one or more canonical previous-session requirements exist: - If one or more canonical previous-session requirements exist:
- clears managed `previous/` state; - clears managed `previous/` state;
- hydrates required/optional previous artifacts from the configured previous sessions committed archive current state; - hydrates required/optional previous artifacts from the configured previous sessions committed archive current state;
- writes `previous/manifest.json` and hydrated `previous/artifacts/**`; - writes `previous/manifest.json` and hydrated `previous/artifacts/**`;
- stores archive-relative artifact paths such as `artifacts/session_recap.md` as `previous/artifacts/session_recap.md`, not `previous/artifacts/artifacts/session_recap.md`;
- records hydrated previous inputs in `manifest.Inputs` with source `previous_session_archive.current`. - records hydrated previous inputs in `manifest.Inputs` with source `previous_session_archive.current`.
- If no canonical previous-session requirements exist, prepare does not manage `previous/`. - If no canonical previous-session requirements exist, prepare does not manage `previous/`.
- `manifest.Inputs` is sorted deterministically by `(kind, path)`. - `manifest.Inputs` is sorted deterministically by `(kind, path)`.
- S3 audio `manifest.Inputs` retain S3 provenance and include `cache_path`; `spool_path` is present only when the current prepare invocation downloaded the file.
## Required and optional previous-session behavior ## Required and optional previous-session behavior
- `previous_session_id` unset: - `previous_session_id` unset:

View File

@@ -6,7 +6,8 @@ Document Narratio's remote storage backend contracts and implementations under `
## Inputs and outputs ## Inputs and outputs
Inputs: Inputs:
- Resolved storage config (`pipeline.storage.*`). - Resolved storage config (`pipeline.storage.*`).
- Bucket-relative object keys and local file paths from stage/app orchestration. - Already-loaded environment variables for configured S3 credentials.
- Bucket-relative object keys and local file paths from app/stage orchestration.
Outputs: Outputs:
- Listed/downloaded/uploaded object metadata (`ObjectInfo`). - Listed/downloaded/uploaded object metadata (`ObjectInfo`).
@@ -22,6 +23,7 @@ Does not own:
- Session/run prefix semantics. - Session/run prefix semantics.
- Archive commit order semantics. - Archive commit order semantics.
- Manifest updates. - Manifest updates.
- Filesystem secret loading from `pipeline.secrets.env_dir`.
## Config fields used ## Config fields used
- `pipeline.storage.backend` - `pipeline.storage.backend`
@@ -55,8 +57,10 @@ Implementations:
## Failure behavior ## Failure behavior
- `NewObjectStoreFromConfig` fails when no remote backend is configured or required S3 config is missing. - `NewObjectStoreFromConfig` fails when no remote backend is configured or required S3 config is missing.
- `S3Backend` constructor fails when required bucket is missing or AWS client setup fails. - `S3Backend` constructor fails when required bucket is missing or AWS client setup fails.
- App command orchestration loads configured filesystem secrets before calling the object-store factory.
- CRUD operations return contextual errors (including not-found behavior via `Exists`). - CRUD operations return contextual errors (including not-found behavior via `Exists`).
- Key normalization is applied before operations (`\\` to `/`, leading slash trimmed). - Key normalization is applied before operations (`\\` to `/`, leading slash trimmed).
- Remote session loading uses `List` to find the exact `session.yml` key and `Download` to materialize it to a local temp file.
## Tests to inspect before changing ## Tests to inspect before changing
- `internal/adapters/storage/factory_test.go` - `internal/adapters/storage/factory_test.go`

View File

@@ -32,6 +32,8 @@ Does not own:
- `pipeline.workspace.cleanup_after_archive` - `pipeline.workspace.cleanup_after_archive`
- `pipeline.spool.root` - `pipeline.spool.root`
- `pipeline.spool.delete_audio_after_archive` - `pipeline.spool.delete_audio_after_archive`
- `pipeline.cache.root`
- `pipeline.cache.s3_audio`
- `session.campaign` - `session.campaign`
- `session.session_id` - `session.session_id`
@@ -43,7 +45,11 @@ None directly in this subsystem. Stages may use object storage adapters and then
- Invocation history is persisted per run in run manifests under `runs/{run_id}/manifest.json`. - Invocation history is persisted per run in run manifests under `runs/{run_id}/manifest.json`.
- During each run, stage outputs are often written run-local first (`runs/{run_id}/{stage}/outputs/...`) and promoted to canonical session paths after stage success. - During each run, stage outputs are often written run-local first (`runs/{run_id}/{stage}/outputs/...`) and promoted to canonical session paths after stage success.
- `manifest.Artifacts` entries record `ProducerRunID` for durable outputs. - `manifest.Artifacts` entries record `ProducerRunID` for durable outputs.
- For S3 audio sessions, `prepare` records spool/work paths and S3 provenance in `manifest.Inputs`. - For S3 audio sessions, `prepare` records work/cache paths, S3 provenance, and spool path when the invocation downloaded the object.
- Durable cache state under `pipeline.cache.root` is not workspace state and is preserved by default by `narratio clean`.
- `narratio clean --session-id <id>` removes the session work root and session spool root.
- `narratio clean --all` removes all local session work under `workspace.root/work` and spool children under `spool.root`.
- `narratio clean --clear-cache` is the explicit opt-in for deleting matching S3 audio cache entries.
## Skip and Resume Behavior ## Skip and Resume Behavior
- Skip/resume decisions are made in `internal/app` (`run_control.go`, `resume.go`) using stage status in the session manifest. - Skip/resume decisions are made in `internal/app` (`run_control.go`, `resume.go`) using stage status in the session manifest.
@@ -67,4 +73,5 @@ None directly in this subsystem. Stages may use object storage adapters and then
- Run roots are always nested: `runs/{run_id}` under the session root. - Run roots are always nested: `runs/{run_id}` under the session root.
- Run-local output promotion must end in canonical session paths. - Run-local output promotion must end in canonical session paths.
- `previous/**` is session-durable state and must not be treated as run-local output scratch state. - `previous/**` is session-durable state and must not be treated as run-local output scratch state.
- Cleanup only targets run-scoped directories and must never delete configured root directories. - Automatic post-archive cleanup only targets run-scoped directories and must never delete configured root directories.
- Manual `clean` may delete session-scoped directories or the `workspace.root/work` directory, but it must preserve configured root directories and reject unsafe targets.

View File

@@ -6,21 +6,37 @@ For field-level configuration, see [docs/config.md](./config.md). For full comma
## Normal workflow (S3-first path) ## Normal workflow (S3-first path)
1. Upload session `.flac` files to object storage under the configured session audio prefix. 1. Create or upload `session.yml`, or pass a local `session.yml` explicitly.
2. Run Narratio: 2. Upload session `.flac` files to object storage under the configured session audio prefix.
3. Run Narratio:
```bash ```bash
narratio run --session-id 2026-04-04 narratio run --session-id 2026-04-04
``` ```
3. Read success output: 4. Read success output:
- `narratio run: session <session_id>; executed=<n> skipped=<n>; manifest=<path>` - `narratio run: session <session_id>; executed=<n> skipped=<n>; manifest=<path>`
- use `manifest=<path>` with `status` for inspection. - use `manifest=<path>` with `status` for inspection.
Notes: Notes:
- default config/session discovery applies unless `--config` and `--session` are passed. - default config/campaign/session discovery checks system config locations unless `--config`, `--campaign`, and `--session` are passed.
- when local `session.yml` discovery misses, `--session-id` loads remote `session.yml` from `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`.
- S3 audio mode requires `session.inputs.audio_s3.prefix` and valid object-store access. - S3 audio mode requires `session.inputs.audio_s3.prefix` and valid object-store access.
Initialize a remote session skeleton:
```bash
narratio session init --config /etc/narratio/pipeline.yml --campaign /etc/narratio/campaign.yml --session-id 2026-04-04 --remote
```
Remote init writes `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`. It fails if the object already exists unless `--force` is passed.
Validate before running:
```bash
narratio session validate --session-id 2026-04-04
```
## Restore workflow ## Restore workflow
Use restore when local durable session state is missing or stale and archive current state is authoritative. Use restore when local durable session state is missing or stale and archive current state is authoritative.
@@ -40,7 +56,7 @@ narratio restore --session-id 2026-04-04
Post-restore analyze rerun pattern: Post-restore analyze rerun pattern:
```bash ```bash
narratio run-stage --session-id 2026-04-04 --force analyze narratio analyze --session-id 2026-04-04
``` ```
Restore source-of-truth: Restore source-of-truth:
@@ -52,6 +68,16 @@ Restore default scope:
- includes `audio/**` only with `--include-audio` - includes `audio/**` only with `--include-audio`
- excludes `runs/**`, `logs/**`, `reports/**`, `config/**`, `inputs/**`, and `current/**` (except remote `current/manifest.json` as source) - excludes `runs/**`, `logs/**`, `reports/**`, `config/**`, `inputs/**`, and `current/**` (except remote `current/manifest.json` as source)
Reset local state before restore testing:
```bash
narratio clean --session-id 2026-04-04 --dry-run
narratio clean --session-id 2026-04-04
narratio restore --session-id 2026-04-04 --include-audio
```
`clean` removes the local session work directory and session spool directory. It preserves the durable S3 audio cache by default, so repeated restore or forced prepare tests do not re-download large audio files.
## Local filesystem layout and state artifacts ## Local filesystem layout and state artifacts
Session root: Session root:
@@ -61,6 +87,7 @@ Primary state:
- `manifest.json`: session-level stage state. - `manifest.json`: session-level stage state.
- `runs/{run_id}/manifest.json`: invocation-level state. - `runs/{run_id}/manifest.json`: invocation-level state.
- `.lock`: session lock while a modifying command is active. - `.lock`: session lock while a modifying command is active.
- `inputs/campaign.yml`, `inputs/session.yml`, and `inputs/pipeline.resolved.yml`: materialized config inputs for the run.
Canonical session directories: Canonical session directories:
- `inputs/` - `inputs/`
@@ -97,8 +124,9 @@ Configured artifact source reuse:
- reused configured artifact provenance is `filesystem.disabled_artifact_output`. - reused configured artifact provenance is `filesystem.disabled_artifact_output`.
`--artifacts` behavior: `--artifacts` behavior:
- accepted on `run`, `resume`, and `run-stage analyze`. - accepted on `run`, `resume`, `run-stage analyze`, and `analyze`.
- filters analyze execution only; does not force stage rerun. - filters analyze execution only.
- does not imply force on `run`, `resume`, or `run-stage`; `narratio analyze` is force-by-design.
Canonical previous-session input behavior: Canonical previous-session input behavior:
- canonical sources use `narratio.previous_session.artifact.<artifact_key>`. - canonical sources use `narratio.previous_session.artifact.<artifact_key>`.
@@ -116,6 +144,7 @@ When archive is enabled and run upload is enabled, archive publishes under:
Archive uploads: Archive uploads:
- run record files from run root (excluding `audio/`). - run record files from run root (excluding `audio/`).
- promoted files from explicit `archive.promote_artifacts` rules. - promoted files from explicit `archive.promote_artifacts` rules.
- mutable session locks from helper commands live at `{session_prefix}/locks.yml`.
Publish order: Publish order:
1. upload `current/manifest.json` 1. upload `current/manifest.json`
@@ -129,6 +158,17 @@ Archive promotion is explicit and source-based:
- missing required promotion sources fail archive stage. - missing required promotion sources fail archive stage.
- missing optional promotion sources are skipped. - missing optional promotion sources are skipped.
- invalid resolved artifacts fail archive stage. - invalid resolved artifacts fail archive stage.
- `archive.locks` skips top-level promotion overwrites for locked sources while run-local uploads still publish.
- remote locks from `{session_prefix}/locks.yml` are merged with static `archive.locks`; static locks win on duplicate sources.
- locked required promotions are treated as intentional successful skips and are recorded in archive metadata.
Lock helper behavior:
- `narratio locks --session-id <id>` lists effective static and remote locks.
- `narratio locks add --session-id <id> --reason <text> <source>` writes a remote lock.
- `narratio locks add --session-id <id> --force --reason <text> <source>` updates an existing remote lock reason.
- `narratio locks remove --session-id <id> <source>` removes only a remote lock.
- `locks remove` cannot remove static pipeline locks.
- remote lock writes check whether the lock store exists, but are not compare-and-swap atomic.
## Resume, retry, restore, and safe rerun behavior ## Resume, retry, restore, and safe rerun behavior
@@ -146,6 +186,7 @@ Restore conflict policy:
Forced reruns: Forced reruns:
- force-rerunning an upstream succeeded stage marks downstream succeeded stages as `stale`. - force-rerunning an upstream succeeded stage marks downstream succeeded stages as `stale`.
- ordinary `--force` does not override archive locks.
Safe rerun pattern: Safe rerun pattern:
1. rerun the changed stage with `--force`. 1. rerun the changed stage with `--force`.
@@ -153,12 +194,20 @@ Safe rerun pattern:
## Cleanup behavior ## Cleanup behavior
Cleanup is considered only when archive stage executed and succeeded. Automatic post-archive cleanup is considered only when archive stage executed and succeeded.
Cleanup toggles: Automatic cleanup toggles:
- `pipeline.spool.delete_audio_after_archive=true` deletes run-scoped spool audio. - `pipeline.spool.delete_audio_after_archive=true` deletes run-scoped spool audio.
- `pipeline.workspace.cleanup_after_archive=true` deletes run-scoped local run directory. - `pipeline.workspace.cleanup_after_archive=true` deletes run-scoped local run directory.
Manual cleanup:
- `narratio clean --session-id <id>` deletes `{workspace.root}/work/{campaign}/{session_id}` and `{spool.root}/{campaign}/{session_id}`.
- `narratio clean --all` deletes all local session work under `{workspace.root}/work` and all spool children under `{spool.root}`.
- `--dry-run` prints targets without deleting.
- `--clear-cache` also removes matching S3 audio cache files. Without it, cache is preserved.
The S3 audio cache under `pipeline.cache.root` is durable input cache state, not workspace or spool state. Automatic cleanup and default manual cleanup do not delete it.
Cleanup eligibility gates: Cleanup eligibility gates:
- archive enabled - archive enabled
- archive run upload enabled - archive run upload enabled
@@ -185,18 +234,26 @@ Recommended recovery:
1. inspect state: 1. inspect state:
```bash
narratio status --session-id 2026-04-04
```
This reports local manifest state, committed remote current state, expected remote transcript/artifact availability, and archive locks.
2. for one manifest file, run:
```bash ```bash
narratio status --manifest <manifest-path> narratio status --manifest <manifest-path>
``` ```
2. for restore-specific checks, run: 3. for restore-specific checks, run:
```bash ```bash
narratio restore --session-id 2026-04-04 --dry-run narratio restore --session-id 2026-04-04 --dry-run
``` ```
3. fix root cause (config/input/credentials/storage/service availability). 4. fix root cause (config/input/credentials/storage/service availability).
4. continue with `resume`, or targeted `run-stage --force` followed by `resume`. 5. continue with `resume`, or targeted `run-stage --force` followed by `resume`.
## Restore report ## Restore report
@@ -213,7 +270,9 @@ Dry-run does not write restore report files.
## Operational caveats ## Operational caveats
- `status` requires explicit `--manifest`; there is no session-id lookup command. - `status` with no config/session flags still requires explicit `--manifest`.
- `status --session-id <id>` uses normal config/session loading, including remote session fallback.
- `status --session-id <id>` includes the same promoted remote output availability view as `artifacts list --remote` when storage is configured.
- local and S3 audio input modes are mutually exclusive. - local and S3 audio input modes are mutually exclusive.
- archive publish requires upstream stages through `analyze` to be `succeeded`. - archive publish requires upstream stages through `analyze` to be `succeeded`.
- required promotion rules can fail when selected analyze artifacts did not generate a required file path. - required promotion rules can fail when selected analyze artifacts did not generate a required file path.

View File

@@ -0,0 +1,52 @@
# Roadmap: Operator Helper Commands
## Status
Implemented.
The operator helper command set is no longer conceptual. Current behavior is documented in:
- `docs/cli.md`
- `docs/operations.md`
- `docs/config.md`
- `docs/internal/artifacts.md`
- `docs/internal/stage-archive.md`
## Implemented Commands
- `narratio session validate`
- `narratio status --manifest <path>`
- `narratio status --session-id <id>`
- `narratio session init --output <path>`
- `narratio session init --remote`
- `narratio artifacts list`
- `narratio artifacts list --remote`
- `narratio locks`
- `narratio locks add <source>`
- `narratio locks remove <source>`
## Implemented Decisions
- Helper output is text-only. No JSON schema exists yet.
- `status` remains a top-level command.
- `session validate`, `session init`, and `artifacts list` are nested helper commands.
- `locks` is the single top-level command for listing, adding, and removing archive promotion locks.
- Remote session initialization requires explicit `--remote`.
- Local session initialization requires `--output`.
- Remote artifact availability is opt-in with `artifacts list --remote`.
- Mutable locks are source-based and stored at `{session_prefix}/locks.yml`.
- The remote lock store uses strict YAML with top-level `locks`.
- Static `pipeline.archive.locks` and remote locks are merged; static locks win on duplicate sources.
- `locks remove` removes only remote locks.
- Ordinary execution `--force` does not override locks.
- Remote lock writes use existence checks and `--force` for updates; there is no compare-and-swap protection.
## Remaining Future Enhancements
These are intentionally not implemented:
- `--json` output for helper commands.
- Optimistic concurrency or ETag compare-and-swap for remote lock mutations.
- Rich remote artifact availability across historical run-local objects.
- Session-lock acquisition for remote mutation helpers.
- Broader campaign helper commands such as `campaign validate` or `campaign publish`.

View File

@@ -1,629 +0,0 @@
# Roadmap: Previous-Session Artifacts
## Status
Completed.
This roadmap describes the implementation strategy for first-class previous-session artifact support in Narratio. It belongs under `docs/roadmap/previous.md` until the feature is implemented. After implementation, current behavior should be documented in the appropriate user-facing and internal documentation files, and this roadmap should be removed or marked complete according to the documentation policy.
## Summary
Narratio should support using artifacts from a previous session as inputs to artifacts generated for the current session.
The primary use case is session recap continuity: a current session recap should be able to consume the previous session recap. The design should support arbitrary previous-session artifacts from the start, not just `session_recap`.
The canonical source syntax should be:
```yaml
source: narratio.previous_session.artifact.<artifact_name>
```
For example:
```yaml
source: narratio.previous_session.artifact.session_recap
```
Previous-session artifacts are materialized during the `prepare` stage into a current-session top-level `previous/` directory. Downstream stages consume only the local `previous/` copies. The S3 backend is authoritative for previous-session state.
## Design Decisions
### 1. Add `previous_session_id` to `session.yml`
Add an optional top-level session key:
```yaml
session_id: "{{ session_id }}"
previous_session_id: "{{ previous_session_id }}"
campaign: sample-campaign
```
Rules:
- `previous_session_id` is optional.
- If present, it identifies the previous session within the same campaign.
- It must not equal `session_id`.
- It should use the same validation rules as `session_id`.
- It may be supplied through session templating.
- Add a CLI/template value such as `--previous-session-id <id>` if required by the existing session templating implementation.
- If a template placeholder for `previous_session_id` is present and no value is supplied, loading should fail with a clear unresolved-template error.
### 2. Use canonical previous-session artifact source IDs
Support this source pattern in Scriptorium artifact input definitions:
```text
narratio.previous_session.artifact.<artifact_name>
```
Examples:
```yaml
inputs:
previous_recap:
source: narratio.previous_session.artifact.session_recap
required: false
previous_quest_log:
source: narratio.previous_session.artifact.quest_log
required: true
```
Rules:
- `<artifact_name>` must be a valid configured artifact key.
- Use the same artifact key validation rules as current-session runtime artifacts.
- Do not special-case `session_recap`.
- Do not limit implementation to a fixed list of previous artifacts.
### 3. Add a top-level `previous/` workspace directory
Extend the session workspace layout with:
```text
previous/
manifest.json
artifacts/
<artifact outputs copied from the previous session>
```
The `previous/` directory is current-session state. It is a prepared input cache, not a full mirror of the previous session workspace.
Conceptually:
```text
work/<campaign>/<current_session_id>/
previous/
manifest.json
artifacts/
session_recap.md
quest_log.json
```
The current session should not read directly from the previous session's local workspace during ordinary operation.
### 4. S3 is authoritative for previous-session state
For this initial implementation, previous-session artifacts should be downloaded from the configured S3 backend.
Do not compare local and remote copies.
Do not prefer local previous-session workspace state.
Do not implement a `--local` override in this roadmap. That can be considered later.
The simplified stage behavior is:
1. If the local manifest indicates `prepare` already succeeded and `--force` is not supplied, the runner skips `prepare`. No previous-session download occurs.
2. If `prepare` has not succeeded, `prepare` runs and downloads referenced previous-session artifacts from S3.
3. If `prepare` previously succeeded but `--force` is supplied, `prepare` runs again and overwrites local `previous/` state from S3.
### 5. Materialize only referenced previous-session artifacts
During `prepare`, scan the current resolved pipeline/session configuration for Scriptorium artifact inputs whose source matches:
```text
narratio.previous_session.artifact.<artifact_name>
```
Only those referenced previous-session artifacts need to be downloaded.
Do not blindly download every previous-session artifact.
If no previous-session artifact sources are referenced, `prepare` should not require `previous_session_id` and should not touch `previous/`.
### 6. Preserve stage boundaries
`prepare` owns previous-session artifact materialization because these files are inputs to later stages.
`analyze` should not talk to S3.
The Scriptorium adapter should not know about previous sessions.
The storage adapter should not infer campaign, session, run, or root-prefix semantics. Callers should continue to provide explicit bucket-relative keys.
### 7. Archive and restore `previous/`
After implementation:
- `archive` should include `previous/` as durable current-session prepared input state.
- `restore` should restore `previous/` along with the rest of the durable session state it already restores.
- `previous/` should not be treated as current-session generated artifacts.
- `previous/` entries should be recorded as inputs/provenance, not as outputs produced by the current session.
## Target User Workflow
A typical session config:
```yaml
session_id: "{{ session_id }}"
previous_session_id: "{{ previous_session_id }}"
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./examples/speakers.yml
autocorrect_file: ./examples/autocorrect.yml
glossary_file: ./examples/glossary.yml
```
A typical artifact config:
```yaml
scriptorium:
artifacts:
session_recap:
enabled: true
prompt_id: dnd_session.session_recap
output_path: artifacts/session_recap.md
inputs:
transcript:
source: narratio.transcript.trimmed
required: true
previous_recap:
source: narratio.previous_session.artifact.session_recap
required: false
```
Typical command:
```bash
narratio run --session-id 2026-04-11 --previous-session-id 2026-04-04
```
Expected behavior:
1. `prepare` sees a referenced previous-session artifact: `session_recap`.
2. `prepare` downloads the previous session's `narratio.artifact.session_recap` from S3.
3. `prepare` writes it under the current session workspace, for example `previous/artifacts/session_recap.md`.
4. `prepare` records provenance in the current session manifest.
5. `analyze` resolves `narratio.previous_session.artifact.session_recap` from the local `previous/` directory.
6. Scriptorium receives the previous recap as a normal input file.
## Implementation Plan
### Phase 1: Session config and templating
Update session configuration structs to include:
```yaml
previous_session_id: ""
```
Implementation steps:
1. Add `PreviousSessionID` or equivalent to the session config type.
2. Add validation:
- optional;
- same format constraints as `session_id`;
- must not equal `session_id`.
3. Extend session templating support to include:
- `{{previous_session_id}}`
- `{{ previous_session_id }}`
4. Add a CLI flag if required by current templating flow:
- `--previous-session-id <id>`
5. Ensure unresolved `previous_session_id` placeholders fail clearly.
6. Update config tests for:
- no previous session;
- valid previous session;
- previous session equal to current session;
- unresolved placeholder;
- CLI/template rendering.
Do not add future workflow flags in this phase.
### Phase 2: Workspace path helpers
Add centralized path helpers for current-session previous-state paths.
Suggested helpers:
```text
SessionPreviousDir()
SessionPreviousManifestPath()
SessionPreviousArtifactsDir()
SessionPreviousArtifactPath(name or relative output path)
```
The exact names should match existing path-helper style.
Rules:
- Do not construct `previous/` paths through scattered string concatenation.
- Keep paths session-relative where possible.
- Ensure workspace layout creation includes `previous/` only when appropriate, or creates it idempotently with the rest of the layout if simpler.
Tests:
- path helper tests;
- workspace layout tests;
- ensure cleanup logic does not accidentally delete configured roots;
- ensure `previous/` is treated as session-durable state, not run-local state.
### Phase 3: Previous-session source parsing
Add parsing/recognition for:
```text
narratio.previous_session.artifact.<artifact_name>
```
Implementation steps:
1. Add constants/helpers in the artifact/source parsing layer.
2. Validate artifact names using the same rules as current runtime artifact keys.
3. Add helpers such as:
- `IsPreviousSessionArtifactSource(source string) bool`
- `PreviousSessionArtifactName(source string) (string, bool)`
4. Ensure config validation accepts this source pattern.
5. Ensure invalid sources fail clearly.
Tests:
- valid previous-session artifact source;
- invalid/missing artifact name;
- invalid artifact key characters;
- ordinary current-session sources still validate;
- unknown sources still fail.
### Phase 4: Scan configured artifacts for previous-session inputs
Add a helper that inspects resolved Scriptorium artifact definitions and returns the set of referenced previous-session artifact names.
Rules:
- Scan enabled artifacts according to current artifact-enable semantics.
- Include all inputs whose source matches `narratio.previous_session.artifact.<name>`.
- Deduplicate artifact names.
- Sort results deterministically.
- Preserve required/optional information per reference.
- If the same previous artifact is referenced both required and optional, treat it as required.
Suggested output model:
```go
type PreviousArtifactRequirement struct {
Name string
Required bool
Sources []string // optional diagnostics
}
```
Tests:
- no artifacts;
- no previous inputs;
- one optional previous input;
- one required previous input;
- duplicate references;
- required plus optional reference to the same artifact;
- deterministic ordering.
### Phase 5: Resolve previous-session archive keys
Implement a narrow service/helper used by `prepare` to resolve previous-session artifact files from S3.
Responsibilities:
1. Locate the previous session's current remote state.
2. Download the previous session manifest, or the minimum remote metadata needed to resolve artifact source IDs.
3. Resolve `narratio.artifact.<name>` inside the previous session's artifact catalog/manifest.
4. Download the resolved artifact into the current session's `previous/` directory.
5. Download/store the previous session manifest as `previous/manifest.json`.
6. Return provenance records for manifest input recording.
Important archive invariant:
- Remote current state must be based on the committed archive marker.
- Do not treat incomplete archive uploads as current state.
- Use the existing archive/current remote layout and commit-marker rules.
- `current/run_id.txt` is the final remote commit marker and should be respected when locating current remote state.
Do not put prefix semantics into the storage adapter. Compute explicit bucket-relative keys in app/stage/archive helper code, then call the storage adapter.
Required vs optional behavior:
- Required previous artifact missing from S3/current manifest: fail `prepare`.
- Optional previous artifact missing from S3/current manifest: continue without materializing that input.
- Previous session missing entirely:
- fail if any referenced previous artifact is required;
- continue if all referenced previous artifacts are optional.
- If previous_session_id is unset:
- fail if any referenced previous artifact is required;
- continue and omit all previous-session inputs if all are optional.
Validation behavior:
- Downloaded artifacts should pass the same validation rules as current-session artifacts where practical.
- Generic Scriptorium artifacts should at least be non-empty.
- Invalid required artifact: fail.
- Invalid optional artifact: prefer fail if the object exists but is invalid, because invalid archived data is usually an operator problem rather than absence.
Tests:
- downloads previous manifest;
- downloads required previous artifact;
- skips missing optional previous artifact;
- fails missing required previous artifact;
- fails required previous artifact when previous_session_id is unset;
- optional previous artifact with no previous_session_id does not fail;
- respects current remote commit marker;
- does not use local previous-session workspace state;
- uses storage adapter with explicit keys.
### Phase 6: Integrate with `prepare`
Extend the `prepare` stage:
1. Run existing input materialization as before.
2. Detect previous-session artifact requirements.
3. If requirements exist, hydrate `previous/` from S3 according to the rules above.
4. Record hydrated previous artifacts in `manifest.Inputs`.
5. Preserve existing prepare outputs and provenance behavior.
Overwrite behavior:
- If `prepare` runs, it owns `previous/`.
- Before hydrating, clear the managed `previous/` directory, or clear the managed previous artifact paths.
- Prefer clearing the whole `previous/` directory if no other feature writes there.
- Under `--force`, this naturally overwrites local `previous/` state.
- Do not compare local and remote copies.
Skip behavior:
- Do not add stage-local skip logic.
- Runner-level skip remains authoritative.
- If the manifest says `prepare` succeeded and `--force` is not supplied, `prepare` does not run and no S3 downloads occur.
- If users add a new previous-session input after `prepare` already succeeded, they must rerun prepare with `--force`.
Error message requirement:
If an analyze-stage input cannot be resolved because `previous/` is missing or stale, the error should tell the operator to run:
```bash
narratio run-stage --force prepare
```
or the appropriate existing CLI command shape.
Tests:
- ordinary prepare without previous_session_id remains unchanged;
- prepare with optional previous artifact and no previous_session_id succeeds;
- prepare with required previous artifact and no previous_session_id fails;
- prepare with required previous artifact downloads to `previous/`;
- force prepare overwrites `previous/`;
- prepare records manifest inputs for previous artifacts;
- prepare skip behavior remains controlled by runner tests;
- no regression in S3 audio prepare behavior.
### Phase 7: Artifact resolver support
Update artifact resolution so Scriptorium inputs can resolve:
```text
narratio.previous_session.artifact.<artifact_name>
```
from the current session's `previous/` directory.
Rules:
- The resolver should not call S3.
- The resolver should not read the previous session's local workspace.
- The resolver should map the previous-session source ID to the local prepared copy under `previous/`.
- Resolution should use manifest input records when available.
- Fallback to the local `previous/` path may be allowed if consistent with existing resolver behavior, but manifest provenance should be preferred.
- Missing required input should fail with a clear prepare-oriented message.
- Optional missing input should be omitted.
Suggested provenance:
```text
previous_session.manifest.outputs
previous_session.archive.current
current_session.previous_cache
```
Use names that fit the existing manifest/resolver vocabulary.
Tests:
- resolves prepared previous artifact through manifest input record;
- resolves or fails appropriately when only filesystem copy exists, depending on chosen fallback policy;
- missing optional previous artifact is omitted;
- missing required previous artifact fails clearly;
- current-session `narratio.artifact.<name>` behavior is unchanged.
### Phase 8: Analyze-stage integration
The analyze stage should require little or no special previous-session logic if the resolver is designed correctly.
Confirm:
- Scriptorium input resolution accepts previous-session source IDs.
- The Scriptorium adapter receives a normal local input path.
- Render-debug and run modes behave the same as for ordinary inputs.
- Stage metadata includes useful input provenance if current structures support it.
Tests:
- configured artifact receives previous recap input;
- optional previous recap omitted when not prepared;
- required previous recap fails when not prepared;
- render-debug path works with previous-session inputs;
- no S3 calls occur from analyze.
### Phase 9: Archive `previous/`
Update archive behavior so the current session's durable `previous/` directory is uploaded/preserved.
Rules:
- `previous/` is current-session input/provenance state.
- It is not a generated current-session artifact.
- Archive it with the current session's durable state, alongside other durable session files according to the current archive layout.
- Preserve existing archive commit ordering.
- Do not make `previous/` upload the final commit marker.
- Do not treat missing `previous/` as an error when no previous-session inputs were prepared.
Tests:
- archive includes `previous/manifest.json` and prepared previous artifacts when present;
- archive omits or tolerates absent `previous/` when unused;
- archive commit ordering remains valid;
- cleanup behavior does not delete durable `previous/` before archive commit.
### Phase 10: Restore `previous/`
Update `narratio restore` so it restores the current session's archived `previous/` directory.
Rules:
- Restore `previous/` as durable current-session state.
- Do not infer or restore the previous session workspace merely because `previous_session_id` exists.
- Do not redownload previous-session artifacts from the previous session archive during restore; restore the current session's archived `previous/` cache.
- Respect existing restore flags and overwrite behavior.
Tests:
- restore downloads `previous/` when present;
- restore succeeds when `previous/` is absent;
- restore with force overwrites local `previous/` according to existing restore semantics;
- restored current session can run analyze using `previous/` without needing previous session archive access.
### Phase 11: Documentation updates after implementation
After implementation, update current-behavior docs. Do not document implemented behavior only in this roadmap.
Likely files:
- `docs/config.md`
- `docs/cli.md`
- `docs/operations.md`
- `docs/internal/stage-prepare.md`
- `docs/internal/artifacts.md`
- `docs/internal/storage.md` only if storage contracts change
- `docs/internal/workspace.md`
- relevant maintained examples under `examples/`
Docs should explain:
- `session.previous_session_id`;
- `--previous-session-id` if added;
- `narratio.previous_session.artifact.<artifact_name>`;
- `previous/` workspace directory;
- S3-authoritative previous-session behavior;
- required vs optional previous-session artifact handling;
- when to run `narratio run-stage --force prepare`.
Do not document future `--local` support as current behavior.
## Future Work
These items are explicitly out of scope for the initial implementation.
### `--local` previous-session source
A future flag may allow `prepare` to copy previous-session artifacts from a local previous session workspace instead of S3.
Possible future command shape:
```bash
narratio run-stage --force prepare --local
```
or a more specific flag such as:
```bash
narratio run-stage --force prepare --previous-source local
```
Do not implement this now.
### `narratio run --restore`
A future flag may run `narratio restore` before starting the regular pipeline:
```bash
narratio run --restore --session-id 2026-04-11
```
Do not implement this as part of previous-session artifact support unless it already exists and only requires documentation.
### Multi-previous-session support
A future design may support more than one previous/reference session.
Do not implement this now.
### Previous-session artifact version pinning
A future design may pin previous-session artifact inputs to a specific previous run ID or checksum.
Do not implement this now.
## Test Plan Summary
Run at least:
```bash
go test ./internal/config -v
go test ./internal/artifacts -v
go test ./internal/stage -run Prepare -v
go test ./internal/stage -run Analyze -v
go test ./internal/app -run TestExecute -v
go test ./...
```
Add focused tests for:
- session config and templating;
- previous-session source parsing;
- previous artifact requirement scanning;
- S3-backed previous artifact download;
- prepare skip/force semantics;
- manifest input provenance;
- resolver behavior;
- analyze integration;
- archive/restore `previous/` persistence;
- examples load/validate.
## Acceptance Criteria
The feature is complete when:
1. `session.yml` supports optional `previous_session_id`.
2. Session templating supports `previous_session_id`.
3. Scriptorium artifact inputs accept `narratio.previous_session.artifact.<artifact_name>`.
4. `prepare` scans enabled Scriptorium artifacts for previous-session artifact inputs.
5. `prepare` downloads referenced previous-session artifacts from S3 into `previous/`.
6. Required/optional behavior is correct.
7. `prepare` uses stage-level skip/force behavior and does not compare local and remote copies.
8. `analyze` resolves previous-session artifacts only from local prepared `previous/` state.
9. `archive` persists `previous/`.
10. `restore` restores `previous/`.
11. Tests cover config, prepare, resolver, analyze, archive, and restore behavior.
12. Current-behavior docs and maintained examples are updated after implementation.
13. No storage adapter implementation infers campaign/session/root-prefix semantics.
14. No previous-session S3 logic is added to `analyze` or the Scriptorium adapter.

View File

@@ -9,19 +9,19 @@ Symptom:
- `run`, `plan`, `resume`, `run-stage`, or `restore` fails with config/session not found. - `run`, `plan`, `resume`, `run-stage`, or `restore` fails with config/session not found.
Likely Cause: Likely Cause:
- `pipeline.yml` or `session.yml` is missing from discovery paths. - `pipeline.yml`, `campaign.yml`, or `session.yml` is missing from system discovery paths.
- wrong working directory when relying on `./session.yml`. - a local working-directory config file was not passed explicitly.
Diagnostics: Diagnostics:
```bash ```bash
pwd
ls -l ./session.yml
ls -l /usr/local/etc/narratio/pipeline.yml /etc/narratio/pipeline.yml ls -l /usr/local/etc/narratio/pipeline.yml /etc/narratio/pipeline.yml
ls -l /usr/local/etc/narratio/campaign.yml /etc/narratio/campaign.yml
ls -l /usr/local/etc/narratio/session.yml /etc/narratio/session.yml
``` ```
Safe Fix: Safe Fix:
- pass explicit `--config` and `--session`. - pass explicit `--config`, `--campaign`, and `--session`.
- or place files in documented discovery paths. - or place files in documented discovery paths.
Links: Links:
@@ -40,7 +40,7 @@ Likely Cause:
Diagnostics: Diagnostics:
```bash ```bash
narratio plan --session ./session.yml --session-id 2026-04-04 narratio plan --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session ./session.yml --session-id 2026-04-04
``` ```
Safe Fix: Safe Fix:
@@ -62,7 +62,7 @@ Likely Cause:
Diagnostics: Diagnostics:
```bash ```bash
narratio plan --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 narratio plan --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04
``` ```
Safe Fix: Safe Fix:
@@ -84,7 +84,7 @@ Likely Cause:
Diagnostics: Diagnostics:
```bash ```bash
narratio run --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts player_handout narratio run --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts player_handout
``` ```
Safe Fix: Safe Fix:
@@ -106,7 +106,7 @@ Likely Cause:
Diagnostics: Diagnostics:
```bash ```bash
narratio run-stage --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts session_recap polish narratio run-stage --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts session_recap polish
``` ```
Safe Fix: Safe Fix:
@@ -129,7 +129,7 @@ Likely Cause:
Diagnostics: Diagnostics:
```bash ```bash
narratio plan --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 narratio plan --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04
``` ```
Safe Fix: Safe Fix:
@@ -154,7 +154,7 @@ Diagnostics:
```bash ```bash
narratio status --manifest /path/to/manifest.json narratio status --manifest /path/to/manifest.json
narratio run-stage --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts player_handout analyze narratio run-stage --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts player_handout analyze
``` ```
Safe Fix: Safe Fix:
@@ -227,7 +227,7 @@ Likely Cause:
Diagnostics: Diagnostics:
```bash ```bash
narratio restore --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run narratio restore --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
``` ```
Safe Fix: Safe Fix:
@@ -250,7 +250,7 @@ Likely Cause:
Diagnostics: Diagnostics:
```bash ```bash
narratio restore --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run narratio restore --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
``` ```
Safe Fix: Safe Fix:
@@ -272,7 +272,7 @@ Likely Cause:
Diagnostics: Diagnostics:
```bash ```bash
narratio restore --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run narratio restore --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
``` ```
Safe Fix: Safe Fix:
@@ -342,7 +342,7 @@ Likely Cause:
Diagnostics: Diagnostics:
```bash ```bash
narratio run-stage --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 prepare narratio run-stage --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 prepare
``` ```
Safe Fix: Safe Fix:
@@ -366,7 +366,7 @@ Diagnostics:
```bash ```bash
narratio status --manifest /path/to/manifest.json narratio status --manifest /path/to/manifest.json
narratio run-stage --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 archive narratio run-stage --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 archive
``` ```
Safe Fix: Safe Fix:

5
examples/campaign.yml Normal file
View File

@@ -0,0 +1,5 @@
campaign: sample-campaign
inputs:
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml

View File

@@ -1,9 +1,5 @@
session_id: 2026-05-03 session_id: 2026-05-03
campaign: sample-campaign
date: 2026-05-03 date: 2026-05-03
title: Sample Session title: Sample Session
inputs: inputs:
audio_dir: ./audio audio_dir: ./audio
speakers_file: ./examples/speakers.yml
autocorrect_file: ./examples/autocorrect.yml
glossary_file: ./examples/glossary.yml

View File

@@ -1,10 +1,6 @@
session_id: 2026-05-03 session_id: 2026-05-03
campaign: sample-campaign
date: 2026-05-03 date: 2026-05-03
title: Sample Session title: Sample Session
inputs: inputs:
audio_s3: audio_s3:
prefix: audio/ prefix: audio/
speakers_file: ./examples/speakers.yml
autocorrect_file: ./examples/autocorrect.yml
glossary_file: ./examples/glossary.yml

View File

@@ -1,7 +1,3 @@
session_id: "{{ session_id }}" session_id: "{{ session_id }}"
campaign: sample-campaign
inputs: inputs:
audio_dir: ./audio audio_dir: ./audio
speakers_file: ./examples/speakers.yml
autocorrect_file: ./examples/autocorrect.yml
glossary_file: ./examples/glossary.yml

View File

@@ -27,8 +27,9 @@ type FakeBackend struct {
Err error Err error
Result ArchiveResult Result ArchiveResult
Objects map[string]FakeObject Objects map[string]FakeObject
Uploads []FakeUploadCall Uploads []FakeUploadCall
Downloads []FakeDownloadCall
ListErr error ListErr error
DownloadErr error DownloadErr error
@@ -43,6 +44,12 @@ type FakeUploadCall struct {
Options UploadOptions Options UploadOptions
} }
// FakeDownloadCall captures one download invocation in call order.
type FakeDownloadCall struct {
Key string
LocalPath string
}
// Archive records request and returns configured response. // Archive records request and returns configured response.
func (f *FakeBackend) Archive(ctx context.Context, req ArchiveRequest) (ArchiveResult, error) { func (f *FakeBackend) Archive(ctx context.Context, req ArchiveRequest) (ArchiveResult, error) {
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
@@ -130,6 +137,10 @@ func (f *FakeBackend) Download(ctx context.Context, key, localPath string) error
if !ok { if !ok {
return fmt.Errorf("download object %q: %w", key, os.ErrNotExist) return fmt.Errorf("download object %q: %w", key, os.ErrNotExist)
} }
f.Downloads = append(f.Downloads, FakeDownloadCall{
Key: normalizeObjectKey(key),
LocalPath: localPath,
})
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil { if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
return fmt.Errorf("download object %q: create parent directory: %w", key, err) return fmt.Errorf("download object %q: create parent directory: %w", key, err)

View File

@@ -56,7 +56,7 @@ func TestFakeBackendDownload(t *testing.T) {
fake.SeedObject(FakeObject{Key: "audio/a.flac", Data: []byte("audio-a")}) fake.SeedObject(FakeObject{Key: "audio/a.flac", Data: []byte("audio-a")})
dst := filepath.Join(t.TempDir(), "nested", "a.flac") dst := filepath.Join(t.TempDir(), "nested", "a.flac")
if err := fake.Download(context.Background(), "audio/a.flac", dst); err != nil { if err := fake.Download(context.Background(), `audio\a.flac`, dst); err != nil {
t.Fatalf("Download() error = %v", err) t.Fatalf("Download() error = %v", err)
} }
data, err := os.ReadFile(dst) data, err := os.ReadFile(dst)

View File

@@ -9,17 +9,19 @@ import (
"testing" "testing"
"time" "time"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest" "gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/stage"
) )
func TestExecuteRunStageArtifactsNonAnalyzeFails(t *testing.T) { func TestExecuteRunStageArtifactsNonAnalyzeFails(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute( code := Execute(
[]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--artifacts", "session_recap", "polish"}, []string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "session_recap", "polish"},
&stdout, &stdout,
&stderr, &stderr,
) )
@@ -33,12 +35,12 @@ func TestExecuteRunStageArtifactsNonAnalyzeFails(t *testing.T) {
func TestExecuteUnknownArtifactsFailValidation(t *testing.T) { func TestExecuteUnknownArtifactsFailValidation(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute( code := Execute(
[]string{"run", "--config", pipelinePath, "--session", sessionPath, "--artifacts", "unknown_artifact"}, []string{"run", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
&stdout, &stdout,
&stderr, &stderr,
) )
@@ -52,7 +54,7 @@ func TestExecuteUnknownArtifactsFailValidation(t *testing.T) {
func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) { func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json") manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
store := &manifest.LocalStore{} store := &manifest.LocalStore{}
@@ -65,7 +67,7 @@ func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
err := RunStage( err := RunStage(
context.Background(), context.Background(),
[]string{"--config", pipelinePath, "--session", sessionPath, "--artifacts", "session_recap,session_recap", "analyze"}, []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "session_recap,session_recap", "analyze"},
&out, &out,
) )
if err != nil { if err != nil {
@@ -78,7 +80,7 @@ func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) { func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json") manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
store := &manifest.LocalStore{} store := &manifest.LocalStore{}
@@ -93,7 +95,7 @@ func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
err := Resume( err := Resume(
context.Background(), context.Background(),
[]string{"--config", pipelinePath, "--session", sessionPath, "--artifacts", "session_recap"}, []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
&out, &out,
) )
if err != nil { if err != nil {
@@ -104,10 +106,154 @@ func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
} }
} }
func writeValidConfigFilesWithScriptoriumArtifacts(t *testing.T, workspaceRoot string) (string, string) { func TestExecuteAnalyzeForceRunsAnalyze(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
var capturedStages []string
var capturedForce bool
origExecuteStagesFn := executeStagesFn
t.Cleanup(func() {
executeStagesFn = origExecuteStagesFn
})
executeStagesFn = func(_ context.Context, _ *config.Config, stages []stage.Stage, opts RunOptions) (*RunSummary, error) {
for _, s := range stages {
capturedStages = append(capturedStages, s.Name())
}
capturedForce = opts.Force
return &RunSummary{
ManifestPath: filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json"),
Executed: []string{"analyze"},
}, nil
}
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute(
[]string{"analyze", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath},
&stdout,
&stderr,
)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if len(capturedStages) != 1 || capturedStages[0] != "analyze" {
t.Fatalf("captured stages = %#v, want [analyze]", capturedStages)
}
if !capturedForce {
t.Fatal("captured force = false, want true")
}
if !strings.Contains(stdout.String(), "narratio analyze: executed=1 skipped=0 force=true; manifest=") {
t.Fatalf("stdout = %q, want analyze summary", stdout.String())
}
}
func TestExecuteAnalyzePropagatesSelectedArtifacts(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
var capturedArtifacts []string
origExecuteStagesFn := executeStagesFn
t.Cleanup(func() {
executeStagesFn = origExecuteStagesFn
})
executeStagesFn = func(_ context.Context, _ *config.Config, _ []stage.Stage, opts RunOptions) (*RunSummary, error) {
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
return &RunSummary{ManifestPath: filepath.Join(workspaceRoot, "manifest.json"), Executed: []string{"analyze"}}, nil
}
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute(
[]string{
"analyze",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--artifacts", "player_handout,session_recap",
},
&stdout,
&stderr,
)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if strings.Join(capturedArtifacts, ",") != "player_handout,session_recap" {
t.Fatalf("captured artifacts = %#v, want sorted selected artifacts", capturedArtifacts)
}
}
func TestExecuteAnalyzeUnknownArtifactFailsValidation(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute(
[]string{"analyze", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
&stdout,
&stderr,
)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), `analyze: --artifacts includes unknown artifact "unknown_artifact"`) {
t.Fatalf("stderr = %q, want unknown-artifact validation error", stderr.String())
}
}
func TestExecuteAnalyzeRejectsPositionalArgsAndForceFlag(t *testing.T) {
cases := []struct {
name string
args []string
want string
}{
{name: "positional", args: []string{"analyze", "extra"}, want: "analyze: unexpected positional arguments"},
{name: "force flag", args: []string{"analyze", "--force"}, want: "analyze: invalid flags: flag provided but not defined: -force"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute(tc.args, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), tc.want) {
t.Fatalf("stderr = %q, want %q", stderr.String(), tc.want)
}
})
}
}
func TestExecuteAnalyzeMissingConfigUsesRunStageLoadingPath(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"analyze"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "analyze: no pipeline config path provided and no default pipeline config found; searched:") {
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
}
}
func TestExecuteUsageIncludesAnalyze(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute(nil, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "analyze") {
t.Fatalf("stderr = %q, want usage to include analyze", stderr.String())
}
}
func writeValidConfigFilesWithScriptoriumArtifacts(t *testing.T, workspaceRoot string) (string, string, string) {
t.Helper() t.Helper()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
f, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0) f, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0)
if err != nil { if err != nil {
t.Fatalf("open pipeline config for append: %v", err) t.Fatalf("open pipeline config for append: %v", err)
@@ -136,5 +282,5 @@ scriptorium:
if _, err := f.WriteString(extra); err != nil { if _, err := f.WriteString(extra); err != nil {
t.Fatalf("append scriptorium config: %v", err) t.Fatalf("append scriptorium config: %v", err)
} }
return pipelinePath, sessionPath return pipelinePath, campaignPath, sessionPath
} }

View File

@@ -0,0 +1,49 @@
package app
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func resolveCampaignConfigPath(flagValue string) (string, error) {
return resolveCampaignConfigPathWithCandidates(flagValue, config.DefaultCampaignConfigSearchPaths)
}
func resolveCampaignConfigPathWithCandidates(flagValue string, candidates []string) (string, error) {
if explicit := strings.TrimSpace(flagValue); explicit != "" {
return explicit, nil
}
ordered := make([]string, 0, len(candidates))
for _, raw := range candidates {
path := strings.TrimSpace(raw)
if path == "" {
continue
}
ordered = append(ordered, path)
info, err := os.Stat(path)
if err == nil {
if info.IsDir() {
continue
}
return filepath.Clean(path), nil
}
if errors.Is(err, os.ErrNotExist) {
continue
}
return "", fmt.Errorf("check default campaign config %q: %w", path, err)
}
if len(ordered) == 0 {
return "", fmt.Errorf("no campaign config path provided and no default locations configured")
}
return "", fmt.Errorf(
"no campaign config path provided and no default campaign config found; searched: %s; pass --campaign to use an explicit path",
strings.Join(ordered, ", "),
)
}

View File

@@ -0,0 +1,49 @@
package app
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestResolveCampaignConfigPathExplicitWins(t *testing.T) {
explicit := filepath.Join(t.TempDir(), "custom-campaign.yml")
got, err := resolveCampaignConfigPathWithCandidates(explicit, []string{filepath.Join(t.TempDir(), "campaign.yml")})
if err != nil {
t.Fatalf("resolveCampaignConfigPathWithCandidates() error = %v", err)
}
if got != explicit {
t.Fatalf("path = %q, want explicit path %q", got, explicit)
}
}
func TestResolveCampaignConfigPathUsesFirstExistingDefault(t *testing.T) {
dir := t.TempDir()
missing := filepath.Join(dir, "missing.yml")
found := filepath.Join(dir, "campaign.yml")
if err := os.WriteFile(found, []byte("campaign: sample-campaign\n"), 0o644); err != nil {
t.Fatalf("write campaign.yml: %v", err)
}
got, err := resolveCampaignConfigPathWithCandidates("", []string{missing, found})
if err != nil {
t.Fatalf("resolveCampaignConfigPathWithCandidates() error = %v", err)
}
if got != filepath.Clean(found) {
t.Fatalf("path = %q, want %q", got, filepath.Clean(found))
}
}
func TestResolveCampaignConfigPathErrorIncludesSearchedPaths(t *testing.T) {
_, err := resolveCampaignConfigPathWithCandidates("", []string{"/usr/local/etc/narratio/campaign.yml", "/etc/narratio/campaign.yml"})
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "searched") {
t.Fatalf("error = %q, want searched paths", err.Error())
}
if !strings.Contains(err.Error(), "pass --campaign") {
t.Fatalf("error = %q, want explicit-campaign guidance", err.Error())
}
}

336
internal/app/clean.go Normal file
View File

@@ -0,0 +1,336 @@
package app
import (
"context"
"flag"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
// Clean removes local workspace/spool state while preserving durable cache
// state unless cache cleanup is explicitly requested.
func Clean(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("clean", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var flags commonConfigFlags
var all bool
var dryRun bool
var clearCache bool
addCommonConfigFlags(fs, &flags)
fs.BoolVar(&all, "all", false, "clean all local session work/spool state")
fs.BoolVar(&dryRun, "dry-run", false, "print cleanup targets without deleting")
fs.BoolVar(&clearCache, "clear-cache", false, "also clear durable S3 audio cache entries")
if err := fs.Parse(args); err != nil {
return fmt.Errorf("clean: invalid flags: %w", err)
}
if fs.NArg() != 0 {
return fmt.Errorf("clean: unexpected positional arguments")
}
if all {
return cleanAllLocal(flags, dryRun, clearCache, out)
}
return cleanSession(ctx, flags, dryRun, clearCache, out)
}
func cleanSession(ctx context.Context, flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
if strings.TrimSpace(flags.sessionID) == "" {
return fmt.Errorf("clean: --session-id is required unless --all is set")
}
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.sessionPath, flags.sessionOptions())
if err != nil {
return fmt.Errorf("clean: %w", err)
}
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
return fmt.Errorf("clean: resolved pipeline and session config are required")
}
campaign := strings.TrimSpace(cfg.Session.Campaign)
sessionID := strings.TrimSpace(cfg.Session.SessionID)
if campaign == "" || sessionID == "" {
return fmt.Errorf("clean: campaign and session_id are required")
}
if dryRun {
fmt.Fprintf(out, "Clean plan for %s/%s\n", campaign, sessionID)
} else {
fmt.Fprintf(out, "Cleaned %s/%s\n", campaign, sessionID)
}
workDir := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, campaign, sessionID)
spoolDir := artifacts.SessionSpoolDir(cfg.Pipeline.Spool.Root, campaign, sessionID)
if err := reportCleanScopedDir(out, cfg.Pipeline.Workspace.Root, workDir, "clean.workspace.session", dryRun); err != nil {
return fmt.Errorf("clean: %w", err)
}
if err := reportCleanScopedDir(out, cfg.Pipeline.Spool.Root, spoolDir, "clean.spool.session", dryRun); err != nil {
return fmt.Errorf("clean: %w", err)
}
if clearCache {
if err := cleanSessionAudioCache(ctx, cfg, dryRun, out); err != nil {
return fmt.Errorf("clean: %w", err)
}
} else {
fmt.Fprintln(out, "Cache: preserved")
}
return nil
}
func cleanAllLocal(flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
if strings.TrimSpace(flags.campaignPath) != "" ||
strings.TrimSpace(flags.sessionPath) != "" ||
strings.TrimSpace(flags.sessionID) != "" ||
strings.TrimSpace(flags.previousSessionID) != "" {
return fmt.Errorf("clean: --all cannot be combined with --campaign, --session, --session-id, or --previous-session-id")
}
resolvedPipelinePath, err := resolvePipelineConfigPath(flags.pipelinePath)
if err != nil {
return fmt.Errorf("clean: %w", err)
}
pipelineCfg, err := config.LoadPipeline(resolvedPipelinePath)
if err != nil {
return fmt.Errorf("clean: %w", err)
}
if dryRun {
fmt.Fprintln(out, "Clean plan for all local sessions")
} else {
fmt.Fprintln(out, "Cleaned all local sessions")
}
workRoot := filepath.Join(pipelineCfg.Workspace.Root, config.PathWorkDirSegment)
if err := reportCleanScopedDir(out, pipelineCfg.Workspace.Root, workRoot, "clean.workspace.all", dryRun); err != nil {
return fmt.Errorf("clean: %w", err)
}
if err := reportCleanRootChildren(out, pipelineCfg.Spool.Root, "clean.spool.all", dryRun); err != nil {
return fmt.Errorf("clean: %w", err)
}
if clearCache {
if err := cleanAllAudioCache(pipelineCfg, dryRun, out); err != nil {
return fmt.Errorf("clean: %w", err)
}
} else {
fmt.Fprintln(out, "Cache: preserved")
}
return nil
}
func reportCleanScopedDir(out io.Writer, root, target, policy string, dryRun bool) error {
dir, err := validateScopedDir(root, target, policy)
if err != nil {
return err
}
if dryRun {
if dir.Exists {
fmt.Fprintf(out, "Would delete: %s\n", dir.TargetAbs)
} else {
fmt.Fprintf(out, "Would skip missing: %s\n", dir.TargetAbs)
}
return nil
}
if !dir.Exists {
fmt.Fprintf(out, "Missing: %s\n", dir.TargetAbs)
return nil
}
if err := os.RemoveAll(dir.TargetAbs); err != nil {
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
}
fmt.Fprintf(out, "Deleted: %s\n", dir.TargetAbs)
return nil
}
func reportCleanRootChildren(out io.Writer, root, policy string, dryRun bool) error {
rootAbs, entries, err := cleanableRootChildren(root, policy)
if err != nil {
return err
}
if len(entries) == 0 {
if dryRun {
fmt.Fprintf(out, "Would skip empty: %s\n", rootAbs)
} else {
fmt.Fprintf(out, "Empty: %s\n", rootAbs)
}
return nil
}
for _, entry := range entries {
if dryRun {
fmt.Fprintf(out, "Would delete: %s\n", entry)
continue
}
if err := os.RemoveAll(entry); err != nil {
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, entry, err)
}
fmt.Fprintf(out, "Deleted: %s\n", entry)
}
return nil
}
func cleanableRootChildren(root, policy string) (string, []string, error) {
cleanRoot := strings.TrimSpace(root)
if cleanRoot == "" {
return "", nil, fmt.Errorf("cleanup policy %s: root path is required", policy)
}
rootAbs, err := filepath.Abs(cleanRoot)
if err != nil {
return "", nil, fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
}
info, err := os.Lstat(rootAbs)
if err != nil {
if os.IsNotExist(err) {
return rootAbs, nil, nil
}
return "", nil, fmt.Errorf("cleanup policy %s: stat root %q: %w", policy, rootAbs, err)
}
if info.Mode()&os.ModeSymlink != 0 {
return "", nil, fmt.Errorf("cleanup policy %s: refusing to clean symlink root %q", policy, rootAbs)
}
if !info.IsDir() {
return "", nil, fmt.Errorf("cleanup policy %s: root %q is not a directory", policy, rootAbs)
}
entries, err := os.ReadDir(rootAbs)
if err != nil {
return "", nil, fmt.Errorf("cleanup policy %s: read root %q: %w", policy, rootAbs, err)
}
out := make([]string, 0, len(entries))
for _, entry := range entries {
path := filepath.Join(rootAbs, entry.Name())
info, err := os.Lstat(path)
if err != nil {
return "", nil, fmt.Errorf("cleanup policy %s: stat child %q: %w", policy, path, err)
}
if info.Mode()&os.ModeSymlink != 0 {
return "", nil, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, path)
}
out = append(out, path)
}
return rootAbs, out, nil
}
func cleanSessionAudioCache(ctx context.Context, cfg *config.Config, dryRun bool, out io.Writer) error {
if cfg.Session.Inputs.AudioS3 == nil {
fmt.Fprintln(out, "Cache: skipped (session does not use audio_s3)")
return nil
}
if cfg.Pipeline.Storage.S3 == nil || strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) == "" {
return fmt.Errorf("clear cache requires pipeline.storage.s3.bucket")
}
store, err := newCommandObjectStore(ctx, cfg, nil)
if err != nil {
return fmt.Errorf("initialize object store for cache cleanup: %w", err)
}
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, cfg.Session.Inputs.AudioS3.Prefix)
objects, err := store.List(ctx, audioPrefix)
if err != nil {
return fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
}
count := 0
for _, obj := range objects {
key := strings.TrimSpace(obj.Key)
if key == "" || strings.HasSuffix(key, "/") || !cleanIsFlac(key) {
continue
}
cachePath, err := artifacts.S3AudioCachePath(cfg.Pipeline.Cache.Root, cfg.Pipeline.Storage.S3.Bucket, key)
if err != nil {
return err
}
deleted, err := reportCleanScopedFile(out, cfg.Pipeline.Cache.Root, cachePath, "clean.cache.session", dryRun)
if err != nil {
return err
}
if deleted {
count++
}
}
if count == 0 {
fmt.Fprintf(out, "Cache: no cached S3 audio files found for %s\n", audioPrefix)
}
return nil
}
func cleanAllAudioCache(cfg *config.PipelineConfig, dryRun bool, out io.Writer) error {
if cfg.Storage.S3 == nil || strings.TrimSpace(cfg.Storage.S3.Bucket) == "" {
return fmt.Errorf("clear cache requires pipeline.storage.s3.bucket")
}
namespaceDir, err := artifacts.S3AudioCacheNamespaceDir(cfg.Cache.Root, cfg.Storage.S3.Bucket, cfg.Storage.S3.RootPrefix)
if err != nil {
return err
}
return reportCleanScopedDir(out, cfg.Cache.Root, namespaceDir, "clean.cache.all", dryRun)
}
func reportCleanScopedFile(out io.Writer, root, target, policy string, dryRun bool) (bool, error) {
file, err := validateScopedFile(root, target, policy)
if err != nil {
return false, err
}
if dryRun {
if file.Exists {
fmt.Fprintf(out, "Would delete cache file: %s\n", file.TargetAbs)
return true, nil
}
fmt.Fprintf(out, "Would skip missing cache file: %s\n", file.TargetAbs)
return false, nil
}
if !file.Exists {
fmt.Fprintf(out, "Missing cache file: %s\n", file.TargetAbs)
return false, nil
}
if err := os.Remove(file.TargetAbs); err != nil {
return false, fmt.Errorf("cleanup policy %s: remove %q: %w", policy, file.TargetAbs, err)
}
fmt.Fprintf(out, "Deleted cache file: %s\n", file.TargetAbs)
return true, nil
}
func validateScopedFile(root, target, policy string) (scopedDir, error) {
cleanRoot := strings.TrimSpace(root)
cleanTarget := strings.TrimSpace(target)
if cleanRoot == "" {
return scopedDir{}, fmt.Errorf("cleanup policy %s: root path is required", policy)
}
if cleanTarget == "" {
return scopedDir{}, fmt.Errorf("cleanup policy %s: target path is required", policy)
}
rootAbs, err := filepath.Abs(cleanRoot)
if err != nil {
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
}
targetAbs, err := filepath.Abs(cleanTarget)
if err != nil {
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve target %q: %w", policy, cleanTarget, err)
}
rel, err := filepath.Rel(rootAbs, targetAbs)
if err != nil {
return scopedDir{}, fmt.Errorf("cleanup policy %s: relative path from %q to %q: %w", policy, rootAbs, targetAbs, err)
}
if rel == "." {
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete root directory %q", policy, rootAbs)
}
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete path outside root: root=%q target=%q", policy, rootAbs, targetAbs)
}
info, err := os.Lstat(targetAbs)
if err != nil {
if os.IsNotExist(err) {
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: false}, nil
}
return scopedDir{}, fmt.Errorf("cleanup policy %s: stat target %q: %w", policy, targetAbs, err)
}
if info.Mode()&os.ModeSymlink != 0 {
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, targetAbs)
}
if info.IsDir() {
return scopedDir{}, fmt.Errorf("cleanup policy %s: target %q is a directory", policy, targetAbs)
}
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: true}, nil
}
func cleanIsFlac(path string) bool {
return strings.EqualFold(filepath.Ext(path), ".flac")
}

255
internal/app/clean_test.go Normal file
View File

@@ -0,0 +1,255 @@
package app
import (
"bytes"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
)
func TestExecuteCleanSessionDeletesWorkAndSpoolButPreservesCache(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
workDir := artifacts.SessionWorkDirForCampaign(workspaceRoot, "sample-campaign", "2026-05-03")
spoolDir := artifacts.SessionSpoolDir(filepath.Join(workspaceRoot, "spool"), "sample-campaign", "2026-05-03")
cachePath, err := artifacts.S3AudioCachePath(filepath.Join(workspaceRoot, "cache"), "test-bucket", "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
if err != nil {
t.Fatalf("S3AudioCachePath() error = %v", err)
}
mustWriteTestFile(t, filepath.Join(workDir, "manifest.json"), "{}")
mustWriteTestFile(t, filepath.Join(spoolDir, "run-1", "audio", "alice.flac"), "audio")
mustWriteTestFile(t, cachePath, "cached-audio")
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
cleanAssertMissing(t, workDir)
cleanAssertMissing(t, spoolDir)
cleanAssertExists(t, cachePath)
if !strings.Contains(stdout.String(), "Cache: preserved") {
t.Fatalf("stdout = %q, want cache preserved", stdout.String())
}
}
func TestExecuteCleanSessionDryRunDeletesNothing(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
workDir := artifacts.SessionWorkDirForCampaign(workspaceRoot, "sample-campaign", "2026-05-03")
spoolDir := artifacts.SessionSpoolDir(filepath.Join(workspaceRoot, "spool"), "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workDir, "manifest.json"), "{}")
mustWriteTestFile(t, filepath.Join(spoolDir, "run-1", "audio", "alice.flac"), "audio")
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03", "--dry-run"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
cleanAssertExists(t, workDir)
cleanAssertExists(t, spoolDir)
if !strings.Contains(stdout.String(), "Would delete:") {
t.Fatalf("stdout = %q, want dry-run delete plan", stdout.String())
}
}
func TestExecuteCleanMissingSessionPathsSucceeds(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if !strings.Contains(stdout.String(), "Missing:") {
t.Fatalf("stdout = %q, want missing path output", stdout.String())
}
}
func TestExecuteCleanSessionClearCacheRemovesOnlyS3AudioCache(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
if err := os.WriteFile(sessionPath, []byte(`session_id: 2026-05-03
inputs:
audio_s3:
prefix: audio/
`), 0o644); err != nil {
t.Fatalf("write session: %v", err)
}
audioKey := "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac"
fake := &storage.FakeBackend{}
fake.SeedObject(storage.FakeObject{Key: audioKey, Data: []byte("audio")})
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
cacheRoot := filepath.Join(workspaceRoot, "cache")
cachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", audioKey)
if err != nil {
t.Fatalf("S3AudioCachePath() error = %v", err)
}
otherCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "dnd/campaigns/other/sessions/2026-05-03/audio/bob.flac")
if err != nil {
t.Fatalf("S3AudioCachePath() error = %v", err)
}
mustWriteTestFile(t, cachePath, "cached-audio")
mustWriteTestFile(t, otherCachePath, "other-audio")
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03", "--clear-cache"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
cleanAssertMissing(t, cachePath)
cleanAssertExists(t, otherCachePath)
if storeInitCalls != 1 {
t.Fatalf("object store init calls = %d, want 1", storeInitCalls)
}
}
func TestExecuteCleanLocalAudioClearCacheIsNoop(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03", "--clear-cache"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if !strings.Contains(stdout.String(), "Cache: skipped (session does not use audio_s3)") {
t.Fatalf("stdout = %q, want local audio cache no-op", stdout.String())
}
}
func TestExecuteCleanAllDeletesWorkAndSpoolContentsButPreservesCache(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
workRoot := filepath.Join(workspaceRoot, "work")
spoolRoot := filepath.Join(workspaceRoot, "spool")
cachePath := filepath.Join(workspaceRoot, "cache", "keep.txt")
mustWriteTestFile(t, filepath.Join(workRoot, "sample-campaign", "2026-05-03", "manifest.json"), "{}")
mustWriteTestFile(t, filepath.Join(spoolRoot, "sample-campaign", "2026-05-03", "run-1", "audio", "alice.flac"), "audio")
mustWriteTestFile(t, cachePath, "cache")
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clean", "--config", pipelinePath, "--all"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
cleanAssertMissing(t, workRoot)
cleanAssertExists(t, spoolRoot)
cleanAssertMissing(t, filepath.Join(spoolRoot, "sample-campaign"))
cleanAssertExists(t, cachePath)
}
func TestExecuteCleanAllClearCacheRemovesS3AudioNamespaceOnly(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
cacheRoot := filepath.Join(workspaceRoot, "cache")
audioCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
if err != nil {
t.Fatalf("S3AudioCachePath() error = %v", err)
}
otherCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "other-root/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
if err != nil {
t.Fatalf("S3AudioCachePath() error = %v", err)
}
mustWriteTestFile(t, audioCachePath, "cached-audio")
mustWriteTestFile(t, otherCachePath, "other-cache")
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clean", "--config", pipelinePath, "--all", "--clear-cache"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
cleanAssertMissing(t, audioCachePath)
cleanAssertExists(t, otherCachePath)
}
func TestExecuteCleanAllRejectsSessionScopedFlags(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--all"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "--all cannot be combined") {
t.Fatalf("stderr = %q, want --all conflict", stderr.String())
}
}
func TestCleanRequiresSessionID(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clean"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "--session-id is required unless --all is set") {
t.Fatalf("stderr = %q, want missing session-id", stderr.String())
}
}
func TestCleanRejectsUnsafeTargets(t *testing.T) {
root := t.TempDir()
outside := t.TempDir()
if err := reportCleanScopedDir(&bytes.Buffer{}, root, filepath.Join(outside, "target"), "test.outside", false); err == nil {
t.Fatal("outside target error = nil, want error")
}
if err := reportCleanScopedDir(&bytes.Buffer{}, root, root, "test.root", false); err == nil {
t.Fatal("root target error = nil, want error")
}
filePath := filepath.Join(root, "file.txt")
mustWriteTestFile(t, filePath, "file")
if err := reportCleanScopedDir(&bytes.Buffer{}, root, filePath, "test.file", false); err == nil {
t.Fatal("file target error = nil, want error")
}
symlinkPath := filepath.Join(root, "link")
if err := os.Symlink(filepath.Join(root, "missing"), symlinkPath); err != nil {
t.Fatalf("Symlink() error = %v", err)
}
if err := reportCleanScopedDir(&bytes.Buffer{}, root, symlinkPath, "test.symlink", false); err == nil {
t.Fatal("symlink target error = nil, want error")
}
}
func TestClearIsNotCommandAlias(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clear"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), `unknown command: "clear"`) {
t.Fatalf("stderr = %q, want unknown clear command", stderr.String())
}
}
func cleanAssertExists(t *testing.T, path string) {
t.Helper()
if _, err := os.Stat(path); err != nil {
t.Fatalf("expected %q to exist: %v", path, err)
}
}
func cleanAssertMissing(t *testing.T, path string) {
t.Helper()
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("expected %q to be missing, stat err=%v", path, err)
}
}

View File

@@ -7,7 +7,7 @@ import (
"strings" "strings"
) )
var supportedCommands = []string{"run", "plan", "status", "resume", "run-stage", "restore"} var supportedCommands = []string{"run", "plan", "status", "resume", "run-stage", "analyze", "restore", "session", "artifacts", "locks", "clean"}
// Execute dispatches CLI commands and returns a process exit code. // Execute dispatches CLI commands and returns a process exit code.
func Execute(args []string, stdout, stderr io.Writer) int { func Execute(args []string, stdout, stderr io.Writer) int {
@@ -32,8 +32,18 @@ func Execute(args []string, stdout, stderr io.Writer) int {
err = Resume(ctx, cmdArgs, stdout) err = Resume(ctx, cmdArgs, stdout)
case "run-stage": case "run-stage":
err = RunStage(ctx, cmdArgs, stdout) err = RunStage(ctx, cmdArgs, stdout)
case "analyze":
err = Analyze(ctx, cmdArgs, stdout)
case "restore": case "restore":
err = Restore(ctx, cmdArgs, stdout) err = Restore(ctx, cmdArgs, stdout)
case "session":
err = Session(ctx, cmdArgs, stdout)
case "artifacts":
err = Artifacts(ctx, cmdArgs, stdout)
case "locks":
err = Locks(ctx, cmdArgs, stdout)
case "clean":
err = Clean(ctx, cmdArgs, stdout)
default: default:
fmt.Fprintf(stderr, "unknown command: %q\n\n", cmd) fmt.Fprintf(stderr, "unknown command: %q\n\n", cmd)
printUsage(stderr) printUsage(stderr)

View File

@@ -24,7 +24,7 @@ func TestExecuteValidCommands(t *testing.T) {
})) }))
defer srv.Close() defer srv.Close()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
manifestPath := writeManifestPathForExecute(t) manifestPath := writeManifestPathForExecute(t)
cases := []struct { cases := []struct {
@@ -32,11 +32,11 @@ func TestExecuteValidCommands(t *testing.T) {
args []string args []string
wantOut string wantOut string
}{ }{
{name: "run", args: []string{"run", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=9 skipped=0; manifest="}, {name: "run", args: []string{"run", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=9 skipped=0; manifest="},
{name: "plan", args: []string{"plan", "--config", pipelinePath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nanalyze: skip\narchive: skip\nnotify: skip"}, {name: "plan", args: []string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nanalyze: skip\narchive: skip\nnotify: skip"},
{name: "status", args: []string{"status", "--manifest", manifestPath}, wantOut: "session_id: 2026-05-03"}, {name: "status", args: []string{"status", "--manifest", manifestPath}, wantOut: "session_id: 2026-05-03"},
{name: "resume", args: []string{"resume", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio resume: session 2026-05-03 has no remaining stages"}, {name: "resume", args: []string{"resume", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, wantOut: "narratio resume: session 2026-05-03 has no remaining stages"},
{name: "run-stage", args: []string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "polish"}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="}, {name: "run-stage", args: []string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "polish"}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
} }
for _, tc := range cases { for _, tc := range cases {
@@ -94,12 +94,12 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
func TestExecuteRunStageUnknownFails(t *testing.T) { func TestExecuteRunStageUnknownFails(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe") pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "unknown"}, &stdout, &stderr) code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "unknown"}, &stdout, &stderr)
if code == 0 { if code == 0 {
t.Fatal("exit code = 0, want non-zero") t.Fatal("exit code = 0, want non-zero")
} }
@@ -110,14 +110,14 @@ func TestExecuteRunStageUnknownFails(t *testing.T) {
func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) { func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe") pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03") workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1}]}`) mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1}]}`)
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "normalize"}, &stdout, &stderr) code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "normalize"}, &stdout, &stderr)
if code != 0 { if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String()) t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
} }
@@ -136,19 +136,19 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) {
})) }))
defer srv.Close() defer srv.Close()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "prepare"}, &stdout, &stderr) code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "prepare"}, &stdout, &stderr)
if code != 0 { if code != 0 {
t.Fatalf("prepare exit code = %d, want 0; stderr=%q", code, stderr.String()) t.Fatalf("prepare exit code = %d, want 0; stderr=%q", code, stderr.String())
} }
stdout.Reset() stdout.Reset()
stderr.Reset() stderr.Reset()
code = Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--force", "transcribe"}, &stdout, &stderr) code = Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "transcribe"}, &stdout, &stderr)
if code != 0 { if code != 0 {
t.Fatalf("transcribe exit code = %d, want 0; stderr=%q", code, stderr.String()) t.Fatalf("transcribe exit code = %d, want 0; stderr=%q", code, stderr.String())
} }
@@ -188,6 +188,7 @@ func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1") t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1")
pipelinePath := filepath.Join(configDir, "pipeline.yml") pipelinePath := filepath.Join(configDir, "pipeline.yml")
campaignPath := writeAppTestCampaignConfig(t, configDir)
sessionPath := filepath.Join(configDir, "session.yml") sessionPath := filepath.Join(configDir, "session.yml")
pipelineYAML := `workspace: pipelineYAML := `workspace:
root: ` + workspaceRoot + ` root: ` + workspaceRoot + `
@@ -239,7 +240,7 @@ inputs:
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &stdout, &stderr) code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "polish"}, &stdout, &stderr)
if code != 0 { if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String()) t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
} }
@@ -252,6 +253,7 @@ func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
configDir := t.TempDir() configDir := t.TempDir()
pipelinePath := filepath.Join(configDir, "pipeline.yml") pipelinePath := filepath.Join(configDir, "pipeline.yml")
campaignPath := writeAppTestCampaignConfig(t, configDir)
sessionPath := filepath.Join(configDir, "session.yml") sessionPath := filepath.Join(configDir, "session.yml")
pipelineYAML := `workspace: pipelineYAML := `workspace:
@@ -288,7 +290,7 @@ inputs:
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute([]string{"run", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr) code := Execute([]string{"run", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
if code == 0 { if code == 0 {
t.Fatal("exit code = 0, want non-zero") t.Fatal("exit code = 0, want non-zero")
} }
@@ -305,11 +307,14 @@ func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T)
})) }))
defer srv.Close() defer srv.Close()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
originalDefaults := append([]string(nil), config.DefaultPipelineConfigSearchPaths...) originalDefaults := append([]string(nil), config.DefaultPipelineConfigSearchPaths...)
originalCampaignDefaults := append([]string(nil), config.DefaultCampaignConfigSearchPaths...)
config.DefaultPipelineConfigSearchPaths = []string{pipelinePath} config.DefaultPipelineConfigSearchPaths = []string{pipelinePath}
config.DefaultCampaignConfigSearchPaths = []string{campaignPath}
defer func() { defer func() {
config.DefaultPipelineConfigSearchPaths = originalDefaults config.DefaultPipelineConfigSearchPaths = originalDefaults
config.DefaultCampaignConfigSearchPaths = originalCampaignDefaults
}() }()
var stdout bytes.Buffer var stdout bytes.Buffer
@@ -323,6 +328,33 @@ func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T)
} }
} }
func TestExecuteMissingCampaignConfigReportsSearchedPaths(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, _, sessionPath := writeValidConfigFiles(t, workspaceRoot)
missingCampaignPath := filepath.Join(t.TempDir(), "campaign.yml")
originalCampaignDefaults := append([]string(nil), config.DefaultCampaignConfigSearchPaths...)
config.DefaultCampaignConfigSearchPaths = []string{missingCampaignPath}
defer func() {
config.DefaultCampaignConfigSearchPaths = originalCampaignDefaults
}()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"run", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if stdout.Len() != 0 {
t.Fatalf("stdout = %q, want empty", stdout.String())
}
if !strings.Contains(stderr.String(), "no campaign config path provided and no default campaign config found; searched:") {
t.Fatalf("stderr = %q, want campaign discovery failure", stderr.String())
}
if !strings.Contains(stderr.String(), "pass --campaign") {
t.Fatalf("stderr = %q, want explicit campaign guidance", stderr.String())
}
}
func TestExecuteInvalidCommand(t *testing.T) { func TestExecuteInvalidCommand(t *testing.T) {
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
@@ -359,11 +391,12 @@ func TestExecuteMissingCommand(t *testing.T) {
} }
} }
func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...string) (string, string) { func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...string) (string, string, string) {
t.Helper() t.Helper()
dir := t.TempDir() dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml") pipelinePath := filepath.Join(dir, "pipeline.yml")
campaignPath := filepath.Join(dir, "campaign.yml")
sessionPath := filepath.Join(dir, "session.yml") sessionPath := filepath.Join(dir, "session.yml")
url := "https://example.com/transcribe" url := "https://example.com/transcribe"
if len(transcribeURL) > 0 && strings.TrimSpace(transcribeURL[0]) != "" { if len(transcribeURL) > 0 && strings.TrimSpace(transcribeURL[0]) != "" {
@@ -377,6 +410,10 @@ func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...
pipelineYAML := `workspace: pipelineYAML := `workspace:
root: ` + workspaceRoot + ` root: ` + workspaceRoot + `
cache:
root: ` + filepath.Join(workspaceRoot, "cache") + `
spool:
root: ` + filepath.Join(workspaceRoot, "spool") + `
storage: storage:
backend: s3 backend: s3
s3: s3:
@@ -407,9 +444,11 @@ notification:
` `
sessionYAML := `session_id: 2026-05-03 sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign
inputs: inputs:
audio_dir: ./audio audio_dir: ./audio
`
campaignYAML := `campaign: sample-campaign
inputs:
speakers_file: ./speakers.yml speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml glossary_file: ./glossary.yml
@@ -418,6 +457,9 @@ inputs:
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil { if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
t.Fatalf("write pipeline config: %v", err) t.Fatalf("write pipeline config: %v", err)
} }
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
t.Fatalf("write campaign config: %v", err)
}
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil { if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session config: %v", err) t.Fatalf("write session config: %v", err)
} }
@@ -427,7 +469,22 @@ inputs:
mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n") mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n")
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes") mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
return pipelinePath, sessionPath return pipelinePath, campaignPath, sessionPath
}
func writeAppTestCampaignConfig(t *testing.T, dir string) string {
t.Helper()
campaignPath := filepath.Join(dir, "campaign.yml")
campaignYAML := `campaign: sample-campaign
inputs:
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
t.Fatalf("write campaign.yml: %v", err)
}
return campaignPath
} }
func writeManifestPathForExecute(t *testing.T) string { func writeManifestPathForExecute(t *testing.T) string {

View File

@@ -0,0 +1,133 @@
package app
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, sessionFlag string, sessionOpts config.SessionLoadOptions) (*config.Config, error) {
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelineFlag)
if err != nil {
return nil, err
}
resolvedCampaignPath, err := resolveCampaignConfigPath(campaignFlag)
if err != nil {
return nil, err
}
if explicitSession := strings.TrimSpace(sessionFlag); explicitSession != "" {
return config.LoadWithSessionOptions(resolvedPipelinePath, resolvedCampaignPath, explicitSession, sessionOpts)
}
discoveredSession, err := discoverSessionConfigPathWithCandidates(config.DefaultSessionConfigSearchPaths)
if err != nil {
return nil, err
}
if discoveredSession.Path != "" {
return config.LoadWithSessionOptions(resolvedPipelinePath, resolvedCampaignPath, discoveredSession.Path, sessionOpts)
}
pipelineCfg, err := config.LoadPipeline(resolvedPipelinePath)
if err != nil {
return nil, err
}
campaignCfg, err := config.LoadCampaign(resolvedCampaignPath)
if err != nil {
return nil, err
}
sessionID := strings.TrimSpace(sessionOpts.SessionID)
if sessionID == "" {
return nil, missingSessionConfigError(discoveredSession.Searched, "remote session loading requires --session-id")
}
sessionPrefix := artifacts.S3SessionPrefix(pipelineCfg.Storage.S3.RootPrefix, campaignCfg.Campaign, sessionID)
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
partialCfg := &config.Config{
Pipeline: pipelineCfg,
Campaign: campaignCfg,
PipelinePath: resolvedPipelinePath,
CampaignPath: resolvedCampaignPath,
}
store, err := newCommandObjectStore(ctx, partialCfg, nil)
if err != nil {
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q unavailable: %v", remoteKey, err))
}
sessionInfo, err := findRemoteSessionConfig(ctx, store, sessionPrefix, remoteKey)
if err != nil {
return nil, missingSessionConfigError(discoveredSession.Searched, err.Error())
}
sessionTempPath, err := downloadRemoteSessionConfig(ctx, store, remoteKey)
if err != nil {
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q download failed: %v", remoteKey, err))
}
sessionBytes, err := os.ReadFile(sessionTempPath)
if err != nil {
return nil, fmt.Errorf("read downloaded remote session %q: %w", sessionTempPath, err)
}
sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(pipelineCfg)+"/"+remoteKey, sessionBytes, sessionOpts)
if err != nil {
return nil, err
}
return config.Resolve(
resolvedPipelinePath,
pipelineCfg,
resolvedCampaignPath,
campaignCfg,
sessionTempPath,
sessionCfg,
config.SessionSource{
Source: "session_config.s3",
LocalPath: sessionTempPath,
S3Bucket: s3BucketName(pipelineCfg),
S3Key: remoteKey,
S3Size: sessionInfo.Size,
S3ETag: sessionInfo.ETag,
SpoolPath: sessionTempPath,
},
)
}
func findRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, sessionPrefix, remoteKey string) (storage.ObjectInfo, error) {
objects, err := store.List(ctx, sessionPrefix)
if err != nil {
return storage.ObjectInfo{}, fmt.Errorf("remote session %q list failed: %w", remoteKey, err)
}
for _, obj := range objects {
if obj.Key == remoteKey {
return obj, nil
}
}
return storage.ObjectInfo{}, fmt.Errorf("remote session %q not found", remoteKey)
}
func downloadRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, remoteKey string) (string, error) {
f, err := os.CreateTemp("", "narratio-session-*.yml")
if err != nil {
return "", fmt.Errorf("create temp file: %w", err)
}
path := f.Name()
if err := f.Close(); err != nil {
return "", fmt.Errorf("close temp file %q: %w", path, err)
}
if err := store.Download(ctx, remoteKey, path); err != nil {
return "", err
}
return filepath.Clean(path), nil
}
func s3BucketName(cfg *config.PipelineConfig) string {
if cfg == nil || cfg.Storage.S3 == nil {
return ""
}
return strings.TrimSpace(cfg.Storage.S3.Bucket)
}

View File

@@ -0,0 +1,21 @@
package app
import (
"context"
"fmt"
"log/slog"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func newCommandObjectStore(ctx context.Context, cfg *config.Config, logger *slog.Logger) (storage.ObjectStore, error) {
if _, err := loadSecretsFromConfig(cfg, logger); err != nil {
return nil, fmt.Errorf("load secrets from files: %w", err)
}
store, err := newObjectStoreFromConfigFn(ctx, cfg)
if err != nil {
return nil, fmt.Errorf("initialize object store backend: %w", err)
}
return store, nil
}

View File

@@ -0,0 +1,165 @@
package app
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func TestNewCommandObjectStoreLoadsSecretsBeforeFactory(t *testing.T) {
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_KEY_ID"
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_SECRET"
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
secretsDir := t.TempDir()
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "loaded-key-id\n")
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "loaded-secret\n")
cfg := commandObjectStoreTestConfig(secretsDir)
fake := &storage.FakeBackend{}
called := false
origStoreFn := newObjectStoreFromConfigFn
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
called = true
if got := os.Getenv(accessKeyEnv); got != "loaded-key-id" {
return nil, errors.New("access key was not loaded before object store init")
}
if got := os.Getenv(secretKeyEnv); got != "loaded-secret" {
return nil, errors.New("secret key was not loaded before object store init")
}
return fake, nil
}
t.Cleanup(func() {
newObjectStoreFromConfigFn = origStoreFn
})
store, err := newCommandObjectStore(context.Background(), cfg, nil)
if err != nil {
t.Fatalf("newCommandObjectStore() error = %v", err)
}
if store != fake {
t.Fatalf("store = %#v, want fake backend", store)
}
if !called {
t.Fatal("object store factory was not called")
}
}
func TestNewCommandObjectStorePreservesExistingEnv(t *testing.T) {
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_KEY_ID"
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_SECRET"
t.Setenv(accessKeyEnv, "existing-key-id")
t.Setenv(secretKeyEnv, "existing-secret")
secretsDir := t.TempDir()
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "file-key-id\n")
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "file-secret\n")
cfg := commandObjectStoreTestConfig(secretsDir)
origStoreFn := newObjectStoreFromConfigFn
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
if got := os.Getenv(accessKeyEnv); got != "existing-key-id" {
return nil, errors.New("existing access key was overwritten")
}
if got := os.Getenv(secretKeyEnv); got != "existing-secret" {
return nil, errors.New("existing secret key was overwritten")
}
return &storage.FakeBackend{}, nil
}
t.Cleanup(func() {
newObjectStoreFromConfigFn = origStoreFn
})
if _, err := newCommandObjectStore(context.Background(), cfg, nil); err != nil {
t.Fatalf("newCommandObjectStore() error = %v", err)
}
}
func TestNewCommandObjectStoreSecretErrorStopsFactory(t *testing.T) {
cfg := commandObjectStoreTestConfig(filepath.Join(t.TempDir(), "missing"))
called := false
origStoreFn := newObjectStoreFromConfigFn
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
called = true
return &storage.FakeBackend{}, nil
}
t.Cleanup(func() {
newObjectStoreFromConfigFn = origStoreFn
})
_, err := newCommandObjectStore(context.Background(), cfg, nil)
if err == nil {
t.Fatal("expected error, got nil")
}
if called {
t.Fatal("object store factory was called after secret load failure")
}
if !strings.Contains(err.Error(), "load secrets from files") {
t.Fatalf("error = %q, want secret loading context", err.Error())
}
}
func TestNewCommandObjectStoreFactoryErrorIsContextual(t *testing.T) {
cfg := commandObjectStoreTestConfig("")
origStoreFn := newObjectStoreFromConfigFn
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
return nil, errors.New("factory boom")
}
t.Cleanup(func() {
newObjectStoreFromConfigFn = origStoreFn
})
_, err := newCommandObjectStore(context.Background(), cfg, nil)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "initialize object store backend") || !strings.Contains(err.Error(), "factory boom") {
t.Fatalf("error = %q, want factory context", err.Error())
}
}
func commandObjectStoreTestConfig(secretsDir string) *config.Config {
cfg := &config.Config{
Pipeline: &config.PipelineConfig{
Storage: config.StorageConfig{
Backend: "s3",
S3: &config.StorageS3Config{
Bucket: "test-bucket",
AccessKeyIDEnv: "NARRATIO_TEST_COMMAND_STORE_KEY_ID",
SecretKeyEnv: "NARRATIO_TEST_COMMAND_STORE_SECRET",
},
},
},
}
if strings.TrimSpace(secretsDir) != "" {
cfg.Pipeline.Secrets = &config.SecretsConfig{EnvDir: secretsDir}
}
return cfg
}
func restoreEnvAfterTest(t *testing.T, names ...string) {
t.Helper()
originals := make(map[string]string, len(names))
present := make(map[string]bool, len(names))
for _, name := range names {
value, ok := os.LookupEnv(name)
originals[name] = value
present[name] = ok
_ = os.Unsetenv(name)
}
t.Cleanup(func() {
for _, name := range names {
if present[name] {
_ = os.Setenv(name, originals[name])
} else {
_ = os.Unsetenv(name)
}
}
})
}

View File

@@ -0,0 +1,958 @@
package app
import (
"context"
"flag"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gopkg.in/yaml.v3"
)
type commonConfigFlags struct {
pipelinePath string
campaignPath string
sessionPath string
sessionID string
previousSessionID string
}
type finding struct {
Severity string
Category string
Message string
}
type findingError struct {
count int
}
func (e findingError) Error() string {
return fmt.Sprintf("%d validation error(s)", e.count)
}
func addCommonConfigFlags(fs *flag.FlagSet, flags *commonConfigFlags) {
fs.StringVar(&flags.pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&flags.campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
fs.StringVar(&flags.sessionPath, "session", "", "path to session.yml")
fs.StringVar(&flags.sessionID, "session-id", "", "session identifier for session.yml templates")
fs.StringVar(&flags.previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
}
func (f commonConfigFlags) sessionOptions() config.SessionLoadOptions {
return config.SessionLoadOptions{
SessionID: f.sessionID,
PreviousSessionID: f.previousSessionID,
}
}
// Session dispatches session helper subcommands.
func Session(ctx context.Context, args []string, out io.Writer) error {
if len(args) == 0 {
return fmt.Errorf("session: expected subcommand: validate|init")
}
switch args[0] {
case "validate":
return SessionValidate(ctx, args[1:], out)
case "init":
return SessionInit(ctx, args[1:], out)
default:
return fmt.Errorf("session: unknown subcommand %q", args[0])
}
}
// Artifacts dispatches artifact helper subcommands.
func Artifacts(ctx context.Context, args []string, out io.Writer) error {
if len(args) == 0 {
return fmt.Errorf("artifacts: expected subcommand: list")
}
switch args[0] {
case "list":
return ArtifactsList(ctx, args[1:], out)
default:
return fmt.Errorf("artifacts: unknown subcommand %q", args[0])
}
}
// SessionValidate performs a read-only session preflight.
func SessionValidate(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("session validate", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var flags commonConfigFlags
addCommonConfigFlags(fs, &flags)
if err := fs.Parse(args); err != nil {
return fmt.Errorf("session validate: invalid flags: %w", err)
}
if fs.NArg() != 0 {
return fmt.Errorf("session validate: unexpected positional arguments")
}
findings := []finding{}
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.sessionPath, flags.sessionOptions())
if err != nil {
findings = append(findings, errorFinding("config", err.Error()))
return renderFindings(out, "", "", findings)
}
if err := config.Validate(cfg); err != nil {
findings = append(findings, errorFinding("config", err.Error()))
} else {
findings = append(findings, okFinding("config", "resolved pipeline, campaign, and session config"))
}
findings = append(findings, okFinding("session", fmt.Sprintf("session source: %s", sessionSourceSummary(cfg))))
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
findings = append(findings, validateStableInputFindings(cfg)...)
findings = append(findings, validateLocalAudioFindings(cfg)...)
store, storeErr := objectStoreIfConfigured(ctx, cfg)
if storeErr != nil {
findings = append(findings, errorFinding("storage", storeErr.Error()))
}
if cfg.Session.Inputs.AudioS3 != nil {
if storeErr != nil {
findings = append(findings, errorFinding("audio", "remote audio cannot be checked because storage is unavailable"))
} else {
findings = append(findings, validateRemoteAudioFinding(ctx, cfg, store))
}
}
requirements := artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg))
if len(requirements) == 0 {
findings = append(findings, okFinding("previous", "no previous-session artifacts required"))
} else if strings.TrimSpace(cfg.Session.PreviousSessionID) == "" {
findings = append(findings, errorFinding("previous", "previous_session_id is required by configured previous-session artifacts"))
} else if storeErr != nil {
findings = append(findings, errorFinding("previous", "previous-session artifacts cannot be checked because storage is unavailable"))
} else {
findings = append(findings, validatePreviousArtifactFindings(ctx, cfg, store, requirements)...)
}
locks, lockErr := loadEffectiveLocks(ctx, cfg, store)
if lockErr != nil {
findings = append(findings, errorFinding("locks", lockErr.Error()))
} else if len(locks.All) == 0 {
findings = append(findings, okFinding("locks", "no effective archive locks"))
} else {
for _, lock := range locks.All {
findings = append(findings, warnFinding("locks", fmt.Sprintf("%s locked: %s", lock.Source, strings.TrimSpace(lock.Reason))))
}
}
if paths.ManifestPath != "" {
findings = append(findings, infoFinding("workspace", "manifest path: "+paths.ManifestPath))
}
return renderFindings(out, cfg.Session.Campaign, cfg.Session.SessionID, findings)
}
// Status reports either a requested manifest or effective local/remote session state.
func Status(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("status", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var manifestPath string
var flags commonConfigFlags
fs.StringVar(&manifestPath, "manifest", "", "path to manifest.json")
addCommonConfigFlags(fs, &flags)
if err := fs.Parse(args); err != nil {
return fmt.Errorf("status: invalid flags: %w", err)
}
if fs.NArg() != 0 {
return fmt.Errorf("status: unexpected positional arguments")
}
if strings.TrimSpace(manifestPath) != "" {
return statusManifest(ctx, manifestPath, out)
}
if flags.pipelinePath == "" && flags.campaignPath == "" && flags.sessionPath == "" && flags.sessionID == "" && flags.previousSessionID == "" {
return fmt.Errorf("status: --manifest is required")
}
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.sessionPath, flags.sessionOptions())
if err != nil {
return fmt.Errorf("status: %w", err)
}
if err := config.Validate(cfg); err != nil {
return fmt.Errorf("status: %w", err)
}
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
fmt.Fprintf(out, "Session: %s\n", cfg.Session.SessionID)
fmt.Fprintf(out, "Campaign: %s\n", cfg.Session.Campaign)
fmt.Fprintf(out, "Workspace: %s\n", paths.Root)
fmt.Fprintf(out, "Session config: %s\n", sessionSourceSummary(cfg))
if m, err := loadLocalManifest(ctx, paths.ManifestPath); err != nil {
fmt.Fprintf(out, "Local manifest: error: %v\n", err)
} else if m == nil {
fmt.Fprintln(out, "Local manifest: missing")
} else {
fmt.Fprintf(out, "Local manifest: %s\n", paths.ManifestPath)
writeStageStatuses(out, m)
}
store, storeErr := objectStoreIfConfigured(ctx, cfg)
if storeErr != nil {
fmt.Fprintf(out, "Remote archive: unavailable: %v\n", storeErr)
} else if store != nil {
current, err := discoverRemoteCurrentStateFn(ctx, cfg, store)
if err != nil {
fmt.Fprintf(out, "Remote archive: missing or unavailable: %v\n", err)
} else {
fmt.Fprintf(out, "Remote archive: current run %s\n", current.RunID)
fmt.Fprintf(out, "Remote manifest: %s\n", current.CurrentManifestKey)
}
}
locks, err := loadEffectiveLocks(ctx, cfg, store)
if catalog, catalogErr := buildHelperArtifactCatalog(cfg); catalogErr != nil {
fmt.Fprintf(out, "Remote outputs: error: %v\n", catalogErr)
} else if storeErr == nil {
catalogLocks := locks
if err != nil {
catalogLocks = &effectiveLocks{
Static: staticArchiveLocks(cfg),
All: staticArchiveLocks(cfg),
}
}
promotedRemoteState := map[string]string{}
if store != nil {
promotedRemoteState = remotePromotionAvailability(ctx, cfg, store, catalog)
}
fmt.Fprintln(out, "Remote outputs:")
writeArtifactList(out, cfg, catalog, catalogLocks, promotedRemoteState)
}
if err != nil {
fmt.Fprintf(out, "Archive locks: error: %v\n", err)
} else {
writeLocks(out, cfg, locks)
}
fmt.Fprintln(out, "Next actions:")
fmt.Fprintf(out, "- narratio session validate --session-id %s\n", cfg.Session.SessionID)
fmt.Fprintf(out, "- narratio restore --session-id %s --dry-run\n", cfg.Session.SessionID)
return nil
}
func statusManifest(ctx context.Context, manifestPath string, out io.Writer) error {
store := &manifest.LocalStore{}
m, err := store.Load(ctx, manifestPath)
if err != nil {
return fmt.Errorf("status: %w", err)
}
if _, err := fmt.Fprintf(out, "session_id: %s\n", m.SessionID); err != nil {
return err
}
if _, err := fmt.Fprintf(out, "updated_at: %s\n", m.UpdatedAt.UTC().Format("2006-01-02T15:04:05Z07:00")); err != nil {
return err
}
writeStageStatuses(out, m)
return nil
}
// SessionInit creates a local or remote session.yml skeleton.
func SessionInit(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("session init", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var pipelinePath, campaignPath, sessionID, previousSessionID, date, title, output, audioS3Prefix, audioDir string
var remote, force bool
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml")
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier")
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier")
fs.StringVar(&date, "date", "", "session date")
fs.StringVar(&title, "title", "", "session title")
fs.StringVar(&output, "output", "", "local output session.yml path")
fs.StringVar(&audioS3Prefix, "audio-s3-prefix", "", "session audio S3 prefix")
fs.StringVar(&audioDir, "audio-dir", "", "local audio directory")
fs.BoolVar(&remote, "remote", false, "write session.yml to S3 session prefix")
fs.BoolVar(&force, "force", false, "overwrite existing target")
if err := fs.Parse(args); err != nil {
return fmt.Errorf("session init: invalid flags: %w", err)
}
if fs.NArg() != 0 {
return fmt.Errorf("session init: unexpected positional arguments")
}
if strings.TrimSpace(pipelinePath) == "" || strings.TrimSpace(campaignPath) == "" || strings.TrimSpace(sessionID) == "" {
return fmt.Errorf("session init: --config, --campaign, and --session-id are required")
}
if (strings.TrimSpace(output) == "") == !remote {
return fmt.Errorf("session init: specify exactly one target: --output <path> or --remote")
}
if strings.TrimSpace(audioDir) != "" && strings.TrimSpace(audioS3Prefix) != "" {
return fmt.Errorf("session init: --audio-dir and --audio-s3-prefix are mutually exclusive")
}
resolvedPipeline, err := resolvePipelineConfigPath(pipelinePath)
if err != nil {
return fmt.Errorf("session init: %w", err)
}
resolvedCampaign, err := resolveCampaignConfigPath(campaignPath)
if err != nil {
return fmt.Errorf("session init: %w", err)
}
pipelineCfg, err := config.LoadPipeline(resolvedPipeline)
if err != nil {
return fmt.Errorf("session init: %w", err)
}
campaignCfg, err := config.LoadCampaign(resolvedCampaign)
if err != nil {
return fmt.Errorf("session init: %w", err)
}
data, err := buildSessionYAML(campaignCfg.Campaign, sessionID, previousSessionID, date, title, audioS3Prefix, audioDir)
if err != nil {
return fmt.Errorf("session init: %w", err)
}
label := strings.TrimSpace(output)
if label == "" {
label = "remote session.yml"
}
sessionCfg, err := config.LoadSessionBytesWithOptions(label, data, config.SessionLoadOptions{
SessionID: sessionID,
PreviousSessionID: previousSessionID,
})
if err != nil {
return fmt.Errorf("session init: %w", err)
}
cfg, err := config.Resolve(resolvedPipeline, pipelineCfg, resolvedCampaign, campaignCfg, label, sessionCfg, config.SessionSource{Source: "session_config", LocalPath: label})
if err != nil {
return fmt.Errorf("session init: %w", err)
}
if err := config.Validate(cfg); err != nil {
return fmt.Errorf("session init: %w", err)
}
if !remote {
if err := writeLocalFile(output, data, force); err != nil {
return fmt.Errorf("session init: %w", err)
}
_, err := fmt.Fprintf(out, "narratio session init: wrote %s\n", filepath.Clean(output))
return err
}
store, err := newCommandObjectStore(ctx, cfg, nil)
if err != nil {
return fmt.Errorf("session init: %w", err)
}
sessionPrefix := artifacts.S3SessionPrefix(pipelineCfg.Storage.S3.RootPrefix, campaignCfg.Campaign, sessionID)
key := artifacts.S3SessionConfigKey(sessionPrefix)
exists, err := store.Exists(ctx, key)
if err != nil {
return fmt.Errorf("session init: check remote session %q: %w", key, err)
}
if exists && !force {
return fmt.Errorf("session init: remote session %q already exists; pass --force to overwrite", key)
}
tmp, err := os.CreateTemp("", "narratio-session-init-*.yml")
if err != nil {
return fmt.Errorf("session init: create temp file: %w", err)
}
tmpPath := tmp.Name()
defer func() { _ = os.Remove(tmpPath) }()
if _, err := tmp.Write(data); err != nil {
_ = tmp.Close()
return fmt.Errorf("session init: write temp file: %w", err)
}
if err := tmp.Close(); err != nil {
return fmt.Errorf("session init: close temp file: %w", err)
}
if _, err := store.Upload(ctx, tmpPath, key, storage.UploadOptions{ContentType: "application/x-yaml; charset=utf-8"}); err != nil {
return fmt.Errorf("session init: upload remote session %q: %w", key, err)
}
_, err = fmt.Fprintf(out, "narratio session init: wrote s3://%s/%s\n", s3BucketName(pipelineCfg), key)
return err
}
// ArtifactsList lists effective artifact sources.
func ArtifactsList(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("artifacts list", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var flags commonConfigFlags
var remote bool
addCommonConfigFlags(fs, &flags)
fs.BoolVar(&remote, "remote", false, "inspect remote archive availability")
if err := fs.Parse(args); err != nil {
return fmt.Errorf("artifacts list: invalid flags: %w", err)
}
if fs.NArg() != 0 {
return fmt.Errorf("artifacts list: unexpected positional arguments")
}
cfg, store, locks, _, err := loadHelperContext(ctx, flags, remote)
if err != nil {
return fmt.Errorf("artifacts list: %w", err)
}
catalog, err := buildHelperArtifactCatalog(cfg)
if err != nil {
return fmt.Errorf("artifacts list: %w", err)
}
promotedRemoteState := map[string]string{}
if remote && store != nil {
promotedRemoteState = remotePromotionAvailability(ctx, cfg, store, catalog)
}
writeArtifactList(out, cfg, catalog, locks, promotedRemoteState)
return nil
}
// Locks dispatches archive lock list and mutation helpers.
func Locks(ctx context.Context, args []string, out io.Writer) error {
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
switch args[0] {
case "add":
return LocksAdd(ctx, args[1:], out)
case "remove":
return LocksRemove(ctx, args[1:], out)
default:
return fmt.Errorf("locks: unknown subcommand %q", args[0])
}
}
return LocksList(ctx, args, out)
}
// LocksList lists effective archive locks.
func LocksList(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("locks", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var flags commonConfigFlags
addCommonConfigFlags(fs, &flags)
if err := fs.Parse(args); err != nil {
return fmt.Errorf("locks: invalid flags: %w", err)
}
if fs.NArg() != 0 {
return fmt.Errorf("locks: unexpected positional arguments")
}
if strings.TrimSpace(flags.sessionID) == "" {
return fmt.Errorf("locks: --session-id is required")
}
cfg, _, locks, _, err := loadHelperContext(ctx, flags, true)
if err != nil {
return fmt.Errorf("locks: %w", err)
}
writeLocks(out, cfg, locks)
return nil
}
// LocksAdd adds or updates one remote lock.
func LocksAdd(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("locks add", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var flags commonConfigFlags
var reason string
var force bool
addCommonConfigFlags(fs, &flags)
fs.StringVar(&reason, "reason", "", "lock reason")
fs.BoolVar(&force, "force", false, "update existing remote lock")
if err := fs.Parse(args); err != nil {
return fmt.Errorf("locks add: invalid flags: %w", err)
}
if fs.NArg() != 1 {
return fmt.Errorf("locks add: expected exactly one source id")
}
if strings.TrimSpace(flags.sessionID) == "" {
return fmt.Errorf("locks add: --session-id is required")
}
source := strings.TrimSpace(fs.Arg(0))
cfg, store, locks, _, err := loadHelperContext(ctx, flags, true)
if err != nil {
return fmt.Errorf("locks add: %w", err)
}
if _, err := config.ValidateArchiveLockRules([]config.ArchiveLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks add"); err != nil {
return fmt.Errorf("locks add: %w", err)
}
if _, ok := lockSourceSet(locks.Static)[source]; ok {
return fmt.Errorf("locks add: source %q is locked by pipeline config and cannot be modified remotely", source)
}
remoteSet := lockSourceSet(locks.Remote)
if _, exists := remoteSet[source]; exists && !force {
return fmt.Errorf("locks add: remote lock for %q already exists; pass --force to update", source)
}
remoteSet[source] = config.ArchiveLockRule{Source: source, Reason: strings.TrimSpace(reason)}
remoteLocks := lockMapValues(remoteSet)
if _, err := config.ValidateArchiveLockRules(remoteLocks, cfg.Pipeline.Scriptorium, "locks"); err != nil {
return fmt.Errorf("locks add: %w", err)
}
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.ArchiveLockStore{Locks: remoteLocks}); err != nil {
return fmt.Errorf("locks add: %w", err)
}
_, err = fmt.Fprintf(out, "narratio locks add: locked %s\n", source)
return err
}
// LocksRemove removes one remote lock.
func LocksRemove(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("locks remove", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var flags commonConfigFlags
addCommonConfigFlags(fs, &flags)
if err := fs.Parse(args); err != nil {
return fmt.Errorf("locks remove: invalid flags: %w", err)
}
if fs.NArg() != 1 {
return fmt.Errorf("locks remove: expected exactly one source id")
}
if strings.TrimSpace(flags.sessionID) == "" {
return fmt.Errorf("locks remove: --session-id is required")
}
source := strings.TrimSpace(fs.Arg(0))
cfg, store, locks, _, err := loadHelperContext(ctx, flags, true)
if err != nil {
return fmt.Errorf("locks remove: %w", err)
}
if _, err := config.ValidateArchiveLockRules([]config.ArchiveLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks remove"); err != nil {
return fmt.Errorf("locks remove: %w", err)
}
remoteSet := lockSourceSet(locks.Remote)
if _, ok := remoteSet[source]; !ok {
if _, static := lockSourceSet(locks.Static)[source]; static {
return fmt.Errorf("locks remove: source %q is locked by pipeline config and cannot be unlocked remotely", source)
}
return fmt.Errorf("locks remove: remote lock for %q does not exist", source)
}
delete(remoteSet, source)
remoteLocks := lockMapValues(remoteSet)
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.ArchiveLockStore{Locks: remoteLocks}); err != nil {
return fmt.Errorf("locks remove: %w", err)
}
_, err = fmt.Fprintf(out, "narratio locks remove: unlocked %s\n", source)
return err
}
func loadHelperContext(ctx context.Context, flags commonConfigFlags, needStore bool) (*config.Config, storage.ObjectStore, *effectiveLocks, *manifest.Manifest, error) {
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.sessionPath, flags.sessionOptions())
if err != nil {
return nil, nil, nil, nil, err
}
if err := config.Validate(cfg); err != nil {
return nil, nil, nil, nil, err
}
var store storage.ObjectStore
if needStore {
store, err = newCommandObjectStore(ctx, cfg, nil)
if err != nil {
return nil, nil, nil, nil, err
}
} else {
store, _ = objectStoreIfConfigured(ctx, cfg)
}
locks, err := loadEffectiveLocks(ctx, cfg, store)
if err != nil {
return nil, nil, nil, nil, err
}
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
m, err := loadLocalManifest(ctx, paths.ManifestPath)
if err != nil {
return nil, nil, nil, nil, err
}
return cfg, store, locks, m, nil
}
func objectStoreIfConfigured(ctx context.Context, cfg *config.Config) (storage.ObjectStore, error) {
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Storage.S3 == nil || strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) == "" {
return nil, nil
}
store, err := newCommandObjectStore(ctx, cfg, nil)
if err != nil {
return nil, err
}
return store, nil
}
func buildSessionYAML(campaign, sessionID, previousSessionID, date, title, audioS3Prefix, audioDir string) ([]byte, error) {
if strings.TrimSpace(date) == "" && regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`).MatchString(strings.TrimSpace(sessionID)) {
date = strings.TrimSpace(sessionID)
}
type audioS3 struct {
Prefix string `yaml:"prefix"`
}
type inputs struct {
AudioDir string `yaml:"audio_dir,omitempty"`
AudioS3 *audioS3 `yaml:"audio_s3,omitempty"`
}
type sessionYAML struct {
Campaign string `yaml:"campaign"`
SessionID string `yaml:"session_id"`
PreviousSessionID string `yaml:"previous_session_id,omitempty"`
Date string `yaml:"date,omitempty"`
Title string `yaml:"title,omitempty"`
Inputs inputs `yaml:"inputs"`
}
in := inputs{AudioDir: strings.TrimSpace(audioDir)}
if in.AudioDir == "" {
prefix := strings.TrimSpace(audioS3Prefix)
if prefix == "" {
prefix = "audio/"
}
in.AudioS3 = &audioS3{Prefix: prefix}
}
data, err := yaml.Marshal(sessionYAML{
Campaign: strings.TrimSpace(campaign),
SessionID: strings.TrimSpace(sessionID),
PreviousSessionID: strings.TrimSpace(previousSessionID),
Date: strings.TrimSpace(date),
Title: strings.TrimSpace(title),
Inputs: in,
})
if err != nil {
return nil, err
}
return data, nil
}
func renderFindings(out io.Writer, campaign, sessionID string, findings []finding) error {
if campaign != "" || sessionID != "" {
fmt.Fprintf(out, "Campaign: %s\n", campaign)
fmt.Fprintf(out, "Session: %s\n\n", sessionID)
}
errorsCount := 0
for _, f := range findings {
if f.Severity == "ERROR" {
errorsCount++
}
fmt.Fprintf(out, "%-5s %-10s %s\n", f.Severity, f.Category, f.Message)
}
if errorsCount > 0 {
return findingError{count: errorsCount}
}
return nil
}
func okFinding(category, msg string) finding { return finding{"OK", category, msg} }
func infoFinding(category, msg string) finding { return finding{"INFO", category, msg} }
func warnFinding(category, msg string) finding { return finding{"WARN", category, msg} }
func errorFinding(category, msg string) finding { return finding{"ERROR", category, msg} }
func sessionSourceSummary(cfg *config.Config) string {
source := cfg.SessionSource.Source
if source == "" {
source = "session_config"
}
if cfg.SessionSource.S3Key != "" {
return source + " " + cfg.SessionSource.S3Key
}
return source + " " + cfg.SessionPath
}
func validateStableInputFindings(cfg *config.Config) []finding {
items := []struct {
name string
in config.ResolvedInputFile
}{
{"speakers", cfg.StableInputs.SpeakersFile},
{"autocorrect", cfg.StableInputs.AutocorrectFile},
{"glossary", cfg.StableInputs.GlossaryFile},
}
out := make([]finding, 0, len(items))
for _, item := range items {
path, err := resolveHelperConfigRelativePath(item.in)
if err != nil {
out = append(out, errorFinding("inputs", item.name+": "+err.Error()))
continue
}
if _, err := os.Stat(path); err != nil {
out = append(out, errorFinding("inputs", fmt.Sprintf("%s missing: %v", item.name, err)))
} else {
out = append(out, okFinding("inputs", item.name+": "+path))
}
}
return out
}
func resolveHelperConfigRelativePath(input config.ResolvedInputFile) (string, error) {
if strings.TrimSpace(input.ConfigPath) == "" {
return "", fmt.Errorf("source config path is required")
}
path := strings.TrimSpace(input.Path)
if path == "" {
return "", fmt.Errorf("path is required")
}
if filepath.IsAbs(path) {
return filepath.Clean(path), nil
}
return filepath.Clean(filepath.Join(filepath.Dir(input.ConfigPath), path)), nil
}
func validateLocalAudioFindings(cfg *config.Config) []finding {
if cfg.Session.Inputs.AudioS3 != nil {
return nil
}
audioDir := strings.TrimSpace(cfg.Session.Inputs.AudioDir)
if audioDir == "" && len(cfg.Session.Inputs.AudioFiles) == 0 {
return []finding{errorFinding("audio", "audio_dir, audio_files, or audio_s3 is required")}
}
base := filepath.Dir(cfg.SessionPath)
paths := []string{}
if audioDir != "" {
dir := audioDir
if !filepath.IsAbs(dir) {
dir = filepath.Join(base, dir)
}
matches, err := filepath.Glob(filepath.Join(dir, "*.flac"))
if err != nil || len(matches) == 0 {
return []finding{errorFinding("audio", "no .flac files found in "+dir)}
}
paths = append(paths, matches...)
}
for _, file := range cfg.Session.Inputs.AudioFiles {
p := file
if !filepath.IsAbs(p) {
p = filepath.Join(base, p)
}
paths = append(paths, p)
}
for _, p := range paths {
if _, err := os.Stat(p); err != nil {
return []finding{errorFinding("audio", fmt.Sprintf("audio file missing: %v", err))}
}
}
return []finding{okFinding("audio", fmt.Sprintf("%d local audio file(s)", len(paths)))}
}
func validateRemoteAudioFinding(ctx context.Context, cfg *config.Config, store storage.ObjectStore) finding {
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, cfg.Session.Inputs.AudioS3.Prefix)
objects, err := store.List(ctx, audioPrefix)
if err != nil {
return errorFinding("audio", err.Error())
}
count := 0
for _, obj := range objects {
if strings.HasSuffix(strings.ToLower(obj.Key), ".flac") {
count++
}
}
if count == 0 {
return errorFinding("audio", "no remote .flac objects found under "+audioPrefix)
}
return okFinding("audio", fmt.Sprintf("%d remote .flac object(s)", count))
}
func validatePreviousArtifactFindings(ctx context.Context, cfg *config.Config, store storage.ObjectStore, requirements []artifacts.PreviousArtifactRequirement) []finding {
out := []finding{}
prefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.PreviousSessionID)
manifestKey, runIDKey := artifacts.ResolveArchiveCurrentStateKeys(prefix)
for _, key := range []string{runIDKey, manifestKey} {
exists, err := store.Exists(ctx, key)
if err != nil {
out = append(out, errorFinding("previous", fmt.Sprintf("check %s: %v", key, err)))
return out
}
if !exists {
out = append(out, errorFinding("previous", "missing "+key))
return out
}
}
for _, req := range requirements {
out = append(out, okFinding("previous", fmt.Sprintf("%s required=%t", req.Name, req.Required)))
}
return out
}
func loadLocalManifest(ctx context.Context, path string) (*manifest.Manifest, error) {
if _, err := os.Stat(path); err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
store := &manifest.LocalStore{}
return store.Load(ctx, path)
}
func writeStageStatuses(out io.Writer, m *manifest.Manifest) {
if m == nil || len(m.Stages) == 0 {
fmt.Fprintln(out, "stages: no stages recorded")
return
}
fmt.Fprintln(out, "stages:")
names := make([]string, 0, len(m.Stages))
for name := range m.Stages {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
fmt.Fprintf(out, "- %s: %s\n", name, m.Stages[name].Status)
}
}
func buildHelperArtifactCatalog(cfg *config.Config) (*artifacts.ArtifactCatalog, error) {
catalog := artifacts.NewArtifactCatalog()
if err := catalog.RegisterBuiltIns(); err != nil {
return nil, err
}
configured := map[string]artifacts.ConfiguredArtifactDefinition{}
if cfg.Pipeline.Scriptorium != nil {
for key, item := range cfg.Pipeline.Scriptorium.Artifacts {
configured[key] = artifacts.ConfiguredArtifactDefinition{Enabled: item.Enabled, OutputPath: item.OutputPath}
}
}
if err := catalog.RegisterConfiguredArtifacts(configured, nil); err != nil {
return nil, err
}
return catalog, nil
}
func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.ArtifactCatalog, locks *effectiveLocks, promotedRemoteState map[string]string) {
lockSet := lockSourceSet(locks.All)
fmt.Fprintln(out, "Built-in:")
for _, id := range []string{
artifacts.ArtifactTranscriptMerged,
artifacts.ArtifactTranscriptPolished,
artifacts.ArtifactTranscriptFull,
artifacts.ArtifactTranscriptTrimmed,
artifacts.ArtifactBoundsSession,
} {
writeArtifactLine(out, id, lockSet)
}
fmt.Fprintln(out, "Configured:")
for _, entry := range catalog.ListConfigured() {
writeArtifactLine(out, entry.SourceID, lockSet)
}
fmt.Fprintln(out, "Previous-session:")
for _, req := range artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg)) {
fmt.Fprintf(out, "- narratio.previous_session.artifact.%s required=%t\n", req.Name, req.Required)
}
fmt.Fprintln(out, "Promoted:")
for _, rule := range cfg.Pipeline.Archive.PromoteArtifacts {
writePromotedArtifactLine(out, rule, catalog, lockSet, promotedRemoteState)
}
}
func writeArtifactLine(out io.Writer, source string, lockSet map[string]config.ArchiveLockRule) {
parts := []string{source}
if _, ok := lockSet[source]; ok {
parts = append(parts, "locked")
}
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
}
func writePromotedArtifactLine(out io.Writer, rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog, lockSet map[string]config.ArchiveLockRule, remoteState map[string]string) {
source := strings.TrimSpace(rule.Source)
parts := []string{source}
if _, ok := lockSet[source]; ok {
parts = append(parts, "locked")
}
dest, showDest, err := helperPromotionDest(rule, catalog)
if err != nil {
parts = append(parts, "remote=error")
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
return
}
if showDest {
parts = append(parts, "dest="+dest)
}
if state := remoteState[promotionRemoteStateKey(source, dest)]; state != "" {
parts = append(parts, state)
}
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
}
func remotePromotionAvailability(ctx context.Context, cfg *config.Config, store storage.ObjectStore, catalog *artifacts.ArtifactCatalog) map[string]string {
out := map[string]string{}
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
for _, rule := range cfg.Pipeline.Archive.PromoteArtifacts {
source := strings.TrimSpace(rule.Source)
dest, _, err := helperPromotionDest(rule, catalog)
if err != nil {
out[promotionRemoteStateKey(source, "")] = "remote=error"
continue
}
key := artifacts.S3PromotedArtifactKey(sessionPrefix, dest)
if exists, err := store.Exists(ctx, key); err == nil && exists {
out[promotionRemoteStateKey(source, dest)] = "remote=promoted"
} else if err != nil {
out[promotionRemoteStateKey(source, dest)] = "remote=error"
} else {
out[promotionRemoteStateKey(source, dest)] = "remote=missing"
}
}
return out
}
func helperPromotionDest(rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog) (string, bool, error) {
source := strings.TrimSpace(rule.Source)
dest := strings.TrimSpace(rule.Dest)
if dest == "" {
entry, ok := catalog.Lookup(source)
if !ok {
return "", false, fmt.Errorf("destination omitted and source is unknown")
}
dest = strings.TrimSpace(entry.CanonicalRelPath)
if dest == "" {
return "", false, fmt.Errorf("destination omitted and no canonical destination is available")
}
}
normalized, err := normalizeHelperArchiveRelativePath(dest)
if err != nil {
return "", false, err
}
entry, ok := catalog.Lookup(source)
showDest := !ok || strings.TrimSpace(entry.CanonicalRelPath) != normalized
return normalized, showDest, nil
}
func normalizeHelperArchiveRelativePath(rel string) (string, error) {
trimmed := strings.TrimSpace(rel)
if trimmed == "" {
return "", fmt.Errorf("relative path is required")
}
cleaned := filepath.ToSlash(filepath.Clean(filepath.FromSlash(trimmed)))
if cleaned == "." || cleaned == "" {
return "", fmt.Errorf("relative path is required")
}
if filepath.IsAbs(trimmed) || strings.HasPrefix(cleaned, "/") || cleaned == ".." || strings.HasPrefix(cleaned, "../") {
return "", fmt.Errorf("path must be a clean relative path")
}
return cleaned, nil
}
func promotionRemoteStateKey(source, dest string) string {
return strings.TrimSpace(source) + "\x00" + strings.TrimSpace(dest)
}
func writeLocks(out io.Writer, cfg *config.Config, locks *effectiveLocks) {
if locks == nil || len(locks.All) == 0 {
fmt.Fprintln(out, "Archive locks: none")
return
}
fmt.Fprintln(out, "Archive locks:")
promoted := map[string]config.ArchivePromotionRule{}
if cfg != nil && cfg.Pipeline != nil && cfg.Pipeline.Archive != nil {
for _, rule := range cfg.Pipeline.Archive.PromoteArtifacts {
promoted[strings.TrimSpace(rule.Source)] = rule
}
}
staticSet := lockSourceSet(locks.Static)
for _, lock := range locks.All {
origin := "remote"
if _, ok := staticSet[lock.Source]; ok {
origin = "pipeline"
}
promo := "not-promoted"
if _, ok := promoted[lock.Source]; ok {
promo = "promoted"
}
reason := strings.TrimSpace(lock.Reason)
if reason == "" {
reason = "(no reason)"
}
fmt.Fprintf(out, "- %s origin=%s %s reason=%s\n", lock.Source, origin, promo, reason)
}
}
func lockMapValues(in map[string]config.ArchiveLockRule) []config.ArchiveLockRule {
keys := make([]string, 0, len(in))
for key := range in {
keys = append(keys, key)
}
sort.Strings(keys)
out := make([]config.ArchiveLockRule, 0, len(keys))
for _, key := range keys {
item := in[key]
item.Source = key
item.Reason = strings.TrimSpace(item.Reason)
out = append(out, item)
}
return out
}

View File

@@ -0,0 +1,589 @@
package app
import (
"bytes"
"context"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
func TestExecuteSessionInitRemoteWritesCanonicalSessionConfig(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{
"session", "init",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session-id", "2026-06-07",
"--title", "The Black Cabin",
"--remote",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
key := artifacts.S3SessionConfigKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-06-07"))
obj, ok := fake.Objects[key]
if !ok {
t.Fatalf("remote session key %q not uploaded; objects=%v", key, fake.Objects)
}
if !strings.Contains(string(obj.Data), `session_id: "2026-06-07"`) || !strings.Contains(string(obj.Data), "prefix: audio/") {
t.Fatalf("remote session data = %q", string(obj.Data))
}
if storeInitCalls != 1 {
t.Fatalf("object store init calls = %d, want 1", storeInitCalls)
}
}
func TestExecuteSessionValidateLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
accessKeyEnv := "NARRATIO_TEST_VALIDATE_OBJECT_KEY_ID"
secretKeyEnv := "NARRATIO_TEST_VALIDATE_OBJECT_SECRET"
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
secretsDir := t.TempDir()
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-key-id\n")
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret\n")
addSecretsToPipelineConfig(t, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv)
if err := os.WriteFile(sessionPath, []byte(`session_id: 2026-05-03
inputs:
audio_s3:
prefix: audio/
`), 0o644); err != nil {
t.Fatalf("write session: %v", err)
}
fake := &storage.FakeBackend{}
audioKey := artifacts.S3PromotedArtifactKey(artifacts.S3AudioPrefix(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"), "audio/"), "alice.flac")
fake.SeedObject(storage.FakeObject{Key: audioKey, Data: []byte("audio")})
origStoreFn := newObjectStoreFromConfigFn
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
if os.Getenv(accessKeyEnv) != "test-key-id" || os.Getenv(secretKeyEnv) != "test-secret" {
return nil, fmt.Errorf("secrets were not loaded before object store init")
}
return fake, nil
}
t.Cleanup(func() {
newObjectStoreFromConfigFn = origStoreFn
})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"session", "validate", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
if !strings.Contains(stdout.String(), "OK audio") {
t.Fatalf("stdout = %q, want OK audio", stdout.String())
}
}
func TestExecuteLocksAddListAndRemoveUseRemoteLockStore(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{
"locks", "add",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
"--reason", "manual edit",
"narratio.transcript.trimmed",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("locks add exit code = %d, want 0; stderr=%q", code, stderr.String())
}
key := artifacts.S3SessionLocksKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"))
obj, ok := fake.Objects[key]
if !ok {
t.Fatalf("remote locks key %q not uploaded", key)
}
if !strings.Contains(string(obj.Data), "source: narratio.transcript.trimmed") || !strings.Contains(string(obj.Data), "reason: manual edit") {
t.Fatalf("lock store data = %q", string(obj.Data))
}
stdout.Reset()
stderr.Reset()
code = Execute([]string{
"locks",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("locks list exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if !strings.Contains(stdout.String(), "- narratio.transcript.trimmed origin=remote") {
t.Fatalf("stdout = %q, want remote lock", stdout.String())
}
stdout.Reset()
stderr.Reset()
code = Execute([]string{
"locks", "remove",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
"narratio.transcript.trimmed",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("locks remove exit code = %d, want 0; stderr=%q", code, stderr.String())
}
store, err := config.LoadArchiveLockStoreBytes("locks.yml", fake.Objects[key].Data, nil)
if err != nil {
t.Fatalf("LoadArchiveLockStoreBytes() error = %v", err)
}
if len(store.Locks) != 0 {
t.Fatalf("locks after remove = %#v, want empty", store.Locks)
}
if storeInitCalls != 3 {
t.Fatalf("object store init calls = %d, want 3", storeInitCalls)
}
}
func TestExecuteLocksAddDuplicateRequiresForce(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{
"locks", "add",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
"--reason", "first",
"narratio.transcript.trimmed",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("initial locks add exit code = %d, want 0; stderr=%q", code, stderr.String())
}
stdout.Reset()
stderr.Reset()
code = Execute([]string{
"locks", "add",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
"--reason", "second",
"narratio.transcript.trimmed",
}, &stdout, &stderr)
if code == 0 {
t.Fatal("duplicate locks add exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "pass --force to update") {
t.Fatalf("stderr = %q, want force guidance", stderr.String())
}
stdout.Reset()
stderr.Reset()
code = Execute([]string{
"locks", "add",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
"--reason", "second",
"--force",
"narratio.transcript.trimmed",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("forced locks add exit code = %d, want 0; stderr=%q", code, stderr.String())
}
key := artifacts.S3SessionLocksKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"))
if !strings.Contains(string(fake.Objects[key].Data), "reason: second") {
t.Fatalf("lock store data = %q, want updated reason", string(fake.Objects[key].Data))
}
}
func TestExecuteLocksRequireSessionID(t *testing.T) {
tests := []struct {
name string
args []string
want string
}{
{"list", []string{"locks"}, "locks: --session-id is required"},
{"add", []string{"locks", "add", "narratio.transcript.trimmed"}, "locks add: --session-id is required"},
{"remove", []string{"locks", "remove", "narratio.transcript.trimmed"}, "locks remove: --session-id is required"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute(tt.args, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), tt.want) {
t.Fatalf("stderr = %q, want %q", stderr.String(), tt.want)
}
})
}
}
func TestExecuteLocksCannotModifyStaticLocks(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
addStaticArchiveLockToPipelineConfig(t, pipelinePath, "narratio.transcript.trimmed")
fake := &storage.FakeBackend{}
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{
"locks", "add",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
"narratio.transcript.trimmed",
}, &stdout, &stderr)
if code == 0 {
t.Fatal("locks add static lock exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "locked by pipeline config") {
t.Fatalf("stderr = %q, want static lock error", stderr.String())
}
stdout.Reset()
stderr.Reset()
code = Execute([]string{
"locks", "remove",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
"narratio.transcript.trimmed",
}, &stdout, &stderr)
if code == 0 {
t.Fatal("locks remove static lock exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "locked by pipeline config") {
t.Fatalf("stderr = %q, want static lock error", stderr.String())
}
}
func TestExecuteTopLevelLockAndUnlockAreRemoved(t *testing.T) {
tests := []string{"lock", "unlock"}
for _, cmd := range tests {
t.Run(cmd, func(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{cmd, "narratio.transcript.trimmed"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), `unknown command: "`+cmd+`"`) {
t.Fatalf("stderr = %q, want unknown command", stderr.String())
}
})
}
}
func TestExecuteArtifactsListRemoteReportsPromotedAvailability(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
addArchivePromotionsToPipeline(t, pipelinePath, `
promote_artifacts:
- source: narratio.transcript.trimmed
dest: transcripts/trimmed.json
required: true
`)
fake := &storage.FakeBackend{}
trimmedKey := artifacts.S3PromotedArtifactKey(
artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"),
"transcripts/trimmed.json",
)
fake.SeedObject(storage.FakeObject{Key: trimmedKey, Data: []byte(`{"segments":[]}`)})
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{
"artifacts", "list",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--remote",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if !strings.Contains(stdout.String(), "narratio.transcript.trimmed remote=promoted") {
t.Fatalf("stdout = %q, want promoted remote availability", stdout.String())
}
}
func TestExecuteArtifactsListRemoteUsesPromotionDestinations(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
addArchivePromotionsToPipeline(t, pipelinePath, `
promote_artifacts:
- source: narratio.transcript.full
dest: transcripts/full.json
required: true
- source: narratio.bounds.session
dest: transcripts/bounds.json
required: true
`)
fake := &storage.FakeBackend{}
sessionPrefix := artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03")
fake.SeedObject(storage.FakeObject{Key: artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/full.json"), Data: []byte(`{"segments":[]}`)})
fake.SeedObject(storage.FakeObject{Key: artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/bounds.json"), Data: []byte(`{}`)})
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{
"artifacts", "list",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--remote",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
out := stdout.String()
for _, unwanted := range []string{
"narratio.transcript.full remote=missing",
"narratio.bounds.session remote=missing",
} {
if strings.Contains(out, unwanted) {
t.Fatalf("stdout = %q, did not want catalog remote marker %q", out, unwanted)
}
}
for _, want := range []string{
"narratio.transcript.full dest=transcripts/full.json remote=promoted",
"narratio.bounds.session dest=transcripts/bounds.json remote=promoted",
} {
if !strings.Contains(out, want) {
t.Fatalf("stdout = %q, want %q", out, want)
}
}
}
func TestExecuteStatusReportsRemoteArtifactCatalog(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
addArchivePromotionsToPipeline(t, pipelinePath, `
promote_artifacts:
- source: narratio.transcript.trimmed
dest: transcripts/trimmed.json
required: true
- source: narratio.transcript.full
dest: transcripts/full.json
required: true
`)
fake := &storage.FakeBackend{}
sessionPrefix := artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03")
manifestKey, runIDKey := artifacts.ResolveArchiveCurrentStateKeys(sessionPrefix)
trimmedKey := artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/trimmed.json")
fullKey := artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/full.json")
lockKey := artifacts.S3SessionLocksKey(sessionPrefix)
fake.SeedObject(storage.FakeObject{Key: runIDKey, Data: []byte("20260519T010203Z-a1b2c3d4\n")})
fake.SeedObject(storage.FakeObject{Key: manifestKey, Data: restoreManifestJSON(t, "2026-05-03", "sample-campaign")})
fake.SeedObject(storage.FakeObject{Key: trimmedKey, Data: []byte(`{"segments":[]}`)})
fake.SeedObject(storage.FakeObject{Key: fullKey, Data: []byte(`{"segments":[]}`)})
fake.SeedObject(storage.FakeObject{Key: lockKey, Data: []byte("locks:\n - source: narratio.transcript.trimmed\n reason: remote review\n")})
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{
"status",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
out := stdout.String()
for _, want := range []string{
"Remote outputs:",
"Built-in:",
"Configured:",
"Previous-session:",
"Promoted:",
"narratio.transcript.trimmed locked",
"narratio.transcript.trimmed locked remote=promoted",
"narratio.transcript.full dest=transcripts/full.json remote=promoted",
} {
if !strings.Contains(out, want) {
t.Fatalf("stdout = %q, want %q", out, want)
}
}
if strings.Contains(out, "narratio.transcript.merged remote=missing") {
t.Fatalf("stdout = %q, did not want catalog remote marker", out)
}
}
func TestExecuteStatusReportsRemoteArtifactCatalogErrorsWithoutFailing(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
addArchivePromotionsToPipeline(t, pipelinePath, `
promote_artifacts:
- source: narratio.transcript.trimmed
dest: transcripts/trimmed.json
required: true
`)
fake := &storage.FakeBackend{ExistsErr: fmt.Errorf("exists failed")}
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{
"status",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
out := stdout.String()
if !strings.Contains(out, "Remote archive: missing or unavailable:") {
t.Fatalf("stdout = %q, want remote archive unavailable state", out)
}
if !strings.Contains(out, "Remote outputs:") || !strings.Contains(out, "narratio.transcript.trimmed remote=error") {
t.Fatalf("stdout = %q, want remote output error state", out)
}
if !strings.Contains(out, "Archive locks: error:") {
t.Fatalf("stdout = %q, want archive locks error", out)
}
}
func TestExecuteArchiveLoadsRemoteLocks(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidArchiveConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
lockKey := artifacts.S3SessionLocksKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"))
fake.SeedObject(storage.FakeObject{Key: lockKey, Data: []byte("locks:\n - source: narratio.transcript.trimmed\n reason: remote review\n")})
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
// The archive stage only checks the manifest statuses and source files.
_ = stageName
}
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "trimmed.json"), `{"segments":[]}`)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "archive"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
promotedKey := artifacts.S3PromotedArtifactKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"), "transcripts/trimmed.json")
if _, ok := fake.Objects[promotedKey]; ok {
t.Fatalf("locked promoted key %q was uploaded", promotedKey)
}
}
func addArchivePromotionsToPipeline(t *testing.T, pipelinePath, archiveYAML string) {
t.Helper()
data, err := os.ReadFile(pipelinePath)
if err != nil {
t.Fatalf("read pipeline: %v", err)
}
updated := strings.Replace(string(data), " upload_run: false\n", " upload_run: false\n"+archiveYAML, 1)
if updated == string(data) {
t.Fatalf("pipeline %q did not contain archive upload_run marker", pipelinePath)
}
if err := os.WriteFile(pipelinePath, []byte(updated), 0o644); err != nil {
t.Fatalf("write pipeline: %v", err)
}
}
func writeValidArchiveConfigFiles(t *testing.T, workspaceRoot string) (string, string, string) {
t.Helper()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
data, err := os.ReadFile(pipelinePath)
if err != nil {
t.Fatalf("read pipeline: %v", err)
}
updated := strings.Replace(string(data), "upload_run: false", "upload_run: true", 1)
if err := os.WriteFile(pipelinePath, []byte(updated), 0o644); err != nil {
t.Fatalf("write pipeline: %v", err)
}
ctx := context.Background()
cfg, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{})
if err != nil {
t.Fatalf("LoadWithSessionOptions() error = %v", err)
}
store := &manifest.LocalStore{}
m := manifest.New("2026-05-03", nowUTC())
m.Campaign = "sample-campaign"
m.RunID = "20260521T160000Z-test"
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
m.MarkStageSucceeded(name, nowUTC(), nil)
}
path := artifacts.SessionManifestPathForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
if err := store.Save(ctx, path, m); err != nil {
t.Fatalf("save manifest: %v", err)
}
runManifestPath := artifacts.SessionRunManifestPathForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, m.RunID)
if err := os.MkdirAll(filepath.Dir(runManifestPath), 0o755); err != nil {
t.Fatalf("mkdir run manifest: %v", err)
}
if err := os.WriteFile(runManifestPath, []byte("{}\n"), 0o644); err != nil {
t.Fatalf("write run manifest: %v", err)
}
return pipelinePath, campaignPath, sessionPath
}
func addStaticArchiveLockToPipelineConfig(t *testing.T, pipelinePath, source string) {
t.Helper()
data, err := os.ReadFile(pipelinePath)
if err != nil {
t.Fatalf("read pipeline: %v", err)
}
updated := strings.Replace(
string(data),
"archive:\n enabled: true\n upload_run: false\n",
"archive:\n enabled: true\n upload_run: false\n locks:\n - source: "+source+"\n reason: static review\n",
1,
)
if updated == string(data) {
t.Fatalf("archive section not found in pipeline config")
}
if err := os.WriteFile(pipelinePath, []byte(updated), 0o644); err != nil {
t.Fatalf("write pipeline: %v", err)
}
}

View File

@@ -20,11 +20,13 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
fs.SetOutput(io.Discard) fs.SetOutput(io.Discard)
var pipelinePath string var pipelinePath string
var campaignPath string
var sessionPath string var sessionPath string
var sessionID string var sessionID string
var previousSessionID string var previousSessionID string
var force bool var force bool
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)") fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml") fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates") fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates") fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
@@ -36,16 +38,7 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
if fs.NArg() != 0 { if fs.NArg() != 0 {
return fmt.Errorf("plan: unexpected positional arguments") return fmt.Errorf("plan: unexpected positional arguments")
} }
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath) cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
if err != nil {
return fmt.Errorf("plan: %w", err)
}
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
if err != nil {
return fmt.Errorf("plan: %w", err)
}
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
SessionID: sessionID, SessionID: sessionID,
PreviousSessionID: previousSessionID, PreviousSessionID: previousSessionID,
}) })

View File

@@ -15,10 +15,10 @@ import (
func TestPlanCreatesAndReusesWorkdir(t *testing.T) { func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var out bytes.Buffer var out bytes.Buffer
args := []string{"--config", pipelinePath, "--session", sessionPath} args := []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}
if err := Plan(context.Background(), args, &out); err != nil { if err := Plan(context.Background(), args, &out); err != nil {
t.Fatalf("first Plan() error = %v", err) t.Fatalf("first Plan() error = %v", err)
@@ -62,7 +62,7 @@ func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
func TestPlanShowsRunAndSkipFromManifest(t *testing.T) { func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json") manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
store := &manifest.LocalStore{} store := &manifest.LocalStore{}
@@ -74,7 +74,7 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
} }
var out bytes.Buffer var out bytes.Buffer
if err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out); err != nil { if err := Plan(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out); err != nil {
t.Fatalf("Plan() error = %v", err) t.Fatalf("Plan() error = %v", err)
} }
got := out.String() got := out.String()
@@ -93,6 +93,7 @@ func TestPlanFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
configDir := t.TempDir() configDir := t.TempDir()
pipelinePath := filepath.Join(configDir, "pipeline.yml") pipelinePath := filepath.Join(configDir, "pipeline.yml")
campaignPath := writeAppTestCampaignConfig(t, configDir)
sessionPath := filepath.Join(configDir, "session.yml") sessionPath := filepath.Join(configDir, "session.yml")
pipelineYAML := `workspace: pipelineYAML := `workspace:
@@ -128,7 +129,7 @@ inputs:
} }
var out bytes.Buffer var out bytes.Buffer
err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out) err := Plan(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
if err == nil { if err == nil {
t.Fatal("expected error, got nil") t.Fatal("expected error, got nil")
} }

View File

@@ -153,53 +153,70 @@ func archiveCleanupEligible(cfg *config.Config, sr *manifest.StageRecord) (bool,
return true, "" return true, ""
} }
type scopedDir struct {
RootAbs string
TargetAbs string
Exists bool
}
func removeRunScopedDir(root, target, policy string) error { func removeRunScopedDir(root, target, policy string) error {
dir, err := validateScopedDir(root, target, policy)
if err != nil {
return err
}
if !dir.Exists {
return nil
}
if err := os.RemoveAll(dir.TargetAbs); err != nil {
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
}
return nil
}
func validateScopedDir(root, target, policy string) (scopedDir, error) {
cleanRoot := strings.TrimSpace(root) cleanRoot := strings.TrimSpace(root)
cleanTarget := strings.TrimSpace(target) cleanTarget := strings.TrimSpace(target)
if cleanRoot == "" { if cleanRoot == "" {
return fmt.Errorf("cleanup policy %s: root path is required", policy) return scopedDir{}, fmt.Errorf("cleanup policy %s: root path is required", policy)
} }
if cleanTarget == "" { if cleanTarget == "" {
return fmt.Errorf("cleanup policy %s: target path is required", policy) return scopedDir{}, fmt.Errorf("cleanup policy %s: target path is required", policy)
} }
rootAbs, err := filepath.Abs(cleanRoot) rootAbs, err := filepath.Abs(cleanRoot)
if err != nil { if err != nil {
return fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err) return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
} }
targetAbs, err := filepath.Abs(cleanTarget) targetAbs, err := filepath.Abs(cleanTarget)
if err != nil { if err != nil {
return fmt.Errorf("cleanup policy %s: resolve target %q: %w", policy, cleanTarget, err) return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve target %q: %w", policy, cleanTarget, err)
} }
rel, err := filepath.Rel(rootAbs, targetAbs) rel, err := filepath.Rel(rootAbs, targetAbs)
if err != nil { if err != nil {
return fmt.Errorf("cleanup policy %s: relative path from %q to %q: %w", policy, rootAbs, targetAbs, err) return scopedDir{}, fmt.Errorf("cleanup policy %s: relative path from %q to %q: %w", policy, rootAbs, targetAbs, err)
} }
if rel == "." { if rel == "." {
return fmt.Errorf("cleanup policy %s: refusing to delete root directory %q", policy, rootAbs) return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete root directory %q", policy, rootAbs)
} }
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return fmt.Errorf("cleanup policy %s: refusing to delete path outside root: root=%q target=%q", policy, rootAbs, targetAbs) return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete path outside root: root=%q target=%q", policy, rootAbs, targetAbs)
} }
info, err := os.Lstat(targetAbs) info, err := os.Lstat(targetAbs)
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
return nil return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: false}, nil
} }
return fmt.Errorf("cleanup policy %s: stat target %q: %w", policy, targetAbs, err) return scopedDir{}, fmt.Errorf("cleanup policy %s: stat target %q: %w", policy, targetAbs, err)
} }
if info.Mode()&os.ModeSymlink != 0 { if info.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, targetAbs) return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, targetAbs)
} }
if !info.IsDir() { if !info.IsDir() {
return fmt.Errorf("cleanup policy %s: target %q is not a directory", policy, targetAbs) return scopedDir{}, fmt.Errorf("cleanup policy %s: target %q is not a directory", policy, targetAbs)
} }
if err := os.RemoveAll(targetAbs); err != nil { return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: true}, nil
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, targetAbs, err)
}
return nil
} }
func asString(v any) string { func asString(v any) string {

View File

@@ -0,0 +1,157 @@
package app
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
type effectiveLocks struct {
Static []config.ArchiveLockRule
Remote []config.ArchiveLockRule
All []config.ArchiveLockRule
Key string
}
func remoteLocksKey(cfg *config.Config) (string, error) {
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
return "", fmt.Errorf("resolved config is required")
}
if cfg.Pipeline.Storage.S3 == nil {
return "", fmt.Errorf("pipeline.storage.s3 configuration is required")
}
sessionPrefix := artifacts.S3SessionPrefix(
cfg.Pipeline.Storage.S3.RootPrefix,
cfg.Session.Campaign,
cfg.Session.SessionID,
)
return artifacts.S3SessionLocksKey(sessionPrefix), nil
}
func loadRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*config.ArchiveLockStore, string, error) {
key, err := remoteLocksKey(cfg)
if err != nil {
return nil, "", err
}
exists, err := store.Exists(ctx, key)
if err != nil {
return nil, key, fmt.Errorf("check remote locks %q: %w", key, err)
}
if !exists {
return &config.ArchiveLockStore{}, key, nil
}
tmp, err := downloadObjectToTemp(ctx, store, key, "narratio-locks-*.yml")
if err != nil {
return nil, key, fmt.Errorf("download remote locks %q: %w", key, err)
}
defer func() { _ = os.Remove(tmp) }()
data, err := os.ReadFile(tmp)
if err != nil {
return nil, key, fmt.Errorf("read remote locks %q: %w", key, err)
}
lockStore, err := config.LoadArchiveLockStoreBytes("s3://"+s3BucketName(cfg.Pipeline)+"/"+key, data, cfg.Pipeline.Scriptorium)
if err != nil {
return nil, key, err
}
return lockStore, key, nil
}
func loadEffectiveLocks(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*effectiveLocks, error) {
staticLocks := staticArchiveLocks(cfg)
if store == nil {
return &effectiveLocks{
Static: staticLocks,
All: append([]config.ArchiveLockRule(nil), staticLocks...),
}, nil
}
lockStore, key, err := loadRemoteLockStore(ctx, cfg, store)
if err != nil {
return nil, err
}
remoteLocks := append([]config.ArchiveLockRule(nil), lockStore.Locks...)
return &effectiveLocks{
Static: staticLocks,
Remote: remoteLocks,
All: config.MergeArchiveLockRules(staticLocks, remoteLocks),
Key: key,
}, nil
}
func staticArchiveLocks(cfg *config.Config) []config.ArchiveLockRule {
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Archive == nil {
return nil
}
return append([]config.ArchiveLockRule(nil), cfg.Pipeline.Archive.Locks...)
}
func applyEffectiveLocks(cfg *config.Config, locks []config.ArchiveLockRule) {
if cfg == nil || cfg.Pipeline == nil {
return
}
if cfg.Pipeline.Archive == nil {
cfg.Pipeline.Archive = &config.ArchiveConfig{}
}
cfg.Pipeline.Archive.Locks = append([]config.ArchiveLockRule(nil), locks...)
}
func uploadRemoteLockStore(ctx context.Context, store storage.ObjectStore, key string, lockStore *config.ArchiveLockStore) error {
data, err := config.MarshalArchiveLockStore(lockStore)
if err != nil {
return err
}
tmp, err := os.CreateTemp("", "narratio-locks-upload-*.yml")
if err != nil {
return fmt.Errorf("create lock store temp file: %w", err)
}
tmpPath := tmp.Name()
defer func() { _ = os.Remove(tmpPath) }()
if _, err := tmp.Write(data); err != nil {
_ = tmp.Close()
return fmt.Errorf("write lock store temp file: %w", err)
}
if err := tmp.Close(); err != nil {
return fmt.Errorf("close lock store temp file: %w", err)
}
if _, err := store.Upload(ctx, tmpPath, key, storage.UploadOptions{ContentType: "application/x-yaml; charset=utf-8"}); err != nil {
return fmt.Errorf("upload remote locks %q: %w", key, err)
}
return nil
}
func lockSourceSet(locks []config.ArchiveLockRule) map[string]config.ArchiveLockRule {
out := make(map[string]config.ArchiveLockRule, len(locks))
for _, lock := range locks {
source := strings.TrimSpace(lock.Source)
if source == "" {
continue
}
lock.Source = source
lock.Reason = strings.TrimSpace(lock.Reason)
out[source] = lock
}
return out
}
func writeLocalFile(path string, data []byte, force bool) error {
cleaned := filepath.Clean(strings.TrimSpace(path))
if cleaned == "" || cleaned == "." {
return fmt.Errorf("output path is required")
}
if !force {
if _, err := os.Stat(cleaned); err == nil {
return fmt.Errorf("output file %q already exists; pass --force to overwrite", cleaned)
} else if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("check output file %q: %w", cleaned, err)
}
}
if err := os.MkdirAll(filepath.Dir(cleaned), 0o755); err != nil {
return fmt.Errorf("create output directory: %w", err)
}
return os.WriteFile(cleaned, data, 0o644)
}

View File

@@ -0,0 +1,252 @@
package app
import (
"bytes"
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func TestExecuteRemoteSessionFallbackLoadsFromObjectStore(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
remoteKey := seedRemoteSessionConfig(t, fake, "2026-05-03", `session_id: "{{ session_id }}"
inputs:
audio_s3:
prefix: audio/
`)
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if storeInitCalls != 1 {
t.Fatalf("object store init calls = %d, want 1", storeInitCalls)
}
if !strings.Contains(stdout.String(), "narratio plan: workdir prepared") {
t.Fatalf("stdout = %q, want plan output", stdout.String())
}
if _, ok := fake.Objects[remoteKey]; !ok {
t.Fatalf("remote session key %q was not seeded", remoteKey)
}
}
func TestExecuteRemoteSessionFallbackLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
accessKeyEnv := "NARRATIO_TEST_REMOTE_SESSION_KEY_ID"
secretKeyEnv := "NARRATIO_TEST_REMOTE_SESSION_SECRET"
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
secretsDir := t.TempDir()
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "remote-session-key-id\n")
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "remote-session-secret\n")
addSecretsToPipelineConfig(t, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv)
fake := &storage.FakeBackend{}
seedRemoteSessionConfig(t, fake, "2026-05-03", `session_id: "{{ session_id }}"
inputs:
audio_s3:
prefix: audio/
`)
origStoreFn := newObjectStoreFromConfigFn
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
config.DefaultSessionConfigSearchPaths = []string{filepath.Join(t.TempDir(), "session.yml")}
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
if os.Getenv(accessKeyEnv) != "remote-session-key-id" || os.Getenv(secretKeyEnv) != "remote-session-secret" {
return nil, fmt.Errorf("secrets were not loaded before remote session object store init")
}
return fake, nil
}
t.Cleanup(func() {
newObjectStoreFromConfigFn = origStoreFn
config.DefaultSessionConfigSearchPaths = origSessionDefaults
})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
}
func TestExecuteExplicitLocalSessionPrecedenceSkipsRemote(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if storeInitCalls != 0 {
t.Fatalf("object store init calls = %d, want 0", storeInitCalls)
}
}
func TestExecuteLocalSessionDiscoveryPrecedenceSkipsRemote(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if storeInitCalls != 0 {
t.Fatalf("object store init calls = %d, want 0", storeInitCalls)
}
}
func TestExecuteRemoteSessionMissingObjectFailsClearly(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
var storeInitCalls int
missingSessionPath := filepath.Join(t.TempDir(), "session.yml")
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{missingSessionPath})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "remote session") || !strings.Contains(stderr.String(), "session.yml") || !strings.Contains(stderr.String(), "not found") {
t.Fatalf("stderr = %q, want remote session not found context", stderr.String())
}
if !strings.Contains(stderr.String(), missingSessionPath) {
t.Fatalf("stderr = %q, want local searched path", stderr.String())
}
}
func TestExecuteRemoteSessionRequiresSessionID(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
var storeInitCalls int
restoreAppConfigTestGlobals(t, &storage.FakeBackend{}, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "remote session loading requires --session-id") {
t.Fatalf("stderr = %q, want session-id guidance", stderr.String())
}
if storeInitCalls != 0 {
t.Fatalf("object store init calls = %d, want 0", storeInitCalls)
}
}
func TestExecuteRemoteSessionStorageInitErrorFailsClearly(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
origStoreFn := newObjectStoreFromConfigFn
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
config.DefaultSessionConfigSearchPaths = []string{filepath.Join(t.TempDir(), "session.yml")}
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
return nil, errors.New("storage unavailable")
}
t.Cleanup(func() {
newObjectStoreFromConfigFn = origStoreFn
config.DefaultSessionConfigSearchPaths = origSessionDefaults
})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "storage unavailable") || !strings.Contains(stderr.String(), "remote session") {
t.Fatalf("stderr = %q, want remote storage context", stderr.String())
}
}
func TestExecuteRemoteSessionMalformedYAMLFailsStrictDecode(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
seedRemoteSessionConfig(t, fake, "2026-05-03", "session_id: 2026-05-03\nunknown: true\n")
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "strict decode failed") {
t.Fatalf("stderr = %q, want strict decode context", stderr.String())
}
}
func restoreAppConfigTestGlobals(t *testing.T, fake *storage.FakeBackend, storeInitCalls *int, sessionDefaults []string) {
t.Helper()
origStoreFn := newObjectStoreFromConfigFn
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
config.DefaultSessionConfigSearchPaths = append([]string(nil), sessionDefaults...)
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
if storeInitCalls != nil {
(*storeInitCalls)++
}
return fake, nil
}
t.Cleanup(func() {
newObjectStoreFromConfigFn = origStoreFn
config.DefaultSessionConfigSearchPaths = origSessionDefaults
})
}
func seedRemoteSessionConfig(t *testing.T, fake *storage.FakeBackend, sessionID, content string) string {
t.Helper()
sessionPrefix := artifacts.S3SessionPrefix("dnd", "sample-campaign", sessionID)
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
fake.SeedObject(storage.FakeObject{
Key: remoteKey,
Data: []byte(content),
ETag: "remote-session-etag",
})
return remoteKey
}
func addSecretsToPipelineConfig(t *testing.T, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv string) {
t.Helper()
pipelineData, err := os.ReadFile(pipelinePath)
if err != nil {
t.Fatalf("read pipeline: %v", err)
}
pipelineYAML := strings.Replace(
string(pipelineData),
"storage:\n backend: s3\n s3:\n bucket: test-bucket\n",
"storage:\n backend: s3\n s3:\n bucket: test-bucket\n access_key_id_env: "+accessKeyEnv+"\n secret_access_key_env: "+secretKeyEnv+"\nsecrets:\n env_dir: "+secretsDir+"\n",
1,
)
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
t.Fatalf("write pipeline: %v", err)
}
}

View File

@@ -26,6 +26,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
fs.SetOutput(out) fs.SetOutput(out)
var pipelinePath string var pipelinePath string
var campaignPath string
var sessionPath string var sessionPath string
var sessionID string var sessionID string
var previousSessionID string var previousSessionID string
@@ -33,6 +34,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
var force bool var force bool
var includeAudio bool var includeAudio bool
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)") fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml") fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates") fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates") fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
@@ -40,7 +42,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
fs.BoolVar(&force, "force", false, "overwrite local conflicts with remote state") fs.BoolVar(&force, "force", false, "overwrite local conflicts with remote state")
fs.BoolVar(&includeAudio, "include-audio", false, "include archived session-level audio objects") fs.BoolVar(&includeAudio, "include-audio", false, "include archived session-level audio objects")
fs.Usage = func() { fs.Usage = func() {
_, _ = fmt.Fprintln(out, "Usage: narratio restore [--config <path>] [--session <path>] [--session-id <value>] [--previous-session-id <value>] [--dry-run] [--force] [--include-audio]") _, _ = fmt.Fprintln(out, "Usage: narratio restore [--config <path>] [--campaign <path>] [--session <path>] [--session-id <value>] [--previous-session-id <value>] [--dry-run] [--force] [--include-audio]")
_, _ = fmt.Fprintln(out) _, _ = fmt.Fprintln(out)
_, _ = fmt.Fprintln(out, "Flags:") _, _ = fmt.Fprintln(out, "Flags:")
fs.PrintDefaults() fs.PrintDefaults()
@@ -55,16 +57,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
if fs.NArg() != 0 { if fs.NArg() != 0 {
return fmt.Errorf("restore: unexpected positional arguments") return fmt.Errorf("restore: unexpected positional arguments")
} }
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath) cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
if err != nil {
return fmt.Errorf("restore: %w", err)
}
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
if err != nil {
return fmt.Errorf("restore: %w", err)
}
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
SessionID: sessionID, SessionID: sessionID,
PreviousSessionID: previousSessionID, PreviousSessionID: previousSessionID,
}) })
@@ -74,11 +67,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
if err := config.Validate(cfg); err != nil { if err := config.Validate(cfg); err != nil {
return fmt.Errorf("restore: %w", err) return fmt.Errorf("restore: %w", err)
} }
if _, err := loadSecretsFromConfig(cfg, logging.NewLogger(os.Stderr, slog.LevelInfo)); err != nil { objectStore, err := newCommandObjectStore(ctx, cfg, logging.NewLogger(os.Stderr, slog.LevelInfo))
return fmt.Errorf("restore: %w", err)
}
objectStore, err := newObjectStoreFromConfigFn(ctx, cfg)
if err != nil { if err != nil {
return fmt.Errorf("restore: %w", err) return fmt.Errorf("restore: %w", err)
} }

View File

@@ -9,6 +9,7 @@ import (
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage" "gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts" "gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/audio"
"gitea.maximumdirect.net/eric/narratio/internal/config" "gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest" "gitea.maximumdirect.net/eric/narratio/internal/manifest"
) )
@@ -92,6 +93,10 @@ func executeRestoreDownloadAction(
return fmt.Errorf("restore plan local path mismatch for %q", action.LocalRelativePath) return fmt.Errorf("restore plan local path mismatch for %q", action.LocalRelativePath)
} }
if restoreActionIsAudio(action) {
return executeRestoreAudioAction(ctx, cfg, safeLocalPath, action, store)
}
tmpPath, err := downloadObjectToSiblingTemp(ctx, store, action.RemoteKey, safeLocalPath) tmpPath, err := downloadObjectToSiblingTemp(ctx, store, action.RemoteKey, safeLocalPath)
if err != nil { if err != nil {
return fmt.Errorf("download to temp file: %w", err) return fmt.Errorf("download to temp file: %w", err)
@@ -120,6 +125,38 @@ func executeRestoreDownloadAction(
return nil return nil
} }
func executeRestoreAudioAction(
ctx context.Context,
cfg *config.Config,
safeLocalPath string,
action RestoreAction,
store storage.ObjectStore,
) error {
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Storage.S3 == nil || cfg.Session == nil {
return fmt.Errorf("resolved s3 config and session are required")
}
spoolDir := artifacts.SessionSpoolRestoreAudioDir(cfg.Pipeline.Spool.Root, cfg.Session.Campaign, cfg.Session.SessionID)
spoolPath := filepath.Join(spoolDir, filepath.Base(safeLocalPath))
cacheEnabled := cfg.Pipeline.Cache.S3Audio == nil || *cfg.Pipeline.Cache.S3Audio
_, err := audio.MaterializeS3Audio(ctx, audio.S3MaterializeRequest{
Store: store,
Object: storage.ObjectInfo{
Key: action.RemoteKey,
Size: action.Size,
ETag: action.ETag,
},
Bucket: strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket),
CacheRoot: strings.TrimSpace(cfg.Pipeline.Cache.Root),
CacheEnabled: cacheEnabled,
SpoolPath: spoolPath,
DestPath: safeLocalPath,
})
if err != nil {
return fmt.Errorf("materialize audio: %w", err)
}
return nil
}
func downloadObjectToSiblingTemp(ctx context.Context, store storage.ObjectStore, remoteKey, destPath string) (string, error) { func downloadObjectToSiblingTemp(ctx context.Context, store storage.ObjectStore, remoteKey, destPath string) (string, error) {
if strings.TrimSpace(destPath) == "" { if strings.TrimSpace(destPath) == "" {
return "", fmt.Errorf("destination path is required") return "", fmt.Errorf("destination path is required")

View File

@@ -18,10 +18,10 @@ import (
func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) { func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{} fake := &storage.FakeBackend{}
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath) cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte(`{"segments":[1,2,3]}`)) seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte(`{"segments":[1,2,3]}`))
seedRestoreObject(fake, sessionPrefix+"artifacts/session_recap.md", []byte("# recap\n")) seedRestoreObject(fake, sessionPrefix+"artifacts/session_recap.md", []byte("# recap\n"))
seedRestoreObject(fake, sessionPrefix+"audio/alice.flac", []byte("remote-audio")) seedRestoreObject(fake, sessionPrefix+"audio/alice.flac", []byte("remote-audio"))
@@ -32,7 +32,7 @@ func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr) code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
if code != 0 { if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String()) t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
} }
@@ -64,17 +64,17 @@ func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) { func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{} fake := &storage.FakeBackend{}
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath) cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(fake, sessionPrefix+"audio/alice.flac", []byte("remote-audio")) seedRestoreObject(fake, sessionPrefix+"audio/alice.flac", []byte("remote-audio"))
restoreWithStoreAndRealPhases(t, fake) restoreWithStoreAndRealPhases(t, fake)
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--include-audio"}, &stdout, &stderr) code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--include-audio"}, &stdout, &stderr)
if code != 0 { if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String()) t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
} }
@@ -87,12 +87,50 @@ func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {
} }
} }
func TestExecuteRestoreRestoresPreviousCacheWhenPresent(t *testing.T) { func TestExecuteRestoreIncludeAudioUsesCacheAfterWorkspaceDeletion(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{} fake := &storage.FakeBackend{}
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath) cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
audioKey := sessionPrefix + "audio/alice.flac"
seedRestoreObject(fake, audioKey, []byte("remote-audio"))
restoreWithStoreAndRealPhases(t, fake)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--include-audio"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("first restore exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if got := fakeDownloadCount(fake, audioKey); got != 1 {
t.Fatalf("audio downloads after first restore = %d, want 1", got)
}
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
mustReadEquals(t, filepath.Join(sessionRoot, "audio", "alice.flac"), "remote-audio")
if err := os.RemoveAll(sessionRoot); err != nil {
t.Fatalf("remove session root: %v", err)
}
stdout.Reset()
stderr.Reset()
code = Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--include-audio"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("second restore exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if got := fakeDownloadCount(fake, audioKey); got != 1 {
t.Fatalf("audio downloads after cached restore = %d, want still 1", got)
}
mustReadEquals(t, filepath.Join(sessionRoot, "audio", "alice.flac"), "remote-audio")
}
func TestExecuteRestoreRestoresPreviousCacheWhenPresent(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(fake, sessionPrefix+"previous/manifest.json", []byte(`{"session_id":"2026-04-26"}`)) seedRestoreObject(fake, sessionPrefix+"previous/manifest.json", []byte(`{"session_id":"2026-04-26"}`))
seedRestoreObject(fake, sessionPrefix+"previous/artifacts/session_recap.md", []byte("# previous recap\n")) seedRestoreObject(fake, sessionPrefix+"previous/artifacts/session_recap.md", []byte("# previous recap\n"))
@@ -100,7 +138,7 @@ func TestExecuteRestoreRestoresPreviousCacheWhenPresent(t *testing.T) {
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr) code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
if code != 0 { if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String()) t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
} }
@@ -116,10 +154,10 @@ func TestExecuteRestoreRestoresPreviousCacheWhenPresent(t *testing.T) {
func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) { func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{} fake := &storage.FakeBackend{}
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath) cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript")) seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID) sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
@@ -129,7 +167,7 @@ func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) {
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr) code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
if code == 0 { if code == 0 {
t.Fatal("exit code = 0, want non-zero") t.Fatal("exit code = 0, want non-zero")
} }
@@ -148,10 +186,10 @@ func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) {
func TestExecuteRestoreForceOverwritesDifferingFile(t *testing.T) { func TestExecuteRestoreForceOverwritesDifferingFile(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{} fake := &storage.FakeBackend{}
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath) cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript")) seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID) sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
@@ -161,7 +199,7 @@ func TestExecuteRestoreForceOverwritesDifferingFile(t *testing.T) {
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--force"}, &stdout, &stderr) code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
if code != 0 { if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String()) t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
} }
@@ -174,10 +212,10 @@ func TestExecuteRestoreForceOverwritesDifferingFile(t *testing.T) {
func TestExecuteRestoreForceOverwritesDifferingPreviousCacheFile(t *testing.T) { func TestExecuteRestoreForceOverwritesDifferingPreviousCacheFile(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{} fake := &storage.FakeBackend{}
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath) cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(fake, sessionPrefix+"previous/artifacts/session_recap.md", []byte("# remote previous recap\n")) seedRestoreObject(fake, sessionPrefix+"previous/artifacts/session_recap.md", []byte("# remote previous recap\n"))
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID) sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
@@ -187,7 +225,7 @@ func TestExecuteRestoreForceOverwritesDifferingPreviousCacheFile(t *testing.T) {
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--force"}, &stdout, &stderr) code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
if code != 0 { if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String()) t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
} }
@@ -196,10 +234,10 @@ func TestExecuteRestoreForceOverwritesDifferingPreviousCacheFile(t *testing.T) {
func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) { func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{} fake := &storage.FakeBackend{}
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath) cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript")) seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
store := artifacts.NewLocalStore(workspaceRoot) store := artifacts.NewLocalStore(workspaceRoot)
@@ -213,7 +251,7 @@ func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) {
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr) code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
if code == 0 { if code == 0 {
t.Fatal("exit code = 0, want non-zero") t.Fatal("exit code = 0, want non-zero")
} }
@@ -229,10 +267,10 @@ func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) {
func TestExecuteRestoreInvalidManifestDoesNotCorruptExistingManifest(t *testing.T) { func TestExecuteRestoreInvalidManifestDoesNotCorruptExistingManifest(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
base := &storage.FakeBackend{} base := &storage.FakeBackend{}
cfg, sessionPrefix, manifestKey, _ := seedRestoreCommittedState(t, base, pipelinePath, sessionPath) cfg, sessionPrefix, manifestKey, _ := seedRestoreCommittedState(t, base, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(base, sessionPrefix+"transcripts/full.json", []byte("remote-transcript")) seedRestoreObject(base, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
toggled := &stagedManifestDownloadStore{ toggled := &stagedManifestDownloadStore{
@@ -260,7 +298,7 @@ func TestExecuteRestoreInvalidManifestDoesNotCorruptExistingManifest(t *testing.
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--force"}, &stdout, &stderr) code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
if code == 0 { if code == 0 {
t.Fatal("exit code = 0, want non-zero") t.Fatal("exit code = 0, want non-zero")
} }
@@ -344,9 +382,9 @@ func restoreWithStoreAndRealPhases(t *testing.T, objectStore storage.ObjectStore
executeRestorePlanFn = executeRestorePlan executeRestorePlanFn = executeRestorePlan
} }
func seedRestoreCommittedState(t *testing.T, fake *storage.FakeBackend, pipelinePath, sessionPath string) (*config.Config, string, string, string) { func seedRestoreCommittedState(t *testing.T, fake *storage.FakeBackend, pipelinePath, campaignPath, sessionPath string) (*config.Config, string, string, string) {
t.Helper() t.Helper()
cfg, err := config.LoadWithSessionOptions(pipelinePath, sessionPath, config.SessionLoadOptions{}) cfg, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{})
if err != nil { if err != nil {
t.Fatalf("LoadWithSessionOptions() error = %v", err) t.Fatalf("LoadWithSessionOptions() error = %v", err)
} }
@@ -374,6 +412,16 @@ func mustReadEquals(t *testing.T, path, want string) {
} }
} }
func fakeDownloadCount(fake *storage.FakeBackend, key string) int {
count := 0
for _, call := range fake.Downloads {
if call.Key == key {
count++
}
}
return count
}
type stagedManifestDownloadStore struct { type stagedManifestDownloadStore struct {
delegate *storage.FakeBackend delegate *storage.FakeBackend
manifestKey string manifestKey string

View File

@@ -257,6 +257,35 @@ func classifyRestoreAction(
return action, nil return action, nil
} }
if restoreRelativePathIsAudio(localRelPath) {
if object.Size > 0 {
if info.Size() == object.Size {
action.Kind = RestoreActionSkipSame
action.SameLocal = true
action.Reason = "local audio size matches remote content"
return action, nil
}
if force {
action.Kind = RestoreActionDownload
action.Reason = "local audio differs (size mismatch); overwrite with --force"
return action, nil
}
action.Kind = RestoreActionConflict
action.Conflict = true
action.Reason = "local audio differs (size mismatch)"
return action, nil
}
if force {
action.Kind = RestoreActionDownload
action.Reason = "local audio exists; remote size unavailable; overwrite with --force"
return action, nil
}
action.Kind = RestoreActionConflict
action.Conflict = true
action.Reason = "local audio exists; remote size unavailable"
return action, nil
}
if object.Size > 0 && info.Size() != object.Size { if object.Size > 0 && info.Size() != object.Size {
if force { if force {
action.Kind = RestoreActionDownload action.Kind = RestoreActionDownload
@@ -303,6 +332,15 @@ func classifyRestoreAction(
return action, nil return action, nil
} }
func restoreActionIsAudio(action RestoreAction) bool {
return restoreRelativePathIsAudio(action.LocalRelativePath)
}
func restoreRelativePathIsAudio(rel string) bool {
cleanRel := path.Clean(strings.TrimSpace(rel))
return cleanRel == config.PathAudioDirSegment || strings.HasPrefix(cleanRel, config.PathAudioDirSegment+"/")
}
func writeRestorePlan(out io.Writer, current *RemoteCurrentState, plan *RestorePlan, opts RestorePlanOptions) error { func writeRestorePlan(out io.Writer, current *RemoteCurrentState, plan *RestorePlan, opts RestorePlanOptions) error {
if out == nil { if out == nil {
return fmt.Errorf("output writer is required") return fmt.Errorf("output writer is required")

View File

@@ -58,6 +58,33 @@ func TestRestorePlanIncludeAudio(t *testing.T) {
} }
} }
func TestRestorePlanExistingAudioUsesSizeWithoutRemoteChecksumDownload(t *testing.T) {
cfg := restorePlanConfig(t)
current := restorePlanCurrentState(t, cfg)
store := &storage.FakeBackend{}
seedRestoreObject(store, current.CurrentManifestKey, []byte(`{"session_id":"2026-05-03"}`))
seedRestoreObject(store, current.SessionPrefix+"audio/alice.flac", []byte("audio"))
sessionRoot := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
mustWriteTestFile(t, filepath.Join(sessionRoot, "audio", "alice.flac"), "local")
plan, err := buildRestorePlan(context.Background(), cfg, current, store, RestorePlanOptions{IncludeAudio: true})
if err != nil {
t.Fatalf("buildRestorePlan() error = %v", err)
}
if len(store.Downloads) != 0 {
t.Fatalf("downloads = %d, want no remote checksum download for audio", len(store.Downloads))
}
actionByRel := map[string]RestoreAction{}
for _, action := range plan.Actions {
actionByRel[action.LocalRelativePath] = action
}
audioAction := actionByRel["audio/alice.flac"]
if audioAction.Kind != RestoreActionSkipSame {
t.Fatalf("audio action kind = %q, want %q", audioAction.Kind, RestoreActionSkipSame)
}
}
func TestRestorePlanIncludesPreviousCacheByDefault(t *testing.T) { func TestRestorePlanIncludesPreviousCacheByDefault(t *testing.T) {
cfg := restorePlanConfig(t) cfg := restorePlanConfig(t)
current := restorePlanCurrentState(t, cfg) current := restorePlanCurrentState(t, cfg)

View File

@@ -32,6 +32,9 @@ func TestExecuteRestoreHelp(t *testing.T) {
if !strings.Contains(out, "--include-audio") { if !strings.Contains(out, "--include-audio") {
t.Fatalf("stdout = %q, want --include-audio flag", out) t.Fatalf("stdout = %q, want --include-audio flag", out)
} }
if !strings.Contains(out, "--campaign") {
t.Fatalf("stdout = %q, want --campaign flag", out)
}
} }
func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) { func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
@@ -70,7 +73,7 @@ func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
} }
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
@@ -78,6 +81,7 @@ func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
[]string{ []string{
"restore", "restore",
"--config", pipelinePath, "--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath, "--session", sessionPath,
"--session-id", "2026-05-03", "--session-id", "2026-05-03",
"--dry-run", "--dry-run",
@@ -116,11 +120,11 @@ func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
func TestExecuteRestoreRejectsUnexpectedPositionalArguments(t *testing.T) { func TestExecuteRestoreRejectsUnexpectedPositionalArguments(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "extra"}, &stdout, &stderr) code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "extra"}, &stdout, &stderr)
if code == 0 { if code == 0 {
t.Fatal("exit code = 0, want non-zero") t.Fatal("exit code = 0, want non-zero")
} }
@@ -138,11 +142,11 @@ func TestExecuteRestoreFailsWhenStorageBackendNotConfigured(t *testing.T) {
}) })
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeRestoreConfigWithoutStorage(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeRestoreConfigWithoutStorage(t, workspaceRoot)
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr) code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
if code == 0 { if code == 0 {
t.Fatal("exit code = 0, want non-zero") t.Fatal("exit code = 0, want non-zero")
} }
@@ -166,11 +170,11 @@ func TestExecuteRestoreDiscoveryErrorSurfaced(t *testing.T) {
} }
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr) code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
if code == 0 { if code == 0 {
t.Fatal("exit code = 0, want non-zero") t.Fatal("exit code = 0, want non-zero")
} }
@@ -208,7 +212,7 @@ func TestExecuteRestoreLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
restoreEnv(secretKeyEnv) restoreEnv(secretKeyEnv)
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
secretsDir := filepath.Join(t.TempDir(), "secrets") secretsDir := filepath.Join(t.TempDir(), "secrets")
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-access-key-id\n") mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-access-key-id\n")
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret-key\n") mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret-key\n")
@@ -261,6 +265,7 @@ func TestExecuteRestoreLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
[]string{ []string{
"restore", "restore",
"--config", pipelinePath, "--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath, "--session", sessionPath,
"--session-id", "2026-05-03", "--session-id", "2026-05-03",
"--dry-run", "--dry-run",
@@ -307,10 +312,10 @@ func TestExecuteRestoreNonDryRunConflictFailsBeforeNYI(t *testing.T) {
} }
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr) code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
if code == 0 { if code == 0 {
t.Fatal("exit code = 0, want non-zero") t.Fatal("exit code = 0, want non-zero")
} }
@@ -359,10 +364,10 @@ func TestExecuteRestoreNonDryRunForceExecutesPlan(t *testing.T) {
} }
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var stdout bytes.Buffer var stdout bytes.Buffer
var stderr bytes.Buffer var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--force"}, &stdout, &stderr) code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
if code != 0 { if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String()) t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
} }
@@ -374,11 +379,12 @@ func TestExecuteRestoreNonDryRunForceExecutesPlan(t *testing.T) {
} }
} }
func writeRestoreConfigWithoutStorage(t *testing.T, workspaceRoot string) (string, string) { func writeRestoreConfigWithoutStorage(t *testing.T, workspaceRoot string) (string, string, string) {
t.Helper() t.Helper()
dir := t.TempDir() dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml") pipelinePath := filepath.Join(dir, "pipeline.yml")
campaignPath := writeAppTestCampaignConfig(t, dir)
sessionPath := filepath.Join(dir, "session.yml") sessionPath := filepath.Join(dir, "session.yml")
pipelineYAML := `workspace: pipelineYAML := `workspace:
@@ -405,5 +411,5 @@ inputs:
mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n") mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n")
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes") mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
return pipelinePath, sessionPath return pipelinePath, campaignPath, sessionPath
} }

View File

@@ -19,10 +19,10 @@ import (
func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) { func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
fake := &storage.FakeBackend{} fake := &storage.FakeBackend{}
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath) cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(fake, runIDKey, []byte("20260519T010203Z-a1b2c3d4\n")) seedRestoreObject(fake, runIDKey, []byte("20260519T010203Z-a1b2c3d4\n"))
seedRestoreObject(fake, manifestKey, restoreWorkflowManifestJSON(t, cfg.Session.SessionID, cfg.Session.Campaign)) seedRestoreObject(fake, manifestKey, restoreWorkflowManifestJSON(t, cfg.Session.SessionID, cfg.Session.Campaign))
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte(`{"segments":[1,2,3]}`+"\n")) seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte(`{"segments":[1,2,3]}`+"\n"))
@@ -48,6 +48,7 @@ func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
[]string{ []string{
"restore", "restore",
"--config", pipelinePath, "--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath, "--session", sessionPath,
"--session-id", cfg.Session.SessionID, "--session-id", cfg.Session.SessionID,
}, },
@@ -86,6 +87,7 @@ func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
[]string{ []string{
"run-stage", "run-stage",
"--config", pipelinePath, "--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath, "--session", sessionPath,
"--session-id", cfg.Session.SessionID, "--session-id", cfg.Session.SessionID,
"--force", "--force",
@@ -157,7 +159,7 @@ func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
func TestRestoreThenAnalyzeUsesRestoredPreviousCacheWithoutObjectStore(t *testing.T) { func TestRestoreThenAnalyzeUsesRestoredPreviousCacheWithoutObjectStore(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
appendRestoreWorkflowScriptoriumConfig(t, pipelinePath, ` appendRestoreWorkflowScriptoriumConfig(t, pipelinePath, `
scriptorium: scriptorium:
binary: scriptorium binary: scriptorium
@@ -176,7 +178,7 @@ scriptorium:
`) `)
fakeStore := &storage.FakeBackend{} fakeStore := &storage.FakeBackend{}
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fakeStore, pipelinePath, sessionPath) cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fakeStore, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(fakeStore, runIDKey, []byte("20260519T010203Z-a1b2c3d4\n")) seedRestoreObject(fakeStore, runIDKey, []byte("20260519T010203Z-a1b2c3d4\n"))
seedRestoreObject(fakeStore, manifestKey, restoreWorkflowManifestJSON(t, cfg.Session.SessionID, cfg.Session.Campaign)) seedRestoreObject(fakeStore, manifestKey, restoreWorkflowManifestJSON(t, cfg.Session.SessionID, cfg.Session.Campaign))
seedRestoreObject(fakeStore, sessionPrefix+"transcripts/trimmed.json", []byte(`{"segments":[]}`+"\n")) seedRestoreObject(fakeStore, sessionPrefix+"transcripts/trimmed.json", []byte(`{"segments":[]}`+"\n"))
@@ -191,6 +193,7 @@ scriptorium:
[]string{ []string{
"restore", "restore",
"--config", pipelinePath, "--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath, "--session", sessionPath,
"--session-id", cfg.Session.SessionID, "--session-id", cfg.Session.SessionID,
}, },
@@ -235,6 +238,7 @@ scriptorium:
[]string{ []string{
"run-stage", "run-stage",
"--config", pipelinePath, "--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath, "--session", sessionPath,
"--session-id", cfg.Session.SessionID, "--session-id", cfg.Session.SessionID,
"--force", "--force",

View File

@@ -17,12 +17,14 @@ func Resume(ctx context.Context, args []string, out io.Writer) error {
fs.SetOutput(io.Discard) fs.SetOutput(io.Discard)
var pipelinePath string var pipelinePath string
var campaignPath string
var sessionPath string var sessionPath string
var sessionID string var sessionID string
var previousSessionID string var previousSessionID string
var force bool var force bool
var selectedArtifacts artifactSelectionFlag var selectedArtifacts artifactSelectionFlag
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)") fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml") fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates") fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates") fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
@@ -35,16 +37,7 @@ func Resume(ctx context.Context, args []string, out io.Writer) error {
if fs.NArg() != 0 { if fs.NArg() != 0 {
return fmt.Errorf("resume: unexpected positional arguments") return fmt.Errorf("resume: unexpected positional arguments")
} }
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath) cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
if err != nil {
return fmt.Errorf("resume: %w", err)
}
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
if err != nil {
return fmt.Errorf("resume: %w", err)
}
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
SessionID: sessionID, SessionID: sessionID,
PreviousSessionID: previousSessionID, PreviousSessionID: previousSessionID,
}) })

View File

@@ -15,7 +15,7 @@ import (
func TestResumeStartsAfterCompletedStages(t *testing.T) { func TestResumeStartsAfterCompletedStages(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json") manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
store := &manifest.LocalStore{} store := &manifest.LocalStore{}
@@ -32,7 +32,7 @@ func TestResumeStartsAfterCompletedStages(t *testing.T) {
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n") mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
var out bytes.Buffer var out bytes.Buffer
err := Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out) err := Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
if err != nil { if err != nil {
t.Fatalf("Resume() error = %v", err) t.Fatalf("Resume() error = %v", err)
} }
@@ -51,7 +51,7 @@ func TestResumeStartsAfterCompletedStages(t *testing.T) {
func TestResumeNoRemainingStages(t *testing.T) { func TestResumeNoRemainingStages(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json") manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
store := &manifest.LocalStore{} store := &manifest.LocalStore{}
@@ -64,7 +64,7 @@ func TestResumeNoRemainingStages(t *testing.T) {
} }
var out bytes.Buffer var out bytes.Buffer
err := Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out) err := Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
if err != nil { if err != nil {
t.Fatalf("Resume() error = %v", err) t.Fatalf("Resume() error = %v", err)
} }
@@ -80,7 +80,7 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
_, _ = w.Write([]byte(`{"source":"resume-force-test","segments":[{"speaker":"alice"}]}`)) _, _ = w.Write([]byte(`{"source":"resume-force-test","segments":[{"speaker":"alice"}]}`))
})) }))
defer srv.Close() defer srv.Close()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json") manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
store := &manifest.LocalStore{} store := &manifest.LocalStore{}
@@ -93,7 +93,7 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
} }
var out bytes.Buffer var out bytes.Buffer
err := Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--force"}, &out) err := Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &out)
if err != nil { if err != nil {
t.Fatalf("Resume() error = %v", err) t.Fatalf("Resume() error = %v", err)
} }
@@ -104,14 +104,14 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
func TestRunStageExecutesOnlySelectedStage(t *testing.T) { func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json") manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03") workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`) mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n") mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
var out bytes.Buffer var out bytes.Buffer
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "polish"}, &out) err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "polish"}, &out)
if err != nil { if err != nil {
t.Fatalf("RunStage() error = %v", err) t.Fatalf("RunStage() error = %v", err)
} }
@@ -134,7 +134,7 @@ func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
func TestRunStageSkipAndForce(t *testing.T) { func TestRunStageSkipAndForce(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json") manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03") workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`) mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
@@ -148,7 +148,7 @@ func TestRunStageSkipAndForce(t *testing.T) {
} }
var out bytes.Buffer var out bytes.Buffer
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "polish"}, &out) err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "polish"}, &out)
if err != nil { if err != nil {
t.Fatalf("RunStage() error = %v", err) t.Fatalf("RunStage() error = %v", err)
} }
@@ -157,7 +157,7 @@ func TestRunStageSkipAndForce(t *testing.T) {
} }
out.Reset() out.Reset()
err = RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &out) err = RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "polish"}, &out)
if err != nil { if err != nil {
t.Fatalf("RunStage(force) error = %v", err) t.Fatalf("RunStage(force) error = %v", err)
} }
@@ -168,7 +168,7 @@ func TestRunStageSkipAndForce(t *testing.T) {
func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing.T) { func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json") manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03") workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`) mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
@@ -184,7 +184,7 @@ func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing
} }
var out bytes.Buffer var out bytes.Buffer
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &out) err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "polish"}, &out)
if err != nil { if err != nil {
t.Fatalf("RunStage(force) error = %v", err) t.Fatalf("RunStage(force) error = %v", err)
} }
@@ -203,7 +203,7 @@ func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing
} }
out.Reset() out.Reset()
err = Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out) err = Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
if err != nil { if err != nil {
t.Fatalf("Resume() error = %v", err) t.Fatalf("Resume() error = %v", err)
} }
@@ -214,13 +214,13 @@ func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing
func TestRunStageTrimExecutes(t *testing.T) { func TestRunStageTrimExecutes(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json") manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03") workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`) mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
var out bytes.Buffer var out bytes.Buffer
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "trim"}, &out) err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "trim"}, &out)
if err != nil { if err != nil {
t.Fatalf("RunStage(trim) error = %v", err) t.Fatalf("RunStage(trim) error = %v", err)
} }
@@ -243,13 +243,13 @@ func TestRunStageTrimExecutes(t *testing.T) {
func TestRunStageNormalizeExecutes(t *testing.T) { func TestRunStageNormalizeExecutes(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json") manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03") workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1},{"id":2}]}`) mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
var out bytes.Buffer var out bytes.Buffer
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "normalize"}, &out) err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "normalize"}, &out)
if err != nil { if err != nil {
t.Fatalf("RunStage(normalize) error = %v", err) t.Fatalf("RunStage(normalize) error = %v", err)
} }

View File

@@ -15,12 +15,14 @@ func Run(ctx context.Context, args []string, out io.Writer) error {
fs.SetOutput(io.Discard) fs.SetOutput(io.Discard)
var pipelinePath string var pipelinePath string
var campaignPath string
var sessionPath string var sessionPath string
var sessionID string var sessionID string
var previousSessionID string var previousSessionID string
var force bool var force bool
var selectedArtifacts artifactSelectionFlag var selectedArtifacts artifactSelectionFlag
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)") fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml") fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates") fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates") fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
@@ -33,16 +35,7 @@ func Run(ctx context.Context, args []string, out io.Writer) error {
if fs.NArg() != 0 { if fs.NArg() != 0 {
return fmt.Errorf("run: unexpected positional arguments") return fmt.Errorf("run: unexpected positional arguments")
} }
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath) cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
if err != nil {
return fmt.Errorf("run: %w", err)
}
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
if err != nil {
return fmt.Errorf("run: %w", err)
}
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
SessionID: sessionID, SessionID: sessionID,
PreviousSessionID: previousSessionID, PreviousSessionID: previousSessionID,
}) })

View File

@@ -15,12 +15,14 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
fs.SetOutput(io.Discard) fs.SetOutput(io.Discard)
var pipelinePath string var pipelinePath string
var campaignPath string
var sessionPath string var sessionPath string
var sessionID string var sessionID string
var previousSessionID string var previousSessionID string
var force bool var force bool
var selectedArtifacts artifactSelectionFlag var selectedArtifacts artifactSelectionFlag
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)") fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml") fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates") fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates") fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
@@ -33,54 +35,34 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
if fs.NArg() != 1 { if fs.NArg() != 1 {
return fmt.Errorf("run-stage: expected exactly one stage name") return fmt.Errorf("run-stage: expected exactly one stage name")
} }
stageName := fs.Arg(0)
normalizedArtifacts, err := selectedArtifacts.Normalize() normalizedArtifacts, err := selectedArtifacts.Normalize()
if err != nil { if err != nil {
return fmt.Errorf("run-stage: invalid --artifacts: %w", err) return fmt.Errorf("run-stage: invalid --artifacts: %w", err)
} }
stageName := fs.Arg(0)
if len(normalizedArtifacts) > 0 && stageName != "analyze" { if len(normalizedArtifacts) > 0 && stageName != "analyze" {
return fmt.Errorf("run-stage: --artifacts is only supported for stage \"analyze\"") return fmt.Errorf("run-stage: --artifacts is only supported for stage \"analyze\"")
} }
stages, err := BuildSingleStagePlan(stageName)
if err != nil {
return fmt.Errorf("run-stage: %w", err)
}
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath) summary, err := runSingleStageCommand(ctx, singleStageCommand{
if err != nil { CommandName: "run-stage",
return fmt.Errorf("run-stage: %w", err) StageName: stageName,
} PipelinePath: pipelinePath,
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath) CampaignPath: campaignPath,
if err != nil { SessionPath: sessionPath,
return fmt.Errorf("run-stage: %w", err)
}
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
SessionID: sessionID, SessionID: sessionID,
PreviousSessionID: previousSessionID, PreviousSessionID: previousSessionID,
})
if err != nil {
return fmt.Errorf("run-stage: %w", err)
}
if err := config.Validate(cfg); err != nil {
return fmt.Errorf("run-stage: %w", err)
}
if err := validateSelectedAnalyzeArtifacts(cfg, normalizedArtifacts); err != nil {
return fmt.Errorf("run-stage: %w", err)
}
summary, err := executeStagesFn(ctx, cfg, stages, RunOptions{
Force: force, Force: force,
SelectedArtifacts: normalizedArtifacts, SelectedArtifacts: normalizedArtifacts,
}) })
if err != nil { if err != nil {
return fmt.Errorf("run-stage: %w", err) return err
} }
_, err = fmt.Fprintf( _, err = fmt.Fprintf(
out, out,
"narratio run-stage: stage=%s executed=%d skipped=%d force=%t; manifest=%s\n", "narratio run-stage: stage=%s executed=%d skipped=%d force=%t; manifest=%s\n",
stages[0].Name(), stageName,
len(summary.Executed), len(summary.Executed),
len(summary.Skipped), len(summary.Skipped),
force, force,
@@ -88,3 +70,99 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
) )
return err return err
} }
// Analyze force-runs the analyze stage.
func Analyze(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("analyze", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var pipelinePath string
var campaignPath string
var sessionPath string
var sessionID string
var previousSessionID string
var selectedArtifacts artifactSelectionFlag
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
fs.Var(&selectedArtifacts, "artifacts", "artifact names to execute during analyze (comma-separated or repeatable)")
if err := fs.Parse(args); err != nil {
return fmt.Errorf("analyze: invalid flags: %w", err)
}
if fs.NArg() != 0 {
return fmt.Errorf("analyze: unexpected positional arguments")
}
normalizedArtifacts, err := selectedArtifacts.Normalize()
if err != nil {
return fmt.Errorf("analyze: invalid --artifacts: %w", err)
}
summary, err := runSingleStageCommand(ctx, singleStageCommand{
CommandName: "analyze",
StageName: "analyze",
PipelinePath: pipelinePath,
CampaignPath: campaignPath,
SessionPath: sessionPath,
SessionID: sessionID,
PreviousSessionID: previousSessionID,
Force: true,
SelectedArtifacts: normalizedArtifacts,
})
if err != nil {
return err
}
_, err = fmt.Fprintf(
out,
"narratio analyze: executed=%d skipped=%d force=true; manifest=%s\n",
len(summary.Executed),
len(summary.Skipped),
summary.ManifestPath,
)
return err
}
type singleStageCommand struct {
CommandName string
StageName string
PipelinePath string
CampaignPath string
SessionPath string
SessionID string
PreviousSessionID string
Force bool
SelectedArtifacts []string
}
func runSingleStageCommand(ctx context.Context, req singleStageCommand) (*RunSummary, error) {
stages, err := BuildSingleStagePlan(req.StageName)
if err != nil {
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
}
cfg, err := loadCommandConfig(ctx, req.PipelinePath, req.CampaignPath, req.SessionPath, config.SessionLoadOptions{
SessionID: req.SessionID,
PreviousSessionID: req.PreviousSessionID,
})
if err != nil {
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
}
if err := config.Validate(cfg); err != nil {
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
}
if err := validateSelectedAnalyzeArtifacts(cfg, req.SelectedArtifacts); err != nil {
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
}
summary, err := executeStagesFn(ctx, cfg, stages, RunOptions{
Force: req.Force,
SelectedArtifacts: req.SelectedArtifacts,
})
if err != nil {
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
}
return summary, nil
}

View File

@@ -87,12 +87,19 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
env.Storage = &storage.NoopBackend{} env.Storage = &storage.NoopBackend{}
} }
if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages) { if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages) {
objectStore, err := storage.NewObjectStoreFromConfig(ctx, env.Config) objectStore, err := newCommandObjectStore(ctx, env.Config, nil)
if err != nil { if err != nil {
return nil, fmt.Errorf("initialize object store backend: %w", err) return nil, err
} }
env.ObjectStore = objectStore env.ObjectStore = objectStore
} }
if needsRemoteLocksForRun(env.Config, stages) {
locks, err := loadEffectiveLocks(ctx, env.Config, env.ObjectStore)
if err != nil {
return nil, fmt.Errorf("load remote archive locks: %w", err)
}
applyEffectiveLocks(env.Config, locks.All)
}
if env.Notifier == nil { if env.Notifier == nil {
env.Notifier = &notify.NoopSender{} env.Notifier = &notify.NoopSender{}
} }
@@ -576,6 +583,32 @@ func needsObjectStoreForRun(cfg *config.Config, stages []stage.Stage) bool {
return true return true
} }
func needsRemoteLocksForRun(cfg *config.Config, stages []stage.Stage) bool {
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
return false
}
archiveRequested := false
for _, s := range stages {
if s != nil && s.Name() == "archive" {
archiveRequested = true
break
}
}
if !archiveRequested {
return false
}
if cfg.Pipeline.Archive == nil {
return false
}
if cfg.Pipeline.Archive.Enabled != nil && !*cfg.Pipeline.Archive.Enabled {
return false
}
if cfg.Pipeline.Archive.UploadRun != nil && !*cfg.Pipeline.Archive.UploadRun {
return false
}
return cfg.Pipeline.Storage.S3 != nil
}
func configuredScriptoriumArtifacts(cfg *config.Config) map[string]config.ScriptoriumArtifactConfig { func configuredScriptoriumArtifacts(cfg *config.Config) map[string]config.ScriptoriumArtifactConfig {
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Scriptorium == nil { if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Scriptorium == nil {
return nil return nil

View File

@@ -994,10 +994,12 @@ func testConfig(t *testing.T) *config.Config {
workspace := t.TempDir() workspace := t.TempDir()
cfgDir := t.TempDir() cfgDir := t.TempDir()
sessionPath := filepath.Join(cfgDir, "session.yml") sessionPath := filepath.Join(cfgDir, "session.yml")
campaignPath := filepath.Join(cfgDir, "campaign.yml")
pipelinePath := filepath.Join(cfgDir, "pipeline.yml") pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
mustWriteFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n") mustWriteFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
mustWriteFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n") mustWriteFile(t, campaignPath, "campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n")
mustWriteFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\ninputs:\n audio_dir: ./audio\n")
mustWriteFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n") mustWriteFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
mustWriteFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n") mustWriteFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
mustWriteFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n") mustWriteFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
@@ -1005,8 +1007,27 @@ func testConfig(t *testing.T) *config.Config {
return &config.Config{ return &config.Config{
Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: workspace}}, Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: workspace}},
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
PipelinePath: pipelinePath, PipelinePath: pipelinePath,
CampaignPath: campaignPath,
SessionPath: sessionPath, SessionPath: sessionPath,
StableInputs: config.ResolvedStableInputs{
SpeakersFile: config.ResolvedInputFile{
Path: "./speakers.yml",
ConfigPath: campaignPath,
Source: "campaign_config",
},
AutocorrectFile: config.ResolvedInputFile{
Path: "./autocorrect.yml",
ConfigPath: campaignPath,
Source: "campaign_config",
},
GlossaryFile: config.ResolvedInputFile{
Path: "./glossary.yml",
ConfigPath: campaignPath,
Source: "campaign_config",
},
},
Session: &config.SessionConfig{ Session: &config.SessionConfig{
SessionID: "2026-05-03", SessionID: "2026-05-03",
Campaign: "sample-campaign", Campaign: "sample-campaign",
@@ -1023,6 +1044,7 @@ func testConfig(t *testing.T) *config.Config {
func TestBuildDefaultRunnersWithOmittedToolSections(t *testing.T) { func TestBuildDefaultRunnersWithOmittedToolSections(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml") pipelinePath := filepath.Join(dir, "pipeline.yml")
campaignPath := filepath.Join(dir, "campaign.yml")
sessionPath := filepath.Join(dir, "session.yml") sessionPath := filepath.Join(dir, "session.yml")
pipelineYAML := `workspace: pipelineYAML := `workspace:
root: ` + t.TempDir() + ` root: ` + t.TempDir() + `
@@ -1032,6 +1054,12 @@ analyzer:
timeout: 20m timeout: 20m
notification: notification:
timeout: 10s timeout: 10s
`
campaignYAML := `campaign: sample-campaign
inputs:
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
` `
sessionYAML := `session_id: 2026-05-03 sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign campaign: sample-campaign
@@ -1042,6 +1070,7 @@ inputs:
glossary_file: ./glossary.yml glossary_file: ./glossary.yml
` `
mustWriteFile(t, pipelinePath, pipelineYAML) mustWriteFile(t, pipelinePath, pipelineYAML)
mustWriteFile(t, campaignPath, campaignYAML)
mustWriteFile(t, sessionPath, sessionYAML) mustWriteFile(t, sessionPath, sessionYAML)
cfg, err := config.Load(pipelinePath, sessionPath) cfg, err := config.Load(pipelinePath, sessionPath)

View File

@@ -4,14 +4,15 @@ import (
"bytes" "bytes"
"context" "context"
"os" "os"
"path/filepath"
"strings" "strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/narratio/internal/config"
) )
func TestPlanUsesDiscoveredSessionTemplateWithSessionIDs(t *testing.T) { func TestPlanUsesDiscoveredSessionTemplateWithSessionIDs(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
sessionTemplate := `session_id: "{{ session_id }}" sessionTemplate := `session_id: "{{ session_id }}"
previous_session_id: "{{ previous_session_id }}" previous_session_id: "{{ previous_session_id }}"
@@ -26,19 +27,14 @@ inputs:
t.Fatalf("write session template: %v", err) t.Fatalf("write session template: %v", err)
} }
cwd := filepath.Dir(sessionPath) origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
originalWD, err := os.Getwd() config.DefaultSessionConfigSearchPaths = []string{sessionPath}
if err != nil { t.Cleanup(func() { config.DefaultSessionConfigSearchPaths = origSessionDefaults })
t.Fatalf("Getwd(): %v", err)
}
if err := os.Chdir(cwd); err != nil {
t.Fatalf("Chdir(%q): %v", cwd, err)
}
t.Cleanup(func() { _ = os.Chdir(originalWD) })
var out bytes.Buffer var out bytes.Buffer
if err := Plan(context.Background(), []string{ if err := Plan(context.Background(), []string{
"--config", pipelinePath, "--config", pipelinePath,
"--campaign", campaignPath,
"--session-id", "2026-04-04", "--session-id", "2026-04-04",
"--previous-session-id", "2026-03-28", "--previous-session-id", "2026-03-28",
}, &out); err != nil { }, &out); err != nil {
@@ -51,10 +47,10 @@ inputs:
func TestPlanFailsWhenSessionIDMismatchesConcreteSession(t *testing.T) { func TestPlanFailsWhenSessionIDMismatchesConcreteSession(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var out bytes.Buffer var out bytes.Buffer
err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--session-id", "2026-04-04"}, &out) err := Plan(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-04-04"}, &out)
if err == nil { if err == nil {
t.Fatal("expected error, got nil") t.Fatal("expected error, got nil")
} }
@@ -65,7 +61,7 @@ func TestPlanFailsWhenSessionIDMismatchesConcreteSession(t *testing.T) {
func TestPlanFailsWhenPreviousSessionIDMismatchesConcreteSession(t *testing.T) { func TestPlanFailsWhenPreviousSessionIDMismatchesConcreteSession(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
sessionYAML := `session_id: 2026-05-03 sessionYAML := `session_id: 2026-05-03
previous_session_id: 2026-04-26 previous_session_id: 2026-04-26
@@ -83,6 +79,7 @@ inputs:
var out bytes.Buffer var out bytes.Buffer
err := Plan(context.Background(), []string{ err := Plan(context.Background(), []string{
"--config", pipelinePath, "--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath, "--session", sessionPath,
"--session-id", "2026-05-03", "--session-id", "2026-05-03",
"--previous-session-id", "2026-04-25", "--previous-session-id", "2026-04-25",
@@ -97,10 +94,10 @@ inputs:
func TestRunStageAcceptsSessionIDFlagAndParsesStageName(t *testing.T) { func TestRunStageAcceptsSessionIDFlagAndParsesStageName(t *testing.T) {
workspaceRoot := t.TempDir() workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot) pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var out bytes.Buffer var out bytes.Buffer
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--session-id", "2026-05-03", "prepare"}, &out) err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03", "prepare"}, &out)
if err != nil { if err != nil {
t.Fatalf("RunStage() error = %v", err) t.Fatalf("RunStage() error = %v", err)
} }
@@ -110,7 +107,7 @@ func TestRunStageAcceptsSessionIDFlagAndParsesStageName(t *testing.T) {
} }
func TestResolveSessionConfigPathErrorIncludesSearchedPaths(t *testing.T) { func TestResolveSessionConfigPathErrorIncludesSearchedPaths(t *testing.T) {
_, err := resolveSessionConfigPathWithCandidates("", []string{"./session.yml", "/usr/local/etc/narratio/session.yml", "/etc/narratio/session.yml"}) _, err := resolveSessionConfigPathWithCandidates("", []string{"/usr/local/etc/narratio/session.yml", "/etc/narratio/session.yml"})
if err == nil { if err == nil {
t.Fatal("expected error, got nil") t.Fatal("expected error, got nil")
} }

View File

@@ -19,6 +19,22 @@ func resolveSessionConfigPathWithCandidates(flagValue string, candidates []strin
return explicit, nil return explicit, nil
} }
resolved, err := discoverSessionConfigPathWithCandidates(candidates)
if err != nil {
return "", err
}
if resolved.Path != "" {
return resolved.Path, nil
}
return "", missingSessionConfigError(resolved.Searched, "")
}
type sessionConfigDiscovery struct {
Path string
Searched []string
}
func discoverSessionConfigPathWithCandidates(candidates []string) (sessionConfigDiscovery, error) {
ordered := make([]string, 0, len(candidates)) ordered := make([]string, 0, len(candidates))
for _, raw := range candidates { for _, raw := range candidates {
path := strings.TrimSpace(raw) path := strings.TrimSpace(raw)
@@ -31,19 +47,32 @@ func resolveSessionConfigPathWithCandidates(flagValue string, candidates []strin
if info.IsDir() { if info.IsDir() {
continue continue
} }
return filepath.Clean(path), nil return sessionConfigDiscovery{Path: filepath.Clean(path), Searched: ordered}, nil
} }
if errors.Is(err, os.ErrNotExist) { if errors.Is(err, os.ErrNotExist) {
continue continue
} }
return "", fmt.Errorf("check default session config %q: %w", path, err) return sessionConfigDiscovery{}, fmt.Errorf("check default session config %q: %w", path, err)
} }
return sessionConfigDiscovery{Searched: ordered}, nil
}
func missingSessionConfigError(searched []string, remoteDetail string) error {
ordered := append([]string(nil), searched...)
if len(ordered) == 0 { if len(ordered) == 0 {
return "", fmt.Errorf("no session config path provided and no default locations configured") if strings.TrimSpace(remoteDetail) != "" {
return fmt.Errorf("no session config path provided and no default locations configured; %s", remoteDetail)
}
return fmt.Errorf("no session config path provided and no default locations configured")
} }
return "", fmt.Errorf( msg := fmt.Sprintf(
"no session config path provided and no default session config found; searched: %s; pass --session to use an explicit path", "no session config path provided and no default session config found; searched: %s",
strings.Join(ordered, ", "), strings.Join(ordered, ", "),
) )
if strings.TrimSpace(remoteDetail) != "" {
msg += "; " + strings.TrimSpace(remoteDetail)
}
msg += "; pass --session to use an explicit path"
return fmt.Errorf("%s", msg)
} }

View File

@@ -1,67 +0,0 @@
package app
import (
"context"
"flag"
"fmt"
"io"
"sort"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
// Status reads and prints stage statuses from an existing manifest.
func Status(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("status", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var manifestPath string
fs.StringVar(&manifestPath, "manifest", "", "path to manifest.json")
if err := fs.Parse(args); err != nil {
return fmt.Errorf("status: invalid flags: %w", err)
}
if fs.NArg() != 0 {
return fmt.Errorf("status: unexpected positional arguments")
}
if manifestPath == "" {
return fmt.Errorf("status: --manifest is required")
}
store := &manifest.LocalStore{}
m, err := store.Load(ctx, manifestPath)
if err != nil {
return fmt.Errorf("status: %w", err)
}
if _, err := fmt.Fprintf(out, "session_id: %s\n", m.SessionID); err != nil {
return err
}
if _, err := fmt.Fprintf(out, "updated_at: %s\n", m.UpdatedAt.UTC().Format("2006-01-02T15:04:05Z07:00")); err != nil {
return err
}
if len(m.Stages) == 0 {
_, err := fmt.Fprintln(out, "stages: no stages recorded")
return err
}
if _, err := fmt.Fprintln(out, "stages:"); err != nil {
return err
}
names := make([]string, 0, len(m.Stages))
for name := range m.Stages {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
status := m.Stages[name].Status
if _, err := fmt.Fprintf(out, "- %s: %s\n", name, status); err != nil {
return err
}
}
return nil
}

View File

@@ -4,6 +4,7 @@ import (
"errors" "errors"
"os" "os"
"path/filepath" "path/filepath"
"reflect"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -409,6 +410,17 @@ func TestResolvePreviousSessionArtifactWithCatalogFallsBackToPreparedCachePath(t
} }
} }
func TestPreviousSessionCacheCandidatePathsStripsArtifactsPrefix(t *testing.T) {
workspace := t.TempDir()
paths := buildSessionPaths(workspace, "campaign", "session")
got := previousSessionCacheCandidatePaths(paths, "artifacts/session_recap.md")
want := []string{filepath.Join(paths.PreviousArtifactsDir, "session_recap.md")}
if !reflect.DeepEqual(got, want) {
t.Fatalf("previousSessionCacheCandidatePaths() = %#v, want %#v", got, want)
}
}
func TestResolvePreviousSessionArtifactWithCatalogMissingReturnsTypedError(t *testing.T) { func TestResolvePreviousSessionArtifactWithCatalogMissingReturnsTypedError(t *testing.T) {
workspace := t.TempDir() workspace := t.TempDir()
paths := buildSessionPaths(workspace, "campaign", "session") paths := buildSessionPaths(workspace, "campaign", "session")

View File

@@ -1,7 +1,10 @@
package artifacts package artifacts
import ( import (
"fmt"
"path"
"path/filepath" "path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/config" "gitea.maximumdirect.net/eric/narratio/internal/config"
) )
@@ -64,7 +67,7 @@ func SessionPreviousArtifactsDirForCampaign(rootDir, campaign, sessionID string)
func SessionPreviousArtifactPathForCampaign(rootDir, campaign, sessionID, artifactRelativePath string) string { func SessionPreviousArtifactPathForCampaign(rootDir, campaign, sessionID, artifactRelativePath string) string {
return filepath.Join( return filepath.Join(
SessionPreviousArtifactsDirForCampaign(rootDir, campaign, sessionID), SessionPreviousArtifactsDirForCampaign(rootDir, campaign, sessionID),
filepath.FromSlash(artifactRelativePath), filepath.FromSlash(previousArtifactCacheRelativePath(artifactRelativePath)),
) )
} }
@@ -88,6 +91,60 @@ func SessionSpoolAudioDir(spoolRoot, campaign, sessionID, runID string) string {
return filepath.Join(spoolRoot, campaign, sessionID, runID, config.PathAudioDirSegment) return filepath.Join(spoolRoot, campaign, sessionID, runID, config.PathAudioDirSegment)
} }
// SessionSpoolDir returns the campaign/session scoped local spool root.
func SessionSpoolDir(spoolRoot, campaign, sessionID string) string {
return filepath.Join(spoolRoot, campaign, sessionID)
}
// SessionSpoolRestoreAudioDir returns the local spool audio path for restore downloads.
func SessionSpoolRestoreAudioDir(spoolRoot, campaign, sessionID string) string {
return filepath.Join(SessionSpoolDir(spoolRoot, campaign, sessionID), "restore", config.PathAudioDirSegment)
}
// S3AudioCachePath returns the durable local cache path for one S3 audio object.
func S3AudioCachePath(cacheRoot, bucket, key string) (string, error) {
root := filepath.Clean(strings.TrimSpace(cacheRoot))
if root == "." || root == "" {
return "", fmt.Errorf("cache root is required")
}
bucket = strings.Trim(strings.TrimSpace(bucket), "/")
if bucket == "" || bucket == "." || bucket == ".." || strings.Contains(bucket, "/") || strings.Contains(bucket, `\`) {
return "", fmt.Errorf("bucket is required and must be a single path segment")
}
rawKey := strings.ReplaceAll(strings.TrimSpace(key), `\`, "/")
if strings.HasPrefix(rawKey, "/") {
return "", fmt.Errorf("s3 key must not be absolute")
}
cleanKey := cleanCacheS3Key(rawKey)
if cleanKey == "" {
return "", fmt.Errorf("s3 key is required")
}
if cleanKey == ".." || strings.HasPrefix(cleanKey, "../") || strings.HasPrefix(cleanKey, "/") {
return "", fmt.Errorf("s3 key must not escape cache root")
}
return filepath.Join(root, "s3", bucket, filepath.FromSlash(cleanKey)), nil
}
// S3AudioCacheNamespaceDir returns the durable local cache namespace for all
// Narratio S3 audio objects under one bucket/root prefix.
func S3AudioCacheNamespaceDir(cacheRoot, bucket, rootPrefix string) (string, error) {
sentinelKey := path.Join(cleanS3PathPart(rootPrefix), config.S3CampaignsSegment, ".narratio-cache-sentinel")
sentinelPath, err := S3AudioCachePath(cacheRoot, bucket, sentinelKey)
if err != nil {
return "", err
}
return filepath.Dir(sentinelPath), nil
}
func cleanCacheS3Key(key string) string {
normalized := strings.ReplaceAll(strings.TrimSpace(key), `\`, "/")
normalized = strings.Trim(normalized, "/")
if normalized == "" {
return ""
}
return path.Clean(normalized)
}
// SessionPreviousDir returns the previous-session state directory for already-resolved session paths. // SessionPreviousDir returns the previous-session state directory for already-resolved session paths.
func SessionPreviousDir(paths SessionPaths) string { func SessionPreviousDir(paths SessionPaths) string {
return paths.PreviousDir return paths.PreviousDir
@@ -105,7 +162,19 @@ func SessionPreviousArtifactsDir(paths SessionPaths) string {
// SessionPreviousArtifactPath returns a path under previous/artifacts for already-resolved session paths. // SessionPreviousArtifactPath returns a path under previous/artifacts for already-resolved session paths.
func SessionPreviousArtifactPath(paths SessionPaths, artifactRelativePath string) string { func SessionPreviousArtifactPath(paths SessionPaths, artifactRelativePath string) string {
return filepath.Join(paths.PreviousArtifactsDir, filepath.FromSlash(artifactRelativePath)) return filepath.Join(paths.PreviousArtifactsDir, filepath.FromSlash(previousArtifactCacheRelativePath(artifactRelativePath)))
}
func previousArtifactCacheRelativePath(artifactRelativePath string) string {
rel := filepath.ToSlash(filepath.Clean(filepath.FromSlash(strings.TrimSpace(artifactRelativePath))))
if rel == "." {
return ""
}
const artifactsPrefix = "artifacts/"
if strings.HasPrefix(rel, artifactsPrefix) && len(rel) > len(artifactsPrefix) {
return strings.TrimPrefix(rel, artifactsPrefix)
}
return rel
} }
func buildSessionPaths(workspaceRoot, campaign, sessionID string) SessionPaths { func buildSessionPaths(workspaceRoot, campaign, sessionID string) SessionPaths {

View File

@@ -74,6 +74,11 @@ func TestSessionPreviousPathsForCampaign(t *testing.T) {
if artifactPath != wantArtifactPath { if artifactPath != wantArtifactPath {
t.Fatalf("SessionPreviousArtifactPathForCampaign() = %q, want %q", artifactPath, wantArtifactPath) t.Fatalf("SessionPreviousArtifactPathForCampaign() = %q, want %q", artifactPath, wantArtifactPath)
} }
archiveRelativeArtifactPath := SessionPreviousArtifactPathForCampaign(root, "forsaken", "2026-04-19", "artifacts/session_recap.md")
if archiveRelativeArtifactPath != wantArtifactPath {
t.Fatalf("SessionPreviousArtifactPathForCampaign(archive-relative) = %q, want %q", archiveRelativeArtifactPath, wantArtifactPath)
}
} }
func TestSessionPreviousPathsFromSessionPaths(t *testing.T) { func TestSessionPreviousPathsFromSessionPaths(t *testing.T) {
@@ -93,6 +98,12 @@ func TestSessionPreviousPathsFromSessionPaths(t *testing.T) {
if got != want { if got != want {
t.Fatalf("SessionPreviousArtifactPath() = %q, want %q", got, want) t.Fatalf("SessionPreviousArtifactPath() = %q, want %q", got, want)
} }
got = SessionPreviousArtifactPath(paths, "artifacts/session_recap.md")
want = filepath.Join(paths.PreviousArtifactsDir, "session_recap.md")
if got != want {
t.Fatalf("SessionPreviousArtifactPath(archive-relative) = %q, want %q", got, want)
}
} }
func TestSessionSpoolAudioDir(t *testing.T) { func TestSessionSpoolAudioDir(t *testing.T) {
@@ -103,3 +114,66 @@ func TestSessionSpoolAudioDir(t *testing.T) {
t.Fatalf("SessionSpoolAudioDir() = %q, want %q", got, want) t.Fatalf("SessionSpoolAudioDir() = %q, want %q", got, want)
} }
} }
func TestSessionSpoolDir(t *testing.T) {
root := "/var/spool/narratio"
got := SessionSpoolDir(root, "forsaken", "2026-04-19")
want := filepath.Join(root, "forsaken", "2026-04-19")
if got != want {
t.Fatalf("SessionSpoolDir() = %q, want %q", got, want)
}
}
func TestSessionSpoolRestoreAudioDir(t *testing.T) {
root := "/var/spool/narratio"
got := SessionSpoolRestoreAudioDir(root, "forsaken", "2026-04-19")
want := filepath.Join(root, "forsaken", "2026-04-19", "restore", "audio")
if got != want {
t.Fatalf("SessionSpoolRestoreAudioDir() = %q, want %q", got, want)
}
}
func TestS3AudioCachePath(t *testing.T) {
got, err := S3AudioCachePath("/var/cache/narratio", "my-dnd-archive", "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac")
if err != nil {
t.Fatalf("S3AudioCachePath() error = %v", err)
}
want := filepath.Join("/var/cache/narratio", "s3", "my-dnd-archive", "dnd", "campaigns", "forsaken", "sessions", "2026-04-19", "audio", "alice.flac")
if got != want {
t.Fatalf("S3AudioCachePath() = %q, want %q", got, want)
}
}
func TestS3AudioCacheNamespaceDir(t *testing.T) {
got, err := S3AudioCacheNamespaceDir("/var/cache/narratio", "my-dnd-archive", "dnd")
if err != nil {
t.Fatalf("S3AudioCacheNamespaceDir() error = %v", err)
}
want := filepath.Join("/var/cache/narratio", "s3", "my-dnd-archive", "dnd", "campaigns")
if got != want {
t.Fatalf("S3AudioCacheNamespaceDir() = %q, want %q", got, want)
}
}
func TestS3AudioCachePathRejectsUnsafeInputs(t *testing.T) {
tests := []struct {
name string
root string
bucket string
key string
}{
{name: "empty root", root: "", bucket: "bucket", key: "audio/a.flac"},
{name: "empty bucket", root: "/cache", bucket: "", key: "audio/a.flac"},
{name: "bucket slash", root: "/cache", bucket: "bad/bucket", key: "audio/a.flac"},
{name: "empty key", root: "/cache", bucket: "bucket", key: ""},
{name: "escaping key", root: "/cache", bucket: "bucket", key: "../audio/a.flac"},
{name: "absolute key", root: "/cache", bucket: "bucket", key: "/audio/a.flac"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got, err := S3AudioCachePath(tt.root, tt.bucket, tt.key); err == nil {
t.Fatalf("S3AudioCachePath() = %q, want error", got)
}
})
}
}

View File

@@ -34,6 +34,18 @@ func S3AudioPrefix(sessionPrefix, audioPrefix string) string {
return ensureS3TrailingSlash(key) return ensureS3TrailingSlash(key)
} }
// S3SessionConfigKey returns the session config key.
// Format: {session_prefix}/session.yml
func S3SessionConfigKey(sessionPrefix string) string {
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), "session.yml")
}
// S3SessionLocksKey returns the mutable session lock store key.
// Format: {session_prefix}/locks.yml
func S3SessionLocksKey(sessionPrefix string) string {
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), "locks.yml")
}
// S3CurrentManifestKey returns the current manifest pointer key. // S3CurrentManifestKey returns the current manifest pointer key.
// Format: {session_prefix}/current/manifest.json // Format: {session_prefix}/current/manifest.json
func S3CurrentManifestKey(sessionPrefix string) string { func S3CurrentManifestKey(sessionPrefix string) string {

View File

@@ -17,6 +17,16 @@ func TestS3KeyConstruction(t *testing.T) {
t.Fatalf("audioPrefix = %q", audioPrefix) t.Fatalf("audioPrefix = %q", audioPrefix)
} }
sessionConfigKey := S3SessionConfigKey(`dnd\campaigns\forsaken\sessions\2026-04-19\`)
if sessionConfigKey != "dnd/campaigns/forsaken/sessions/2026-04-19/session.yml" {
t.Fatalf("session config key = %q", sessionConfigKey)
}
locksKey := S3SessionLocksKey(`dnd\campaigns\forsaken\sessions\2026-04-19\`)
if locksKey != "dnd/campaigns/forsaken/sessions/2026-04-19/locks.yml" {
t.Fatalf("locks key = %q", locksKey)
}
runPrefix := S3RunPrefix(sessionPrefix, runID) runPrefix := S3RunPrefix(sessionPrefix, runID)
wantRunPrefix := "dnd/campaigns/forsaken/sessions/2026-04-19/runs/" + runID + "/" wantRunPrefix := "dnd/campaigns/forsaken/sessions/2026-04-19/runs/" + runID + "/"
if runPrefix != wantRunPrefix { if runPrefix != wantRunPrefix {

223
internal/audio/s3_audio.go Normal file
View File

@@ -0,0 +1,223 @@
package audio
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
)
// S3MaterializeRequest describes one S3-backed audio materialization.
type S3MaterializeRequest struct {
Store storage.ObjectStore
Object storage.ObjectInfo
Bucket string
CacheRoot string
CacheEnabled bool
SpoolPath string
DestPath string
}
// S3MaterializeResult captures local materialization provenance.
type S3MaterializeResult struct {
Checksum string
CachePath string
SpoolPath string
CacheHit bool
Downloaded bool
}
// MaterializeS3Audio installs one S3 audio object into the destination path,
// reusing and refreshing the durable local cache when enabled.
func MaterializeS3Audio(ctx context.Context, req S3MaterializeRequest) (S3MaterializeResult, error) {
if err := ctx.Err(); err != nil {
return S3MaterializeResult{}, err
}
if req.Store == nil {
return S3MaterializeResult{}, fmt.Errorf("object store is required")
}
key := strings.TrimSpace(req.Object.Key)
if key == "" {
return S3MaterializeResult{}, fmt.Errorf("s3 object key is required")
}
if strings.TrimSpace(req.DestPath) == "" {
return S3MaterializeResult{}, fmt.Errorf("destination path is required")
}
result := S3MaterializeResult{}
if req.CacheEnabled {
cachePath, err := artifacts.S3AudioCachePath(req.CacheRoot, req.Bucket, key)
if err != nil {
return S3MaterializeResult{}, fmt.Errorf("resolve audio cache path: %w", err)
}
result.CachePath = cachePath
if ok, err := validCachedAudio(cachePath, req.Object.Size); err != nil {
return S3MaterializeResult{}, err
} else if ok {
checksum, err := copyFileAtomicWithChecksum(cachePath, req.DestPath, 0o644)
if err != nil {
return S3MaterializeResult{}, fmt.Errorf("materialize cached audio %q: %w", cachePath, err)
}
result.Checksum = checksum
result.CacheHit = true
return result, nil
}
}
spoolPath := strings.TrimSpace(req.SpoolPath)
if spoolPath == "" {
return S3MaterializeResult{}, fmt.Errorf("spool path is required for s3 audio download")
}
if err := downloadObjectAtomic(ctx, req.Store, key, spoolPath); err != nil {
return S3MaterializeResult{}, fmt.Errorf("download s3 audio object %q: %w", key, err)
}
if err := validateLocalAudio(spoolPath, req.Object.Size); err != nil {
return S3MaterializeResult{}, fmt.Errorf("validate downloaded audio %q: %w", spoolPath, err)
}
checksum, err := copyFileAtomicWithChecksum(spoolPath, req.DestPath, 0o644)
if err != nil {
return S3MaterializeResult{}, fmt.Errorf("materialize downloaded audio %q: %w", filepath.Base(req.DestPath), err)
}
result.Checksum = checksum
result.SpoolPath = spoolPath
result.Downloaded = true
if result.CachePath != "" {
if _, err := copyFileAtomicWithChecksum(spoolPath, result.CachePath, 0o644); err != nil {
return S3MaterializeResult{}, fmt.Errorf("populate audio cache %q: %w", result.CachePath, err)
}
}
return result, nil
}
func validCachedAudio(path string, expectedSize int64) (bool, error) {
info, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, fmt.Errorf("stat cached audio %q: %w", path, err)
}
if info.IsDir() {
return false, fmt.Errorf("cached audio path is a directory: %q", path)
}
if info.Size() <= 0 {
return false, nil
}
if expectedSize > 0 && info.Size() != expectedSize {
return false, nil
}
return true, nil
}
func validateLocalAudio(path string, expectedSize int64) error {
info, err := os.Stat(path)
if err != nil {
return fmt.Errorf("stat: %w", err)
}
if info.IsDir() {
return fmt.Errorf("path is a directory")
}
if info.Size() <= 0 {
return fmt.Errorf("file is empty")
}
if expectedSize > 0 && info.Size() != expectedSize {
return fmt.Errorf("size %d does not match remote size %d", info.Size(), expectedSize)
}
return nil
}
func downloadObjectAtomic(ctx context.Context, store storage.ObjectStore, key, destPath string) error {
if strings.TrimSpace(destPath) == "" {
return fmt.Errorf("destination path is required")
}
dir := filepath.Dir(destPath)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("create destination directory: %w", err)
}
base := filepath.Base(destPath)
tmp, err := os.CreateTemp(dir, "."+base+".download-*.tmp")
if err != nil {
return fmt.Errorf("create temp file: %w", err)
}
tmpPath := tmp.Name()
if err := tmp.Close(); err != nil {
_ = os.Remove(tmpPath)
return fmt.Errorf("close temp file: %w", err)
}
removeTmp := true
defer func() {
if removeTmp {
_ = os.Remove(tmpPath)
}
}()
if err := store.Download(ctx, key, tmpPath); err != nil {
return err
}
if err := os.Chmod(tmpPath, 0o644); err != nil {
return fmt.Errorf("set temp file permissions: %w", err)
}
if err := os.Rename(tmpPath, destPath); err != nil {
return fmt.Errorf("install downloaded file: %w", err)
}
removeTmp = false
return nil
}
func copyFileAtomicWithChecksum(src, dst string, perm os.FileMode) (string, error) {
if strings.TrimSpace(src) == "" || strings.TrimSpace(dst) == "" {
return "", fmt.Errorf("source and destination paths are required")
}
in, err := os.Open(src)
if err != nil {
return "", err
}
defer func() { _ = in.Close() }()
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
return "", fmt.Errorf("create destination directory: %w", err)
}
base := filepath.Base(dst)
tmp, err := os.CreateTemp(filepath.Dir(dst), "."+base+".tmp-*")
if err != nil {
return "", fmt.Errorf("create temp file: %w", err)
}
tmpPath := tmp.Name()
removeTmp := true
defer func() {
if removeTmp {
_ = os.Remove(tmpPath)
}
}()
digest := sha256.New()
if _, err := io.Copy(io.MultiWriter(tmp, digest), in); err != nil {
_ = tmp.Close()
return "", fmt.Errorf("copy file: %w", err)
}
if err := tmp.Sync(); err != nil {
_ = tmp.Close()
return "", fmt.Errorf("sync temp file: %w", err)
}
if err := tmp.Close(); err != nil {
return "", fmt.Errorf("close temp file: %w", err)
}
if err := os.Chmod(tmpPath, perm); err != nil {
return "", fmt.Errorf("chmod temp file: %w", err)
}
if err := os.Rename(tmpPath, dst); err != nil {
return "", fmt.Errorf("install temp file: %w", err)
}
removeTmp = false
return hex.EncodeToString(digest.Sum(nil)), nil
}

View File

@@ -0,0 +1,145 @@
package audio
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
)
func TestMaterializeS3AudioCacheMissDownloadsAndPopulatesCache(t *testing.T) {
root := t.TempDir()
fake := &storage.FakeBackend{}
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("audio")})
req := testMaterializeRequest(t, root, fake, key, int64(len("audio")))
result, err := MaterializeS3Audio(context.Background(), req)
if err != nil {
t.Fatalf("MaterializeS3Audio() error = %v", err)
}
if !result.Downloaded || result.CacheHit {
t.Fatalf("result = %#v, want downloaded miss", result)
}
if len(fake.Downloads) != 1 {
t.Fatalf("downloads = %d, want 1", len(fake.Downloads))
}
assertFileEquals(t, req.DestPath, "audio")
assertFileEquals(t, req.SpoolPath, "audio")
assertFileEquals(t, result.CachePath, "audio")
if result.Checksum == "" {
t.Fatalf("checksum is empty")
}
}
func TestMaterializeS3AudioCacheHitSkipsDownload(t *testing.T) {
root := t.TempDir()
fake := &storage.FakeBackend{}
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
req := testMaterializeRequest(t, root, fake, key, int64(len("audio")))
cachePath, err := artifacts.S3AudioCachePath(req.CacheRoot, req.Bucket, key)
if err != nil {
t.Fatalf("cache path: %v", err)
}
writeAudioTestFile(t, cachePath, "audio")
result, err := MaterializeS3Audio(context.Background(), req)
if err != nil {
t.Fatalf("MaterializeS3Audio() error = %v", err)
}
if !result.CacheHit || result.Downloaded {
t.Fatalf("result = %#v, want cache hit", result)
}
if len(fake.Downloads) != 0 {
t.Fatalf("downloads = %d, want 0", len(fake.Downloads))
}
assertFileEquals(t, req.DestPath, "audio")
if result.SpoolPath != "" {
t.Fatalf("spool path = %q, want empty on cache hit", result.SpoolPath)
}
}
func TestMaterializeS3AudioInvalidCacheRefreshesFromS3(t *testing.T) {
root := t.TempDir()
fake := &storage.FakeBackend{}
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("fresh-audio")})
req := testMaterializeRequest(t, root, fake, key, int64(len("fresh-audio")))
cachePath, err := artifacts.S3AudioCachePath(req.CacheRoot, req.Bucket, key)
if err != nil {
t.Fatalf("cache path: %v", err)
}
writeAudioTestFile(t, cachePath, "stale")
result, err := MaterializeS3Audio(context.Background(), req)
if err != nil {
t.Fatalf("MaterializeS3Audio() error = %v", err)
}
if !result.Downloaded || result.CacheHit {
t.Fatalf("result = %#v, want refreshed miss", result)
}
if len(fake.Downloads) != 1 {
t.Fatalf("downloads = %d, want 1", len(fake.Downloads))
}
assertFileEquals(t, req.DestPath, "fresh-audio")
assertFileEquals(t, cachePath, "fresh-audio")
}
func TestMaterializeS3AudioDownloadFailureLeavesDestinationMissing(t *testing.T) {
root := t.TempDir()
fake := &storage.FakeBackend{DownloadErr: os.ErrPermission}
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("audio")})
req := testMaterializeRequest(t, root, fake, key, int64(len("audio")))
_, err := MaterializeS3Audio(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "download s3 audio object") {
t.Fatalf("error = %v, want download error", err)
}
assertMissing(t, req.DestPath)
}
func testMaterializeRequest(t *testing.T, root string, fake *storage.FakeBackend, key string, size int64) S3MaterializeRequest {
t.Helper()
return S3MaterializeRequest{
Store: fake,
Object: storage.ObjectInfo{Key: key, Size: size, ETag: "etag"},
Bucket: "my-dnd-archive",
CacheRoot: filepath.Join(root, "cache"),
CacheEnabled: true,
SpoolPath: filepath.Join(root, "spool", "alice.flac"),
DestPath: filepath.Join(root, "work", "audio", "alice.flac"),
}
}
func writeAudioTestFile(t *testing.T, path, contents string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("mkdir %q: %v", path, err)
}
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
t.Fatalf("write %q: %v", path, err)
}
}
func assertFileEquals(t *testing.T, path, want string) {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %q: %v", path, err)
}
if string(data) != want {
t.Fatalf("%q = %q, want %q", path, string(data), want)
}
}
func assertMissing(t *testing.T, path string) {
t.Helper()
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("stat %q = %v, want not exists", path, err)
}
}

View File

@@ -0,0 +1,67 @@
package config
import (
"strings"
"testing"
)
func TestCacheDefaults(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, `workspace:
root: /tmp/narratio
whisperx:
transcribe_url: https://example.com/transcribe
analyzer:
timeout: 20m
notification:
timeout: 10s
`, `session_id: 2026-05-03
inputs:
audio_dir: ./audio
`)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Pipeline.Cache.Root != DefaultCacheRoot {
t.Fatalf("cache.root = %q, want %q", cfg.Pipeline.Cache.Root, DefaultCacheRoot)
}
if cfg.Pipeline.Cache.S3Audio == nil || *cfg.Pipeline.Cache.S3Audio != DefaultCacheS3Audio {
t.Fatalf("cache.s3_audio = %v, want %v", cfg.Pipeline.Cache.S3Audio, DefaultCacheS3Audio)
}
}
func TestCacheStrictDecodeRejectsUnknownFields(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, `workspace:
root: /tmp/narratio
cache:
root: /var/cache/narratio
unknown: true
`, `session_id: 2026-05-03
inputs:
audio_dir: ./audio
`)
_, err := Load(pipelinePath, sessionPath)
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("Load() error = %v, want strict decode failed", err)
}
}
func TestCacheValidationRequiresRootWhenS3AudioEnabled(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, `workspace:
root: /tmp/narratio
cache:
root: " "
s3_audio: true
`, `session_id: 2026-05-03
inputs:
audio_dir: ./audio
`)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if err == nil || !strings.Contains(err.Error(), "pipeline.cache.root is required") {
t.Fatalf("Validate() error = %v, want cache root error", err)
}
}

View File

@@ -0,0 +1,137 @@
package config
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestCampaignConfigDefaultSearchOrder(t *testing.T) {
want := []string{
"/usr/local/etc/narratio/campaign.yml",
"/etc/narratio/campaign.yml",
}
if len(DefaultCampaignConfigSearchPaths) != len(want) {
t.Fatalf("DefaultCampaignConfigSearchPaths = %#v, want %#v", DefaultCampaignConfigSearchPaths, want)
}
for i := range want {
if DefaultCampaignConfigSearchPaths[i] != want[i] {
t.Fatalf("DefaultCampaignConfigSearchPaths[%d] = %q, want %q", i, DefaultCampaignConfigSearchPaths[i], want[i])
}
}
}
func TestCampaignStrictDecodeRejectsUnknownFields(t *testing.T) {
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
"campaign: sample-campaign\nunknown: true\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n",
"session_id: 2026-05-03\ninputs:\n audio_dir: ./audio\n",
)
_, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
if err == nil {
t.Fatal("expected load error, got nil")
}
if !strings.Contains(err.Error(), "campaign file") || !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("error = %q, want campaign strict decode context", err.Error())
}
}
func TestCampaignSessionMergeFillsStableInputs(t *testing.T) {
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
"campaign: sample-campaign\ninputs:\n speakers_file: ./campaign-speakers.yml\n autocorrect_file: ./campaign-autocorrect.yml\n glossary_file: ./campaign-glossary.yml\n",
"session_id: 2026-05-03\ninputs:\n audio_dir: ./audio\n",
)
cfg, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
if err != nil {
t.Fatalf("LoadWithSessionOptions() error = %v", err)
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
if cfg.Session.Campaign != "sample-campaign" {
t.Fatalf("session campaign = %q, want campaign config value", cfg.Session.Campaign)
}
assertResolvedStableInput(t, cfg.StableInputs.SpeakersFile, "./campaign-speakers.yml", campaignPath, "campaign_config")
assertResolvedStableInput(t, cfg.StableInputs.AutocorrectFile, "./campaign-autocorrect.yml", campaignPath, "campaign_config")
assertResolvedStableInput(t, cfg.StableInputs.GlossaryFile, "./campaign-glossary.yml", campaignPath, "campaign_config")
}
func TestCampaignSessionMergeSessionOverridesStableInputs(t *testing.T) {
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
"campaign: sample-campaign\ninputs:\n speakers_file: ./campaign-speakers.yml\n autocorrect_file: ./campaign-autocorrect.yml\n glossary_file: ./campaign-glossary.yml\n",
"session_id: 2026-05-03\ncampaign: sample-campaign\ninputs:\n audio_dir: ./audio\n speakers_file: ./session-speakers.yml\n",
)
cfg, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
if err != nil {
t.Fatalf("LoadWithSessionOptions() error = %v", err)
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
assertResolvedStableInput(t, cfg.StableInputs.SpeakersFile, "./session-speakers.yml", sessionPath, "session_config")
assertResolvedStableInput(t, cfg.StableInputs.AutocorrectFile, "./campaign-autocorrect.yml", campaignPath, "campaign_config")
assertResolvedStableInput(t, cfg.StableInputs.GlossaryFile, "./campaign-glossary.yml", campaignPath, "campaign_config")
}
func TestCampaignSessionMismatchFails(t *testing.T) {
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
"campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n",
"session_id: 2026-05-03\ncampaign: other-campaign\ninputs:\n audio_dir: ./audio\n",
)
_, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
if err == nil {
t.Fatal("expected load error, got nil")
}
if !strings.Contains(err.Error(), "does not match campaign config") {
t.Fatalf("error = %q, want campaign mismatch context", err.Error())
}
}
func TestLoadMissingCampaignFileFails(t *testing.T) {
pipelinePath, _, sessionPath := writeCampaignConfigTestFiles(t,
"campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n",
"session_id: 2026-05-03\ninputs:\n audio_dir: ./audio\n",
)
missingCampaignPath := filepath.Join(filepath.Dir(sessionPath), "missing-campaign.yml")
_, err := LoadWithSessionOptions(pipelinePath, missingCampaignPath, sessionPath, SessionLoadOptions{})
if err == nil {
t.Fatal("expected load error, got nil")
}
if !strings.Contains(err.Error(), "load campaign config") {
t.Fatalf("error = %q, want campaign load context", err.Error())
}
}
func writeCampaignConfigTestFiles(t *testing.T, campaignYAML, sessionYAML string) (string, string, string) {
t.Helper()
dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml")
campaignPath := filepath.Join(dir, "campaign.yml")
sessionPath := filepath.Join(dir, "session.yml")
pipelineYAML := "workspace:\n root: " + filepath.ToSlash(filepath.Join(dir, "work")) + "\nwhisperx:\n transcribe_url: https://example.com/transcribe\nanalyzer:\n timeout: 20m\nnotification:\n timeout: 10s\n"
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
t.Fatalf("write pipeline.yml: %v", err)
}
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
t.Fatalf("write campaign.yml: %v", err)
}
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
return pipelinePath, campaignPath, sessionPath
}
func assertResolvedStableInput(t *testing.T, got ResolvedInputFile, wantPath, wantConfigPath, wantSource string) {
t.Helper()
if got.Path != wantPath || got.ConfigPath != wantConfigPath || got.Source != wantSource {
t.Fatalf("resolved input = %#v, want path=%q config_path=%q source=%q", got, wantPath, wantConfigPath, wantSource)
}
}

View File

@@ -1,11 +1,17 @@
package config package config
// Config is the resolved combined configuration from pipeline.yml and session.yml. // Config is the resolved combined configuration from pipeline.yml,
// campaign.yml, and session.yml.
type Config struct { type Config struct {
Pipeline *PipelineConfig Pipeline *PipelineConfig
Campaign *CampaignConfig
Session *SessionConfig Session *SessionConfig
PipelinePath string PipelinePath string
CampaignPath string
SessionPath string SessionPath string
StableInputs ResolvedStableInputs
SessionSource SessionSource
} }
// PipelineConfig contains durable pipeline-level settings. // PipelineConfig contains durable pipeline-level settings.
@@ -13,6 +19,7 @@ type PipelineConfig struct {
Workspace WorkspaceConfig `yaml:"workspace"` Workspace WorkspaceConfig `yaml:"workspace"`
Storage StorageConfig `yaml:"storage"` Storage StorageConfig `yaml:"storage"`
Spool SpoolConfig `yaml:"spool"` Spool SpoolConfig `yaml:"spool"`
Cache CacheConfig `yaml:"cache"`
Archive *ArchiveConfig `yaml:"archive"` Archive *ArchiveConfig `yaml:"archive"`
Secrets *SecretsConfig `yaml:"secrets"` Secrets *SecretsConfig `yaml:"secrets"`
WhisperX WhisperXConfig `yaml:"whisperx"` WhisperX WhisperXConfig `yaml:"whisperx"`
@@ -25,6 +32,19 @@ type PipelineConfig struct {
Notification NotificationConfig `yaml:"notification"` Notification NotificationConfig `yaml:"notification"`
} }
// CampaignConfig contains stable campaign-level identity and input defaults.
type CampaignConfig struct {
Campaign string `yaml:"campaign"`
Inputs CampaignInputsConfig `yaml:"inputs"`
}
// CampaignInputsConfig contains stable campaign-level input file references.
type CampaignInputsConfig struct {
SpeakersFile string `yaml:"speakers_file"`
AutocorrectFile string `yaml:"autocorrect_file"`
GlossaryFile string `yaml:"glossary_file"`
}
// SessionConfig contains per-session inputs and metadata. // SessionConfig contains per-session inputs and metadata.
type SessionConfig struct { type SessionConfig struct {
SessionID string `yaml:"session_id"` SessionID string `yaml:"session_id"`
@@ -71,11 +91,18 @@ type SpoolConfig struct {
DeleteAudioAfterArchive bool `yaml:"delete_audio_after_archive"` DeleteAudioAfterArchive bool `yaml:"delete_audio_after_archive"`
} }
// CacheConfig configures durable local caches for reusable remote inputs.
type CacheConfig struct {
Root string `yaml:"root"`
S3Audio *bool `yaml:"s3_audio"`
}
// ArchiveConfig configures archive behavior and artifact promotions. // ArchiveConfig configures archive behavior and artifact promotions.
type ArchiveConfig struct { type ArchiveConfig struct {
Enabled *bool `yaml:"enabled"` Enabled *bool `yaml:"enabled"`
UploadRun *bool `yaml:"upload_run"` UploadRun *bool `yaml:"upload_run"`
PromoteArtifacts []ArchivePromotionRule `yaml:"promote_artifacts"` PromoteArtifacts []ArchivePromotionRule `yaml:"promote_artifacts"`
Locks []ArchiveLockRule `yaml:"locks"`
} }
// ArchivePromotionRule configures one artifact promotion mapping. // ArchivePromotionRule configures one artifact promotion mapping.
@@ -85,6 +112,18 @@ type ArchivePromotionRule struct {
Required *bool `yaml:"required"` Required *bool `yaml:"required"`
} }
// ArchiveLockRule prevents one source-based promotion from overwriting its
// top-level archive destination.
type ArchiveLockRule struct {
Source string `yaml:"source"`
Reason string `yaml:"reason"`
}
// ArchiveLockStore is the mutable per-session remote lock store.
type ArchiveLockStore struct {
Locks []ArchiveLockRule `yaml:"locks"`
}
// WhisperXConfig configures WhisperX adapter settings. // WhisperXConfig configures WhisperX adapter settings.
type WhisperXConfig struct { type WhisperXConfig struct {
TranscribeURL string `yaml:"transcribe_url"` TranscribeURL string `yaml:"transcribe_url"`
@@ -229,3 +268,29 @@ type SessionInputsConfig struct {
type SessionAudioS3Input struct { type SessionAudioS3Input struct {
Prefix string `yaml:"prefix"` Prefix string `yaml:"prefix"`
} }
// ResolvedStableInputs records where stable input file paths came from after
// campaign/session merge.
type ResolvedStableInputs struct {
SpeakersFile ResolvedInputFile
AutocorrectFile ResolvedInputFile
GlossaryFile ResolvedInputFile
}
// ResolvedInputFile records one merged config path and its source config file.
type ResolvedInputFile struct {
Path string
ConfigPath string
Source string
}
// SessionSource records where session.yml came from before materialization.
type SessionSource struct {
Source string
LocalPath string
S3Bucket string
S3Key string
S3Size int64
S3ETag string
SpoolPath string
}

View File

@@ -1,11 +1,12 @@
package config package config
// Default filesystem locations for pipeline configuration lookup when --config // Default filesystem locations for config lookup when config path flags are
// is omitted. Order is highest to lowest precedence. // omitted. Order is highest to lowest precedence.
const ( const (
DefaultPipelineConfigPathUsrLocal = "/usr/local/etc/narratio/pipeline.yml" DefaultPipelineConfigPathUsrLocal = "/usr/local/etc/narratio/pipeline.yml"
DefaultPipelineConfigPathEtc = "/etc/narratio/pipeline.yml" DefaultPipelineConfigPathEtc = "/etc/narratio/pipeline.yml"
DefaultSessionConfigPathLocal = "./session.yml" DefaultCampaignConfigPathUsrLocal = "/usr/local/etc/narratio/campaign.yml"
DefaultCampaignConfigPathEtc = "/etc/narratio/campaign.yml"
DefaultSessionConfigPathUsrLocal = "/usr/local/etc/narratio/session.yml" DefaultSessionConfigPathUsrLocal = "/usr/local/etc/narratio/session.yml"
DefaultSessionConfigPathEtc = "/etc/narratio/session.yml" DefaultSessionConfigPathEtc = "/etc/narratio/session.yml"
DefaultS3AccessKeyIDEnv = "OBJECT_STORAGE_KEY_ID" DefaultS3AccessKeyIDEnv = "OBJECT_STORAGE_KEY_ID"
@@ -13,6 +14,8 @@ const (
DefaultStorageS3RootPrefix = "dnd" DefaultStorageS3RootPrefix = "dnd"
DefaultWorkspaceRoot = "/var/lib/narratio" DefaultWorkspaceRoot = "/var/lib/narratio"
DefaultSpoolRoot = "/var/spool/narratio" DefaultSpoolRoot = "/var/spool/narratio"
DefaultCacheRoot = "/var/cache/narratio"
DefaultCacheS3Audio = true
DefaultWhisperXLanguage = "en" DefaultWhisperXLanguage = "en"
DefaultWhisperXTimeout = "30m" DefaultWhisperXTimeout = "30m"
@@ -88,13 +91,22 @@ var DefaultPipelineConfigSearchPaths = []string{
DefaultPipelineConfigPathEtc, DefaultPipelineConfigPathEtc,
} }
// DefaultCampaignConfigSearchPaths defines the default search order for
// campaign.yml when callers do not provide an explicit path.
//
// Keep this in a variable so future defaults can be extended without changing
// call sites.
var DefaultCampaignConfigSearchPaths = []string{
DefaultCampaignConfigPathUsrLocal,
DefaultCampaignConfigPathEtc,
}
// DefaultSessionConfigSearchPaths defines the default search order for // DefaultSessionConfigSearchPaths defines the default search order for
// session.yml when callers do not provide an explicit path. // session.yml when callers do not provide an explicit path.
// //
// Keep this in a variable so future defaults can be extended without changing // Keep this in a variable so future defaults can be extended without changing
// call sites. // call sites.
var DefaultSessionConfigSearchPaths = []string{ var DefaultSessionConfigSearchPaths = []string{
DefaultSessionConfigPathLocal,
DefaultSessionConfigPathUsrLocal, DefaultSessionConfigPathUsrLocal,
DefaultSessionConfigPathEtc, DefaultSessionConfigPathEtc,
} }

View File

@@ -22,6 +22,15 @@ func LoadPipeline(path string) (*PipelineConfig, error) {
return &cfg, nil return &cfg, nil
} }
// LoadCampaign loads campaign configuration from a YAML file with strict field checking.
func LoadCampaign(path string) (*CampaignConfig, error) {
var cfg CampaignConfig
if err := decodeStrictYAML("campaign", path, &cfg); err != nil {
return nil, fmt.Errorf("load campaign config: %w", err)
}
return &cfg, nil
}
// LoadSession loads session configuration from a YAML file with strict field checking. // LoadSession loads session configuration from a YAML file with strict field checking.
func LoadSession(path string) (*SessionConfig, error) { func LoadSession(path string) (*SessionConfig, error) {
return LoadSessionWithOptions(path, SessionLoadOptions{}) return LoadSessionWithOptions(path, SessionLoadOptions{})
@@ -40,20 +49,25 @@ func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfi
if err != nil { if err != nil {
return nil, fmt.Errorf("load session config: session file %q: open: %w", path, err) return nil, fmt.Errorf("load session config: session file %q: open: %w", path, err)
} }
return LoadSessionBytesWithOptions(path, sessionBytes, opts)
}
rendered, err := renderSessionTemplate(string(sessionBytes), opts) // LoadSessionBytesWithOptions loads session configuration from YAML bytes with
// strict field checking after template rendering.
func LoadSessionBytesWithOptions(label string, data []byte, opts SessionLoadOptions) (*SessionConfig, error) {
rendered, err := renderSessionTemplate(string(data), opts)
if err != nil { if err != nil {
return nil, fmt.Errorf("load session config: %w", err) return nil, fmt.Errorf("load session config: %w", err)
} }
var cfg SessionConfig var cfg SessionConfig
if err := decodeStrictYAMLFromReader("session", path, strings.NewReader(rendered), &cfg); err != nil { if err := decodeStrictYAMLFromReader("session", label, strings.NewReader(rendered), &cfg); err != nil {
return nil, fmt.Errorf("load session config: %w", err) return nil, fmt.Errorf("load session config: %w", err)
} }
if strings.TrimSpace(opts.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != strings.TrimSpace(opts.SessionID) { if strings.TrimSpace(opts.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != strings.TrimSpace(opts.SessionID) {
return nil, fmt.Errorf( return nil, fmt.Errorf(
"load session config: session file %q: session_id mismatch: --session-id %q does not match rendered session_id %q", "load session config: session file %q: session_id mismatch: --session-id %q does not match rendered session_id %q",
path, label,
strings.TrimSpace(opts.SessionID), strings.TrimSpace(opts.SessionID),
strings.TrimSpace(cfg.SessionID), strings.TrimSpace(cfg.SessionID),
) )
@@ -63,7 +77,7 @@ func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfi
strings.TrimSpace(cfg.PreviousSessionID) != strings.TrimSpace(opts.PreviousSessionID) { strings.TrimSpace(cfg.PreviousSessionID) != strings.TrimSpace(opts.PreviousSessionID) {
return nil, fmt.Errorf( return nil, fmt.Errorf(
"load session config: session file %q: previous_session_id mismatch: --previous-session-id %q does not match rendered previous_session_id %q", "load session config: session file %q: previous_session_id mismatch: --previous-session-id %q does not match rendered previous_session_id %q",
path, label,
strings.TrimSpace(opts.PreviousSessionID), strings.TrimSpace(opts.PreviousSessionID),
strings.TrimSpace(cfg.PreviousSessionID), strings.TrimSpace(cfg.PreviousSessionID),
) )
@@ -71,32 +85,171 @@ func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfi
return &cfg, nil return &cfg, nil
} }
// Load loads and resolves combined pipeline and session configuration. // LoadArchiveLockStoreBytes loads a mutable session lock store with strict
func Load(pipelinePath, sessionPath string) (*Config, error) { // field checking and source validation.
return LoadWithSessionOptions(pipelinePath, sessionPath, SessionLoadOptions{}) func LoadArchiveLockStoreBytes(label string, data []byte, scriptorium *ScriptoriumConfig) (*ArchiveLockStore, error) {
var store ArchiveLockStore
if err := decodeStrictYAMLFromReader("archive lock store", label, strings.NewReader(string(data)), &store); err != nil {
return nil, fmt.Errorf("load archive lock store: %w", err)
}
locks, err := ValidateArchiveLockRules(store.Locks, scriptorium, "locks")
if err != nil {
return nil, fmt.Errorf("load archive lock store: %w", err)
}
store.Locks = locks
return &store, nil
} }
// LoadWithSessionOptions loads and resolves combined pipeline and session // MarshalArchiveLockStore serializes a mutable lock store as strict-compatible YAML.
// configuration with session template options. func MarshalArchiveLockStore(store *ArchiveLockStore) ([]byte, error) {
func LoadWithSessionOptions(pipelinePath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) { if store == nil {
store = &ArchiveLockStore{}
}
data, err := yaml.Marshal(store)
if err != nil {
return nil, fmt.Errorf("marshal archive lock store: %w", err)
}
return data, nil
}
// Load loads and resolves combined pipeline, campaign, and session configuration.
// Passing only a session path is supported for package-internal compatibility;
// in that form campaign.yml is expected next to the session file.
func Load(pipelinePath string, paths ...string) (*Config, error) {
campaignPath, sessionPath, err := campaignSessionPaths(paths...)
if err != nil {
return nil, err
}
return LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
}
// LoadWithSessionOptions loads and resolves combined pipeline, campaign, and
// session configuration with session template options.
func LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) {
pipelineCfg, err := LoadPipeline(pipelinePath) pipelineCfg, err := LoadPipeline(pipelinePath)
if err != nil { if err != nil {
return nil, err return nil, err
} }
campaignCfg, err := LoadCampaign(campaignPath)
if err != nil {
return nil, err
}
sessionCfg, err := LoadSessionWithOptions(sessionPath, sessionOpts) sessionCfg, err := LoadSessionWithOptions(sessionPath, sessionOpts)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return Resolve(pipelinePath, pipelineCfg, campaignPath, campaignCfg, sessionPath, sessionCfg, SessionSource{
Source: "session_config",
LocalPath: sessionPath,
})
}
// Resolve builds final stage-facing configuration from already loaded
// pipeline, campaign, and session documents.
func Resolve(pipelinePath string, pipelineCfg *PipelineConfig, campaignPath string, campaignCfg *CampaignConfig, sessionPath string, sessionCfg *SessionConfig, sessionSource SessionSource) (*Config, error) {
stableInputs, err := mergeCampaignSession(campaignCfg, sessionCfg, campaignPath, sessionPath)
if err != nil {
return nil, err
}
if strings.TrimSpace(sessionSource.Source) == "" {
sessionSource.Source = "session_config"
}
if strings.TrimSpace(sessionSource.LocalPath) == "" {
sessionSource.LocalPath = sessionPath
}
return &Config{ return &Config{
Pipeline: pipelineCfg, Pipeline: pipelineCfg,
Session: sessionCfg, Campaign: campaignCfg,
PipelinePath: pipelinePath, Session: sessionCfg,
SessionPath: sessionPath, PipelinePath: pipelinePath,
CampaignPath: campaignPath,
SessionPath: sessionPath,
StableInputs: stableInputs,
SessionSource: sessionSource,
}, nil }, nil
} }
func campaignSessionPaths(paths ...string) (campaignPath, sessionPath string, err error) {
switch len(paths) {
case 1:
sessionPath = paths[0]
campaignPath = filepath.Join(filepath.Dir(sessionPath), "campaign.yml")
case 2:
campaignPath = paths[0]
sessionPath = paths[1]
default:
return "", "", fmt.Errorf("load config: expected session path or campaign and session paths")
}
return campaignPath, sessionPath, nil
}
func mergeCampaignSession(campaignCfg *CampaignConfig, sessionCfg *SessionConfig, campaignPath, sessionPath string) (ResolvedStableInputs, error) {
if campaignCfg == nil {
return ResolvedStableInputs{}, fmt.Errorf("campaign config is required")
}
if sessionCfg == nil {
return ResolvedStableInputs{}, fmt.Errorf("session config is required")
}
campaignName := strings.TrimSpace(campaignCfg.Campaign)
sessionCampaign := strings.TrimSpace(sessionCfg.Campaign)
if sessionCampaign != "" && campaignName != "" && sessionCampaign != campaignName {
return ResolvedStableInputs{}, fmt.Errorf(
"campaign/session config invalid: session campaign %q does not match campaign config %q",
sessionCampaign,
campaignName,
)
}
if sessionCampaign == "" {
sessionCfg.Campaign = campaignName
}
stable := ResolvedStableInputs{
SpeakersFile: selectStableInput(
campaignCfg.Inputs.SpeakersFile,
sessionCfg.Inputs.SpeakersFile,
campaignPath,
sessionPath,
),
AutocorrectFile: selectStableInput(
campaignCfg.Inputs.AutocorrectFile,
sessionCfg.Inputs.AutocorrectFile,
campaignPath,
sessionPath,
),
GlossaryFile: selectStableInput(
campaignCfg.Inputs.GlossaryFile,
sessionCfg.Inputs.GlossaryFile,
campaignPath,
sessionPath,
),
}
sessionCfg.Inputs.SpeakersFile = stable.SpeakersFile.Path
sessionCfg.Inputs.AutocorrectFile = stable.AutocorrectFile.Path
sessionCfg.Inputs.GlossaryFile = stable.GlossaryFile.Path
return stable, nil
}
func selectStableInput(campaignValue, sessionValue, campaignPath, sessionPath string) ResolvedInputFile {
if strings.TrimSpace(sessionValue) != "" {
return ResolvedInputFile{
Path: sessionValue,
ConfigPath: sessionPath,
Source: "session_config",
}
}
return ResolvedInputFile{
Path: campaignValue,
ConfigPath: campaignPath,
Source: "campaign_config",
}
}
func decodeStrictYAML(kind, path string, out any) error { func decodeStrictYAML(kind, path string, out any) error {
f, err := os.Open(path) f, err := os.Open(path)
if err != nil { if err != nil {
@@ -208,6 +361,7 @@ func applyPipelineDefaults(cfg *PipelineConfig) {
applyWorkspaceDefaults(&cfg.Workspace) applyWorkspaceDefaults(&cfg.Workspace)
applyStorageDefaults(&cfg.Storage) applyStorageDefaults(&cfg.Storage)
applySpoolDefaults(&cfg.Spool) applySpoolDefaults(&cfg.Spool)
applyCacheDefaults(&cfg.Cache)
applyArchiveDefaults(&cfg.Archive) applyArchiveDefaults(&cfg.Archive)
applyWhisperXDefaults(&cfg.WhisperX) applyWhisperXDefaults(&cfg.WhisperX)
applySeriatimDefaults(&cfg.Seriatim) applySeriatimDefaults(&cfg.Seriatim)
@@ -256,6 +410,18 @@ func applySpoolDefaults(cfg *SpoolConfig) {
} }
} }
func applyCacheDefaults(cfg *CacheConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultCacheRoot
}
if cfg.S3Audio == nil {
cfg.S3Audio = boolPtr(DefaultCacheS3Audio)
}
}
func applyArchiveDefaults(cfg **ArchiveConfig) { func applyArchiveDefaults(cfg **ArchiveConfig) {
if cfg == nil { if cfg == nil {
return return

View File

@@ -904,6 +904,7 @@ func TestValidateMissingAudioSource(t *testing.T) {
Report: boolPtr(true), Report: boolPtr(true),
}, },
}, },
Campaign: &CampaignConfig{Campaign: "sample-campaign"},
Session: &SessionConfig{ Session: &SessionConfig{
SessionID: "2026-05-03", SessionID: "2026-05-03",
Campaign: "sample-campaign", Campaign: "sample-campaign",
@@ -963,6 +964,7 @@ func TestExamplesLoadAndValidate(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
pipelinePath := filepath.Join(examplesDir, tt.pipelineFile) pipelinePath := filepath.Join(examplesDir, tt.pipelineFile)
campaignPath := filepath.Join(examplesDir, "campaign.yml")
sessionPath := filepath.Join(examplesDir, tt.sessionFile) sessionPath := filepath.Join(examplesDir, tt.sessionFile)
var ( var (
@@ -972,7 +974,7 @@ func TestExamplesLoadAndValidate(t *testing.T) {
if strings.TrimSpace(tt.sessionOpts.SessionID) == "" { if strings.TrimSpace(tt.sessionOpts.SessionID) == "" {
cfg, err = Load(pipelinePath, sessionPath) cfg, err = Load(pipelinePath, sessionPath)
} else { } else {
cfg, err = LoadWithSessionOptions(pipelinePath, sessionPath, tt.sessionOpts) cfg, err = LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, tt.sessionOpts)
} }
if err != nil { if err != nil {
t.Fatalf("load example config error = %v", err) t.Fatalf("load example config error = %v", err)
@@ -1001,14 +1003,34 @@ func writeConfigFiles(t *testing.T, pipelineYAML, sessionYAML string) (string, s
dir := t.TempDir() dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml") pipelinePath := filepath.Join(dir, "pipeline.yml")
campaignPath := filepath.Join(dir, "campaign.yml")
sessionPath := filepath.Join(dir, "session.yml") sessionPath := filepath.Join(dir, "session.yml")
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil { if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
t.Fatalf("write pipeline.yml: %v", err) t.Fatalf("write pipeline.yml: %v", err)
} }
campaignYAML := `campaign: ` + campaignNameFromSessionYAML(sessionYAML) + `
inputs:
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
t.Fatalf("write campaign.yml: %v", err)
}
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil { if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err) t.Fatalf("write session.yml: %v", err)
} }
return pipelinePath, sessionPath return pipelinePath, sessionPath
} }
func campaignNameFromSessionYAML(sessionYAML string) string {
for _, line := range strings.Split(sessionYAML, "\n") {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "campaign:") {
return strings.Trim(strings.TrimSpace(strings.TrimPrefix(trimmed, "campaign:")), `"'`)
}
}
return "sample-campaign"
}

View File

@@ -241,3 +241,38 @@ inputs:
t.Fatalf("SessionID = %q, want 2026-05-03", cfg.SessionID) t.Fatalf("SessionID = %q, want 2026-05-03", cfg.SessionID)
} }
} }
func TestLoadSessionBytesWithOptionsUsesSameTemplateAndStrictDecode(t *testing.T) {
sessionYAML := []byte(`session_id: "{{ session_id }}"
campaign: sample-campaign
inputs:
audio_s3:
prefix: audio/
`)
cfg, err := LoadSessionBytesWithOptions("s3://bucket/session.yml", sessionYAML, SessionLoadOptions{SessionID: "2026-05-03"})
if err != nil {
t.Fatalf("LoadSessionBytesWithOptions() error = %v", err)
}
if cfg.SessionID != "2026-05-03" {
t.Fatalf("SessionID = %q, want 2026-05-03", cfg.SessionID)
}
_, err = LoadSessionBytesWithOptions("s3://bucket/session.yml", []byte("session_id: 2026-05-03\nunknown: true\n"), SessionLoadOptions{})
if err == nil {
t.Fatal("expected strict decode error, got nil")
}
if !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("error = %q, want strict decode context", err.Error())
}
}
func TestLoadSessionBytesWithOptionsMismatchFails(t *testing.T) {
_, err := LoadSessionBytesWithOptions("s3://bucket/session.yml", []byte("session_id: 2026-05-03\n"), SessionLoadOptions{SessionID: "2026-04-04"})
if err == nil {
t.Fatal("expected mismatch error, got nil")
}
if !strings.Contains(err.Error(), "session_id mismatch") {
t.Fatalf("error = %q, want mismatch context", err.Error())
}
}

View File

@@ -287,6 +287,100 @@ archive:
} }
} }
func TestArchiveLockValidation(t *testing.T) {
tests := []struct {
name string
pipelineYML string
wantErr string
}{
{
name: "valid built in source",
pipelineYML: testPipelineBaseYAML + `
archive:
locks:
- source: narratio.transcript.trimmed
reason: reviewed transcript
`,
},
{
name: "valid configured source",
pipelineYML: testPipelineBaseYAML + `
scriptorium:
artifacts:
session_recap:
enabled: true
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
archive:
locks:
- source: narratio.artifact.session_recap
`,
},
{
name: "missing source rejected",
pipelineYML: testPipelineBaseYAML + `
archive:
locks:
- reason: no source
`,
wantErr: "pipeline.archive.locks[0].source is required",
},
{
name: "invalid source rejected",
pipelineYML: testPipelineBaseYAML + `
archive:
locks:
- source: narratio.unknown
`,
wantErr: "pipeline.archive.locks[0].source \"narratio.unknown\" is unsupported",
},
{
name: "duplicate source rejected",
pipelineYML: testPipelineBaseYAML + `
archive:
locks:
- source: narratio.transcript.trimmed
- source: " narratio.transcript.trimmed "
`,
wantErr: "duplicates another archive lock source",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, tt.pipelineYML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if tt.wantErr == "" {
if err != nil {
t.Fatalf("Validate() error = %v", err)
}
return
}
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
}
})
}
}
func TestArchiveLockUnknownFieldFailsStrictDecode(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
archive:
locks:
- source: narratio.transcript.trimmed
dest: transcripts/trimmed.json
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
_, err := Load(pipelinePath, sessionPath)
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("Load() error = %v, want strict decode failed", err)
}
}
func TestArchiveLegacyFromToFailsStrictDecode(t *testing.T) { func TestArchiveLegacyFromToFailsStrictDecode(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + ` pipelineYAML := testPipelineBaseYAML + `
archive: archive:
@@ -301,6 +395,54 @@ archive:
} }
} }
func TestArchiveLockStoreBytesStrictDecodeAndValidation(t *testing.T) {
store, err := LoadArchiveLockStoreBytes("locks.yml", []byte(`locks:
- source: narratio.transcript.trimmed
reason: reviewed
`), nil)
if err != nil {
t.Fatalf("LoadArchiveLockStoreBytes() error = %v", err)
}
if len(store.Locks) != 1 || store.Locks[0].Source != "narratio.transcript.trimmed" || store.Locks[0].Reason != "reviewed" {
t.Fatalf("locks = %#v", store.Locks)
}
_, err = LoadArchiveLockStoreBytes("locks.yml", []byte(`locks:
- source: narratio.transcript.trimmed
dest: transcripts/trimmed.json
`), nil)
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("unknown field error = %v, want strict decode failed", err)
}
_, err = LoadArchiveLockStoreBytes("locks.yml", []byte(`locks:
- source: narratio.transcript.trimmed
- source: narratio.transcript.trimmed
`), nil)
if err == nil || !strings.Contains(err.Error(), "duplicates another archive lock source") {
t.Fatalf("duplicate error = %v", err)
}
}
func TestMergeArchiveLockRulesStaticWins(t *testing.T) {
merged := MergeArchiveLockRules(
[]ArchiveLockRule{{Source: "narratio.transcript.trimmed", Reason: "static"}},
[]ArchiveLockRule{
{Source: "narratio.transcript.trimmed", Reason: "remote"},
{Source: "narratio.transcript.full", Reason: "remote full"},
},
)
if len(merged) != 2 {
t.Fatalf("merged len = %d, want 2: %#v", len(merged), merged)
}
if merged[0].Source != "narratio.transcript.trimmed" || merged[0].Reason != "static" {
t.Fatalf("merged[0] = %#v, want static lock", merged[0])
}
if merged[1].Source != "narratio.transcript.full" {
t.Fatalf("merged[1] = %#v, want remote full lock", merged[1])
}
}
func TestSessionAudioS3Validation(t *testing.T) { func TestSessionAudioS3Validation(t *testing.T) {
tests := []struct { tests := []struct {
name string name string

View File

@@ -17,6 +17,9 @@ func Validate(cfg *Config) error {
if cfg.Pipeline == nil { if cfg.Pipeline == nil {
return fmt.Errorf("pipeline config is required") return fmt.Errorf("pipeline config is required")
} }
if cfg.Campaign == nil {
return fmt.Errorf("campaign config is required")
}
if cfg.Session == nil { if cfg.Session == nil {
return fmt.Errorf("session config is required") return fmt.Errorf("session config is required")
} }
@@ -24,6 +27,9 @@ func Validate(cfg *Config) error {
if err := validatePipeline(cfg.Pipeline); err != nil { if err := validatePipeline(cfg.Pipeline); err != nil {
return fmt.Errorf("pipeline config %q invalid: %w", shortName(cfg.PipelinePath, "pipeline.yml"), err) return fmt.Errorf("pipeline config %q invalid: %w", shortName(cfg.PipelinePath, "pipeline.yml"), err)
} }
if err := validateCampaign(cfg.Campaign); err != nil {
return fmt.Errorf("campaign config %q invalid: %w", shortName(cfg.CampaignPath, "campaign.yml"), err)
}
if err := validateSession(cfg.Session); err != nil { if err := validateSession(cfg.Session); err != nil {
return fmt.Errorf("session config %q invalid: %w", shortName(cfg.SessionPath, "session.yml"), err) return fmt.Errorf("session config %q invalid: %w", shortName(cfg.SessionPath, "session.yml"), err)
} }
@@ -34,6 +40,16 @@ func Validate(cfg *Config) error {
return nil return nil
} }
func validateCampaign(cfg *CampaignConfig) error {
if cfg == nil {
return fmt.Errorf("campaign config is required")
}
if strings.TrimSpace(cfg.Campaign) == "" {
return fmt.Errorf("campaign.campaign is required")
}
return nil
}
func validatePipeline(cfg *PipelineConfig) error { func validatePipeline(cfg *PipelineConfig) error {
if strings.TrimSpace(cfg.Workspace.Root) == "" { if strings.TrimSpace(cfg.Workspace.Root) == "" {
return fmt.Errorf("pipeline.workspace.root is required") return fmt.Errorf("pipeline.workspace.root is required")
@@ -47,6 +63,9 @@ func validatePipeline(cfg *PipelineConfig) error {
if err := validateSpool(cfg.Spool); err != nil { if err := validateSpool(cfg.Spool); err != nil {
return err return err
} }
if err := validateCache(cfg.Cache); err != nil {
return err
}
if err := validateArchive(cfg.Archive, cfg.Scriptorium); err != nil { if err := validateArchive(cfg.Archive, cfg.Scriptorium); err != nil {
return err return err
} }
@@ -104,6 +123,13 @@ func validateSpool(cfg SpoolConfig) error {
return nil return nil
} }
func validateCache(cfg CacheConfig) error {
if cfg.S3Audio != nil && *cfg.S3Audio && strings.TrimSpace(cfg.Root) == "" {
return fmt.Errorf("pipeline.cache.root is required when pipeline.cache.s3_audio is true")
}
return nil
}
func validateArchive(cfg *ArchiveConfig, scriptorium *ScriptoriumConfig) error { func validateArchive(cfg *ArchiveConfig, scriptorium *ScriptoriumConfig) error {
if cfg == nil { if cfg == nil {
return nil return nil
@@ -136,9 +162,69 @@ func validateArchive(cfg *ArchiveConfig, scriptorium *ScriptoriumConfig) error {
} }
seenDest[normalizedDest] = struct{}{} seenDest[normalizedDest] = struct{}{}
} }
locks, err := ValidateArchiveLockRules(cfg.Locks, scriptorium, "pipeline.archive.locks")
if err != nil {
return err
}
cfg.Locks = locks
return nil return nil
} }
// ValidateArchiveLockRules validates and normalizes source-based archive locks.
func ValidateArchiveLockRules(locks []ArchiveLockRule, scriptorium *ScriptoriumConfig, label string) ([]ArchiveLockRule, error) {
seenLocks := map[string]struct{}{}
out := make([]ArchiveLockRule, 0, len(locks))
if strings.TrimSpace(label) == "" {
label = "archive.locks"
}
for i, item := range locks {
prefix := fmt.Sprintf("%s[%d]", label, i)
source := strings.TrimSpace(item.Source)
if source == "" {
return nil, fmt.Errorf("%s.source is required", prefix)
}
if _, err := archiveSourceKnown(source, scriptorium); err != nil {
return nil, fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
}
if _, ok := seenLocks[source]; ok {
return nil, fmt.Errorf("%s.source %q duplicates another archive lock source", prefix, source)
}
seenLocks[source] = struct{}{}
out = append(out, ArchiveLockRule{
Source: source,
Reason: strings.TrimSpace(item.Reason),
})
}
return out, nil
}
// MergeArchiveLockRules returns the union of static and remote locks. Static
// locks win when both sources contain the same lock.
func MergeArchiveLockRules(staticLocks, remoteLocks []ArchiveLockRule) []ArchiveLockRule {
out := make([]ArchiveLockRule, 0, len(staticLocks)+len(remoteLocks))
seen := map[string]struct{}{}
for _, item := range staticLocks {
source := strings.TrimSpace(item.Source)
if source == "" {
continue
}
out = append(out, ArchiveLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)})
seen[source] = struct{}{}
}
for _, item := range remoteLocks {
source := strings.TrimSpace(item.Source)
if source == "" {
continue
}
if _, ok := seen[source]; ok {
continue
}
out = append(out, ArchiveLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)})
seen[source] = struct{}{}
}
return out
}
func archiveSourceKnown(source string, scriptorium *ScriptoriumConfig) (string, error) { func archiveSourceKnown(source string, scriptorium *ScriptoriumConfig) (string, error) {
trimmed := strings.TrimSpace(source) trimmed := strings.TrimSpace(source)
switch trimmed { switch trimmed {

View File

@@ -23,6 +23,7 @@ type InputRecord struct {
S3Size int64 `json:"s3_size,omitempty"` S3Size int64 `json:"s3_size,omitempty"`
S3ETag string `json:"s3_etag,omitempty"` S3ETag string `json:"s3_etag,omitempty"`
SpoolPath string `json:"spool_path,omitempty"` SpoolPath string `json:"spool_path,omitempty"`
CachePath string `json:"cache_path,omitempty"`
} }
// ArtifactRecord captures one produced artifact and optional remote metadata. // ArtifactRecord captures one produced artifact and optional remote metadata.

View File

@@ -126,7 +126,14 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
if err != nil { if err != nil {
return nil, fmt.Errorf("archive: build runtime artifact catalog: %w", err) return nil, fmt.Errorf("archive: build runtime artifact catalog: %w", err)
} }
promotions, skippedOptional, err := resolveArchivePromotions(sessionPaths, m, runtimeCatalog, env.Config.Pipeline.Archive.PromoteArtifacts) promotions, skippedOptional, lockedPromotions, err := resolveArchivePromotions(
sessionPaths,
m,
runtimeCatalog,
env.Config.Pipeline.Archive.PromoteArtifacts,
env.Config.Pipeline.Archive.Locks,
sessionPrefix,
)
if err != nil { if err != nil {
return nil, fmt.Errorf("archive: resolve promotion rules: %w", err) return nil, fmt.Errorf("archive: resolve promotion rules: %w", err)
} }
@@ -166,6 +173,7 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
promotedUploaded, promotedUploaded,
previousUploaded, previousUploaded,
skippedOptional, skippedOptional,
lockedPromotions,
currentManifestKey, currentManifestKey,
)) ))
if err != nil { if err != nil {
@@ -204,6 +212,8 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
"previous_files_uploaded": len(previousUploaded), "previous_files_uploaded": len(previousUploaded),
"previous_uploaded_paths": previousUploaded, "previous_uploaded_paths": previousUploaded,
"skipped_optional_promotions": skippedOptional, "skipped_optional_promotions": skippedOptional,
"locked_promotion_count": len(lockedPromotions),
"locked_promotions": lockedPromotionMetadata(lockedPromotions),
"current_manifest_key": currentManifestKey, "current_manifest_key": currentManifestKey,
"current_run_id_key": currentRunPointerKey, "current_run_id_key": currentRunPointerKey,
"current_pointer_written": true, "current_pointer_written": true,
@@ -220,6 +230,16 @@ type archivePromotion struct {
Provenance string Provenance string
} }
type archiveLockedPromotion struct {
Source string
Dest string
RemoteKey string
Reason string
Required bool
LocalPath string
Provenance string
}
func archiveDisabled(env *Env) bool { func archiveDisabled(env *Env) bool {
cfg := env.Config.Pipeline.Archive cfg := env.Config.Pipeline.Archive
if cfg == nil { if cfg == nil {
@@ -319,26 +339,53 @@ func resolveArchivePromotions(
m *manifest.Manifest, m *manifest.Manifest,
catalog *artifacts.ArtifactCatalog, catalog *artifacts.ArtifactCatalog,
rules []config.ArchivePromotionRule, rules []config.ArchivePromotionRule,
) ([]archivePromotion, []string, error) { locks []config.ArchiveLockRule,
sessionPrefix string,
) ([]archivePromotion, []string, []archiveLockedPromotion, error) {
out := make([]archivePromotion, 0, len(rules)) out := make([]archivePromotion, 0, len(rules))
skippedOptional := make([]string, 0) skippedOptional := make([]string, 0)
lockedPromotions := make([]archiveLockedPromotion, 0)
lockSet := archiveLockSet(locks)
for _, rule := range rules { for _, rule := range rules {
source := strings.TrimSpace(rule.Source) source := strings.TrimSpace(rule.Source)
required := rule.Required == nil || *rule.Required required := rule.Required == nil || *rule.Required
dest, err := resolveArchivePromotionDest(rule, catalog) dest, err := resolveArchivePromotionDest(rule, catalog)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("source %q: %w", source, err) return nil, nil, nil, fmt.Errorf("source %q: %w", source, err)
} }
lock, locked := lockSet[source]
resolved, err := artifacts.ResolveSessionArtifactWithCatalog(paths, m, source, catalog) resolved, err := artifacts.ResolveSessionArtifactWithCatalog(paths, m, source, catalog)
if err != nil { if err != nil {
if locked {
lockedPromotions = append(lockedPromotions, archiveLockedPromotion{
Source: source,
Dest: dest,
RemoteKey: artifacts.S3PromotedArtifactKey(sessionPrefix, dest),
Reason: strings.TrimSpace(lock.Reason),
Required: required,
})
continue
}
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) && !required { if errors.Is(err, artifacts.ErrSessionArtifactNotFound) && !required {
skippedOptional = append(skippedOptional, dest) skippedOptional = append(skippedOptional, dest)
continue continue
} }
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) { if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
return nil, nil, fmt.Errorf("required promotion source unavailable: %q", source) return nil, nil, nil, fmt.Errorf("required promotion source unavailable: %q", source)
} }
return nil, nil, fmt.Errorf("resolve source %q: %w", source, err) return nil, nil, nil, fmt.Errorf("resolve source %q: %w", source, err)
}
if locked {
lockedPromotions = append(lockedPromotions, archiveLockedPromotion{
Source: source,
Dest: dest,
RemoteKey: artifacts.S3PromotedArtifactKey(sessionPrefix, dest),
Reason: strings.TrimSpace(lock.Reason),
Required: required,
LocalPath: resolved.Path,
Provenance: resolved.Provenance,
})
continue
} }
out = append(out, archivePromotion{ out = append(out, archivePromotion{
Source: source, Source: source,
@@ -348,7 +395,21 @@ func resolveArchivePromotions(
Provenance: resolved.Provenance, Provenance: resolved.Provenance,
}) })
} }
return out, skippedOptional, nil return out, skippedOptional, lockedPromotions, nil
}
func archiveLockSet(locks []config.ArchiveLockRule) map[string]config.ArchiveLockRule {
out := make(map[string]config.ArchiveLockRule, len(locks))
for _, lock := range locks {
source := strings.TrimSpace(lock.Source)
if source == "" {
continue
}
lock.Source = source
lock.Reason = strings.TrimSpace(lock.Reason)
out[source] = lock
}
return out
} }
func resolveArchivePromotionDest(rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog) (string, error) { func resolveArchivePromotionDest(rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog) (string, error) {
@@ -663,6 +724,7 @@ func archiveMetadataPreview(
promotedUploaded []string, promotedUploaded []string,
previousUploaded []string, previousUploaded []string,
skippedOptional []string, skippedOptional []string,
lockedPromotions []archiveLockedPromotion,
currentManifestKey string, currentManifestKey string,
) map[string]any { ) map[string]any {
return map[string]any{ return map[string]any{
@@ -677,9 +739,27 @@ func archiveMetadataPreview(
"previous_files_uploaded": len(previousUploaded), "previous_files_uploaded": len(previousUploaded),
"previous_uploaded_paths": append([]string(nil), previousUploaded...), "previous_uploaded_paths": append([]string(nil), previousUploaded...),
"skipped_optional_promotions": append([]string(nil), skippedOptional...), "skipped_optional_promotions": append([]string(nil), skippedOptional...),
"locked_promotion_count": len(lockedPromotions),
"locked_promotions": lockedPromotionMetadata(lockedPromotions),
"current_manifest_key": currentManifestKey, "current_manifest_key": currentManifestKey,
"current_run_id_key": artifacts.S3CurrentRunPointerKey(sessionPrefix), "current_run_id_key": artifacts.S3CurrentRunPointerKey(sessionPrefix),
"current_pointer_written": false, "current_pointer_written": false,
"audio_upload_skipped": true, "audio_upload_skipped": true,
} }
} }
func lockedPromotionMetadata(locked []archiveLockedPromotion) []map[string]any {
out := make([]map[string]any, 0, len(locked))
for _, item := range locked {
out = append(out, map[string]any{
"source": item.Source,
"dest": item.Dest,
"remote_key": item.RemoteKey,
"reason": item.Reason,
"required": item.Required,
"local_path": item.LocalPath,
"provenance": item.Provenance,
})
}
return out
}

View File

@@ -2,6 +2,7 @@ package stage
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"os" "os"
"path/filepath" "path/filepath"
@@ -215,6 +216,128 @@ func TestArchiveSkipsOptionalMissingPromotion(t *testing.T) {
} }
} }
func TestArchiveSkipsLockedRequiredPromotionAndCommits(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
{Source: "narratio.transcript.trimmed", Reason: "human reviewed"},
}
fake := env.ObjectStore.(*storage.FakeBackend)
result, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
trimmedKey := m.S3SessionPrefix + "transcripts/trimmed.json"
if _, ok := fake.Objects[trimmedKey]; ok {
t.Fatalf("locked promotion key %q should not be uploaded", trimmedKey)
}
recapKey := m.S3SessionPrefix + "artifacts/session_recap.md"
if _, ok := fake.Objects[recapKey]; !ok {
t.Fatalf("unlocked promotion key %q should be uploaded", recapKey)
}
runTrimmedKey := m.S3RunPrefix + "trim/outputs/transcripts/trimmed.json"
if _, ok := fake.Objects[runTrimmedKey]; !ok {
t.Fatalf("run-local locked source output %q should still be uploaded", runTrimmedKey)
}
currentRunIDKey := m.S3SessionPrefix + "current/run_id.txt"
if len(fake.Uploads) == 0 || fake.Uploads[len(fake.Uploads)-1].Key != currentRunIDKey {
t.Fatalf("last upload = %#v, want current run pointer %q", fake.Uploads, currentRunIDKey)
}
if result.Metadata["promoted_files_uploaded"] != 1 {
t.Fatalf("metadata promoted_files_uploaded = %#v, want 1", result.Metadata["promoted_files_uploaded"])
}
if result.Metadata["locked_promotion_count"] != 1 {
t.Fatalf("metadata locked_promotion_count = %#v, want 1", result.Metadata["locked_promotion_count"])
}
locked := result.Metadata["locked_promotions"].([]map[string]any)
if len(locked) != 1 {
t.Fatalf("locked_promotions = %#v, want one item", locked)
}
if locked[0]["source"] != "narratio.transcript.trimmed" ||
locked[0]["dest"] != "transcripts/trimmed.json" ||
locked[0]["remote_key"] != trimmedKey ||
locked[0]["reason"] != "human reviewed" ||
locked[0]["required"] != true ||
locked[0]["local_path"] == "" ||
locked[0]["provenance"] == "" {
t.Fatalf("locked promotion metadata = %#v", locked[0])
}
currentManifestKey := m.S3SessionPrefix + "current/manifest.json"
var current map[string]any
if err := json.Unmarshal(fake.Objects[currentManifestKey].Data, &current); err != nil {
t.Fatalf("unmarshal current manifest: %v", err)
}
stages := current["stages"].(map[string]any)
archive := stages["archive"].(map[string]any)
meta := archive["metadata"].(map[string]any)
if meta["locked_promotion_count"] != float64(1) {
t.Fatalf("current manifest locked_promotion_count = %#v, want 1", meta["locked_promotion_count"])
}
items := meta["locked_promotions"].([]any)
if len(items) != 1 {
t.Fatalf("current manifest locked_promotions = %#v, want one item", items)
}
}
func TestArchiveLockedRequiredMissingPromotionSucceeds(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
{Source: "narratio.transcript.merged", Dest: "transcripts/merged.json", Required: boolPtr(true)},
}
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
{Source: "narratio.transcript.merged", Reason: "manual merge is locked"},
}
result, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
fake := env.ObjectStore.(*storage.FakeBackend)
mergedKey := m.S3SessionPrefix + "transcripts/merged.json"
if _, ok := fake.Objects[mergedKey]; ok {
t.Fatalf("locked missing promotion key %q should not be uploaded", mergedKey)
}
if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; !ok {
t.Fatalf("current run pointer should be written for locked missing promotion")
}
locked := result.Metadata["locked_promotions"].([]map[string]any)
if len(locked) != 1 {
t.Fatalf("locked_promotions = %#v, want one item", locked)
}
if locked[0]["local_path"] != "" || locked[0]["provenance"] != "" {
t.Fatalf("locked missing promotion metadata = %#v, want empty local path/provenance", locked[0])
}
}
func TestArchiveLockDoesNotOverwriteExistingPromotion(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
{Source: "narratio.transcript.trimmed", Reason: "already published"},
}
fake := env.ObjectStore.(*storage.FakeBackend)
trimmedKey := m.S3SessionPrefix + "transcripts/trimmed.json"
fake.SeedObject(storage.FakeObject{Key: trimmedKey, Data: []byte("previously published\n")})
if _, err := (archiveStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("Run() error = %v", err)
}
got := string(fake.Objects[trimmedKey].Data)
if got != "previously published\n" {
t.Fatalf("locked promotion object contents = %q, want existing object preserved", got)
}
for _, upload := range fake.Uploads {
if upload.Key == trimmedKey {
t.Fatalf("locked promotion key %q was uploaded", trimmedKey)
}
}
}
func TestArchiveFailsWhenRequiredPromotionMissing(t *testing.T) { func TestArchiveFailsWhenRequiredPromotionMissing(t *testing.T) {
env, m, _ := archiveFixture(t) env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{ env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{

View File

@@ -30,8 +30,10 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
store := artifacts.NewLocalStore(root) store := artifacts.NewLocalStore(root)
cfgDir := t.TempDir() cfgDir := t.TempDir()
sessionPath := filepath.Join(cfgDir, "session.yml") sessionPath := filepath.Join(cfgDir, "session.yml")
campaignPath := filepath.Join(cfgDir, "campaign.yml")
pipelinePath := filepath.Join(cfgDir, "pipeline.yml") pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
writeStageTestFile(t, sessionPath, "session_id: 2026-05-03\n") writeStageTestFile(t, sessionPath, "session_id: 2026-05-03\n")
writeStageTestFile(t, campaignPath, "campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n")
writeStageTestFile(t, pipelinePath, "workspace:\n root: "+root+"\n") writeStageTestFile(t, pipelinePath, "workspace:\n root: "+root+"\n")
writeStageTestFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n") writeStageTestFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
writeStageTestFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n") writeStageTestFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
@@ -48,7 +50,9 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
env := &Env{ env := &Env{
Config: &config.Config{ Config: &config.Config{
SessionPath: sessionPath, SessionPath: sessionPath,
CampaignPath: campaignPath,
PipelinePath: pipelinePath, PipelinePath: pipelinePath,
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
Pipeline: &config.PipelineConfig{ Pipeline: &config.PipelineConfig{
Workspace: config.WorkspaceConfig{Root: root}, Workspace: config.WorkspaceConfig{Root: root},
Storage: config.StorageConfig{ Storage: config.StorageConfig{
@@ -62,14 +66,28 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
UploadRun: boolPtr(true), UploadRun: boolPtr(true),
}, },
}, },
StableInputs: config.ResolvedStableInputs{
SpeakersFile: config.ResolvedInputFile{
Path: "./speakers.yml",
ConfigPath: campaignPath,
Source: "campaign_config",
},
AutocorrectFile: config.ResolvedInputFile{
Path: "./autocorrect.yml",
ConfigPath: campaignPath,
Source: "campaign_config",
},
GlossaryFile: config.ResolvedInputFile{
Path: "./glossary.yml",
ConfigPath: campaignPath,
Source: "campaign_config",
},
},
Session: &config.SessionConfig{ Session: &config.SessionConfig{
SessionID: "2026-05-03", SessionID: "2026-05-03",
Campaign: "sample-campaign", Campaign: "sample-campaign",
Inputs: config.SessionInputsConfig{ Inputs: config.SessionInputsConfig{
AudioDir: "./audio", AudioDir: "./audio",
SpeakersFile: "./speakers.yml",
AutocorrectFile: "./autocorrect.yml",
GlossaryFile: "./glossary.yml",
}, },
}, },
}, },

View File

@@ -13,6 +13,7 @@ import (
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage" "gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts" "gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/audio"
"gitea.maximumdirect.net/eric/narratio/internal/config" "gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest" "gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
@@ -25,6 +26,7 @@ func (prepareStage) Name() string { return "prepare" }
func (prepareStage) Declares() IODecl { func (prepareStage) Declares() IODecl {
return IODecl{ return IODecl{
Inputs: []artifacts.Ref{ Inputs: []artifacts.Ref{
{Kind: "config", Category: "inputs", RelativePath: "campaign.yml"},
{Kind: "config", Category: "inputs", RelativePath: "session.yml"}, {Kind: "config", Category: "inputs", RelativePath: "session.yml"},
{Kind: "config", Category: "inputs", RelativePath: "pipeline.resolved.yml"}, {Kind: "config", Category: "inputs", RelativePath: "pipeline.resolved.yml"},
{Kind: "config", Category: "inputs", RelativePath: "speakers.yml"}, {Kind: "config", Category: "inputs", RelativePath: "speakers.yml"},
@@ -59,21 +61,30 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
return nil, fmt.Errorf("prepare: ensure workdir layout: %w", err) return nil, fmt.Errorf("prepare: ensure workdir layout: %w", err)
} }
campaignSrc := env.Config.CampaignPath
if err := requireFile(campaignSrc, "campaign.yml"); err != nil {
return nil, fmt.Errorf("prepare: %w", err)
}
sessionSrc := env.Config.SessionPath sessionSrc := env.Config.SessionPath
if err := requireFile(sessionSrc, "session.yml"); err != nil { if err := requireFile(sessionSrc, "session.yml"); err != nil {
return nil, fmt.Errorf("prepare: %w", err) return nil, fmt.Errorf("prepare: %w", err)
} }
sessionDir := filepath.Dir(sessionSrc) sessionDir := filepath.Dir(sessionSrc)
speakersSrc, err := resolvePath(sessionDir, env.Config.Session.Inputs.SpeakersFile) speakersInput := stableInputSource(env.Config.StableInputs.SpeakersFile, env.Config.Session.Inputs.SpeakersFile, sessionSrc)
autocorrectInput := stableInputSource(env.Config.StableInputs.AutocorrectFile, env.Config.Session.Inputs.AutocorrectFile, sessionSrc)
glossaryInput := stableInputSource(env.Config.StableInputs.GlossaryFile, env.Config.Session.Inputs.GlossaryFile, sessionSrc)
speakersSrc, err := resolveConfigRelativePath(speakersInput)
if err != nil { if err != nil {
return nil, fmt.Errorf("prepare: speakers path: %w", err) return nil, fmt.Errorf("prepare: speakers path: %w", err)
} }
autocorrectSrc, err := resolvePath(sessionDir, env.Config.Session.Inputs.AutocorrectFile) autocorrectSrc, err := resolveConfigRelativePath(autocorrectInput)
if err != nil { if err != nil {
return nil, fmt.Errorf("prepare: autocorrect path: %w", err) return nil, fmt.Errorf("prepare: autocorrect path: %w", err)
} }
glossarySrc, err := resolvePath(sessionDir, env.Config.Session.Inputs.GlossaryFile) glossarySrc, err := resolveConfigRelativePath(glossaryInput)
if err != nil { if err != nil {
return nil, fmt.Errorf("prepare: glossary path: %w", err) return nil, fmt.Errorf("prepare: glossary path: %w", err)
} }
@@ -96,17 +107,44 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
return nil, fmt.Errorf("prepare: resolve audio inputs: %w", err) return nil, fmt.Errorf("prepare: resolve audio inputs: %w", err)
} }
inputs := make([]manifest.InputRecord, 0, 5+len(resolvedLocalAudio)) inputs := make([]manifest.InputRecord, 0, 6+len(resolvedLocalAudio))
registerInput := func(kind, path, checksum string) { registerInput := func(kind, path, checksum string) {
inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum}) inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum})
} }
registerConfigInput := func(kind, path, checksum, source string) {
inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum, Source: source})
}
registerSessionConfigInput := func(path, checksum string) {
source := env.Config.SessionSource
if strings.TrimSpace(source.Source) == "" {
source.Source = "session_config"
}
inputs = append(inputs, manifest.InputRecord{
Kind: "session_config",
Path: path,
Checksum: checksum,
Source: source.Source,
S3Bucket: source.S3Bucket,
S3Key: source.S3Key,
S3Size: source.S3Size,
S3ETag: source.S3ETag,
SpoolPath: source.SpoolPath,
})
}
campaignDst := filepath.Join(paths.InputsDir, "campaign.yml")
campaignChecksum, err := copyFileIfChanged(env.ArtifactStore, campaignSrc, campaignDst)
if err != nil {
return nil, fmt.Errorf("prepare: materialize campaign.yml: %w", err)
}
registerConfigInput("campaign_config", campaignDst, campaignChecksum, "campaign_config")
sessionDst := filepath.Join(paths.InputsDir, "session.yml") sessionDst := filepath.Join(paths.InputsDir, "session.yml")
sessionChecksum, err := copyFileIfChanged(env.ArtifactStore, sessionSrc, sessionDst) sessionChecksum, err := copyFileIfChanged(env.ArtifactStore, sessionSrc, sessionDst)
if err != nil { if err != nil {
return nil, fmt.Errorf("prepare: materialize session.yml: %w", err) return nil, fmt.Errorf("prepare: materialize session.yml: %w", err)
} }
registerInput("session_config", sessionDst, sessionChecksum) registerSessionConfigInput(sessionDst, sessionChecksum)
pipelineResolvedBytes, err := renderResolvedPipeline(env.Config.Pipeline) pipelineResolvedBytes, err := renderResolvedPipeline(env.Config.Pipeline)
if err != nil { if err != nil {
@@ -120,25 +158,29 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
registerInput("pipeline_resolved", pipelineDst, pipelineChecksum) registerInput("pipeline_resolved", pipelineDst, pipelineChecksum)
for _, cfgFile := range []struct { for _, cfgFile := range []struct {
kind string kind string
src string src string
dst string dst string
source string
}{ }{
{kind: "speakers", src: speakersSrc, dst: filepath.Join(paths.InputsDir, "speakers.yml")}, {kind: "speakers", src: speakersSrc, dst: filepath.Join(paths.InputsDir, "speakers.yml"), source: speakersInput.Source},
{kind: "autocorrect", src: autocorrectSrc, dst: filepath.Join(paths.InputsDir, "autocorrect.yml")}, {kind: "autocorrect", src: autocorrectSrc, dst: filepath.Join(paths.InputsDir, "autocorrect.yml"), source: autocorrectInput.Source},
{kind: "glossary", src: glossarySrc, dst: filepath.Join(paths.InputsDir, "glossary.yml")}, {kind: "glossary", src: glossarySrc, dst: filepath.Join(paths.InputsDir, "glossary.yml"), source: glossaryInput.Source},
} { } {
checksum, err := copyFileIfChanged(env.ArtifactStore, cfgFile.src, cfgFile.dst) checksum, err := copyFileIfChanged(env.ArtifactStore, cfgFile.src, cfgFile.dst)
if err != nil { if err != nil {
return nil, fmt.Errorf("prepare: materialize %s: %w", cfgFile.kind, err) return nil, fmt.Errorf("prepare: materialize %s: %w", cfgFile.kind, err)
} }
registerInput(cfgFile.kind, cfgFile.dst, checksum) registerConfigInput(cfgFile.kind, cfgFile.dst, checksum, cfgFile.source)
} }
var audioCacheStats s3AudioMaterializationStats
if useS3Audio { if useS3Audio {
if err := materializeS3AudioInputs(ctx, env, m, sessionID, &inputs); err != nil { stats, err := materializeS3AudioInputs(ctx, env, m, sessionID, &inputs)
if err != nil {
return nil, fmt.Errorf("prepare: materialize s3 audio: %w", err) return nil, fmt.Errorf("prepare: materialize s3 audio: %w", err)
} }
audioCacheStats = stats
} else { } else {
if err := materializeLocalAudioInputs(env, paths, resolvedLocalAudio, registerInput); err != nil { if err := materializeLocalAudioInputs(env, paths, resolvedLocalAudio, registerInput); err != nil {
return nil, fmt.Errorf("prepare: %w", err) return nil, fmt.Errorf("prepare: %w", err)
@@ -173,6 +215,11 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
"inputs_count": len(inputs), "inputs_count": len(inputs),
"audio_files_resolved": countAudioInputs(inputs), "audio_files_resolved": countAudioInputs(inputs),
} }
if useS3Audio {
metadata["audio_cache_hits"] = audioCacheStats.CacheHits
metadata["audio_cache_misses"] = audioCacheStats.CacheMisses
metadata["audio_s3_downloads"] = audioCacheStats.Downloads
}
if len(previousRequirements) > 0 { if len(previousRequirements) > 0 {
metadata["previous_requirements_count"] = len(previousRequirements) metadata["previous_requirements_count"] = len(previousRequirements)
if previousHydration != nil { if previousHydration != nil {
@@ -195,6 +242,31 @@ func renderResolvedPipeline(cfg *config.PipelineConfig) ([]byte, error) {
return yaml.Marshal(cfg) return yaml.Marshal(cfg)
} }
func stableInputSource(resolved config.ResolvedInputFile, fallbackPath, fallbackConfigPath string) config.ResolvedInputFile {
if strings.TrimSpace(resolved.Path) != "" || strings.TrimSpace(resolved.ConfigPath) != "" || strings.TrimSpace(resolved.Source) != "" {
if strings.TrimSpace(resolved.ConfigPath) == "" {
resolved.ConfigPath = fallbackConfigPath
}
if strings.TrimSpace(resolved.Source) == "" {
resolved.Source = "session_config"
}
return resolved
}
return config.ResolvedInputFile{
Path: fallbackPath,
ConfigPath: fallbackConfigPath,
Source: "session_config",
}
}
func resolveConfigRelativePath(input config.ResolvedInputFile) (string, error) {
basePath := strings.TrimSpace(input.ConfigPath)
if basePath == "" {
return "", fmt.Errorf("source config path is required")
}
return resolvePath(filepath.Dir(basePath), input.Path)
}
func resolveAudioInputs(sessionDir string, inputs config.SessionInputsConfig) ([]string, bool, error) { func resolveAudioInputs(sessionDir string, inputs config.SessionInputsConfig) ([]string, bool, error) {
hasLocal := strings.TrimSpace(inputs.AudioDir) != "" || len(inputs.AudioFiles) > 0 hasLocal := strings.TrimSpace(inputs.AudioDir) != "" || len(inputs.AudioFiles) > 0
if inputs.AudioS3 != nil { if inputs.AudioS3 != nil {
@@ -282,28 +354,34 @@ func materializeLocalAudioInputs(env *Env, paths artifacts.SessionPaths, resolve
return nil return nil
} }
func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifest, sessionID string, inputs *[]manifest.InputRecord) error { type s3AudioMaterializationStats struct {
CacheHits int
CacheMisses int
Downloads int
}
func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifest, sessionID string, inputs *[]manifest.InputRecord) (s3AudioMaterializationStats, error) {
if env.ObjectStore == nil { if env.ObjectStore == nil {
return fmt.Errorf("s3 audio input requires object store backend") return s3AudioMaterializationStats{}, fmt.Errorf("s3 audio input requires object store backend")
} }
if env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil || env.Config.Pipeline.Storage.S3 == nil || env.Config.Session.Inputs.AudioS3 == nil { if env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil || env.Config.Pipeline.Storage.S3 == nil || env.Config.Session.Inputs.AudioS3 == nil {
return fmt.Errorf("s3 audio input requires pipeline.storage.s3 and session.inputs.audio_s3 configuration") return s3AudioMaterializationStats{}, fmt.Errorf("s3 audio input requires pipeline.storage.s3 and session.inputs.audio_s3 configuration")
} }
campaign := strings.TrimSpace(env.Config.Session.Campaign) campaign := strings.TrimSpace(env.Config.Session.Campaign)
if campaign == "" { if campaign == "" {
return fmt.Errorf("session campaign is required for s3 audio input") return s3AudioMaterializationStats{}, fmt.Errorf("session campaign is required for s3 audio input")
} }
runID := strings.TrimSpace(m.RunID) runID := strings.TrimSpace(m.RunID)
if runID == "" { if runID == "" {
return fmt.Errorf("run id is required for s3 audio input") return s3AudioMaterializationStats{}, fmt.Errorf("run id is required for s3 audio input")
} }
sessionPrefix := artifacts.S3SessionPrefix(env.Config.Pipeline.Storage.S3.RootPrefix, campaign, sessionID) sessionPrefix := artifacts.S3SessionPrefix(env.Config.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, env.Config.Session.Inputs.AudioS3.Prefix) audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, env.Config.Session.Inputs.AudioS3.Prefix)
objects, err := env.ObjectStore.List(ctx, audioPrefix) objects, err := env.ObjectStore.List(ctx, audioPrefix)
if err != nil { if err != nil {
return fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err) return s3AudioMaterializationStats{}, fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
} }
audioObjects := make([]storage.ObjectInfo, 0, len(objects)) audioObjects := make([]storage.ObjectInfo, 0, len(objects))
@@ -321,7 +399,7 @@ func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifes
return audioObjects[i].Key < audioObjects[j].Key return audioObjects[i].Key < audioObjects[j].Key
}) })
if len(audioObjects) == 0 { if len(audioObjects) == 0 {
return fmt.Errorf("no .flac files found under s3 audio prefix %q", audioPrefix) return s3AudioMaterializationStats{}, fmt.Errorf("no .flac files found under s3 audio prefix %q", audioPrefix)
} }
spoolAudioDir := strings.TrimSpace(m.LocalSpoolDir) spoolAudioDir := strings.TrimSpace(m.LocalSpoolDir)
@@ -331,45 +409,60 @@ func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifes
workAudioDir := filepath.Join(pathsWorkDirForManifest(env, m, sessionID), "audio") workAudioDir := filepath.Join(pathsWorkDirForManifest(env, m, sessionID), "audio")
if err := os.MkdirAll(spoolAudioDir, 0o755); err != nil { if err := os.MkdirAll(spoolAudioDir, 0o755); err != nil {
return fmt.Errorf("create spool audio directory %q: %w", spoolAudioDir, err) return s3AudioMaterializationStats{}, fmt.Errorf("create spool audio directory %q: %w", spoolAudioDir, err)
} }
if err := os.MkdirAll(workAudioDir, 0o755); err != nil { if err := os.MkdirAll(workAudioDir, 0o755); err != nil {
return fmt.Errorf("create work audio directory %q: %w", workAudioDir, err) return s3AudioMaterializationStats{}, fmt.Errorf("create work audio directory %q: %w", workAudioDir, err)
} }
seenBase := map[string]string{} seenBase := map[string]string{}
stats := s3AudioMaterializationStats{}
cacheEnabled := env.Config.Pipeline.Cache.S3Audio == nil || *env.Config.Pipeline.Cache.S3Audio
for _, obj := range audioObjects { for _, obj := range audioObjects {
base := path.Base(obj.Key) base := path.Base(obj.Key)
if prev, exists := seenBase[base]; exists && prev != obj.Key { if prev, exists := seenBase[base]; exists && prev != obj.Key {
return fmt.Errorf("duplicate s3 audio basename %q from %q and %q", base, prev, obj.Key) return s3AudioMaterializationStats{}, fmt.Errorf("duplicate s3 audio basename %q from %q and %q", base, prev, obj.Key)
} }
seenBase[base] = obj.Key seenBase[base] = obj.Key
spoolPath := filepath.Join(spoolAudioDir, base) spoolPath := filepath.Join(spoolAudioDir, base)
if err := env.ObjectStore.Download(ctx, obj.Key, spoolPath); err != nil {
return fmt.Errorf("download s3 audio object %q: %w", obj.Key, err)
}
workPath := filepath.Join(workAudioDir, base) workPath := filepath.Join(workAudioDir, base)
checksum, err := copyFileIfChanged(env.ArtifactStore, spoolPath, workPath) result, err := audio.MaterializeS3Audio(ctx, audio.S3MaterializeRequest{
Store: env.ObjectStore,
Object: obj,
Bucket: strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket),
CacheRoot: strings.TrimSpace(env.Config.Pipeline.Cache.Root),
CacheEnabled: cacheEnabled,
SpoolPath: spoolPath,
DestPath: workPath,
})
if err != nil { if err != nil {
return fmt.Errorf("materialize downloaded audio %q: %w", base, err) return s3AudioMaterializationStats{}, err
}
if result.CacheHit {
stats.CacheHits++
} else {
stats.CacheMisses++
}
if result.Downloaded {
stats.Downloads++
} }
*inputs = append(*inputs, manifest.InputRecord{ *inputs = append(*inputs, manifest.InputRecord{
Kind: "audio", Kind: "audio",
Path: workPath, Path: workPath,
Checksum: checksum, Checksum: result.Checksum,
Source: "s3", Source: "s3",
S3Bucket: strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket), S3Bucket: strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket),
S3Key: obj.Key, S3Key: obj.Key,
S3Size: obj.Size, S3Size: obj.Size,
S3ETag: obj.ETag, S3ETag: obj.ETag,
SpoolPath: spoolPath, SpoolPath: result.SpoolPath,
CachePath: result.CachePath,
}) })
} }
return nil return stats, nil
} }
func countAudioInputs(inputs []manifest.InputRecord) int { func countAudioInputs(inputs []manifest.InputRecord) int {

View File

@@ -50,6 +50,10 @@ func TestHydratePreviousSessionArtifactsDownloadsManifestAndRequiredArtifact(t *
if _, err := os.Stat(recapPath); err != nil { if _, err := os.Stat(recapPath); err != nil {
t.Fatalf("previous artifact missing: %v", err) t.Fatalf("previous artifact missing: %v", err)
} }
nestedRecapPath := filepath.Join(sessionPaths.PreviousArtifactsDir, "artifacts", "session_recap.md")
if _, err := os.Stat(nestedRecapPath); !os.IsNotExist(err) {
t.Fatalf("nested previous artifact should not be created, stat err = %v", err)
}
manifestInput := findInputByKind(result.Inputs, preparePreviousInputKindManifest) manifestInput := findInputByKind(result.Inputs, preparePreviousInputKindManifest)
if manifestInput == nil { if manifestInput == nil {

View File

@@ -35,6 +35,7 @@ func TestPrepareStageExplicitAudioFiles(t *testing.T) {
paths := sessionPathsForEnv(env, m.SessionID) paths := sessionPathsForEnv(env, m.SessionID)
for _, p := range []string{ for _, p := range []string{
filepath.Join(paths.InputsDir, "campaign.yml"),
filepath.Join(paths.InputsDir, "session.yml"), filepath.Join(paths.InputsDir, "session.yml"),
filepath.Join(paths.InputsDir, "pipeline.resolved.yml"), filepath.Join(paths.InputsDir, "pipeline.resolved.yml"),
filepath.Join(paths.InputsDir, "speakers.yml"), filepath.Join(paths.InputsDir, "speakers.yml"),
@@ -48,8 +49,8 @@ func TestPrepareStageExplicitAudioFiles(t *testing.T) {
} }
} }
if len(m.Inputs) != 7 { if len(m.Inputs) != 8 {
t.Fatalf("manifest inputs len = %d, want 7", len(m.Inputs)) t.Fatalf("manifest inputs len = %d, want 8", len(m.Inputs))
} }
for _, in := range m.Inputs { for _, in := range m.Inputs {
if in.Checksum == "" { if in.Checksum == "" {
@@ -151,6 +152,64 @@ func TestPrepareStageIdempotent(t *testing.T) {
} }
} }
func TestPrepareStageRecordsLocalSessionProvenance(t *testing.T) {
env, m := setupPrepareEnv(t)
root := filepath.Dir(env.Config.SessionPath)
writeFile(t, filepath.Join(root, "audio", "a.flac"), "a")
env.Config.Session.Inputs.AudioFiles = []string{"./audio/a.flac"}
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
sessionInput := findManifestInput(t, m.Inputs, "session_config")
if sessionInput.Source != "session_config" {
t.Fatalf("session source = %q, want session_config", sessionInput.Source)
}
if sessionInput.S3Bucket != "" || sessionInput.S3Key != "" || sessionInput.SpoolPath != "" {
t.Fatalf("local session input has unexpected remote provenance: %#v", sessionInput)
}
}
func TestPrepareStageRecordsRemoteSessionProvenance(t *testing.T) {
env, m := setupPrepareEnv(t)
remoteSessionPath := filepath.Join(t.TempDir(), "downloaded-session.yml")
writeFile(t, remoteSessionPath, "session_id: 2026-05-03\ninputs:\n audio_s3:\n prefix: audio/\n")
env.Config.SessionPath = remoteSessionPath
env.Config.Session.Inputs.AudioDir = ""
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
env.Config.SessionSource = config.SessionSource{
Source: "session_config.s3",
LocalPath: remoteSessionPath,
S3Bucket: "my-dnd-archive",
S3Key: "dnd/campaigns/sample-campaign/sessions/2026-05-03/session.yml",
S3Size: 58,
S3ETag: "session-etag",
SpoolPath: remoteSessionPath,
}
env.Config.Pipeline.Spool = config.SpoolConfig{Root: filepath.Join(t.TempDir(), "spool")}
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
m.RunID = "20260515T031522Z-a1b2c3d4"
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "sample-campaign", m.SessionID, m.RunID)
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "sample-campaign", m.SessionID, m.RunID)
fake := &storage.FakeBackend{}
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac", Data: []byte("alice")})
env.ObjectStore = fake
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
sessionInput := findManifestInput(t, m.Inputs, "session_config")
if sessionInput.Source != "session_config.s3" {
t.Fatalf("session source = %q, want session_config.s3", sessionInput.Source)
}
if sessionInput.S3Bucket != "my-dnd-archive" || sessionInput.S3Key != "dnd/campaigns/sample-campaign/sessions/2026-05-03/session.yml" {
t.Fatalf("remote session input missing bucket/key: %#v", sessionInput)
}
if sessionInput.S3Size != 58 || sessionInput.S3ETag != "session-etag" || sessionInput.SpoolPath != remoteSessionPath {
t.Fatalf("remote session input missing metadata: %#v", sessionInput)
}
}
func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) { func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) {
env, m := setupPrepareEnv(t) env, m := setupPrepareEnv(t)
env.Config.Session.Campaign = "forsaken" env.Config.Session.Campaign = "forsaken"
@@ -186,6 +245,15 @@ func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) {
t.Fatalf("expected file %q: %v", p, err) t.Fatalf("expected file %q: %v", p, err)
} }
} }
if got := result.Metadata["audio_cache_hits"]; got != 0 {
t.Fatalf("audio_cache_hits = %#v, want 0", got)
}
if got := result.Metadata["audio_cache_misses"]; got != 2 {
t.Fatalf("audio_cache_misses = %#v, want 2", got)
}
if got := result.Metadata["audio_s3_downloads"]; got != 2 {
t.Fatalf("audio_s3_downloads = %#v, want 2", got)
}
audioInputs := 0 audioInputs := 0
for _, in := range m.Inputs { for _, in := range m.Inputs {
@@ -199,15 +267,75 @@ func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) {
if in.S3Bucket != "my-dnd-archive" { if in.S3Bucket != "my-dnd-archive" {
t.Fatalf("audio input bucket = %q", in.S3Bucket) t.Fatalf("audio input bucket = %q", in.S3Bucket)
} }
if in.S3Key == "" || in.SpoolPath == "" || in.Checksum == "" { if in.S3Key == "" || in.SpoolPath == "" || in.CachePath == "" || in.Checksum == "" {
t.Fatalf("audio input missing provenance: %#v", in) t.Fatalf("audio input missing provenance: %#v", in)
} }
if _, err := os.Stat(in.CachePath); err != nil {
t.Fatalf("expected cache path %q: %v", in.CachePath, err)
}
} }
if audioInputs != 2 { if audioInputs != 2 {
t.Fatalf("audio input count = %d, want 2", audioInputs) t.Fatalf("audio input count = %d, want 2", audioInputs)
} }
} }
func TestPrepareStageS3AudioUsesCacheOnRerun(t *testing.T) {
env, m := setupPrepareEnv(t)
env.Config.Session.Campaign = "forsaken"
env.Config.Session.Inputs.AudioDir = ""
env.Config.Session.Inputs.AudioFiles = nil
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
env.Config.Pipeline.Spool = config.SpoolConfig{Root: filepath.Join(t.TempDir(), "spool")}
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
m.RunID = "20260515T031522Z-a1b2c3d4"
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "forsaken", m.SessionID, m.RunID)
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "forsaken", m.SessionID, m.RunID)
fake := &storage.FakeBackend{}
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/alice.flac", Data: []byte("alice")})
env.ObjectStore = fake
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("first prepare.Run() error = %v", err)
}
if len(fake.Downloads) != 1 {
t.Fatalf("downloads = %d, want 1", len(fake.Downloads))
}
if err := os.RemoveAll(filepath.Join(m.LocalWorkDir, "audio")); err != nil {
t.Fatalf("remove work audio: %v", err)
}
if err := os.RemoveAll(m.LocalSpoolDir); err != nil {
t.Fatalf("remove spool audio: %v", err)
}
fake.DownloadErr = os.ErrPermission
m2 := manifest.New("2026-05-03", time.Now().UTC())
m2.RunID = "20260515T041522Z-a1b2c3d4"
m2.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "forsaken", m2.SessionID, m2.RunID)
m2.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "forsaken", m2.SessionID, m2.RunID)
result, err := (prepareStage{}).Run(context.Background(), env, m2)
if err != nil {
t.Fatalf("cached prepare.Run() error = %v", err)
}
if len(fake.Downloads) != 1 {
t.Fatalf("downloads = %d, want cached rerun to avoid new download", len(fake.Downloads))
}
if got := result.Metadata["audio_cache_hits"]; got != 1 {
t.Fatalf("audio_cache_hits = %#v, want 1", got)
}
if got := result.Metadata["audio_s3_downloads"]; got != 0 {
t.Fatalf("audio_s3_downloads = %#v, want 0", got)
}
audioInput := findManifestInput(t, m2.Inputs, "audio")
if audioInput.CachePath == "" {
t.Fatalf("audio input missing cache path: %#v", audioInput)
}
if audioInput.SpoolPath != "" {
t.Fatalf("audio input spool path = %q, want empty on cache hit", audioInput.SpoolPath)
}
mustReadFileEquals(t, filepath.Join(m2.LocalWorkDir, "audio", "alice.flac"), "alice")
}
func TestPrepareStageS3AudioFailures(t *testing.T) { func TestPrepareStageS3AudioFailures(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
@@ -382,6 +510,10 @@ func TestPrepareStageHydratesRequiredPreviousArtifactAndRecordsInputs(t *testing
if _, err := os.Stat(recapPath); err != nil { if _, err := os.Stat(recapPath); err != nil {
t.Fatalf("expected previous artifact: %v", err) t.Fatalf("expected previous artifact: %v", err)
} }
nestedRecapPath := filepath.Join(paths.PreviousArtifactsDir, "artifacts", "session_recap.md")
if _, err := os.Stat(nestedRecapPath); !os.IsNotExist(err) {
t.Fatalf("nested previous artifact should not be created, stat err = %v", err)
}
var hasPreviousManifest, hasPreviousArtifact bool var hasPreviousManifest, hasPreviousArtifact bool
for _, in := range m.Inputs { for _, in := range m.Inputs {
@@ -473,25 +605,51 @@ func setupPrepareEnv(t *testing.T) (*Env, *manifest.Manifest) {
sessionPath := filepath.Join(cfgDir, "session.yml") sessionPath := filepath.Join(cfgDir, "session.yml")
pipelinePath := filepath.Join(cfgDir, "pipeline.yml") pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
campaignPath := filepath.Join(cfgDir, "campaign.yml")
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n") writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
writeFile(t, campaignPath, `campaign: sample-campaign
inputs:
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`)
writeFile(t, sessionPath, "session_id: 2026-05-03\n") writeFile(t, sessionPath, "session_id: 2026-05-03\n")
writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n") writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n") writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
writeFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n") writeFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
cfg := &config.Config{ cfg := &config.Config{
Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: workspace}}, Pipeline: &config.PipelineConfig{
Workspace: config.WorkspaceConfig{Root: workspace},
Cache: config.CacheConfig{Root: filepath.Join(t.TempDir(), "cache"), S3Audio: boolPtr(true)},
},
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
SessionPath: sessionPath, SessionPath: sessionPath,
CampaignPath: campaignPath,
PipelinePath: pipelinePath, PipelinePath: pipelinePath,
Session: &config.SessionConfig{ Session: &config.SessionConfig{
SessionID: "2026-05-03", SessionID: "2026-05-03",
Campaign: "sample-campaign", Campaign: "sample-campaign",
Inputs: config.SessionInputsConfig{ Inputs: config.SessionInputsConfig{
AudioDir: "./audio", AudioDir: "./audio",
SpeakersFile: "./speakers.yml", },
AutocorrectFile: "./autocorrect.yml", },
GlossaryFile: "./glossary.yml", StableInputs: config.ResolvedStableInputs{
SpeakersFile: config.ResolvedInputFile{
Path: "./speakers.yml",
ConfigPath: campaignPath,
Source: "campaign_config",
},
AutocorrectFile: config.ResolvedInputFile{
Path: "./autocorrect.yml",
ConfigPath: campaignPath,
Source: "campaign_config",
},
GlossaryFile: config.ResolvedInputFile{
Path: "./glossary.yml",
ConfigPath: campaignPath,
Source: "campaign_config",
}, },
}, },
} }
@@ -519,3 +677,25 @@ func snapshotInputs(inputs []manifest.InputRecord) map[string]string {
} }
return out return out
} }
func findManifestInput(t *testing.T, inputs []manifest.InputRecord, kind string) manifest.InputRecord {
t.Helper()
for _, input := range inputs {
if input.Kind == kind {
return input
}
}
t.Fatalf("manifest input kind %q not found in %#v", kind, inputs)
return manifest.InputRecord{}
}
func mustReadFileEquals(t *testing.T, path, want string) {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %q: %v", path, err)
}
if string(data) != want {
t.Fatalf("%q = %q, want %q", path, string(data), want)
}
}

View File

@@ -233,8 +233,10 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
sessionPath := filepath.Join(cfgDir, "session.yml") sessionPath := filepath.Join(cfgDir, "session.yml")
pipelinePath := filepath.Join(cfgDir, "pipeline.yml") pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
campaignPath := filepath.Join(cfgDir, "campaign.yml")
writeFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n") writeFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n")
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n") writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
writeFile(t, campaignPath, "campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n")
writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n") writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n") writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
writeFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n") writeFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
@@ -243,7 +245,9 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
concurrency := 2 concurrency := 2
cfg := &config.Config{ cfg := &config.Config{
PipelinePath: pipelinePath, PipelinePath: pipelinePath,
CampaignPath: campaignPath,
SessionPath: sessionPath, SessionPath: sessionPath,
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
Pipeline: &config.PipelineConfig{ Pipeline: &config.PipelineConfig{
Workspace: config.WorkspaceConfig{Root: workspace}, Workspace: config.WorkspaceConfig{Root: workspace},
WhisperX: config.WhisperXConfig{ WhisperX: config.WhisperXConfig{
@@ -265,6 +269,11 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
GlossaryFile: "./glossary.yml", GlossaryFile: "./glossary.yml",
}, },
}, },
StableInputs: config.ResolvedStableInputs{
SpeakersFile: config.ResolvedInputFile{Path: "./speakers.yml", ConfigPath: campaignPath, Source: "campaign_config"},
AutocorrectFile: config.ResolvedInputFile{Path: "./autocorrect.yml", ConfigPath: campaignPath, Source: "campaign_config"},
GlossaryFile: config.ResolvedInputFile{Path: "./glossary.yml", ConfigPath: campaignPath, Source: "campaign_config"},
},
} }
env := &Env{ env := &Env{